letta-nightly 0.7.6.dev20250430104233__py3-none-any.whl → 0.7.7.dev20250430205840__py3-none-any.whl

letta/server/server.py

@@ -44,7 +44,7 @@ from letta.schemas.embedding_config import EmbeddingConfig
 # openai schemas
 from letta.schemas.enums import JobStatus, MessageStreamStatus
 from letta.schemas.environment_variables import SandboxEnvironmentVariableCreate
-from letta.schemas.group import GroupCreate, SleeptimeManager
+from letta.schemas.group import GroupCreate, ManagerType, SleeptimeManager, VoiceSleeptimeManager
 from letta.schemas.job import Job, JobUpdate
 from letta.schemas.letta_message import LegacyLettaMessage, LettaMessage, ToolReturnMessage
 from letta.schemas.letta_message_content import TextContent
@@ -363,29 +363,29 @@ class SyncServer(Server):
 
         # For MCP
         """Initialize the MCP clients (there may be multiple)"""
-        # mcp_server_configs = self.get_mcp_servers()
+        mcp_server_configs = self.get_mcp_servers()
         self.mcp_clients: Dict[str, BaseMCPClient] = {}
-        #
-        # for server_name, server_config in mcp_server_configs.items():
-        #     if server_config.type == MCPServerType.SSE:
-        #         self.mcp_clients[server_name] = SSEMCPClient(server_config)
-        #     elif server_config.type == MCPServerType.STDIO:
-        #         self.mcp_clients[server_name] = StdioMCPClient(server_config)
-        #     else:
-        #         raise ValueError(f"Invalid MCP server config: {server_config}")
-        #
-        #     try:
-        #         self.mcp_clients[server_name].connect_to_server()
-        #     except Exception as e:
-        #         logger.error(e)
-        #         self.mcp_clients.pop(server_name)
-        #
-        # # Print out the tools that are connected
-        # for server_name, client in self.mcp_clients.items():
-        #     logger.info(f"Attempting to fetch tools from MCP server: {server_name}")
-        #     mcp_tools = client.list_tools()
-        #     logger.info(f"MCP tools connected: {', '.join([t.name for t in mcp_tools])}")
-        #     logger.debug(f"MCP tools: {', '.join([str(t) for t in mcp_tools])}")
+
+        for server_name, server_config in mcp_server_configs.items():
+            if server_config.type == MCPServerType.SSE:
+                self.mcp_clients[server_name] = SSEMCPClient(server_config)
+            elif server_config.type == MCPServerType.STDIO:
+                self.mcp_clients[server_name] = StdioMCPClient(server_config)
+            else:
+                raise ValueError(f"Invalid MCP server config: {server_config}")
+
+            try:
+                self.mcp_clients[server_name].connect_to_server()
+            except Exception as e:
+                logger.error(e)
+                self.mcp_clients.pop(server_name)
+
+        # Print out the tools that are connected
+        for server_name, client in self.mcp_clients.items():
+            logger.info(f"Attempting to fetch tools from MCP server: {server_name}")
+            mcp_tools = client.list_tools()
+            logger.info(f"MCP tools connected: {', '.join([t.name for t in mcp_tools])}")
+            logger.debug(f"MCP tools: {', '.join([str(t) for t in mcp_tools])}")
 
         # TODO: Remove these in memory caches
         self._llm_config_cache = {}
@@ -397,7 +397,9 @@ class SyncServer(Server):
     def load_agent(self, agent_id: str, actor: User, interface: Union[AgentInterface, None] = None) -> Agent:
         """Updated method to load agents from persisted storage"""
         agent_state = self.agent_manager.get_agent_by_id(agent_id=agent_id, actor=actor)
-        if agent_state.multi_agent_group:
+        # TODO: Think about how to integrate voice sleeptime into sleeptime
+        # TODO: Voice sleeptime agents turn into normal agents when being messaged
+        if agent_state.multi_agent_group and agent_state.multi_agent_group.manager_type != ManagerType.voice_sleeptime:
             return load_multi_agent(
                 group=agent_state.multi_agent_group, agent_state=agent_state, actor=actor, interface=interface, mcp_clients=self.mcp_clients
             )
@@ -769,7 +771,10 @@ class SyncServer(Server):
         log_event(name="end create_agent db")
 
         if request.enable_sleeptime:
-            main_agent = self.create_sleeptime_agent(main_agent=main_agent, actor=actor)
+            if request.agent_type == AgentType.voice_convo_agent:
+                main_agent = self.create_voice_sleeptime_agent(main_agent=main_agent, actor=actor)
+            else:
+                main_agent = self.create_sleeptime_agent(main_agent=main_agent, actor=actor)
 
         return main_agent
 
@@ -788,7 +793,10 @@ class SyncServer(Server):
         if request.enable_sleeptime:
             agent = self.agent_manager.get_agent_by_id(agent_id=agent_id, actor=actor)
             if agent.multi_agent_group is None:
-                self.create_sleeptime_agent(main_agent=agent, actor=actor)
+                if agent.agent_type == AgentType.voice_convo_agent:
+                    self.create_voice_sleeptime_agent(main_agent=agent, actor=actor)
+                else:
+                    self.create_sleeptime_agent(main_agent=agent, actor=actor)
 
         return self.agent_manager.update_agent(
             agent_id=agent_id,
@@ -828,6 +836,38 @@ class SyncServer(Server):
         )
         return self.agent_manager.get_agent_by_id(agent_id=main_agent.id, actor=actor)
 
+    def create_voice_sleeptime_agent(self, main_agent: AgentState, actor: User) -> AgentState:
+        # TODO: Inject system
+        request = CreateAgent(
+            name=main_agent.name + "-sleeptime",
+            agent_type=AgentType.voice_sleeptime_agent,
+            block_ids=[block.id for block in main_agent.memory.blocks],
+            memory_blocks=[
+                CreateBlock(
+                    label="memory_persona",
+                    value=get_persona_text("voice_memory_persona"),
+                ),
+            ],
+            llm_config=main_agent.llm_config,
+            embedding_config=main_agent.embedding_config,
+            project_id=main_agent.project_id,
+        )
+        voice_sleeptime_agent = self.agent_manager.create_agent(
+            agent_create=request,
+            actor=actor,
+        )
+        self.group_manager.create_group(
+            group=GroupCreate(
+                description="Low latency voice chat with async memory management.",
+                agent_ids=[voice_sleeptime_agent.id],
+                manager_config=VoiceSleeptimeManager(
+                    manager_agent_id=main_agent.id,
+                ),
+            ),
+            actor=actor,
+        )
+        return self.agent_manager.get_agent_by_id(agent_id=main_agent.id, actor=actor)
+
     # convert name->id
 
     # TODO: These can be moved to agent_manager

letta/services/agent_manager.py

@@ -11,6 +11,8 @@ from letta.constants import (
     BASE_SLEEPTIME_CHAT_TOOLS,
     BASE_SLEEPTIME_TOOLS,
     BASE_TOOLS,
+    BASE_VOICE_SLEEPTIME_CHAT_TOOLS,
+    BASE_VOICE_SLEEPTIME_TOOLS,
     DATA_SOURCE_ATTACH_ALERT,
     MAX_EMBEDDING_DIM,
     MULTI_AGENT_TOOLS,
@@ -179,7 +181,11 @@ class AgentManager:
         # tools
         tool_names = set(agent_create.tools or [])
         if agent_create.include_base_tools:
-            if agent_create.agent_type == AgentType.sleeptime_agent:
+            if agent_create.agent_type == AgentType.voice_sleeptime_agent:
+                tool_names |= set(BASE_VOICE_SLEEPTIME_TOOLS)
+            elif agent_create.agent_type == AgentType.voice_convo_agent:
+                tool_names |= set(BASE_VOICE_SLEEPTIME_CHAT_TOOLS)
+            elif agent_create.agent_type == AgentType.sleeptime_agent:
                 tool_names |= set(BASE_SLEEPTIME_TOOLS)
             elif agent_create.enable_sleeptime:
                 tool_names |= set(BASE_SLEEPTIME_CHAT_TOOLS)
@@ -603,12 +609,13 @@ class AgentManager:
             # Delete sleeptime agent and group (TODO this is flimsy pls fix)
             if agent.multi_agent_group:
                 participant_agent_ids = agent.multi_agent_group.agent_ids
-                if agent.multi_agent_group.manager_type == ManagerType.sleeptime and len(participant_agent_ids) == 1:
-                    try:
-                        sleeptime_agent = AgentModel.read(db_session=session, identifier=participant_agent_ids[0], actor=actor)
-                        agents_to_delete.append(sleeptime_agent)
-                    except NoResultFound:
-                        pass  # agent already deleted
+                if agent.multi_agent_group.manager_type in {ManagerType.sleeptime, ManagerType.voice_sleeptime} and participant_agent_ids:
+                    for participant_agent_id in participant_agent_ids:
+                        try:
+                            sleeptime_agent = AgentModel.read(db_session=session, identifier=participant_agent_id, actor=actor)
+                            agents_to_delete.append(sleeptime_agent)
+                        except NoResultFound:
+                            pass  # agent already deleted
                     sleeptime_agent_group = GroupModel.read(db_session=session, identifier=agent.multi_agent_group.id, actor=actor)
                     sleeptime_group_to_delete = sleeptime_agent_group
 

letta/services/group_manager.py

@@ -77,6 +77,9 @@ class GroupManager:
                     new_group.sleeptime_agent_frequency = group.manager_config.sleeptime_agent_frequency
                     if new_group.sleeptime_agent_frequency:
                         new_group.turns_counter = -1
+                case ManagerType.voice_sleeptime:
+                    new_group.manager_type = ManagerType.voice_sleeptime
+                    new_group.manager_agent_id = group.manager_config.manager_agent_id
                 case _:
                     raise ValueError(f"Unsupported manager type: {group.manager_config.manager_type}")
 

letta/services/helpers/agent_manager_helper.py

@@ -20,7 +20,7 @@ from letta.schemas.message import Message, MessageCreate
 from letta.schemas.passage import Passage as PydanticPassage
 from letta.schemas.tool_rule import ToolRule
 from letta.schemas.user import User
-from letta.system import get_initial_boot_messages, get_login_event
+from letta.system import get_initial_boot_messages, get_login_event, package_function_response
 from letta.tracing import trace_method
 
 
@@ -94,7 +94,11 @@ def _process_tags(agent: AgentModel, tags: List[str], replace=True):
 def derive_system_message(agent_type: AgentType, enable_sleeptime: Optional[bool] = None, system: Optional[str] = None):
     if system is None:
         # TODO: don't hardcode
-        if agent_type == AgentType.memgpt_agent and not enable_sleeptime:
+        if agent_type == AgentType.voice_convo_agent:
+            system = gpt_system.get_system_text("voice_chat")
+        elif agent_type == AgentType.voice_sleeptime_agent:
+            system = gpt_system.get_system_text("voice_sleeptime")
+        elif agent_type == AgentType.memgpt_agent and not enable_sleeptime:
             system = gpt_system.get_system_text("memgpt_chat")
         elif agent_type == AgentType.memgpt_agent and enable_sleeptime:
             system = gpt_system.get_system_text("memgpt_sleeptime_chat")
@@ -278,23 +282,76 @@ def package_initial_message_sequence(
             packed_message = system.package_user_message(
                 user_message=message_create.content,
             )
+            init_messages.append(
+                Message(
+                    role=message_create.role,
+                    content=[TextContent(text=packed_message)],
+                    name=message_create.name,
+                    organization_id=actor.organization_id,
+                    agent_id=agent_id,
+                    model=model,
+                )
+            )
         elif message_create.role == MessageRole.system:
             packed_message = system.package_system_message(
                 system_message=message_create.content,
             )
+            init_messages.append(
+                Message(
+                    role=message_create.role,
+                    content=[TextContent(text=packed_message)],
+                    name=message_create.name,
+                    organization_id=actor.organization_id,
+                    agent_id=agent_id,
+                    model=model,
+                )
+            )
+        elif message_create.role == MessageRole.assistant:
+            # append tool call to send_message
+            import json
+            import uuid
+
+            from openai.types.chat.chat_completion_message_tool_call import ChatCompletionMessageToolCall as OpenAIToolCall
+            from openai.types.chat.chat_completion_message_tool_call import Function as OpenAIFunction
+
+            from letta.constants import DEFAULT_MESSAGE_TOOL
+
+            tool_call_id = str(uuid.uuid4())
+            init_messages.append(
+                Message(
+                    role=MessageRole.assistant,
+                    content=None,
+                    name=message_create.name,
+                    organization_id=actor.organization_id,
+                    agent_id=agent_id,
+                    model=model,
+                    tool_calls=[
+                        OpenAIToolCall(
+                            id=tool_call_id,
+                            type="function",
+                            function=OpenAIFunction(name=DEFAULT_MESSAGE_TOOL, arguments=json.dumps({"message": message_create.content})),
+                        )
+                    ],
+                )
+            )
+
+            # add tool return
+            function_response = package_function_response(True, "None")
+            init_messages.append(
+                Message(
+                    role=MessageRole.tool,
+                    content=[TextContent(text=function_response)],
+                    name=message_create.name,
+                    organization_id=actor.organization_id,
+                    agent_id=agent_id,
+                    model=model,
+                    tool_call_id=tool_call_id,
+                )
+            )
         else:
+            # TODO: add tool call and tool return
             raise ValueError(f"Invalid message role: {message_create.role}")
 
-        init_messages.append(
-            Message(
-                role=message_create.role,
-                content=[TextContent(text=packed_message)],
-                name=message_create.name,
-                organization_id=actor.organization_id,
-                agent_id=agent_id,
-                model=model,
-            )
-        )
     return init_messages
 
 

letta/services/message_manager.py

@@ -122,7 +122,7 @@ class MessageManager:
         message = self.update_message_by_id(message_id=message_id, message_update=update_message, actor=actor)
 
         # convert back to LettaMessage
-        for letta_msg in message.to_letta_message(use_assistant_message=True):
+        for letta_msg in message.to_letta_messages(use_assistant_message=True):
             if letta_msg.message_type == letta_message_update.message_type:
                 return letta_msg
 
@@ -160,7 +160,7 @@ class MessageManager:
         message = self.update_message_by_id(message_id=message_id, message_update=update_message, actor=actor)
 
         # convert back to LettaMessage
-        for letta_msg in message.to_letta_message(use_assistant_message=True):
+        for letta_msg in message.to_letta_messages(use_assistant_message=True):
             if letta_msg.message_type == letta_message_update.message_type:
                 return letta_msg
 

letta/services/passage_manager.py

@@ -220,15 +220,24 @@ class PassageManager:
         with self.session_maker() as session:
             return AgentPassage.size(db_session=session, actor=actor, agent_id=agent_id)
 
-    def estimate_embeddings_size_GB(
+    def estimate_embeddings_size(
         self,
         actor: PydanticUser,
         agent_id: Optional[str] = None,
+        storage_unit: str = "GB",
     ) -> float:
         """
-        Estimate the size of the embeddings in GB.
+        Estimate the size of the embeddings. Defaults to GB.
         """
+        BYTES_PER_STORAGE_UNIT = {
+            "B": 1,
+            "KB": 1024,
+            "MB": 1024**2,
+            "GB": 1024**3,
+            "TB": 1024**4,
+        }
+        if storage_unit not in BYTES_PER_STORAGE_UNIT:
+            raise ValueError(f"Invalid storage unit: {storage_unit}. Must be one of {list(BYTES_PER_STORAGE_UNIT.keys())}.")
         BYTES_PER_EMBEDDING_DIM = 4
-        BYTES_PER_GB = 1024 * 1024 * 1024
-        GB_PER_EMBEDDING = BYTES_PER_EMBEDDING_DIM / BYTES_PER_GB * MAX_EMBEDDING_DIM
+        GB_PER_EMBEDDING = BYTES_PER_EMBEDDING_DIM / BYTES_PER_STORAGE_UNIT[storage_unit] * MAX_EMBEDDING_DIM
         return self.size(actor=actor, agent_id=agent_id) * GB_PER_EMBEDDING

letta/services/summarizer/summarizer.py

@@ -3,7 +3,7 @@ import json
 import traceback
 from typing import List, Tuple
 
-from letta.agents.ephemeral_memory_agent import EphemeralMemoryAgent
+from letta.agents.voice_sleeptime_agent import VoiceSleeptimeAgent
 from letta.log import get_logger
 from letta.schemas.enums import MessageRole
 from letta.schemas.letta_message_content import TextContent
@@ -21,7 +21,7 @@ class Summarizer:
     """
 
     def __init__(
-        self, mode: SummarizationMode, summarizer_agent: EphemeralMemoryAgent, message_buffer_limit: int = 10, message_buffer_min: int = 3
+        self, mode: SummarizationMode, summarizer_agent: VoiceSleeptimeAgent, message_buffer_limit: int = 10, message_buffer_min: int = 3
     ):
         self.mode = mode
 
@@ -109,17 +109,14 @@ class Summarizer:
 
         evicted_messages_str = "\n".join(formatted_evicted_messages)
         in_context_messages_str = "\n".join(formatted_in_context_messages)
-        summary_request_text = f"""You are a specialized memory recall agent assisting another AI agent by asynchronously reorganizing its memory storage. The LLM agent you are helping maintains a limited context window that retains only the most recent {self.message_buffer_min} messages from its conversations. The provided conversation history includes messages that are about to be evicted from its context window, as well as some additional recent messages for extra clarity and context.
+        summary_request_text = f"""You’re a memory-recall helper for an AI that can only keep the last {self.message_buffer_min} messages. Scan the conversation history, focusing on messages about to drop out of that window, and write crisp notes that capture any important facts or insights about the human so they aren’t lost.
 
-Your task is to carefully review the provided conversation history and proactively generate detailed, relevant memories about the human participant, specifically targeting information contained in messages that are about to be evicted from the context window. Your notes will help preserve critical insights, events, or facts that would otherwise be forgotten.
-
-(Older) Evicted Messages:
+(Older) Evicted Messages:\n
 {evicted_messages_str}
 
-(Newer) In-Context Messages:
+(Newer) In-Context Messages:\n
 {in_context_messages_str}
 """
-
         # Fire-and-forget the summarization task
         self.fire_and_forget(
             self.summarizer_agent.step([MessageCreate(role=MessageRole.user, content=[TextContent(text=summary_request_text)])])

letta/services/tool_manager.py

@@ -7,6 +7,8 @@ from letta.constants import (
     BASE_MEMORY_TOOLS,
     BASE_SLEEPTIME_TOOLS,
     BASE_TOOLS,
+    BASE_VOICE_SLEEPTIME_CHAT_TOOLS,
+    BASE_VOICE_SLEEPTIME_TOOLS,
     LETTA_TOOL_SET,
     MCP_TOOL_TAG_NAME_PREFIX,
     MULTI_AGENT_TOOLS,
@@ -39,18 +41,24 @@ class ToolManager:
     @enforce_types
     def create_or_update_tool(self, pydantic_tool: PydanticTool, actor: PydanticUser) -> PydanticTool:
         """Create a new tool based on the ToolCreate schema."""
-        tool = self.get_tool_by_name(tool_name=pydantic_tool.name, actor=actor)
-        if tool:
+        tool_id = self.get_tool_id_by_name(tool_name=pydantic_tool.name, actor=actor)
+        if tool_id:
             # Put to dict and remove fields that should not be reset
             update_data = pydantic_tool.model_dump(exclude_unset=True, exclude_none=True)
 
             # If there's anything to update
             if update_data:
-                tool = self.update_tool_by_id(tool.id, ToolUpdate(**update_data), actor)
+                # In case we want to update the tool type
+                # Useful if we are shuffling around base tools
+                updated_tool_type = None
+                if "tool_type" in update_data:
+                    updated_tool_type = update_data.get("tool_type")
+                tool = self.update_tool_by_id(tool_id, ToolUpdate(**update_data), actor, updated_tool_type=updated_tool_type)
             else:
                 printd(
                     f"`create_or_update_tool` was called with user_id={actor.id}, organization_id={actor.organization_id}, name={pydantic_tool.name}, but found existing tool with nothing to update."
                 )
+                tool = self.get_tool_by_id(tool_id, actor=actor)
         else:
             tool = self.create_tool(pydantic_tool, actor=actor)
 
@@ -112,6 +120,16 @@ class ToolManager:
         except NoResultFound:
             return None
 
+    @enforce_types
+    def get_tool_id_by_name(self, tool_name: str, actor: PydanticUser) -> Optional[str]:
+        """Retrieve a tool by its name and a user. We derive the organization from the user, and retrieve that tool."""
+        try:
+            with self.session_maker() as session:
+                tool = ToolModel.read(db_session=session, name=tool_name, actor=actor)
+                return tool.id
+        except NoResultFound:
+            return None
+
     @enforce_types
     def list_tools(self, actor: PydanticUser, after: Optional[str] = None, limit: Optional[int] = 50) -> List[PydanticTool]:
         """List all tools with optional pagination."""
@@ -154,7 +172,9 @@ class ToolManager:
             return ToolModel.size(db_session=session, actor=actor, name=LETTA_TOOL_SET)
 
     @enforce_types
-    def update_tool_by_id(self, tool_id: str, tool_update: ToolUpdate, actor: PydanticUser) -> PydanticTool:
+    def update_tool_by_id(
+        self, tool_id: str, tool_update: ToolUpdate, actor: PydanticUser, updated_tool_type: Optional[ToolType] = None
+    ) -> PydanticTool:
         """Update a tool by its ID with the given ToolUpdate object."""
         with self.session_maker() as session:
             # Fetch the tool by ID
@@ -173,6 +193,9 @@ class ToolManager:
                 tool.json_schema = new_schema
                 tool.name = new_schema["name"]
 
+            if updated_tool_type:
+                tool.tool_type = updated_tool_type
+
             # Save the updated tool to the database
             return tool.update(db_session=session, actor=actor).to_pydantic()
 
@@ -190,7 +213,7 @@ class ToolManager:
     def upsert_base_tools(self, actor: PydanticUser) -> List[PydanticTool]:
         """Add default tools in base.py and multi_agent.py"""
         functions_to_schema = {}
-        module_names = ["base", "multi_agent"]
+        module_names = ["base", "multi_agent", "voice"]
 
         for module_name in module_names:
             full_module_name = f"letta.functions.function_sets.{module_name}"
@@ -223,9 +246,12 @@ class ToolManager:
                 elif name in BASE_SLEEPTIME_TOOLS:
                     tool_type = ToolType.LETTA_SLEEPTIME_CORE
                     tags = [tool_type.value]
+                elif name in BASE_VOICE_SLEEPTIME_TOOLS or name in BASE_VOICE_SLEEPTIME_CHAT_TOOLS:
+                    tool_type = ToolType.LETTA_VOICE_SLEEPTIME_CORE
+                    tags = [tool_type.value]
                 else:
                     raise ValueError(
-                        f"Tool name {name} is not in the list of base tool names: {BASE_TOOLS + BASE_MEMORY_TOOLS + MULTI_AGENT_TOOLS + BASE_SLEEPTIME_TOOLS}"
+                        f"Tool name {name} is not in the list of base tool names: {BASE_TOOLS + BASE_MEMORY_TOOLS + MULTI_AGENT_TOOLS + BASE_SLEEPTIME_TOOLS + BASE_VOICE_SLEEPTIME_TOOLS + BASE_VOICE_SLEEPTIME_CHAT_TOOLS}"
                     )
 
                 # create to tool
@@ -243,5 +269,4 @@ class ToolManager:
                 )
 
         # TODO: Delete any base tools that are stale
-
         return tools

{letta_nightly-0.7.6.dev20250430104233.dist-info → letta_nightly-0.7.7.dev20250430205840.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: letta-nightly
-Version: 0.7.6.dev20250430104233
+Version: 0.7.7.dev20250430205840
 Summary: Create LLM agents with long-term memory and custom tools
 License: Apache License
 Author: Letta Team