letta-nightly 0.7.20.dev20250521104258__py3-none-any.whl → 0.7.21.dev20250522104246__py3-none-any.whl

letta/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.7.20"
+__version__ = "0.7.21"
 
 # import clients
 from letta.client.client import LocalClient, RESTClient, create_client

letta/agent.py

@@ -1,4 +1,6 @@
+import asyncio
 import json
+import os
 import time
 import traceback
 import warnings
@@ -7,6 +9,7 @@ from typing import Dict, List, Optional, Tuple, Union
 
 from openai.types.beta.function_tool import FunctionTool as OpenAITool
 
+from letta.agents.helpers import generate_step_id
 from letta.constants import (
     CLI_WARNING_PREFIX,
     COMPOSIO_ENTITY_ENV_VAR_KEY,
@@ -16,6 +19,7 @@ from letta.constants import (
     LETTA_CORE_TOOL_MODULE_NAME,
     LETTA_MULTI_AGENT_TOOL_MODULE_NAME,
     LLM_MAX_TOKENS,
+    READ_ONLY_BLOCK_EDIT_ERROR,
     REQ_HEARTBEAT_MESSAGE,
     SEND_MESSAGE_TOOL_NAME,
 )
@@ -41,7 +45,7 @@ from letta.orm.enums import ToolType
 from letta.schemas.agent import AgentState, AgentStepResponse, UpdateAgent, get_prompt_template_for_agent_type
 from letta.schemas.block import BlockUpdate
 from letta.schemas.embedding_config import EmbeddingConfig
-from letta.schemas.enums import MessageRole
+from letta.schemas.enums import MessageRole, ProviderType
 from letta.schemas.letta_message_content import TextContent
 from letta.schemas.memory import ContextWindowOverview, Memory
 from letta.schemas.message import Message, MessageCreate, ToolReturn
@@ -61,9 +65,10 @@ from letta.services.message_manager import MessageManager
 from letta.services.passage_manager import PassageManager
 from letta.services.provider_manager import ProviderManager
 from letta.services.step_manager import StepManager
+from letta.services.telemetry_manager import NoopTelemetryManager, TelemetryManager
 from letta.services.tool_executor.tool_execution_sandbox import ToolExecutionSandbox
 from letta.services.tool_manager import ToolManager
-from letta.settings import summarizer_settings
+from letta.settings import settings, summarizer_settings
 from letta.streaming_interface import StreamingRefreshCLIInterface
 from letta.system import get_heartbeat, get_token_limit_warning, package_function_response, package_summarize_message, package_user_message
 from letta.tracing import log_event, trace_method
@@ -141,6 +146,7 @@ class Agent(BaseAgent):
         self.agent_manager = AgentManager()
         self.job_manager = JobManager()
         self.step_manager = StepManager()
+        self.telemetry_manager = TelemetryManager() if settings.llm_api_logging else NoopTelemetryManager()
 
         # State needed for heartbeat pausing
 
@@ -298,6 +304,7 @@ class Agent(BaseAgent):
         step_count: Optional[int] = None,
         last_function_failed: bool = False,
         put_inner_thoughts_first: bool = True,
+        step_id: Optional[str] = None,
     ) -> ChatCompletionResponse | None:
         """Get response from LLM API with robust retry mechanism."""
         log_telemetry(self.logger, "_get_ai_reply start")
@@ -347,8 +354,9 @@ class Agent(BaseAgent):
                         messages=message_sequence,
                         llm_config=self.agent_state.llm_config,
                         tools=allowed_functions,
-                        stream=stream,
                         force_tool_call=force_tool_call,
+                        telemetry_manager=self.telemetry_manager,
+                        step_id=step_id,
                     )
                 else:
                     # Fallback to existing flow
@@ -365,6 +373,9 @@ class Agent(BaseAgent):
                         stream_interface=self.interface,
                         put_inner_thoughts_first=put_inner_thoughts_first,
                         name=self.agent_state.name,
+                        telemetry_manager=self.telemetry_manager,
+                        step_id=step_id,
+                        actor=self.user,
                     )
                 log_telemetry(self.logger, "_get_ai_reply create finish")
 
@@ -840,6 +851,9 @@ class Agent(BaseAgent):
             # Extract job_id from metadata if present
             job_id = metadata.get("job_id") if metadata else None
 
+            # Declare step_id for the given step to be used as the step is processing.
+            step_id = generate_step_id()
+
             # Step 0: update core memory
             # only pulling latest block data if shared memory is being used
             current_persisted_memory = Memory(
@@ -870,6 +884,7 @@ class Agent(BaseAgent):
                 step_count=step_count,
                 last_function_failed=last_function_failed,
                 put_inner_thoughts_first=put_inner_thoughts_first,
+                step_id=step_id,
             )
             if not response:
                 # EDGE CASE: Function call failed AND there's no tools left for agent to call -> return early
@@ -944,6 +959,7 @@ class Agent(BaseAgent):
                 actor=self.user,
                 agent_id=self.agent_state.id,
                 provider_name=self.agent_state.llm_config.model_endpoint_type,
+                provider_category=self.agent_state.llm_config.provider_category or "base",
                 model=self.agent_state.llm_config.model,
                 model_endpoint=self.agent_state.llm_config.model_endpoint,
                 context_window_limit=self.agent_state.llm_config.context_window,
@@ -953,6 +969,7 @@ class Agent(BaseAgent):
                     actor=self.user,
                 ),
                 job_id=job_id,
+                step_id=step_id,
             )
             for message in all_new_messages:
                 message.step_id = step.id
@@ -1255,6 +1272,276 @@ class Agent(BaseAgent):
             functions_definitions=available_functions_definitions,
         )
 
+    async def get_context_window_async(self) -> ContextWindowOverview:
+        if os.getenv("LETTA_ENVIRONMENT") == "PRODUCTION":
+            return await self.get_context_window_from_anthropic_async()
+        return await self.get_context_window_from_tiktoken_async()
+
+    async def get_context_window_from_tiktoken_async(self) -> ContextWindowOverview:
+        """Get the context window of the agent"""
+        # Grab the in-context messages
+        # conversion of messages to OpenAI dict format, which is passed to the token counter
+        (in_context_messages, passage_manager_size, message_manager_size) = await asyncio.gather(
+            self.agent_manager.get_in_context_messages_async(agent_id=self.agent_state.id, actor=self.user),
+            self.passage_manager.size_async(actor=self.user, agent_id=self.agent_state.id),
+            self.message_manager.size_async(actor=self.user, agent_id=self.agent_state.id),
+        )
+        in_context_messages_openai = [m.to_openai_dict() for m in in_context_messages]
+
+        # Extract system, memory and external summary
+        if (
+            len(in_context_messages) > 0
+            and in_context_messages[0].role == MessageRole.system
+            and in_context_messages[0].content
+            and len(in_context_messages[0].content) == 1
+            and isinstance(in_context_messages[0].content[0], TextContent)
+        ):
+            system_message = in_context_messages[0].content[0].text
+
+            external_memory_marker_pos = system_message.find("###")
+            core_memory_marker_pos = system_message.find("<", external_memory_marker_pos)
+            if external_memory_marker_pos != -1 and core_memory_marker_pos != -1:
+                system_prompt = system_message[:external_memory_marker_pos].strip()
+                external_memory_summary = system_message[external_memory_marker_pos:core_memory_marker_pos].strip()
+                core_memory = system_message[core_memory_marker_pos:].strip()
+            else:
+                # if no markers found, put everything in system message
+                system_prompt = system_message
+                external_memory_summary = ""
+                core_memory = ""
+        else:
+            # if no system message, fall back on agent's system prompt
+            system_prompt = self.agent_state.system
+            external_memory_summary = ""
+            core_memory = ""
+
+        num_tokens_system = count_tokens(system_prompt)
+        num_tokens_core_memory = count_tokens(core_memory)
+        num_tokens_external_memory_summary = count_tokens(external_memory_summary)
+
+        # Check if there's a summary message in the message queue
+        if (
+            len(in_context_messages) > 1
+            and in_context_messages[1].role == MessageRole.user
+            and in_context_messages[1].content
+            and len(in_context_messages[1].content) == 1
+            and isinstance(in_context_messages[1].content[0], TextContent)
+            # TODO remove hardcoding
+            and "The following is a summary of the previous " in in_context_messages[1].content[0].text
+        ):
+            # Summary message exists
+            text_content = in_context_messages[1].content[0].text
+            assert text_content is not None
+            summary_memory = text_content
+            num_tokens_summary_memory = count_tokens(text_content)
+            # with a summary message, the real messages start at index 2
+            num_tokens_messages = (
+                num_tokens_from_messages(messages=in_context_messages_openai[2:], model=self.model)
+                if len(in_context_messages_openai) > 2
+                else 0
+            )
+
+        else:
+            summary_memory = None
+            num_tokens_summary_memory = 0
+            # with no summary message, the real messages start at index 1
+            num_tokens_messages = (
+                num_tokens_from_messages(messages=in_context_messages_openai[1:], model=self.model)
+                if len(in_context_messages_openai) > 1
+                else 0
+            )
+
+        # tokens taken up by function definitions
+        agent_state_tool_jsons = [t.json_schema for t in self.agent_state.tools]
+        if agent_state_tool_jsons:
+            available_functions_definitions = [OpenAITool(type="function", function=f) for f in agent_state_tool_jsons]
+            num_tokens_available_functions_definitions = num_tokens_from_functions(functions=agent_state_tool_jsons, model=self.model)
+        else:
+            available_functions_definitions = []
+            num_tokens_available_functions_definitions = 0
+
+        num_tokens_used_total = (
+            num_tokens_system  # system prompt
+            + num_tokens_available_functions_definitions  # function definitions
+            + num_tokens_core_memory  # core memory
+            + num_tokens_external_memory_summary  # metadata (statistics) about recall/archival
+            + num_tokens_summary_memory  # summary of ongoing conversation
+            + num_tokens_messages  # tokens taken by messages
+        )
+        assert isinstance(num_tokens_used_total, int)
+
+        return ContextWindowOverview(
+            # context window breakdown (in messages)
+            num_messages=len(in_context_messages),
+            num_archival_memory=passage_manager_size,
+            num_recall_memory=message_manager_size,
+            num_tokens_external_memory_summary=num_tokens_external_memory_summary,
+            external_memory_summary=external_memory_summary,
+            # top-level information
+            context_window_size_max=self.agent_state.llm_config.context_window,
+            context_window_size_current=num_tokens_used_total,
+            # context window breakdown (in tokens)
+            num_tokens_system=num_tokens_system,
+            system_prompt=system_prompt,
+            num_tokens_core_memory=num_tokens_core_memory,
+            core_memory=core_memory,
+            num_tokens_summary_memory=num_tokens_summary_memory,
+            summary_memory=summary_memory,
+            num_tokens_messages=num_tokens_messages,
+            messages=in_context_messages,
+            # related to functions
+            num_tokens_functions_definitions=num_tokens_available_functions_definitions,
+            functions_definitions=available_functions_definitions,
+        )
+
+    async def get_context_window_from_anthropic_async(self) -> ContextWindowOverview:
+        """Get the context window of the agent"""
+        anthropic_client = LLMClient.create(provider_type=ProviderType.anthropic, actor=self.user)
+        model = self.agent_state.llm_config.model if self.agent_state.llm_config.model_endpoint_type == "anthropic" else None
+
+        # Grab the in-context messages
+        # conversion of messages to anthropic dict format, which is passed to the token counter
+        (in_context_messages, passage_manager_size, message_manager_size) = await asyncio.gather(
+            self.agent_manager.get_in_context_messages_async(agent_id=self.agent_state.id, actor=self.user),
+            self.passage_manager.size_async(actor=self.user, agent_id=self.agent_state.id),
+            self.message_manager.size_async(actor=self.user, agent_id=self.agent_state.id),
+        )
+        in_context_messages_anthropic = [m.to_anthropic_dict() for m in in_context_messages]
+
+        # Extract system, memory and external summary
+        if (
+            len(in_context_messages) > 0
+            and in_context_messages[0].role == MessageRole.system
+            and in_context_messages[0].content
+            and len(in_context_messages[0].content) == 1
+            and isinstance(in_context_messages[0].content[0], TextContent)
+        ):
+            system_message = in_context_messages[0].content[0].text
+
+            external_memory_marker_pos = system_message.find("###")
+            core_memory_marker_pos = system_message.find("<", external_memory_marker_pos)
+            if external_memory_marker_pos != -1 and core_memory_marker_pos != -1:
+                system_prompt = system_message[:external_memory_marker_pos].strip()
+                external_memory_summary = system_message[external_memory_marker_pos:core_memory_marker_pos].strip()
+                core_memory = system_message[core_memory_marker_pos:].strip()
+            else:
+                # if no markers found, put everything in system message
+                system_prompt = system_message
+                external_memory_summary = None
+                core_memory = None
+        else:
+            # if no system message, fall back on agent's system prompt
+            system_prompt = self.agent_state.system
+            external_memory_summary = None
+            core_memory = None
+
+        num_tokens_system_coroutine = anthropic_client.count_tokens(model=model, messages=[{"role": "user", "content": system_prompt}])
+        num_tokens_core_memory_coroutine = (
+            anthropic_client.count_tokens(model=model, messages=[{"role": "user", "content": core_memory}])
+            if core_memory
+            else asyncio.sleep(0, result=0)
+        )
+        num_tokens_external_memory_summary_coroutine = (
+            anthropic_client.count_tokens(model=model, messages=[{"role": "user", "content": external_memory_summary}])
+            if external_memory_summary
+            else asyncio.sleep(0, result=0)
+        )
+
+        # Check if there's a summary message in the message queue
+        if (
+            len(in_context_messages) > 1
+            and in_context_messages[1].role == MessageRole.user
+            and in_context_messages[1].content
+            and len(in_context_messages[1].content) == 1
+            and isinstance(in_context_messages[1].content[0], TextContent)
+            # TODO remove hardcoding
+            and "The following is a summary of the previous " in in_context_messages[1].content[0].text
+        ):
+            # Summary message exists
+            text_content = in_context_messages[1].content[0].text
+            assert text_content is not None
+            summary_memory = text_content
+            num_tokens_summary_memory_coroutine = anthropic_client.count_tokens(
+                model=model, messages=[{"role": "user", "content": summary_memory}]
+            )
+            # with a summary message, the real messages start at index 2
+            num_tokens_messages_coroutine = (
+                anthropic_client.count_tokens(model=model, messages=in_context_messages_anthropic[2:])
+                if len(in_context_messages_anthropic) > 2
+                else asyncio.sleep(0, result=0)
+            )
+
+        else:
+            summary_memory = None
+            num_tokens_summary_memory_coroutine = asyncio.sleep(0, result=0)
+            # with no summary message, the real messages start at index 1
+            num_tokens_messages_coroutine = (
+                anthropic_client.count_tokens(model=model, messages=in_context_messages_anthropic[1:])
+                if len(in_context_messages_anthropic) > 1
+                else asyncio.sleep(0, result=0)
+            )
+
+        # tokens taken up by function definitions
+        if self.agent_state.tools and len(self.agent_state.tools) > 0:
+            available_functions_definitions = [OpenAITool(type="function", function=f.json_schema) for f in self.agent_state.tools]
+            num_tokens_available_functions_definitions_coroutine = anthropic_client.count_tokens(
+                model=model,
+                tools=available_functions_definitions,
+            )
+        else:
+            available_functions_definitions = []
+            num_tokens_available_functions_definitions_coroutine = asyncio.sleep(0, result=0)
+
+        (
+            num_tokens_system,
+            num_tokens_core_memory,
+            num_tokens_external_memory_summary,
+            num_tokens_summary_memory,
+            num_tokens_messages,
+            num_tokens_available_functions_definitions,
+        ) = await asyncio.gather(
+            num_tokens_system_coroutine,
+            num_tokens_core_memory_coroutine,
+            num_tokens_external_memory_summary_coroutine,
+            num_tokens_summary_memory_coroutine,
+            num_tokens_messages_coroutine,
+            num_tokens_available_functions_definitions_coroutine,
+        )
+
+        num_tokens_used_total = (
+            num_tokens_system  # system prompt
+            + num_tokens_available_functions_definitions  # function definitions
+            + num_tokens_core_memory  # core memory
+            + num_tokens_external_memory_summary  # metadata (statistics) about recall/archival
+            + num_tokens_summary_memory  # summary of ongoing conversation
+            + num_tokens_messages  # tokens taken by messages
+        )
+        assert isinstance(num_tokens_used_total, int)
+
+        return ContextWindowOverview(
+            # context window breakdown (in messages)
+            num_messages=len(in_context_messages),
+            num_archival_memory=passage_manager_size,
+            num_recall_memory=message_manager_size,
+            num_tokens_external_memory_summary=num_tokens_external_memory_summary,
+            external_memory_summary=external_memory_summary,
+            # top-level information
+            context_window_size_max=self.agent_state.llm_config.context_window,
+            context_window_size_current=num_tokens_used_total,
+            # context window breakdown (in tokens)
+            num_tokens_system=num_tokens_system,
+            system_prompt=system_prompt,
+            num_tokens_core_memory=num_tokens_core_memory,
+            core_memory=core_memory,
+            num_tokens_summary_memory=num_tokens_summary_memory,
+            summary_memory=summary_memory,
+            num_tokens_messages=num_tokens_messages,
+            messages=in_context_messages,
+            # related to functions
+            num_tokens_functions_definitions=num_tokens_available_functions_definitions,
+            functions_definitions=available_functions_definitions,
+        )
+
     def count_tokens(self) -> int:
         """Count the tokens in the current context window"""
         context_window_breakdown = self.get_context_window()

letta/agents/base_agent.py

@@ -72,61 +72,6 @@ class BaseAgent(ABC):
 
         return [{"role": input_message.role.value, "content": get_content(input_message)} for input_message in input_messages]
 
-    def _rebuild_memory(
-        self,
-        in_context_messages: List[Message],
-        agent_state: AgentState,
-        num_messages: int | None = None,  # storing these calculations is specific to the voice agent
-        num_archival_memories: int | None = None,
-    ) -> List[Message]:
-        try:
-            # Refresh Memory
-            # TODO: This only happens for the summary block (voice?)
-            # [DB Call] loading blocks (modifies: agent_state.memory.blocks)
-            self.agent_manager.refresh_memory(agent_state=agent_state, actor=self.actor)
-
-            # TODO: This is a pretty brittle pattern established all over our code, need to get rid of this
-            curr_system_message = in_context_messages[0]
-            curr_memory_str = agent_state.memory.compile()
-            curr_system_message_text = curr_system_message.content[0].text
-            if curr_memory_str in curr_system_message_text:
-                # NOTE: could this cause issues if a block is removed? (substring match would still work)
-                logger.debug(
-                    f"Memory hasn't changed for agent id={agent_state.id} and actor=({self.actor.id}, {self.actor.name}), skipping system prompt rebuild"
-                )
-                return in_context_messages
-
-            memory_edit_timestamp = get_utc_time()
-
-            # [DB Call] size of messages and archival memories
-            num_messages = num_messages or self.message_manager.size(actor=self.actor, agent_id=agent_state.id)
-            num_archival_memories = num_archival_memories or self.passage_manager.size(actor=self.actor, agent_id=agent_state.id)
-
-            new_system_message_str = compile_system_message(
-                system_prompt=agent_state.system,
-                in_context_memory=agent_state.memory,
-                in_context_memory_last_edit=memory_edit_timestamp,
-                previous_message_count=num_messages,
-                archival_memory_size=num_archival_memories,
-            )
-
-            diff = united_diff(curr_system_message_text, new_system_message_str)
-            if len(diff) > 0:
-                logger.debug(f"Rebuilding system with new memory...\nDiff:\n{diff}")
-
-                # [DB Call] Update Messages
-                new_system_message = self.message_manager.update_message_by_id(
-                    curr_system_message.id, message_update=MessageUpdate(content=new_system_message_str), actor=self.actor
-                )
-                # Skip pulling down the agent's memory again to save on a db call
-                return [new_system_message] + in_context_messages[1:]
-
-            else:
-                return in_context_messages
-        except:
-            logger.exception(f"Failed to rebuild memory for agent id={agent_state.id} and actor=({self.actor.id}, {self.actor.name})")
-            raise
-
     async def _rebuild_memory_async(
         self,
         in_context_messages: List[Message],

letta/agents/helpers.py

@@ -1,3 +1,4 @@
+import uuid
 import xml.etree.ElementTree as ET
 from typing import List, Tuple
 
@@ -150,3 +151,7 @@ def deserialize_message_history(xml_str: str) -> Tuple[List[str], str]:
     context = sum_el.text or ""
 
     return messages, context
+
+
+def generate_step_id():
+    return f"step-{uuid.uuid4()}"