letta-nightly 0.8.9.dev20250706104157__py3-none-any.whl → 0.8.10.dev20250707104400__py3-none-any.whl

letta/services/summarizer/summarizer.py

@@ -4,13 +4,19 @@ import traceback
 from typing import List, Optional, Tuple, Union
 
 from letta.agents.ephemeral_summary_agent import EphemeralSummaryAgent
-from letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG
+from letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG, MESSAGE_SUMMARY_REQUEST_ACK
+from letta.helpers.message_helper import convert_message_creates_to_messages
+from letta.llm_api.llm_client import LLMClient
 from letta.log import get_logger
 from letta.otel.tracing import trace_method
+from letta.prompts import gpt_summarize
 from letta.schemas.enums import MessageRole
 from letta.schemas.letta_message_content import TextContent
+from letta.schemas.llm_config import LLMConfig
 from letta.schemas.message import Message, MessageCreate
+from letta.schemas.user import User
 from letta.services.summarizer.enums import SummarizationMode
+from letta.system import package_summarize_message_no_counts
 from letta.templates.template_helper import render_template
 
 logger = get_logger(__name__)
@@ -29,18 +35,24 @@ class Summarizer:
         summarizer_agent: Optional[Union[EphemeralSummaryAgent, "VoiceSleeptimeAgent"]] = None,
         message_buffer_limit: int = 10,
         message_buffer_min: int = 3,
+        partial_evict_summarizer_percentage: float = 0.30,
     ):
         self.mode = mode
 
         # Need to do validation on this
+        # TODO: Move this to config
         self.message_buffer_limit = message_buffer_limit
         self.message_buffer_min = message_buffer_min
         self.summarizer_agent = summarizer_agent
-        # TODO: Move this to config
+        self.partial_evict_summarizer_percentage = partial_evict_summarizer_percentage
 
     @trace_method
-    def summarize(
-        self, in_context_messages: List[Message], new_letta_messages: List[Message], force: bool = False, clear: bool = False
+    async def summarize(
+        self,
+        in_context_messages: List[Message],
+        new_letta_messages: List[Message],
+        force: bool = False,
+        clear: bool = False,
     ) -> Tuple[List[Message], bool]:
         """
         Summarizes or trims in_context_messages according to the chosen mode,
@@ -58,7 +70,19 @@ class Summarizer:
                              (could be appended to the conversation if desired)
         """
         if self.mode == SummarizationMode.STATIC_MESSAGE_BUFFER:
-            return self._static_buffer_summarization(in_context_messages, new_letta_messages, force=force, clear=clear)
+            return self._static_buffer_summarization(
+                in_context_messages,
+                new_letta_messages,
+                force=force,
+                clear=clear,
+            )
+        elif self.mode == SummarizationMode.PARTIAL_EVICT_MESSAGE_BUFFER:
+            return await self._partial_evict_buffer_summarization(
+                in_context_messages,
+                new_letta_messages,
+                force=force,
+                clear=clear,
+            )
         else:
             # Fallback or future logic
             return in_context_messages, False
@@ -75,9 +99,131 @@ class Summarizer:
         task.add_done_callback(callback)
         return task
 
+    async def _partial_evict_buffer_summarization(
+        self,
+        in_context_messages: List[Message],
+        new_letta_messages: List[Message],
+        force: bool = False,
+        clear: bool = False,
+    ) -> Tuple[List[Message], bool]:
+        """Summarization as implemented in the original MemGPT loop, but using message count instead of token count.
+        Evict a partial amount of messages, and replace message[1] with a recursive summary.
+
+        Note that this can't be made sync, because we're waiting on the summary to inject it into the context window,
+        unlike the version that writes it to a block.
+
+        Unless force is True, don't summarize.
+        Ignore clear, we don't use it.
+        """
+        all_in_context_messages = in_context_messages + new_letta_messages
+
+        if not force:
+            logger.debug("Not forcing summarization, returning in-context messages as is.")
+            return all_in_context_messages, False
+
+        # Very ugly code to pull LLMConfig etc from the SummarizerAgent if we're not using it for anything else
+        assert self.summarizer_agent is not None
+
+        # First step: determine how many messages to retain
+        total_message_count = len(all_in_context_messages)
+        assert self.partial_evict_summarizer_percentage >= 0.0 and self.partial_evict_summarizer_percentage <= 1.0
+        target_message_start = round((1.0 - self.partial_evict_summarizer_percentage) * total_message_count)
+        logger.info(f"Target message count: {total_message_count}->{(total_message_count-target_message_start)}")
+
+        # The summary message we'll insert is role 'user' (vs 'assistant', 'tool', or 'system')
+        # We are going to put it at index 1 (index 0 is the system message)
+        # That means that index 2 needs to be role 'assistant', so walk up the list starting at
+        # the target_message_count and find the first assistant message
+        for i in range(target_message_start, total_message_count):
+            if all_in_context_messages[i].role == MessageRole.assistant:
+                assistant_message_index = i
+                break
+        else:
+            raise ValueError(f"No assistant message found from indices {target_message_start} to {total_message_count}")
+
+        # The sequence to summarize is index 1 -> assistant_message_index
+        messages_to_summarize = all_in_context_messages[1:assistant_message_index]
+        logger.info(f"Eviction indices: {1}->{assistant_message_index}(/{total_message_count})")
+
+        # Dynamically get the LLMConfig from the summarizer agent
+        # Pretty cringe code here that we need the agent for this but we don't use it
+        agent_state = await self.summarizer_agent.agent_manager.get_agent_by_id_async(
+            agent_id=self.summarizer_agent.agent_id, actor=self.summarizer_agent.actor
+        )
+
+        # TODO if we do this via the "agent", then we can more easily allow toggling on the memory block version
+        summary_message_str = await simple_summary(
+            messages=messages_to_summarize,
+            llm_config=agent_state.llm_config,
+            actor=self.summarizer_agent.actor,
+            include_ack=True,
+        )
+
+        # TODO add counts back
+        # Recall message count
+        # num_recall_messages_current = await self.message_manager.size_async(actor=self.actor, agent_id=agent_state.id)
+        # num_messages_evicted = len(messages_to_summarize)
+        # num_recall_messages_hidden = num_recall_messages_total - len()
+
+        # Create the summary message
+        summary_message_str_packed = package_summarize_message_no_counts(
+            summary=summary_message_str,
+            timezone=agent_state.timezone,
+        )
+        summary_message_obj = convert_message_creates_to_messages(
+            message_creates=[
+                MessageCreate(
+                    role=MessageRole.user,
+                    content=[TextContent(text=summary_message_str_packed)],
+                )
+            ],
+            agent_id=agent_state.id,
+            timezone=agent_state.timezone,
+            # We already packed, don't pack again
+            wrap_user_message=False,
+            wrap_system_message=False,
+        )[0]
+
+        # Create the message in the DB
+        await self.summarizer_agent.message_manager.create_many_messages_async(
+            pydantic_msgs=[summary_message_obj],
+            actor=self.summarizer_agent.actor,
+        )
+
+        updated_in_context_messages = all_in_context_messages[assistant_message_index:]
+        return [all_in_context_messages[0], summary_message_obj] + updated_in_context_messages, True
+
     def _static_buffer_summarization(
-        self, in_context_messages: List[Message], new_letta_messages: List[Message], force: bool = False, clear: bool = False
+        self,
+        in_context_messages: List[Message],
+        new_letta_messages: List[Message],
+        force: bool = False,
+        clear: bool = False,
     ) -> Tuple[List[Message], bool]:
+        """
+        Implements static buffer summarization by maintaining a fixed-size message buffer (< N messages).
+
+        Logic:
+        1. Combine existing context messages with new messages
+        2. If total messages <= buffer limit and not forced, return unchanged
+        3. Calculate how many messages to retain (0 if clear=True, otherwise message_buffer_min)
+        4. Find the trim index to keep the most recent messages while preserving user message boundaries
+        5. Evict older messages (everything between system message and trim index)
+        6. If summarizer agent is available, trigger background summarization of evicted messages
+        7. Return updated context with system message + retained recent messages
+
+        Args:
+            in_context_messages: Existing conversation context messages
+            new_letta_messages: Newly added messages to append
+            force: Force summarization even if buffer limit not exceeded
+            clear: Clear all messages except system message (retain_count = 0)
+
+        Returns:
+            Tuple of (updated_messages, was_summarized)
+            - updated_messages: New context after trimming/summarization
+            - was_summarized: True if messages were evicted and summarization triggered
+        """
+
         all_in_context_messages = in_context_messages + new_letta_messages
 
         if len(all_in_context_messages) <= self.message_buffer_limit and not force:
@@ -139,6 +285,91 @@ class Summarizer:
         return [all_in_context_messages[0]] + updated_in_context_messages, True
 
 
+def simple_formatter(messages: List[Message], include_system: bool = False) -> str:
+    """Go from an OpenAI-style list of messages to a concatenated string"""
+
+    parsed_messages = [message.to_openai_dict() for message in messages if message.role != MessageRole.system or include_system]
+    return "\n".join(json.dumps(msg) for msg in parsed_messages)
+
+
+def simple_message_wrapper(openai_msg: dict) -> Message:
+    """Extremely simple way to map from role/content to Message object w/ throwaway dummy fields"""
+
+    if "role" not in openai_msg:
+        raise ValueError(f"Missing role in openai_msg: {openai_msg}")
+    if "content" not in openai_msg:
+        raise ValueError(f"Missing content in openai_msg: {openai_msg}")
+
+    if openai_msg["role"] == "user":
+        return Message(
+            role=MessageRole.user,
+            content=[TextContent(text=openai_msg["content"])],
+        )
+    elif openai_msg["role"] == "assistant":
+        return Message(
+            role=MessageRole.assistant,
+            content=[TextContent(text=openai_msg["content"])],
+        )
+    elif openai_msg["role"] == "system":
+        return Message(
+            role=MessageRole.system,
+            content=[TextContent(text=openai_msg["content"])],
+        )
+    else:
+        raise ValueError(f"Unknown role: {openai_msg['role']}")
+
+
+async def simple_summary(messages: List[Message], llm_config: LLMConfig, actor: User, include_ack: bool = True) -> str:
+    """Generate a simple summary from a list of messages.
+
+    Intentionally kept functional due to the simplicity of the prompt.
+    """
+
+    # Create an LLMClient from the config
+    llm_client = LLMClient.create(
+        provider_type=llm_config.model_endpoint_type,
+        put_inner_thoughts_first=True,
+        actor=actor,
+    )
+    assert llm_client is not None
+
+    # Prepare the messages payload to send to the LLM
+    system_prompt = gpt_summarize.SYSTEM
+    summary_transcript = simple_formatter(messages)
+
+    if include_ack:
+        input_messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "assistant", "content": MESSAGE_SUMMARY_REQUEST_ACK},
+            {"role": "user", "content": summary_transcript},
+        ]
+    else:
+        input_messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": summary_transcript},
+        ]
+    print("messages going to summarizer:", input_messages)
+    input_messages_obj = [simple_message_wrapper(msg) for msg in input_messages]
+    print("messages going to summarizer (objs):", input_messages_obj)
+
+    request_data = llm_client.build_request_data(input_messages_obj, llm_config, tools=[])
+    print("request data:", request_data)
+    # NOTE: we should disable the inner_thoughts_in_kwargs here, because we don't use it
+    # I'm leaving it commented it out for now for safety but is fine assuming the var here is a copy not a reference
+    # llm_config.put_inner_thoughts_in_kwargs = False
+    response_data = await llm_client.request_async(request_data, llm_config)
+    response = llm_client.convert_response_to_chat_completion(response_data, input_messages_obj, llm_config)
+    if response.choices[0].message.content is None:
+        logger.warning("No content returned from summarizer")
+        # TODO raise an error error instead?
+        # return "[Summary failed to generate]"
+        raise Exception("Summary failed to generate")
+    else:
+        summary = response.choices[0].message.content.strip()
+
+    return summary
+
+
 def format_transcript(messages: List[Message], include_system: bool = False) -> List[str]:
     """
     Turn a list of Message objects into a human-readable transcript.

letta/services/tool_executor/files_tool_executor.py

@@ -2,8 +2,9 @@ import asyncio
 import re
 from typing import Any, Dict, List, Optional
 
-from letta.constants import MAX_FILES_OPEN
+from letta.constants import MAX_FILES_OPEN, PINECONE_TEXT_FIELD_NAME
 from letta.functions.types import FileOpenRequest
+from letta.helpers.pinecone_utils import search_pinecone_index, should_use_pinecone
 from letta.log import get_logger
 from letta.otel.tracing import trace_method
 from letta.schemas.agent import AgentState
@@ -463,14 +464,15 @@ class LettaFileToolExecutor(ToolExecutor):
         return "\n".join(formatted_results)
 
     @trace_method
-    async def semantic_search_files(self, agent_state: AgentState, query: str, limit: int = 10) -> str:
+    async def semantic_search_files(self, agent_state: AgentState, query: str, limit: int = 5) -> str:
         """
         Search for text within attached files using semantic search and return passages with their source filenames.
+        Uses Pinecone if configured, otherwise falls back to traditional search.
 
         Args:
             agent_state: Current agent state
             query: Search query for semantic matching
-            limit: Maximum number of results to return (default: 10)
+            limit: Maximum number of results to return (default: 5)
 
         Returns:
             Formatted string with search results in IDE/terminal style
@@ -485,6 +487,110 @@ class LettaFileToolExecutor(ToolExecutor):
 
         self.logger.info(f"Semantic search started for agent {agent_state.id} with query '{query}' (limit: {limit})")
 
+        # Check if Pinecone is enabled and use it if available
+        if should_use_pinecone():
+            return await self._search_files_pinecone(agent_state, query, limit)
+        else:
+            return await self._search_files_traditional(agent_state, query, limit)
+
+    async def _search_files_pinecone(self, agent_state: AgentState, query: str, limit: int) -> str:
+        """Search files using Pinecone vector database."""
+
+        # Extract unique source_ids
+        # TODO: Inefficient
+        attached_sources = await self.agent_manager.list_attached_sources_async(agent_id=agent_state.id, actor=self.actor)
+        source_ids = [source.id for source in attached_sources]
+        if not source_ids:
+            return f"No valid source IDs found for attached files"
+
+        # Get all attached files for this agent
+        file_agents = await self.files_agents_manager.list_files_for_agent(agent_id=agent_state.id, actor=self.actor)
+        if not file_agents:
+            return "No files are currently attached to search"
+
+        results = []
+        total_hits = 0
+        files_with_matches = {}
+
+        try:
+            filter = {"source_id": {"$in": source_ids}}
+            search_results = await search_pinecone_index(query, limit, filter, self.actor)
+
+            # Process search results
+            if "result" in search_results and "hits" in search_results["result"]:
+                for hit in search_results["result"]["hits"]:
+                    if total_hits >= limit:
+                        break
+
+                    total_hits += 1
+
+                    # Extract hit information
+                    hit_id = hit.get("_id", "unknown")
+                    score = hit.get("_score", 0.0)
+                    fields = hit.get("fields", {})
+                    text = fields.get(PINECONE_TEXT_FIELD_NAME, "")
+                    file_id = fields.get("file_id", "")
+
+                    # Find corresponding file name
+                    file_name = "Unknown File"
+                    for fa in file_agents:
+                        if fa.file_id == file_id:
+                            file_name = fa.file_name
+                            break
+
+                    # Group by file name
+                    if file_name not in files_with_matches:
+                        files_with_matches[file_name] = []
+                    files_with_matches[file_name].append({"text": text, "score": score, "hit_id": hit_id})
+
+        except Exception as e:
+            self.logger.error(f"Pinecone search failed: {str(e)}")
+            raise e
+
+        if not files_with_matches:
+            return f"No semantic matches found in Pinecone for query: '{query}'"
+
+        # Format results
+        passage_num = 0
+        for file_name, matches in files_with_matches.items():
+            for match in matches:
+                passage_num += 1
+
+                # Format each passage with terminal-style header
+                score_display = f"(score: {match['score']:.3f})"
+                passage_header = f"\n=== {file_name} (passage #{passage_num}) {score_display} ==="
+
+                # Format the passage text
+                passage_text = match["text"].strip()
+                lines = passage_text.splitlines()
+                formatted_lines = []
+                for line in lines[:20]:  # Limit to first 20 lines per passage
+                    formatted_lines.append(f"  {line}")
+
+                if len(lines) > 20:
+                    formatted_lines.append(f"  ... [truncated {len(lines) - 20} more lines]")
+
+                passage_content = "\n".join(formatted_lines)
+                results.append(f"{passage_header}\n{passage_content}")
+
+        # Mark access for files that had matches
+        if files_with_matches:
+            matched_file_names = [name for name in files_with_matches.keys() if name != "Unknown File"]
+            if matched_file_names:
+                await self.files_agents_manager.mark_access_bulk(agent_id=agent_state.id, file_names=matched_file_names, actor=self.actor)
+
+        # Create summary header
+        file_count = len(files_with_matches)
+        summary = f"Found {total_hits} Pinecone matches in {file_count} file{'s' if file_count != 1 else ''} for query: '{query}'"
+
+        # Combine all results
+        formatted_results = [summary, "=" * len(summary)] + results
+
+        self.logger.info(f"Pinecone search completed: {total_hits} matches across {file_count} files")
+        return "\n".join(formatted_results)
+
+    async def _search_files_traditional(self, agent_state: AgentState, query: str, limit: int) -> str:
+        """Traditional search using existing passage manager."""
         # Get semantic search results
         passages = await self.agent_manager.list_source_passages_async(
             actor=self.actor,

letta/services/user_manager.py

@@ -14,7 +14,6 @@ from letta.otel.tracing import trace_method
 from letta.schemas.user import User as PydanticUser
 from letta.schemas.user import UserUpdate
 from letta.server.db import db_registry
-from letta.settings import settings
 from letta.utils import enforce_types
 
 logger = get_logger(__name__)

letta/settings.py

@@ -39,12 +39,17 @@ class ToolSettings(BaseSettings):
 class SummarizerSettings(BaseSettings):
     model_config = SettingsConfigDict(env_prefix="letta_summarizer_", extra="ignore")
 
-    mode: SummarizationMode = SummarizationMode.STATIC_MESSAGE_BUFFER
+    # mode: SummarizationMode = SummarizationMode.STATIC_MESSAGE_BUFFER
+    mode: SummarizationMode = SummarizationMode.PARTIAL_EVICT_MESSAGE_BUFFER
     message_buffer_limit: int = 60
     message_buffer_min: int = 15
     enable_summarization: bool = True
     max_summarization_retries: int = 3
 
+    # partial evict summarizer percentage
+    # eviction based on percentage of message count, not token count
+    partial_evict_summarizer_percentage: float = 0.30
+
     # TODO(cliandy): the below settings are tied to old summarization and should be deprecated or moved
     # Controls if we should evict all messages
     # TODO: Can refactor this into an enum if we have a bunch of different kinds of summarizers
@@ -253,6 +258,13 @@ class Settings(BaseSettings):
     llm_request_timeout_seconds: float = Field(default=60.0, ge=10.0, le=1800.0, description="Timeout for LLM requests in seconds")
     llm_stream_timeout_seconds: float = Field(default=60.0, ge=10.0, le=1800.0, description="Timeout for LLM streaming requests in seconds")
 
+    # For embeddings
+    enable_pinecone: bool = False
+    pinecone_api_key: Optional[str] = None
+    pinecone_source_index: Optional[str] = "sources"
+    pinecone_agent_index: Optional[str] = "recall"
+    upsert_pinecone_indices: bool = False
+
     @property
     def letta_pg_uri(self) -> str:
         if self.pg_uri:

letta/system.py

@@ -188,6 +188,22 @@ def package_summarize_message(summary, summary_message_count, hidden_message_cou
     return json_dumps(packaged_message)
 
 
+def package_summarize_message_no_counts(summary, timezone):
+    context_message = (
+        f"Note: prior messages have been hidden from view due to conversation memory constraints.\n"
+        + f"The following is a summary of the previous messages:\n {summary}"
+    )
+
+    formatted_time = get_local_time(timezone=timezone)
+    packaged_message = {
+        "type": "system_alert",
+        "message": context_message,
+        "time": formatted_time,
+    }
+
+    return json_dumps(packaged_message)
+
+
 def package_summarize_message_no_summary(hidden_message_count, message=None, timezone=None):
     """Add useful metadata to the summary message"""
 

{letta_nightly-0.8.9.dev20250706104157.dist-info → letta_nightly-0.8.10.dev20250707104400.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: letta-nightly
-Version: 0.8.9.dev20250706104157
+Version: 0.8.10.dev20250707104400
 Summary: Create LLM agents with long-term memory and custom tools
 License: Apache License
 Author: Letta Team
@@ -75,6 +75,7 @@ Requires-Dist: pathvalidate (>=3.2.1,<4.0.0)
 Requires-Dist: pexpect (>=4.9.0,<5.0.0) ; extra == "dev" or extra == "all"
 Requires-Dist: pg8000 (>=1.30.3,<2.0.0) ; extra == "postgres" or extra == "desktop" or extra == "all"
 Requires-Dist: pgvector (>=0.2.3,<0.3.0) ; extra == "postgres" or extra == "desktop" or extra == "all"
+Requires-Dist: pinecone[asyncio] (>=7.3.0,<8.0.0)
 Requires-Dist: pre-commit (>=3.5.0,<4.0.0) ; extra == "dev" or extra == "all"
 Requires-Dist: prettytable (>=3.9.0,<4.0.0)
 Requires-Dist: psycopg2 (>=2.9.10,<3.0.0) ; extra == "postgres" or extra == "desktop" or extra == "all"