letta-nightly 0.11.6.dev20250902104140__py3-none-any.whl → 0.11.7.dev20250904045700__py3-none-any.whl

letta/services/passage_manager.py

@@ -1,19 +1,24 @@
+import uuid
 from datetime import datetime, timezone
 from functools import lru_cache
-from typing import List, Optional
+from typing import Dict, List, Optional
 
 from openai import AsyncOpenAI, OpenAI
-from sqlalchemy import select
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
 
 from letta.constants import MAX_EMBEDDING_DIM
 from letta.embeddings import parse_and_chunk_text
 from letta.helpers.decorators import async_redis_cache
 from letta.llm_api.llm_client import LLMClient
+from letta.log import get_logger
 from letta.orm import ArchivesAgents
 from letta.orm.errors import NoResultFound
 from letta.orm.passage import ArchivalPassage, SourcePassage
+from letta.orm.passage_tag import PassageTag
 from letta.otel.tracing import trace_method
 from letta.schemas.agent import AgentState
+from letta.schemas.enums import VectorDBProvider
 from letta.schemas.file import FileMetadata as PydanticFileMetadata
 from letta.schemas.passage import Passage as PydanticPassage
 from letta.schemas.user import User as PydanticUser
@@ -21,6 +26,8 @@ from letta.server.db import db_registry
 from letta.services.archive_manager import ArchiveManager
 from letta.utils import enforce_types
 
+logger = get_logger(__name__)
+
 
 # TODO: Add redis-backed caching for backend
 @lru_cache(maxsize=8192)
@@ -47,6 +54,44 @@ class PassageManager:
     def __init__(self):
         self.archive_manager = ArchiveManager()
 
+    async def _create_tags_for_passage(
+        self,
+        session: AsyncSession,
+        passage_id: str,
+        archive_id: str,
+        organization_id: str,
+        tags: List[str],
+        actor: PydanticUser,
+    ) -> List[PassageTag]:
+        """Create tag entries in junction table (complements tags stored in JSON column).
+
+        Junction table enables efficient DISTINCT queries and tag-based filtering.
+
+        Note: Tags are already deduplicated before being passed to this method.
+        """
+        if not tags:
+            return []
+
+        tag_objects = []
+        for tag in tags:
+            tag_obj = PassageTag(
+                id=f"passage-tag-{uuid.uuid4()}",
+                tag=tag,
+                passage_id=passage_id,
+                archive_id=archive_id,
+                organization_id=organization_id,
+            )
+            tag_objects.append(tag_obj)
+
+        # batch create all tags
+        created_tags = await PassageTag.batch_create_async(
+            items=tag_objects,
+            db_session=session,
+            actor=actor,
+        )
+
+        return created_tags
+
     # AGENT PASSAGE METHODS
     @enforce_types
     @trace_method
@@ -154,6 +199,12 @@ class PassageManager:
             raise ValueError("Agent passage cannot have source_id")
 
         data = pydantic_passage.model_dump(to_orm=True)
+
+        # Deduplicate tags if provided (for dual storage consistency)
+        tags = data.get("tags")
+        if tags:
+            tags = list(set(tags))
+
         common_fields = {
             "id": data.get("id"),
             "text": data["text"],
@@ -161,6 +212,7 @@ class PassageManager:
             "embedding_config": data["embedding_config"],
             "organization_id": data["organization_id"],
             "metadata_": data.get("metadata", {}),
+            "tags": tags,
             "is_deleted": data.get("is_deleted", False),
             "created_at": data.get("created_at", datetime.now(timezone.utc)),
         }
@@ -181,6 +233,12 @@ class PassageManager:
             raise ValueError("Agent passage cannot have source_id")
 
         data = pydantic_passage.model_dump(to_orm=True)
+
+        # Deduplicate tags if provided (for dual storage consistency)
+        tags = data.get("tags")
+        if tags:
+            tags = list(set(tags))
+
         common_fields = {
             "id": data.get("id"),
             "text": data["text"],
@@ -188,6 +246,7 @@ class PassageManager:
             "embedding_config": data["embedding_config"],
             "organization_id": data["organization_id"],
             "metadata_": data.get("metadata", {}),
+            "tags": tags,
             "is_deleted": data.get("is_deleted", False),
             "created_at": data.get("created_at", datetime.now(timezone.utc)),
         }
@@ -196,6 +255,18 @@ class PassageManager:
 
         async with db_registry.async_session() as session:
             passage = await passage.create_async(session, actor=actor)
+
+            # dual storage: save tags to junction table for efficient queries
+            if tags:  # use the deduplicated tags variable
+                await self._create_tags_for_passage(
+                    session=session,
+                    passage_id=passage.id,
+                    archive_id=passage.archive_id,
+                    organization_id=passage.organization_id,
+                    tags=tags,  # pass deduplicated tags
+                    actor=actor,
+                )
+
             return passage.to_pydantic()
 
     @enforce_types
@@ -210,6 +281,12 @@ class PassageManager:
             raise ValueError("Source passage cannot have archive_id")
 
         data = pydantic_passage.model_dump(to_orm=True)
+
+        # Deduplicate tags if provided (for dual storage consistency)
+        tags = data.get("tags")
+        if tags:
+            tags = list(set(tags))
+
         common_fields = {
             "id": data.get("id"),
             "text": data["text"],
@@ -217,6 +294,7 @@ class PassageManager:
             "embedding_config": data["embedding_config"],
             "organization_id": data["organization_id"],
             "metadata_": data.get("metadata", {}),
+            "tags": tags,
             "is_deleted": data.get("is_deleted", False),
             "created_at": data.get("created_at", datetime.now(timezone.utc)),
         }
@@ -243,6 +321,12 @@ class PassageManager:
             raise ValueError("Source passage cannot have archive_id")
 
         data = pydantic_passage.model_dump(to_orm=True)
+
+        # Deduplicate tags if provided (for dual storage consistency)
+        tags = data.get("tags")
+        if tags:
+            tags = list(set(tags))
+
         common_fields = {
             "id": data.get("id"),
             "text": data["text"],
@@ -250,6 +334,7 @@ class PassageManager:
             "embedding_config": data["embedding_config"],
             "organization_id": data["organization_id"],
             "metadata_": data.get("metadata", {}),
+            "tags": tags,
             "is_deleted": data.get("is_deleted", False),
             "created_at": data.get("created_at", datetime.now(timezone.utc)),
         }
@@ -309,6 +394,7 @@ class PassageManager:
             "embedding_config": data["embedding_config"],
             "organization_id": data["organization_id"],
             "metadata_": data.get("metadata", {}),
+            "tags": data.get("tags"),
             "is_deleted": data.get("is_deleted", False),
             "created_at": data.get("created_at", datetime.now(timezone.utc)),
         }
@@ -356,6 +442,7 @@ class PassageManager:
                 "embedding_config": data["embedding_config"],
                 "organization_id": data["organization_id"],
                 "metadata_": data.get("metadata", {}),
+                "tags": data.get("tags"),
                 "is_deleted": data.get("is_deleted", False),
                 "created_at": data.get("created_at", datetime.now(timezone.utc)),
             }
@@ -395,6 +482,7 @@ class PassageManager:
                 "embedding_config": data["embedding_config"],
                 "organization_id": data["organization_id"],
                 "metadata_": data.get("metadata", {}),
+                "tags": data.get("tags"),
                 "is_deleted": data.get("is_deleted", False),
                 "created_at": data.get("created_at", datetime.now(timezone.utc)),
             }
@@ -465,8 +553,21 @@ class PassageManager:
         agent_state: AgentState,
         text: str,
         actor: PydanticUser,
+        tags: Optional[List[str]] = None,
+        created_at: Optional[datetime] = None,
+        strict_mode: bool = False,
     ) -> List[PydanticPassage]:
-        """Insert passage(s) into archival memory"""
+        """Insert passage(s) into archival memory
+
+        Args:
+            agent_state: Agent state for embedding configuration
+            text: Text content to store as passages
+            actor: User performing the operation
+            tags: Optional list of tags to attach to all created passages
+
+        Returns:
+            List of created passage objects
+        """
 
         embedding_chunk_size = agent_state.embedding_config.embedding_chunk_size
         embedding_client = LLMClient.create(
@@ -489,19 +590,53 @@ class PassageManager:
             embeddings = await embedding_client.request_embeddings(text_chunks, agent_state.embedding_config)
 
             passages = []
+
+            # Always write to SQL database first
             for chunk_text, embedding in zip(text_chunks, embeddings):
+                passage_data = {
+                    "organization_id": actor.organization_id,
+                    "archive_id": archive.id,
+                    "text": chunk_text,
+                    "embedding": embedding,
+                    "embedding_config": agent_state.embedding_config,
+                    "tags": tags,
+                }
+                # only include created_at if provided
+                if created_at is not None:
+                    passage_data["created_at"] = created_at
+
                 passage = await self.create_agent_passage_async(
-                    PydanticPassage(
-                        organization_id=actor.organization_id,
-                        archive_id=archive.id,
-                        text=chunk_text,
-                        embedding=embedding,
-                        embedding_config=agent_state.embedding_config,
-                    ),
+                    PydanticPassage(**passage_data),
                     actor=actor,
                 )
                 passages.append(passage)
 
+            # If archive uses Turbopuffer, also write to Turbopuffer (dual-write)
+            if archive.vector_db_provider == VectorDBProvider.TPUF:
+                try:
+                    from letta.helpers.tpuf_client import TurbopufferClient
+
+                    tpuf_client = TurbopufferClient()
+
+                    # Extract IDs and texts from the created passages
+                    passage_ids = [p.id for p in passages]
+                    passage_texts = [p.text for p in passages]
+
+                    # Insert to Turbopuffer with the same IDs as SQL
+                    await tpuf_client.insert_archival_memories(
+                        archive_id=archive.id,
+                        text_chunks=passage_texts,
+                        embeddings=embeddings,
+                        passage_ids=passage_ids,  # Use same IDs as SQL
+                        organization_id=actor.organization_id,
+                        tags=tags,
+                        created_at=passages[0].created_at if passages else None,
+                    )
+                except Exception as e:
+                    logger.error(f"Failed to insert passages to Turbopuffer: {e}")
+                    if strict_mode:
+                        raise  # Re-raise the exception in strict mode
+
             return passages
 
         except Exception as e:
@@ -567,6 +702,34 @@ class PassageManager:
 
             # Update the database record with values from the provided record
             update_data = passage.model_dump(to_orm=True, exclude_unset=True, exclude_none=True)
+
+            # Handle tags update separately for junction table
+            new_tags = update_data.pop("tags", None)
+            if new_tags is not None:
+                # Deduplicate tags
+                if new_tags:
+                    new_tags = list(set(new_tags))
+
+                # Delete existing tags from junction table
+                from sqlalchemy import delete
+
+                await session.execute(delete(PassageTag).where(PassageTag.passage_id == passage_id))
+
+                # Create new tags in junction table
+                if new_tags:
+                    await self._create_tags_for_passage(
+                        session=session,
+                        passage_id=passage_id,
+                        archive_id=curr_passage.archive_id,
+                        organization_id=curr_passage.organization_id,
+                        tags=new_tags,
+                        actor=actor,
+                    )
+
+                # Update the tags on the passage object
+                setattr(curr_passage, "tags", new_tags)
+
+            # Update other fields
             for key, value in update_data.items():
                 setattr(curr_passage, key, value)
 
@@ -647,7 +810,7 @@ class PassageManager:
 
     @enforce_types
     @trace_method
-    async def delete_agent_passage_by_id_async(self, passage_id: str, actor: PydanticUser) -> bool:
+    async def delete_agent_passage_by_id_async(self, passage_id: str, actor: PydanticUser, strict_mode: bool = False) -> bool:
         """Delete an agent passage."""
         if not passage_id:
             raise ValueError("Passage ID must be provided.")
@@ -655,7 +818,25 @@ class PassageManager:
         async with db_registry.async_session() as session:
             try:
                 passage = await ArchivalPassage.read_async(db_session=session, identifier=passage_id, actor=actor)
+                archive_id = passage.archive_id
+
+                # Delete from SQL first
                 await passage.hard_delete_async(session, actor=actor)
+
+                # Check if archive uses Turbopuffer and dual-delete
+                if archive_id:
+                    archive = await self.archive_manager.get_archive_by_id_async(archive_id=archive_id, actor=actor)
+                    if archive.vector_db_provider == VectorDBProvider.TPUF:
+                        try:
+                            from letta.helpers.tpuf_client import TurbopufferClient
+
+                            tpuf_client = TurbopufferClient()
+                            await tpuf_client.delete_passage(archive_id=archive_id, passage_id=passage_id)
+                        except Exception as e:
+                            logger.error(f"Failed to delete passage from Turbopuffer: {e}")
+                            if strict_mode:
+                                raise  # Re-raise the exception in strict mode
+
                 return True
             except NoResultFound:
                 raise NoResultFound(f"Agent passage with id {passage_id} not found.")
@@ -812,12 +993,40 @@ class PassageManager:
     @trace_method
     async def delete_agent_passages_async(
         self,
-        actor: PydanticUser,
         passages: List[PydanticPassage],
+        actor: PydanticUser,
+        strict_mode: bool = False,
     ) -> bool:
         """Delete multiple agent passages."""
+        if not passages:
+            return True
+
         async with db_registry.async_session() as session:
+            # Delete from SQL first
             await ArchivalPassage.bulk_hard_delete_async(db_session=session, identifiers=[p.id for p in passages], actor=actor)
+
+            # Group passages by archive_id for efficient Turbopuffer deletion
+            passages_by_archive = {}
+            for passage in passages:
+                if passage.archive_id:
+                    if passage.archive_id not in passages_by_archive:
+                        passages_by_archive[passage.archive_id] = []
+                    passages_by_archive[passage.archive_id].append(passage.id)
+
+            # Check each archive and delete from Turbopuffer if needed
+            for archive_id, passage_ids in passages_by_archive.items():
+                archive = await self.archive_manager.get_archive_by_id_async(archive_id=archive_id, actor=actor)
+                if archive.vector_db_provider == VectorDBProvider.TPUF:
+                    try:
+                        from letta.helpers.tpuf_client import TurbopufferClient
+
+                        tpuf_client = TurbopufferClient()
+                        await tpuf_client.delete_passages(archive_id=archive_id, passage_ids=passage_ids)
+                    except Exception as e:
+                        logger.error(f"Failed to delete passages from Turbopuffer: {e}")
+                        if strict_mode:
+                            raise  # Re-raise the exception in strict mode
+
             return True
 
     @enforce_types
@@ -1009,3 +1218,69 @@ class PassageManager:
             )
             passages = result.scalars().all()
             return [p.to_pydantic() for p in passages]
+
+    @enforce_types
+    @trace_method
+    async def get_unique_tags_for_archive_async(
+        self,
+        archive_id: str,
+        actor: PydanticUser,
+    ) -> List[str]:
+        """Get all unique tags for an archive.
+
+        Args:
+            archive_id: ID of the archive
+            actor: User performing the operation
+
+        Returns:
+            List of unique tag values
+        """
+        async with db_registry.async_session() as session:
+            stmt = (
+                select(PassageTag.tag)
+                .distinct()
+                .where(
+                    PassageTag.archive_id == archive_id,
+                    PassageTag.organization_id == actor.organization_id,
+                    PassageTag.is_deleted == False,
+                )
+                .order_by(PassageTag.tag)
+            )
+
+            result = await session.execute(stmt)
+            tags = result.scalars().all()
+
+            return list(tags)
+
+    @enforce_types
+    @trace_method
+    async def get_tag_counts_for_archive_async(
+        self,
+        archive_id: str,
+        actor: PydanticUser,
+    ) -> Dict[str, int]:
+        """Get tag counts for an archive.
+
+        Args:
+            archive_id: ID of the archive
+            actor: User performing the operation
+
+        Returns:
+            Dictionary mapping tag values to their counts
+        """
+        async with db_registry.async_session() as session:
+            stmt = (
+                select(PassageTag.tag, func.count(PassageTag.id).label("count"))
+                .where(
+                    PassageTag.archive_id == archive_id,
+                    PassageTag.organization_id == actor.organization_id,
+                    PassageTag.is_deleted == False,
+                )
+                .group_by(PassageTag.tag)
+                .order_by(PassageTag.tag)
+            )
+
+            result = await session.execute(stmt)
+            rows = result.all()
+
+            return {row.tag: row.count for row in rows}

letta/services/provider_manager.py

@@ -3,15 +3,13 @@ from typing import List, Optional, Tuple, Union
 from letta.orm.provider import Provider as ProviderModel
 from letta.otel.tracing import trace_method
 from letta.schemas.enums import ProviderCategory, ProviderType
-from letta.schemas.providers import Provider as PydanticProvider
-from letta.schemas.providers import ProviderCheck, ProviderCreate, ProviderUpdate
+from letta.schemas.providers import Provider as PydanticProvider, ProviderCheck, ProviderCreate, ProviderUpdate
 from letta.schemas.user import User as PydanticUser
 from letta.server.db import db_registry
 from letta.utils import enforce_types
 
 
 class ProviderManager:
-
     @enforce_types
     @trace_method
     def create_provider(self, request: ProviderCreate, actor: PydanticUser) -> PydanticProvider:

letta/services/sandbox_config_manager.py

@@ -3,15 +3,20 @@ from typing import Dict, List, Optional
 from letta.constants import LETTA_TOOL_EXECUTION_DIR
 from letta.log import get_logger
 from letta.orm.errors import NoResultFound
-from letta.orm.sandbox_config import SandboxConfig as SandboxConfigModel
-from letta.orm.sandbox_config import SandboxEnvironmentVariable as SandboxEnvVarModel
+from letta.orm.sandbox_config import SandboxConfig as SandboxConfigModel, SandboxEnvironmentVariable as SandboxEnvVarModel
 from letta.otel.tracing import trace_method
 from letta.schemas.enums import SandboxType
-from letta.schemas.environment_variables import SandboxEnvironmentVariable as PydanticEnvVar
-from letta.schemas.environment_variables import SandboxEnvironmentVariableCreate, SandboxEnvironmentVariableUpdate
-from letta.schemas.sandbox_config import LocalSandboxConfig
-from letta.schemas.sandbox_config import SandboxConfig as PydanticSandboxConfig
-from letta.schemas.sandbox_config import SandboxConfigCreate, SandboxConfigUpdate
+from letta.schemas.environment_variables import (
+    SandboxEnvironmentVariable as PydanticEnvVar,
+    SandboxEnvironmentVariableCreate,
+    SandboxEnvironmentVariableUpdate,
+)
+from letta.schemas.sandbox_config import (
+    LocalSandboxConfig,
+    SandboxConfig as PydanticSandboxConfig,
+    SandboxConfigCreate,
+    SandboxConfigUpdate,
+)
 from letta.schemas.user import User as PydanticUser
 from letta.server.db import db_registry
 from letta.utils import enforce_types, printd

letta/services/source_manager.py

@@ -9,8 +9,7 @@ from letta.orm.source import Source as SourceModel
 from letta.orm.sources_agents import SourcesAgents
 from letta.otel.tracing import trace_method
 from letta.schemas.agent import AgentState as PydanticAgentState
-from letta.schemas.source import Source as PydanticSource
-from letta.schemas.source import SourceUpdate
+from letta.schemas.source import Source as PydanticSource, SourceUpdate
 from letta.schemas.user import User as PydanticUser
 from letta.server.db import db_registry
 from letta.utils import enforce_types, printd

letta/services/step_manager.py

@@ -29,7 +29,6 @@ class FeedbackType(str, Enum):
 
 
 class StepManager:
-
     @enforce_types
     @trace_method
     async def list_steps_async(

letta/services/summarizer/summarizer.py

@@ -137,7 +137,7 @@ class Summarizer:
         total_message_count = len(all_in_context_messages)
         assert self.partial_evict_summarizer_percentage >= 0.0 and self.partial_evict_summarizer_percentage <= 1.0
         target_message_start = round((1.0 - self.partial_evict_summarizer_percentage) * total_message_count)
-        logger.info(f"Target message count: {total_message_count}->{(total_message_count-target_message_start)}")
+        logger.info(f"Target message count: {total_message_count}->{(total_message_count - target_message_start)}")
 
         # The summary message we'll insert is role 'user' (vs 'assistant', 'tool', or 'system')
         # We are going to put it at index 1 (index 0 is the system message)
@@ -295,7 +295,9 @@ class Summarizer:
 def simple_formatter(messages: List[Message], include_system: bool = False) -> str:
     """Go from an OpenAI-style list of messages to a concatenated string"""
 
-    parsed_messages = [message.to_openai_dict() for message in messages if message.role != MessageRole.system or include_system]
+    parsed_messages = Message.to_openai_dicts_from_list(
+        [message for message in messages if message.role != MessageRole.system or include_system]
+    )
     return "\n".join(json.dumps(msg) for msg in parsed_messages)
 
 

letta/services/telemetry_manager.py

@@ -2,8 +2,7 @@ from letta.helpers.json_helpers import json_dumps, json_loads
 from letta.helpers.singleton import singleton
 from letta.orm.provider_trace import ProviderTrace as ProviderTraceModel
 from letta.otel.tracing import trace_method
-from letta.schemas.provider_trace import ProviderTrace as PydanticProviderTrace
-from letta.schemas.provider_trace import ProviderTraceCreate
+from letta.schemas.provider_trace import ProviderTrace as PydanticProviderTrace, ProviderTraceCreate
 from letta.schemas.step import Step as PydanticStep
 from letta.schemas.user import User as PydanticUser
 from letta.server.db import db_registry
@@ -11,7 +10,6 @@ from letta.utils import enforce_types
 
 
 class TelemetryManager:
-
     @enforce_types
     @trace_method
     async def get_provider_trace_by_step_id_async(