letta-nightly 0.11.7.dev20250909104137__py3-none-any.whl → 0.11.7.dev20250910104051__py3-none-any.whl

letta/services/archive_manager.py

@@ -5,6 +5,7 @@ from sqlalchemy import select
 from letta.helpers.tpuf_client import should_use_tpuf
 from letta.log import get_logger
 from letta.orm import ArchivalPassage, Archive as ArchiveModel, ArchivesAgents
+from letta.otel.tracing import trace_method
 from letta.schemas.archive import Archive as PydanticArchive
 from letta.schemas.enums import VectorDBProvider
 from letta.schemas.user import User as PydanticUser
@@ -19,6 +20,7 @@ class ArchiveManager:
     """Manager class to handle business logic related to Archives."""
 
     @enforce_types
+    @trace_method
     def create_archive(
         self,
         name: str,
@@ -44,6 +46,7 @@ class ArchiveManager:
             raise
 
     @enforce_types
+    @trace_method
     async def create_archive_async(
         self,
         name: str,
@@ -69,6 +72,7 @@ class ArchiveManager:
             raise
 
     @enforce_types
+    @trace_method
     async def get_archive_by_id_async(
         self,
         archive_id: str,
@@ -84,6 +88,7 @@ class ArchiveManager:
             return archive.to_pydantic()
 
     @enforce_types
+    @trace_method
     def attach_agent_to_archive(
         self,
         agent_id: str,
@@ -113,6 +118,7 @@ class ArchiveManager:
             session.commit()
 
     @enforce_types
+    @trace_method
     async def attach_agent_to_archive_async(
         self,
         agent_id: str,
@@ -148,6 +154,7 @@ class ArchiveManager:
             await session.commit()
 
     @enforce_types
+    @trace_method
     async def get_default_archive_for_agent_async(
         self,
         agent_id: str,
@@ -179,6 +186,24 @@ class ArchiveManager:
         return None
 
     @enforce_types
+    @trace_method
+    async def delete_archive_async(
+        self,
+        archive_id: str,
+        actor: PydanticUser = None,
+    ) -> None:
+        """Delete an archive permanently."""
+        async with db_registry.async_session() as session:
+            archive_model = await ArchiveModel.read_async(
+                db_session=session,
+                identifier=archive_id,
+                actor=actor,
+            )
+            await archive_model.hard_delete_async(session, actor=actor)
+            logger.info(f"Deleted archive {archive_id}")
+
+    @enforce_types
+    @trace_method
     async def get_or_create_default_archive_for_agent_async(
         self,
         agent_id: str,
@@ -187,6 +212,8 @@ class ArchiveManager:
     ) -> PydanticArchive:
         """Get the agent's default archive, creating one if it doesn't exist."""
         # First check if agent has any archives
+        from sqlalchemy.exc import IntegrityError
+
         from letta.services.agent_manager import AgentManager
 
         agent_manager = AgentManager()
@@ -215,17 +242,38 @@ class ArchiveManager:
             actor=actor,
         )
 
-        # Attach the agent to the archive as owner
-        await self.attach_agent_to_archive_async(
-            agent_id=agent_id,
-            archive_id=archive.id,
-            is_owner=True,
-            actor=actor,
-        )
-
-        return archive
+        try:
+            # Attach the agent to the archive as owner
+            await self.attach_agent_to_archive_async(
+                agent_id=agent_id,
+                archive_id=archive.id,
+                is_owner=True,
+                actor=actor,
+            )
+            return archive
+        except IntegrityError:
+            # race condition: another concurrent request already created and attached an archive
+            # clean up the orphaned archive we just created
+            logger.info(f"Race condition detected for agent {agent_id}, cleaning up orphaned archive {archive.id}")
+            await self.delete_archive_async(archive_id=archive.id, actor=actor)
+
+            # fetch the existing archive that was created by the concurrent request
+            archive_ids = await agent_manager.get_agent_archive_ids_async(
+                agent_id=agent_id,
+                actor=actor,
+            )
+            if archive_ids:
+                archive = await self.get_archive_by_id_async(
+                    archive_id=archive_ids[0],
+                    actor=actor,
+                )
+                return archive
+            else:
+                # this shouldn't happen, but if it does, re-raise
+                raise
 
     @enforce_types
+    @trace_method
     def get_or_create_default_archive_for_agent(
         self,
         agent_id: str,
@@ -269,6 +317,7 @@ class ArchiveManager:
         return archive_model.to_pydantic()
 
     @enforce_types
+    @trace_method
     async def get_agents_for_archive_async(
         self,
         archive_id: str,
@@ -280,6 +329,7 @@ class ArchiveManager:
             return [row[0] for row in result.fetchall()]
 
     @enforce_types
+    @trace_method
     async def get_agent_from_passage_async(
         self,
         passage_id: str,
@@ -309,6 +359,7 @@ class ArchiveManager:
             return agent_ids[0]
 
     @enforce_types
+    @trace_method
     async def get_or_set_vector_db_namespace_async(
         self,
         archive_id: str,

letta/services/block_manager.py

@@ -188,6 +188,7 @@ class BlockManager:
         connected_to_agents_count_lt: Optional[int] = None,
         connected_to_agents_count_eq: Optional[List[int]] = None,
         ascending: bool = True,
+        show_hidden_blocks: Optional[bool] = None,
     ) -> List[PydanticBlock]:
         """Async version of get_blocks method. Retrieve blocks based on various optional filters."""
         from sqlalchemy import select
@@ -228,6 +229,10 @@ class BlockManager:
             if value_search:
                 query = query.where(BlockModel.value.ilike(f"%{value_search}%"))
 
+            # Apply hidden filter
+            if not show_hidden_blocks:
+                query = query.where((BlockModel.hidden.is_(None)) | (BlockModel.hidden == False))
+
             needs_distinct = False
 
             needs_agent_count_join = any(

letta/services/file_processor/embedder/base_embedder.py

@@ -2,6 +2,7 @@ from abc import ABC, abstractmethod
 from typing import List
 
 from letta.log import get_logger
+from letta.schemas.enums import VectorDBProvider
 from letta.schemas.passage import Passage
 from letta.schemas.user import User
 
@@ -11,6 +12,10 @@ logger = get_logger(__name__)
 class BaseEmbedder(ABC):
     """Abstract base class for embedding generation"""
 
+    def __init__(self):
+        # Default to NATIVE, subclasses will override this
+        self.vector_db_type = VectorDBProvider.NATIVE
+
     @abstractmethod
     async def generate_embedded_passages(self, file_id: str, source_id: str, chunks: List[str], actor: User) -> List[Passage]:
         """Generate embeddings for chunks with batching and concurrent processing"""

letta/services/file_processor/embedder/openai_embedder.py

@@ -19,6 +19,10 @@ class OpenAIEmbedder(BaseEmbedder):
     """OpenAI-based embedding generation"""
 
     def __init__(self, embedding_config: Optional[EmbeddingConfig] = None):
+        super().__init__()
+        # OpenAI embedder uses the native vector db (PostgreSQL)
+        # self.vector_db_type already set to VectorDBProvider.NATIVE by parent
+
         self.default_embedding_config = (
             EmbeddingConfig.default_config(model_name="text-embedding-3-small", provider="openai")
             if model_settings.openai_api_key

letta/services/file_processor/embedder/pinecone_embedder.py

@@ -4,6 +4,7 @@ from letta.helpers.pinecone_utils import upsert_file_records_to_pinecone_index
 from letta.log import get_logger
 from letta.otel.tracing import log_event, trace_method
 from letta.schemas.embedding_config import EmbeddingConfig
+from letta.schemas.enums import VectorDBProvider
 from letta.schemas.passage import Passage
 from letta.schemas.user import User
 from letta.services.file_processor.embedder.base_embedder import BaseEmbedder
@@ -20,6 +21,10 @@ class PineconeEmbedder(BaseEmbedder):
     """Pinecone-based embedding generation"""
 
     def __init__(self, embedding_config: Optional[EmbeddingConfig] = None):
+        super().__init__()
+        # set the vector db type for pinecone
+        self.vector_db_type = VectorDBProvider.PINECONE
+
         if not PINECONE_AVAILABLE:
             raise ImportError("Pinecone package is not installed. Install it with: pip install pinecone")
 
@@ -28,7 +33,6 @@ class PineconeEmbedder(BaseEmbedder):
             embedding_config = EmbeddingConfig.default_config(provider="pinecone")
 
         self.embedding_config = embedding_config
-        super().__init__()
 
     @trace_method
     async def generate_embedded_passages(self, file_id: str, source_id: str, chunks: List[str], actor: User) -> List[Passage]:

letta/services/file_processor/embedder/turbopuffer_embedder.py

@@ -0,0 +1,71 @@
+from typing import List, Optional
+
+from letta.helpers.tpuf_client import TurbopufferClient
+from letta.log import get_logger
+from letta.otel.tracing import log_event, trace_method
+from letta.schemas.embedding_config import EmbeddingConfig
+from letta.schemas.enums import VectorDBProvider
+from letta.schemas.passage import Passage
+from letta.schemas.user import User
+from letta.services.file_processor.embedder.base_embedder import BaseEmbedder
+
+logger = get_logger(__name__)
+
+
+class TurbopufferEmbedder(BaseEmbedder):
+    """Turbopuffer-based embedding generation and storage"""
+
+    def __init__(self, embedding_config: Optional[EmbeddingConfig] = None):
+        super().__init__()
+        # set the vector db type for turbopuffer
+        self.vector_db_type = VectorDBProvider.TPUF
+        # use the default embedding config from TurbopufferClient if not provided
+        self.embedding_config = embedding_config or TurbopufferClient.default_embedding_config
+        self.tpuf_client = TurbopufferClient()
+
+    @trace_method
+    async def generate_embedded_passages(self, file_id: str, source_id: str, chunks: List[str], actor: User) -> List[Passage]:
+        """Generate embeddings and store in Turbopuffer, then return Passage objects"""
+        if not chunks:
+            return []
+
+        logger.info(f"Generating embeddings for {len(chunks)} chunks using Turbopuffer")
+        log_event(
+            "turbopuffer_embedder.generation_started",
+            {
+                "total_chunks": len(chunks),
+                "file_id": file_id,
+                "source_id": source_id,
+                "embedding_model": self.embedding_config.embedding_model,
+            },
+        )
+
+        try:
+            # insert passages to Turbopuffer - it will handle embedding generation internally
+            passages = await self.tpuf_client.insert_file_passages(
+                source_id=source_id,
+                file_id=file_id,
+                text_chunks=chunks,
+                organization_id=actor.organization_id,
+                actor=actor,
+            )
+
+            logger.info(f"Successfully generated and stored {len(passages)} passages in Turbopuffer")
+            log_event(
+                "turbopuffer_embedder.generation_completed",
+                {
+                    "passages_created": len(passages),
+                    "total_chunks_processed": len(chunks),
+                    "file_id": file_id,
+                    "source_id": source_id,
+                },
+            )
+            return passages
+
+        except Exception as e:
+            logger.error(f"Failed to generate embeddings with Turbopuffer: {str(e)}")
+            log_event(
+                "turbopuffer_embedder.generation_failed",
+                {"error": str(e), "error_type": type(e).__name__, "file_id": file_id, "source_id": source_id},
+            )
+            raise

letta/services/file_processor/file_processor.py

@@ -6,7 +6,7 @@ from letta.log import get_logger
 from letta.otel.context import get_ctx_attributes
 from letta.otel.tracing import log_event, trace_method
 from letta.schemas.agent import AgentState
-from letta.schemas.enums import FileProcessingStatus
+from letta.schemas.enums import FileProcessingStatus, VectorDBProvider
 from letta.schemas.file import FileMetadata
 from letta.schemas.passage import Passage
 from letta.schemas.user import User
@@ -30,7 +30,6 @@ class FileProcessor:
         file_parser: FileParser,
         embedder: BaseEmbedder,
         actor: User,
-        using_pinecone: bool,
         max_file_size: int = 50 * 1024 * 1024,  # 50MB default
     ):
         self.file_parser = file_parser
@@ -42,7 +41,8 @@ class FileProcessor:
         self.job_manager = JobManager()
         self.agent_manager = AgentManager()
         self.actor = actor
-        self.using_pinecone = using_pinecone
+        # get vector db type from the embedder
+        self.vector_db_type = embedder.vector_db_type
 
     async def _chunk_and_embed_with_fallback(self, file_metadata: FileMetadata, ocr_response, source_id: str) -> List:
         """Chunk text and generate embeddings with fallback to default chunker if needed"""
@@ -218,7 +218,7 @@ class FileProcessor:
                 source_id=source_id,
             )
 
-            if not self.using_pinecone:
+            if self.vector_db_type == VectorDBProvider.NATIVE:
                 all_passages = await self.passage_manager.create_many_source_passages_async(
                     passages=all_passages,
                     file_metadata=file_metadata,
@@ -241,7 +241,8 @@ class FileProcessor:
             )
 
             # update job status
-            if not self.using_pinecone:
+            # pinecone completes slowly, so gets updated later
+            if self.vector_db_type != VectorDBProvider.PINECONE:
                 await self.file_manager.update_file_status(
                     file_id=file_metadata.id,
                     actor=self.actor,
@@ -317,14 +318,15 @@ class FileProcessor:
             )
 
             # Create passages in database (unless using Pinecone)
-            if not self.using_pinecone:
+            if self.vector_db_type == VectorDBProvider.NATIVE:
                 all_passages = await self.passage_manager.create_many_source_passages_async(
                     passages=all_passages, file_metadata=file_metadata, actor=self.actor
                 )
                 log_event("file_processor.import_passages_created", {"filename": filename, "total_passages": len(all_passages)})
 
             # Update file status to completed (valid transition from EMBEDDING)
-            if not self.using_pinecone:
+            # pinecone completes slowly, so gets updated later
+            if self.vector_db_type != VectorDBProvider.PINECONE:
                 await self.file_manager.update_file_status(
                     file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.COMPLETED
                 )

letta/services/group_manager.py

@@ -1,6 +1,7 @@
+from datetime import datetime
 from typing import List, Optional, Union
 
-from sqlalchemy import delete, select
+from sqlalchemy import and_, asc, delete, desc, or_, select
 from sqlalchemy.orm import Session
 
 from letta.orm.agent import Agent as AgentModel
@@ -13,6 +14,7 @@ from letta.schemas.letta_message import LettaMessage
 from letta.schemas.message import Message as PydanticMessage
 from letta.schemas.user import User as PydanticUser
 from letta.server.db import db_registry
+from letta.settings import DatabaseChoice, settings
 from letta.utils import enforce_types
 
 
@@ -27,20 +29,34 @@ class GroupManager:
         before: Optional[str] = None,
         after: Optional[str] = None,
         limit: Optional[int] = 50,
+        show_hidden_groups: Optional[bool] = None,
     ) -> list[PydanticGroup]:
         async with db_registry.async_session() as session:
-            filters = {"organization_id": actor.organization_id}
+            from sqlalchemy import select
+
+            from letta.orm.sqlalchemy_base import AccessType
+
+            query = select(GroupModel)
+            query = GroupModel.apply_access_predicate(query, actor, ["read"], AccessType.ORGANIZATION)
+
+            # Apply filters
             if project_id:
-                filters["project_id"] = project_id
+                query = query.where(GroupModel.project_id == project_id)
             if manager_type:
-                filters["manager_type"] = manager_type
-            groups = await GroupModel.list_async(
-                db_session=session,
-                before=before,
-                after=after,
-                limit=limit,
-                **filters,
-            )
+                query = query.where(GroupModel.manager_type == manager_type)
+
+            # Apply hidden filter
+            if not show_hidden_groups:
+                query = query.where((GroupModel.hidden.is_(None)) | (GroupModel.hidden == False))
+
+            # Apply pagination
+            query = await _apply_group_pagination_async(query, before, after, session, ascending=True)
+
+            if limit:
+                query = query.limit(limit)
+
+            result = await session.execute(query)
+            groups = result.scalars().all()
             return [group.to_pydantic() for group in groups]
 
     @enforce_types
@@ -561,3 +577,50 @@ class GroupManager:
         # 3) ordering
         if max_value <= min_value:
             raise ValueError(f"'{max_name}' must be greater than '{min_name}' (got {max_name}={max_value} <= {min_name}={min_value})")
+
+
+def _cursor_filter(sort_col, id_col, ref_sort_col, ref_id, forward: bool):
+    """
+    Returns a SQLAlchemy filter expression for cursor-based pagination for groups.
+
+    If `forward` is True, returns records after the reference.
+    If `forward` is False, returns records before the reference.
+    """
+    if forward:
+        return or_(
+            sort_col > ref_sort_col,
+            and_(sort_col == ref_sort_col, id_col > ref_id),
+        )
+    else:
+        return or_(
+            sort_col < ref_sort_col,
+            and_(sort_col == ref_sort_col, id_col < ref_id),
+        )
+
+
+async def _apply_group_pagination_async(query, before: Optional[str], after: Optional[str], session, ascending: bool = True) -> any:
+    """Apply cursor-based pagination to group queries."""
+    sort_column = GroupModel.created_at
+
+    if after:
+        result = (await session.execute(select(sort_column, GroupModel.id).where(GroupModel.id == after))).first()
+        if result:
+            after_sort_value, after_id = result
+            # SQLite does not support as granular timestamping, so we need to round the timestamp
+            if settings.database_engine is DatabaseChoice.SQLITE and isinstance(after_sort_value, datetime):
+                after_sort_value = after_sort_value.strftime("%Y-%m-%d %H:%M:%S")
+            query = query.where(_cursor_filter(sort_column, GroupModel.id, after_sort_value, after_id, forward=ascending))
+
+    if before:
+        result = (await session.execute(select(sort_column, GroupModel.id).where(GroupModel.id == before))).first()
+        if result:
+            before_sort_value, before_id = result
+            # SQLite does not support as granular timestamping, so we need to round the timestamp
+            if settings.database_engine is DatabaseChoice.SQLITE and isinstance(before_sort_value, datetime):
+                before_sort_value = before_sort_value.strftime("%Y-%m-%d %H:%M:%S")
+            query = query.where(_cursor_filter(sort_column, GroupModel.id, before_sort_value, before_id, forward=not ascending))
+
+    # Apply ordering
+    order_fn = asc if ascending else desc
+    query = query.order_by(order_fn(sort_column), order_fn(GroupModel.id))
+    return query