letta-nightly 0.11.7.dev20250909104137__py3-none-any.whl → 0.11.7.dev20250911104039__py3-none-any.whl

letta/services/agent_manager.py

@@ -720,7 +720,7 @@ class AgentManager:
         # Only create messages if we initialized with messages
         if not _init_with_no_messages:
             await self.message_manager.create_many_messages_async(
-                pydantic_msgs=init_messages, actor=actor, embedding_config=result.embedding_config
+                pydantic_msgs=init_messages, actor=actor, project_id=result.project_id, template_id=result.template_id
             )
         return result
 
@@ -1834,6 +1834,7 @@ class AgentManager:
                     message_id=curr_system_message.id,
                     message_update=MessageUpdate(**temp_message.model_dump()),
                     actor=actor,
+                    project_id=agent_state.project_id,
                 )
             else:
                 curr_system_message = temp_message
@@ -1887,7 +1888,9 @@ class AgentManager:
         self, messages: List[PydanticMessage], agent_id: str, actor: PydanticUser
     ) -> PydanticAgentState:
         agent = await self.get_agent_by_id_async(agent_id=agent_id, actor=actor)
-        messages = await self.message_manager.create_many_messages_async(messages, actor=actor, embedding_config=agent.embedding_config)
+        messages = await self.message_manager.create_many_messages_async(
+            messages, actor=actor, project_id=agent.project_id, template_id=agent.template_id
+        )
         message_ids = agent.message_ids or []
         message_ids += [m.id for m in messages]
         return await self.set_in_context_messages_async(agent_id=agent_id, message_ids=message_ids, actor=actor)
@@ -2655,7 +2658,7 @@ class AgentManager:
         embedding_config: Optional[EmbeddingConfig] = None,
         tags: Optional[List[str]] = None,
         tag_match_mode: Optional[TagMatchMode] = None,
-    ) -> List[PydanticPassage]:
+    ) -> List[Tuple[PydanticPassage, float, dict]]:
         """Lists all passages attached to an agent."""
         # Check if we should use Turbopuffer for vector search
         if embed_query and agent_id and query_text and embedding_config:
@@ -2688,7 +2691,6 @@ class AgentManager:
                     # use hybrid search to combine vector and full-text search
                     passages_with_scores = await tpuf_client.query_passages(
                         archive_id=archive_ids[0],
-                        query_embedding=query_embedding,
                         query_text=query_text,  # pass text for potential hybrid search
                         search_mode="hybrid",  # use hybrid mode for better results
                         top_k=limit,
@@ -2696,10 +2698,11 @@ class AgentManager:
                         tag_match_mode=tag_match_mode or TagMatchMode.ANY,
                         start_date=start_date,
                         end_date=end_date,
+                        actor=actor,
                     )
 
-                    # Return just the passages (without scores)
-                    return [passage for passage, _ in passages_with_scores]
+                    # Return full tuples with metadata
+                    return passages_with_scores
             else:
                 return []
 
@@ -2750,9 +2753,11 @@ class AgentManager:
                             if query_tags.intersection(passage_tags):
                                 filtered_passages.append(passage)
 
-                return filtered_passages
+                # Return as tuples with empty metadata for SQL path
+                return [(p, 0.0, {}) for p in filtered_passages]
 
-            return pydantic_passages
+            # Return as tuples with empty metadata for SQL path
+            return [(p, 0.0, {}) for p in pydantic_passages]
 
     @enforce_types
     @trace_method
@@ -2766,7 +2771,7 @@ class AgentManager:
         top_k: Optional[int] = None,
         start_datetime: Optional[str] = None,
         end_datetime: Optional[str] = None,
-    ) -> Tuple[List[Dict[str, Any]], int]:
+    ) -> List[Dict[str, Any]]:
         """
         Search archival memory using semantic (embedding-based) search with optional temporal filtering.
 
@@ -2783,11 +2788,11 @@ class AgentManager:
             end_datetime: Filter results before this datetime (ISO 8601 format)
 
         Returns:
-            Tuple of (formatted_results, count)
+            List of formatted results with relevance metadata
         """
         # Handle empty or whitespace-only queries
         if not query or not query.strip():
-            return [], 0
+            return []
 
         # Get the agent to access timezone and embedding config
         agent_state = await self.get_agent_by_id_async(agent_id=agent_id, actor=actor)
@@ -2839,7 +2844,7 @@ class AgentManager:
 
         # Get results using existing passage query method
         limit = top_k if top_k is not None else RETRIEVAL_QUERY_DEFAULT_PAGE_SIZE
-        all_results = await self.query_agent_passages_async(
+        passages_with_metadata = await self.query_agent_passages_async(
             actor=actor,
             agent_id=agent_id,
             query_text=query,
@@ -2852,11 +2857,11 @@ class AgentManager:
             end_date=end_date,
         )
 
-        # Format results to include tags with friendly timestamps
+        # Format results to include tags with friendly timestamps and relevance metadata
         formatted_results = []
-        for result in all_results:
+        for passage, score, metadata in passages_with_metadata:
             # Format timestamp in agent's timezone if available
-            timestamp = result.created_at
+            timestamp = passage.created_at
             if timestamp and agent_state.timezone:
                 try:
                     # Convert to agent's timezone
@@ -2871,9 +2876,26 @@ class AgentManager:
                 # Use ISO format if no timezone is set
                 formatted_timestamp = str(timestamp) if timestamp else "Unknown"
 
-            formatted_results.append({"timestamp": formatted_timestamp, "content": result.text, "tags": result.tags or []})
+            result_dict = {"timestamp": formatted_timestamp, "content": passage.text, "tags": passage.tags or []}
+
+            # Add relevance metadata if available
+            if metadata:
+                relevance_info = {
+                    k: v
+                    for k, v in {
+                        "rrf_score": metadata.get("combined_score"),
+                        "vector_rank": metadata.get("vector_rank"),
+                        "fts_rank": metadata.get("fts_rank"),
+                    }.items()
+                    if v is not None
+                }
+
+                if relevance_info:  # Only add if we have metadata
+                    result_dict["relevance"] = relevance_info
+
+            formatted_results.append(result_dict)
 
-        return formatted_results, len(formatted_results)
+        return formatted_results
 
     @enforce_types
     @trace_method
@@ -3698,45 +3720,3 @@ class AgentManager:
             num_archival_memories=num_archival_memories,
             num_messages=num_messages,
         )
-
-    async def get_or_set_vector_db_namespace_async(
-        self,
-        agent_id: str,
-        organization_id: str,
-    ) -> str:
-        """Get the vector database namespace for an agent, creating it if it doesn't exist.
-
-        Args:
-            agent_id: Agent ID to check/store namespace
-            organization_id: Organization ID for namespace generation
-
-        Returns:
-            The org-scoped namespace name
-        """
-        from sqlalchemy import update
-
-        from letta.settings import settings
-
-        async with db_registry.async_session() as session:
-            # check if namespace already exists
-            result = await session.execute(select(AgentModel._vector_db_namespace).where(AgentModel.id == agent_id))
-            row = result.fetchone()
-
-            if row and row[0]:
-                return row[0]
-
-            # TODO: In the future, we might use agent_id for sharding the namespace
-            # For now, all messages in an org share the same namespace
-
-            # generate org-scoped namespace name
-            environment = settings.environment
-            if environment:
-                namespace_name = f"messages_{organization_id}_{environment.lower()}"
-            else:
-                namespace_name = f"messages_{organization_id}"
-
-            # update the agent with the namespace (keeps agent-level tracking for future sharding)
-            await session.execute(update(AgentModel).where(AgentModel.id == agent_id).values(_vector_db_namespace=namespace_name))
-            await session.commit()
-
-            return namespace_name

letta/services/agent_serialization_manager.py

@@ -12,6 +12,7 @@ from letta.errors import (
     AgentNotFoundForExportError,
 )
 from letta.helpers.pinecone_utils import should_use_pinecone
+from letta.helpers.tpuf_client import should_use_tpuf
 from letta.log import get_logger
 from letta.schemas.agent import AgentState, CreateAgent
 from letta.schemas.agent_file import (
@@ -29,7 +30,7 @@ from letta.schemas.agent_file import (
 )
 from letta.schemas.block import Block
 from letta.schemas.embedding_config import EmbeddingConfig
-from letta.schemas.enums import FileProcessingStatus
+from letta.schemas.enums import FileProcessingStatus, VectorDBProvider
 from letta.schemas.file import FileMetadata
 from letta.schemas.group import Group, GroupCreate
 from letta.schemas.mcp import MCPServer
@@ -52,7 +53,7 @@ from letta.services.message_manager import MessageManager
 from letta.services.source_manager import SourceManager
 from letta.services.tool_manager import ToolManager
 from letta.settings import settings
-from letta.utils import get_latest_alembic_revision
+from letta.utils import get_latest_alembic_revision, safe_create_task
 
 logger = get_logger(__name__)
 
@@ -90,7 +91,6 @@ class AgentSerializationManager:
         self.file_agent_manager = file_agent_manager
         self.message_manager = message_manager
         self.file_parser = MistralFileParser() if settings.mistral_api_key else MarkitdownFileParser()
-        self.using_pinecone = should_use_pinecone()
 
         # ID mapping state for export
         self._db_to_file_ids: Dict[str, str] = {}
@@ -208,6 +208,10 @@ class AgentSerializationManager:
         )
         agent_schema.id = agent_file_id
 
+        # wipe the values of tool_exec_environment_variables (they contain secrets)
+        if agent_schema.tool_exec_environment_variables:
+            agent_schema.tool_exec_environment_variables = {key: "" for key in agent_schema.tool_exec_environment_variables}
+
         if agent_schema.messages:
             for message in agent_schema.messages:
                 message_file_id = self._map_db_to_file_id(message.id, MessageSchema.__id_prefix__)
@@ -588,7 +592,12 @@ class AgentSerializationManager:
             if schema.files and any(f.content for f in schema.files):
                 # Use override embedding config if provided, otherwise use agent's config
                 embedder_config = override_embedding_config if override_embedding_config else schema.agents[0].embedding_config
-                if should_use_pinecone():
+                # determine which embedder to use - turbopuffer takes precedence
+                if should_use_tpuf():
+                    from letta.services.file_processor.embedder.turbopuffer_embedder import TurbopufferEmbedder
+
+                    embedder = TurbopufferEmbedder(embedding_config=embedder_config)
+                elif should_use_pinecone():
                     embedder = PineconeEmbedder(embedding_config=embedder_config)
                 else:
                     embedder = OpenAIEmbedder(embedding_config=embedder_config)
@@ -596,7 +605,6 @@ class AgentSerializationManager:
                     file_parser=self.file_parser,
                     embedder=embedder,
                     actor=actor,
-                    using_pinecone=self.using_pinecone,
                 )
 
                 for file_schema in schema.files:
@@ -614,10 +622,11 @@ class AgentSerializationManager:
 
                         # Create background task for file processing
                         # TODO: This can be moved to celery or RQ or something
-                        task = asyncio.create_task(
+                        task = safe_create_task(
                             self._process_file_async(
                                 file_metadata=file_metadata, source_id=source_db_id, file_processor=file_processor, actor=actor
-                            )
+                            ),
+                            label=f"process_file_{file_metadata.file_name}",
                         )
                         background_tasks.append(task)
                         logger.info(f"Started background processing for file {file_metadata.file_name} (ID: {file_db_id})")
@@ -646,9 +655,10 @@ class AgentSerializationManager:
                 if agent_data.get("source_ids"):
                     agent_data["source_ids"] = [file_to_db_ids[file_id] for file_id in agent_data["source_ids"]]
 
-                if env_vars:
-                    for var in agent_data["tool_exec_environment_variables"]:
-                        var["value"] = env_vars.get(var["key"], "")
+                if env_vars and agent_data.get("tool_exec_environment_variables"):
+                    # update environment variable values from the provided env_vars dict
+                    for key in agent_data["tool_exec_environment_variables"]:
+                        agent_data["tool_exec_environment_variables"][key] = env_vars.get(key, "")
 
                 # Override project_id if provided
                 if project_id:
@@ -675,7 +685,12 @@ class AgentSerializationManager:
                     # Map file ID to the generated database ID immediately
                     message_file_to_db_ids[message_schema.id] = message_obj.id
 
-                created_messages = await self.message_manager.create_many_messages_async(pydantic_msgs=messages, actor=actor)
+                created_messages = await self.message_manager.create_many_messages_async(
+                    pydantic_msgs=messages,
+                    actor=actor,
+                    project_id=created_agent.project_id,
+                    template_id=created_agent.template_id,
+                )
                 imported_count += len(created_messages)
 
                 # Remap in_context_message_ids from file IDs to database IDs

letta/services/archive_manager.py

@@ -5,6 +5,7 @@ from sqlalchemy import select
 from letta.helpers.tpuf_client import should_use_tpuf
 from letta.log import get_logger
 from letta.orm import ArchivalPassage, Archive as ArchiveModel, ArchivesAgents
+from letta.otel.tracing import trace_method
 from letta.schemas.archive import Archive as PydanticArchive
 from letta.schemas.enums import VectorDBProvider
 from letta.schemas.user import User as PydanticUser
@@ -19,6 +20,7 @@ class ArchiveManager:
     """Manager class to handle business logic related to Archives."""
 
     @enforce_types
+    @trace_method
     def create_archive(
         self,
         name: str,
@@ -44,6 +46,7 @@ class ArchiveManager:
             raise
 
     @enforce_types
+    @trace_method
     async def create_archive_async(
         self,
         name: str,
@@ -69,6 +72,7 @@ class ArchiveManager:
             raise
 
     @enforce_types
+    @trace_method
     async def get_archive_by_id_async(
         self,
         archive_id: str,
@@ -84,6 +88,7 @@ class ArchiveManager:
             return archive.to_pydantic()
 
     @enforce_types
+    @trace_method
     def attach_agent_to_archive(
         self,
         agent_id: str,
@@ -113,6 +118,7 @@ class ArchiveManager:
             session.commit()
 
     @enforce_types
+    @trace_method
     async def attach_agent_to_archive_async(
         self,
         agent_id: str,
@@ -148,6 +154,7 @@ class ArchiveManager:
             await session.commit()
 
     @enforce_types
+    @trace_method
     async def get_default_archive_for_agent_async(
         self,
         agent_id: str,
@@ -179,6 +186,24 @@ class ArchiveManager:
         return None
 
     @enforce_types
+    @trace_method
+    async def delete_archive_async(
+        self,
+        archive_id: str,
+        actor: PydanticUser = None,
+    ) -> None:
+        """Delete an archive permanently."""
+        async with db_registry.async_session() as session:
+            archive_model = await ArchiveModel.read_async(
+                db_session=session,
+                identifier=archive_id,
+                actor=actor,
+            )
+            await archive_model.hard_delete_async(session, actor=actor)
+            logger.info(f"Deleted archive {archive_id}")
+
+    @enforce_types
+    @trace_method
     async def get_or_create_default_archive_for_agent_async(
         self,
         agent_id: str,
@@ -187,6 +212,8 @@ class ArchiveManager:
     ) -> PydanticArchive:
         """Get the agent's default archive, creating one if it doesn't exist."""
         # First check if agent has any archives
+        from sqlalchemy.exc import IntegrityError
+
         from letta.services.agent_manager import AgentManager
 
         agent_manager = AgentManager()
@@ -215,17 +242,38 @@ class ArchiveManager:
             actor=actor,
         )
 
-        # Attach the agent to the archive as owner
-        await self.attach_agent_to_archive_async(
-            agent_id=agent_id,
-            archive_id=archive.id,
-            is_owner=True,
-            actor=actor,
-        )
-
-        return archive
+        try:
+            # Attach the agent to the archive as owner
+            await self.attach_agent_to_archive_async(
+                agent_id=agent_id,
+                archive_id=archive.id,
+                is_owner=True,
+                actor=actor,
+            )
+            return archive
+        except IntegrityError:
+            # race condition: another concurrent request already created and attached an archive
+            # clean up the orphaned archive we just created
+            logger.info(f"Race condition detected for agent {agent_id}, cleaning up orphaned archive {archive.id}")
+            await self.delete_archive_async(archive_id=archive.id, actor=actor)
+
+            # fetch the existing archive that was created by the concurrent request
+            archive_ids = await agent_manager.get_agent_archive_ids_async(
+                agent_id=agent_id,
+                actor=actor,
+            )
+            if archive_ids:
+                archive = await self.get_archive_by_id_async(
+                    archive_id=archive_ids[0],
+                    actor=actor,
+                )
+                return archive
+            else:
+                # this shouldn't happen, but if it does, re-raise
+                raise
 
     @enforce_types
+    @trace_method
     def get_or_create_default_archive_for_agent(
         self,
         agent_id: str,
@@ -269,6 +317,7 @@ class ArchiveManager:
         return archive_model.to_pydantic()
 
     @enforce_types
+    @trace_method
     async def get_agents_for_archive_async(
         self,
         archive_id: str,
@@ -280,6 +329,7 @@ class ArchiveManager:
             return [row[0] for row in result.fetchall()]
 
     @enforce_types
+    @trace_method
     async def get_agent_from_passage_async(
         self,
         passage_id: str,
@@ -309,6 +359,7 @@ class ArchiveManager:
             return agent_ids[0]
 
     @enforce_types
+    @trace_method
     async def get_or_set_vector_db_namespace_async(
         self,
         archive_id: str,

letta/services/block_manager.py

@@ -188,6 +188,7 @@ class BlockManager:
         connected_to_agents_count_lt: Optional[int] = None,
         connected_to_agents_count_eq: Optional[List[int]] = None,
         ascending: bool = True,
+        show_hidden_blocks: Optional[bool] = None,
     ) -> List[PydanticBlock]:
         """Async version of get_blocks method. Retrieve blocks based on various optional filters."""
         from sqlalchemy import select
@@ -228,6 +229,10 @@ class BlockManager:
             if value_search:
                 query = query.where(BlockModel.value.ilike(f"%{value_search}%"))
 
+            # Apply hidden filter
+            if not show_hidden_blocks:
+                query = query.where((BlockModel.hidden.is_(None)) | (BlockModel.hidden == False))
+
             needs_distinct = False
 
             needs_agent_count_join = any(

letta/services/file_processor/embedder/base_embedder.py

@@ -2,6 +2,7 @@ from abc import ABC, abstractmethod
 from typing import List
 
 from letta.log import get_logger
+from letta.schemas.enums import VectorDBProvider
 from letta.schemas.passage import Passage
 from letta.schemas.user import User
 
@@ -11,6 +12,10 @@ logger = get_logger(__name__)
 class BaseEmbedder(ABC):
     """Abstract base class for embedding generation"""
 
+    def __init__(self):
+        # Default to NATIVE, subclasses will override this
+        self.vector_db_type = VectorDBProvider.NATIVE
+
     @abstractmethod
     async def generate_embedded_passages(self, file_id: str, source_id: str, chunks: List[str], actor: User) -> List[Passage]:
         """Generate embeddings for chunks with batching and concurrent processing"""

letta/services/file_processor/embedder/openai_embedder.py

@@ -19,6 +19,10 @@ class OpenAIEmbedder(BaseEmbedder):
     """OpenAI-based embedding generation"""
 
     def __init__(self, embedding_config: Optional[EmbeddingConfig] = None):
+        super().__init__()
+        # OpenAI embedder uses the native vector db (PostgreSQL)
+        # self.vector_db_type already set to VectorDBProvider.NATIVE by parent
+
         self.default_embedding_config = (
             EmbeddingConfig.default_config(model_name="text-embedding-3-small", provider="openai")
             if model_settings.openai_api_key

letta/services/file_processor/embedder/pinecone_embedder.py

@@ -4,6 +4,7 @@ from letta.helpers.pinecone_utils import upsert_file_records_to_pinecone_index
 from letta.log import get_logger
 from letta.otel.tracing import log_event, trace_method
 from letta.schemas.embedding_config import EmbeddingConfig
+from letta.schemas.enums import VectorDBProvider
 from letta.schemas.passage import Passage
 from letta.schemas.user import User
 from letta.services.file_processor.embedder.base_embedder import BaseEmbedder
@@ -20,6 +21,10 @@ class PineconeEmbedder(BaseEmbedder):
     """Pinecone-based embedding generation"""
 
     def __init__(self, embedding_config: Optional[EmbeddingConfig] = None):
+        super().__init__()
+        # set the vector db type for pinecone
+        self.vector_db_type = VectorDBProvider.PINECONE
+
         if not PINECONE_AVAILABLE:
             raise ImportError("Pinecone package is not installed. Install it with: pip install pinecone")
 
@@ -28,7 +33,6 @@ class PineconeEmbedder(BaseEmbedder):
             embedding_config = EmbeddingConfig.default_config(provider="pinecone")
 
         self.embedding_config = embedding_config
-        super().__init__()
 
     @trace_method
     async def generate_embedded_passages(self, file_id: str, source_id: str, chunks: List[str], actor: User) -> List[Passage]:

letta/services/file_processor/embedder/turbopuffer_embedder.py

@@ -0,0 +1,71 @@
+from typing import List, Optional
+
+from letta.helpers.tpuf_client import TurbopufferClient
+from letta.log import get_logger
+from letta.otel.tracing import log_event, trace_method
+from letta.schemas.embedding_config import EmbeddingConfig
+from letta.schemas.enums import VectorDBProvider
+from letta.schemas.passage import Passage
+from letta.schemas.user import User
+from letta.services.file_processor.embedder.base_embedder import BaseEmbedder
+
+logger = get_logger(__name__)
+
+
+class TurbopufferEmbedder(BaseEmbedder):
+    """Turbopuffer-based embedding generation and storage"""
+
+    def __init__(self, embedding_config: Optional[EmbeddingConfig] = None):
+        super().__init__()
+        # set the vector db type for turbopuffer
+        self.vector_db_type = VectorDBProvider.TPUF
+        # use the default embedding config from TurbopufferClient if not provided
+        self.embedding_config = embedding_config or TurbopufferClient.default_embedding_config
+        self.tpuf_client = TurbopufferClient()
+
+    @trace_method
+    async def generate_embedded_passages(self, file_id: str, source_id: str, chunks: List[str], actor: User) -> List[Passage]:
+        """Generate embeddings and store in Turbopuffer, then return Passage objects"""
+        if not chunks:
+            return []
+
+        logger.info(f"Generating embeddings for {len(chunks)} chunks using Turbopuffer")
+        log_event(
+            "turbopuffer_embedder.generation_started",
+            {
+                "total_chunks": len(chunks),
+                "file_id": file_id,
+                "source_id": source_id,
+                "embedding_model": self.embedding_config.embedding_model,
+            },
+        )
+
+        try:
+            # insert passages to Turbopuffer - it will handle embedding generation internally
+            passages = await self.tpuf_client.insert_file_passages(
+                source_id=source_id,
+                file_id=file_id,
+                text_chunks=chunks,
+                organization_id=actor.organization_id,
+                actor=actor,
+            )
+
+            logger.info(f"Successfully generated and stored {len(passages)} passages in Turbopuffer")
+            log_event(
+                "turbopuffer_embedder.generation_completed",
+                {
+                    "passages_created": len(passages),
+                    "total_chunks_processed": len(chunks),
+                    "file_id": file_id,
+                    "source_id": source_id,
+                },
+            )
+            return passages
+
+        except Exception as e:
+            logger.error(f"Failed to generate embeddings with Turbopuffer: {str(e)}")
+            log_event(
+                "turbopuffer_embedder.generation_failed",
+                {"error": str(e), "error_type": type(e).__name__, "file_id": file_id, "source_id": source_id},
+            )
+            raise

letta/services/file_processor/file_processor.py

@@ -6,7 +6,7 @@ from letta.log import get_logger
 from letta.otel.context import get_ctx_attributes
 from letta.otel.tracing import log_event, trace_method
 from letta.schemas.agent import AgentState
-from letta.schemas.enums import FileProcessingStatus
+from letta.schemas.enums import FileProcessingStatus, VectorDBProvider
 from letta.schemas.file import FileMetadata
 from letta.schemas.passage import Passage
 from letta.schemas.user import User
@@ -30,7 +30,6 @@ class FileProcessor:
         file_parser: FileParser,
         embedder: BaseEmbedder,
         actor: User,
-        using_pinecone: bool,
         max_file_size: int = 50 * 1024 * 1024,  # 50MB default
     ):
         self.file_parser = file_parser
@@ -42,7 +41,8 @@ class FileProcessor:
         self.job_manager = JobManager()
         self.agent_manager = AgentManager()
         self.actor = actor
-        self.using_pinecone = using_pinecone
+        # get vector db type from the embedder
+        self.vector_db_type = embedder.vector_db_type
 
     async def _chunk_and_embed_with_fallback(self, file_metadata: FileMetadata, ocr_response, source_id: str) -> List:
         """Chunk text and generate embeddings with fallback to default chunker if needed"""
@@ -218,7 +218,7 @@ class FileProcessor:
                 source_id=source_id,
             )
 
-            if not self.using_pinecone:
+            if self.vector_db_type == VectorDBProvider.NATIVE:
                 all_passages = await self.passage_manager.create_many_source_passages_async(
                     passages=all_passages,
                     file_metadata=file_metadata,
@@ -241,7 +241,8 @@ class FileProcessor:
             )
 
             # update job status
-            if not self.using_pinecone:
+            # pinecone completes slowly, so gets updated later
+            if self.vector_db_type != VectorDBProvider.PINECONE:
                 await self.file_manager.update_file_status(
                     file_id=file_metadata.id,
                     actor=self.actor,
@@ -317,14 +318,15 @@ class FileProcessor:
             )
 
             # Create passages in database (unless using Pinecone)
-            if not self.using_pinecone:
+            if self.vector_db_type == VectorDBProvider.NATIVE:
                 all_passages = await self.passage_manager.create_many_source_passages_async(
                     passages=all_passages, file_metadata=file_metadata, actor=self.actor
                 )
                 log_event("file_processor.import_passages_created", {"filename": filename, "total_passages": len(all_passages)})
 
             # Update file status to completed (valid transition from EMBEDDING)
-            if not self.using_pinecone:
+            # pinecone completes slowly, so gets updated later
+            if self.vector_db_type != VectorDBProvider.PINECONE:
                 await self.file_manager.update_file_status(
                     file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.COMPLETED
                 )