letta-nightly 0.8.17.dev20250723104501__py3-none-any.whl → 0.9.0.dev20250724081419__py3-none-any.whl

letta/services/block_manager.py

@@ -1,7 +1,8 @@
 import asyncio
+from datetime import datetime
 from typing import Dict, List, Optional
 
-from sqlalchemy import delete, select
+from sqlalchemy import delete, or_, select
 from sqlalchemy.orm import Session
 
 from letta.log import get_logger
@@ -17,6 +18,7 @@ from letta.schemas.block import BlockUpdate
 from letta.schemas.enums import ActorType
 from letta.schemas.user import User as PydanticUser
 from letta.server.db import db_registry
+from letta.settings import DatabaseChoice, settings
 from letta.utils import enforce_types
 
 logger = get_logger(__name__)
@@ -176,7 +178,10 @@ class BlockManager:
         template_name: Optional[str] = None,
         identity_id: Optional[str] = None,
         identifier_keys: Optional[List[str]] = None,
+        before: Optional[str] = None,
+        after: Optional[str] = None,
         limit: Optional[int] = 50,
+        ascending: bool = True,
     ) -> List[PydanticBlock]:
         """Async version of get_blocks method. Retrieve blocks based on various optional filters."""
         from sqlalchemy import select
@@ -205,19 +210,67 @@ class BlockManager:
             if template_name:
                 query = query.where(BlockModel.template_name == template_name)
 
+            needs_distinct = False
+
             if identifier_keys:
-                query = (
-                    query.join(BlockModel.identities)
-                    .filter(BlockModel.identities.property.mapper.class_.identifier_key.in_(identifier_keys))
-                    .distinct(BlockModel.id)
+                query = query.join(BlockModel.identities).filter(
+                    BlockModel.identities.property.mapper.class_.identifier_key.in_(identifier_keys)
                 )
+                needs_distinct = True
 
             if identity_id:
-                query = (
-                    query.join(BlockModel.identities)
-                    .filter(BlockModel.identities.property.mapper.class_.id == identity_id)
-                    .distinct(BlockModel.id)
-                )
+                query = query.join(BlockModel.identities).filter(BlockModel.identities.property.mapper.class_.id == identity_id)
+                needs_distinct = True
+
+            if after:
+                result = (await session.execute(select(BlockModel.created_at, BlockModel.id).where(BlockModel.id == after))).first()
+                if result:
+                    after_sort_value, after_id = result
+                    # SQLite does not support as granular timestamping, so we need to round the timestamp
+                    if settings.database_engine is DatabaseChoice.SQLITE and isinstance(after_sort_value, datetime):
+                        after_sort_value = after_sort_value.strftime("%Y-%m-%d %H:%M:%S")
+
+                    if ascending:
+                        query = query.where(
+                            BlockModel.created_at > after_sort_value,
+                            or_(BlockModel.created_at == after_sort_value, BlockModel.id > after_id),
+                        )
+                    else:
+                        query = query.where(
+                            BlockModel.created_at < after_sort_value,
+                            or_(BlockModel.created_at == after_sort_value, BlockModel.id < after_id),
+                        )
+
+            if before:
+                result = (await session.execute(select(BlockModel.created_at, BlockModel.id).where(BlockModel.id == before))).first()
+                if result:
+                    before_sort_value, before_id = result
+                    # SQLite does not support as granular timestamping, so we need to round the timestamp
+                    if settings.database_engine is DatabaseChoice.SQLITE and isinstance(before_sort_value, datetime):
+                        before_sort_value = before_sort_value.strftime("%Y-%m-%d %H:%M:%S")
+
+                    if ascending:
+                        query = query.where(
+                            BlockModel.created_at < before_sort_value,
+                            or_(BlockModel.created_at == before_sort_value, BlockModel.id < before_id),
+                        )
+                    else:
+                        query = query.where(
+                            BlockModel.created_at > before_sort_value,
+                            or_(BlockModel.created_at == before_sort_value, BlockModel.id > before_id),
+                        )
+
+            # Apply ordering and handle distinct if needed
+            if needs_distinct:
+                if ascending:
+                    query = query.distinct(BlockModel.id).order_by(BlockModel.id.asc(), BlockModel.created_at.asc())
+                else:
+                    query = query.distinct(BlockModel.id).order_by(BlockModel.id.desc(), BlockModel.created_at.desc())
+            else:
+                if ascending:
+                    query = query.order_by(BlockModel.created_at.asc(), BlockModel.id.asc())
+                else:
+                    query = query.order_by(BlockModel.created_at.desc(), BlockModel.id.desc())
 
             # Add limit
             if limit:
@@ -306,19 +359,87 @@ class BlockManager:
         block_id: str,
         actor: PydanticUser,
         include_relationships: Optional[List[str]] = None,
+        before: Optional[str] = None,
+        after: Optional[str] = None,
+        limit: Optional[int] = 50,
+        ascending: bool = True,
     ) -> List[PydanticAgentState]:
         """
-        Retrieve all agents associated with a given block.
+        Retrieve all agents associated with a given block with pagination support.
+
+        Args:
+            block_id: ID of the block to get agents for
+            actor: User performing the operation
+            include_relationships: List of relationships to include in the response
+            before: Cursor for pagination (get items before this ID)
+            after: Cursor for pagination (get items after this ID)
+            limit: Maximum number of items to return
+            ascending: Sort order (True for ascending, False for descending)
+
+        Returns:
+            List of agent states associated with the block
         """
         async with db_registry.async_session() as session:
+            # Start with a basic query
             query = (
                 select(AgentModel)
                 .where(AgentModel.id.in_(select(BlocksAgents.agent_id).where(BlocksAgents.block_id == block_id)))
                 .where(AgentModel.organization_id == actor.organization_id)
             )
 
+            # Apply pagination using cursor-based approach
+            if after:
+                result = (await session.execute(select(AgentModel.created_at, AgentModel.id).where(AgentModel.id == after))).first()
+                if result:
+                    after_sort_value, after_id = result
+                    # SQLite does not support as granular timestamping, so we need to round the timestamp
+                    if settings.database_engine is DatabaseChoice.SQLITE and isinstance(after_sort_value, datetime):
+                        after_sort_value = after_sort_value.strftime("%Y-%m-%d %H:%M:%S")
+
+                    if ascending:
+                        query = query.where(
+                            AgentModel.created_at > after_sort_value,
+                            or_(AgentModel.created_at == after_sort_value, AgentModel.id > after_id),
+                        )
+                    else:
+                        query = query.where(
+                            AgentModel.created_at < after_sort_value,
+                            or_(AgentModel.created_at == after_sort_value, AgentModel.id < after_id),
+                        )
+
+            if before:
+                result = (await session.execute(select(AgentModel.created_at, AgentModel.id).where(AgentModel.id == before))).first()
+                if result:
+                    before_sort_value, before_id = result
+                    # SQLite does not support as granular timestamping, so we need to round the timestamp
+                    if settings.database_engine is DatabaseChoice.SQLITE and isinstance(before_sort_value, datetime):
+                        before_sort_value = before_sort_value.strftime("%Y-%m-%d %H:%M:%S")
+
+                    if ascending:
+                        query = query.where(
+                            AgentModel.created_at < before_sort_value,
+                            or_(AgentModel.created_at == before_sort_value, AgentModel.id < before_id),
+                        )
+                    else:
+                        query = query.where(
+                            AgentModel.created_at > before_sort_value,
+                            or_(AgentModel.created_at == before_sort_value, AgentModel.id > before_id),
+                        )
+
+            # Apply sorting
+            if ascending:
+                query = query.order_by(AgentModel.created_at.asc(), AgentModel.id.asc())
+            else:
+                query = query.order_by(AgentModel.created_at.desc(), AgentModel.id.desc())
+
+            # Apply limit
+            if limit:
+                query = query.limit(limit)
+
+            # Execute the query
             result = await session.execute(query)
             agents_orm = result.scalars().all()
+
             agents = await asyncio.gather(*[agent.to_pydantic_async(include_relationships=include_relationships) for agent in agents_orm])
             return agents
 

letta/services/file_manager.py

@@ -9,7 +9,6 @@ from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import selectinload
 
 from letta.constants import MAX_FILENAME_LENGTH
-from letta.helpers.decorators import async_redis_cache
 from letta.orm.errors import NoResultFound
 from letta.orm.file import FileContent as FileContentModel
 from letta.orm.file import FileMetadata as FileMetadataModel
@@ -38,13 +37,14 @@ class FileManager:
 
     async def _invalidate_file_caches(self, file_id: str, actor: PydanticUser, original_filename: str = None, source_id: str = None):
         """Invalidate all caches related to a file."""
-        # invalidate file content cache (all variants)
-        await self.get_file_by_id.cache_invalidate(self, file_id, actor, include_content=True)
-        await self.get_file_by_id.cache_invalidate(self, file_id, actor, include_content=False)
+        # TEMPORARILY DISABLED - caching is disabled
+        # # invalidate file content cache (all variants)
+        # await self.get_file_by_id.cache_invalidate(self, file_id, actor, include_content=True)
+        # await self.get_file_by_id.cache_invalidate(self, file_id, actor, include_content=False)
 
-        # invalidate filename-based cache if we have the info
-        if original_filename and source_id:
-            await self.get_file_by_original_name_and_source.cache_invalidate(self, original_filename, source_id, actor)
+        # # invalidate filename-based cache if we have the info
+        # if original_filename and source_id:
+        #     await self.get_file_by_original_name_and_source.cache_invalidate(self, original_filename, source_id, actor)
 
     @enforce_types
     @trace_method
@@ -86,12 +86,12 @@ class FileManager:
     # TODO: We make actor optional for now, but should most likely be enforced due to security reasons
     @enforce_types
     @trace_method
-    @async_redis_cache(
-        key_func=lambda self, file_id, actor=None, include_content=False, strip_directory_prefix=False: f"{file_id}:{actor.organization_id if actor else 'none'}:{include_content}:{strip_directory_prefix}",
-        prefix="file_content",
-        ttl_s=3600,
-        model_class=PydanticFileMetadata,
-    )
+    # @async_redis_cache(
+    #     key_func=lambda self, file_id, actor=None, include_content=False, strip_directory_prefix=False: f"{file_id}:{actor.organization_id if actor else 'none'}:{include_content}:{strip_directory_prefix}",
+    #     prefix="file_content",
+    #     ttl_s=3600,
+    #     model_class=PydanticFileMetadata,
+    # )
     async def get_file_by_id(
         self, file_id: str, actor: Optional[PydanticUser] = None, *, include_content: bool = False, strip_directory_prefix: bool = False
     ) -> Optional[PydanticFileMetadata]:
@@ -143,12 +143,31 @@ class FileManager:
         error_message: Optional[str] = None,
         total_chunks: Optional[int] = None,
         chunks_embedded: Optional[int] = None,
-    ) -> PydanticFileMetadata:
+        enforce_state_transitions: bool = True,
+    ) -> Optional[PydanticFileMetadata]:
         """
         Update processing_status, error_message, total_chunks, and/or chunks_embedded on a FileMetadata row.
 
-        * 1st round-trip → UPDATE
-        * 2nd round-trip → SELECT fresh row (same as read_async)
+        Enforces state transition rules (when enforce_state_transitions=True):
+        - PENDING -> PARSING -> EMBEDDING -> COMPLETED (normal flow)
+        - Any non-terminal state -> ERROR
+        - ERROR and COMPLETED are terminal (no transitions allowed)
+
+        Args:
+            file_id: ID of the file to update
+            actor: User performing the update
+            processing_status: New processing status to set
+            error_message: Error message to set (if any)
+            total_chunks: Total number of chunks in the file
+            chunks_embedded: Number of chunks already embedded
+            enforce_state_transitions: Whether to enforce state transition rules (default: True).
+                                     Set to False to bypass validation for testing or special cases.
+
+        Returns:
+            Updated file metadata, or None if the update was blocked
+
+        * 1st round-trip → UPDATE with optional state validation
+        * 2nd round-trip → SELECT fresh row (same as read_async) if update succeeded
         """
 
         if processing_status is None and error_message is None and total_chunks is None and chunks_embedded is None:
@@ -164,23 +183,79 @@ class FileManager:
         if chunks_embedded is not None:
             values["chunks_embedded"] = chunks_embedded
 
+        # validate state transitions before making any database calls
+        if enforce_state_transitions and processing_status == FileProcessingStatus.PENDING:
+            # PENDING cannot be set after initial creation
+            raise ValueError(f"Cannot transition to PENDING state for file {file_id} - PENDING is only valid as initial state")
+
         async with db_registry.async_session() as session:
-            # Fast in-place update – no ORM hydration
+            # build where conditions
+            where_conditions = [
+                FileMetadataModel.id == file_id,
+                FileMetadataModel.organization_id == actor.organization_id,
+            ]
+
+            # only add state transition validation if enforce_state_transitions is True
+            if enforce_state_transitions:
+                # prevent updates to terminal states (ERROR, COMPLETED)
+                where_conditions.append(
+                    FileMetadataModel.processing_status.notin_([FileProcessingStatus.ERROR, FileProcessingStatus.COMPLETED])
+                )
+
+                if processing_status is not None:
+                    # enforce specific transitions based on target status
+                    if processing_status == FileProcessingStatus.PARSING:
+                        where_conditions.append(FileMetadataModel.processing_status == FileProcessingStatus.PENDING)
+                    elif processing_status == FileProcessingStatus.EMBEDDING:
+                        where_conditions.append(FileMetadataModel.processing_status == FileProcessingStatus.PARSING)
+                    elif processing_status == FileProcessingStatus.COMPLETED:
+                        where_conditions.append(FileMetadataModel.processing_status == FileProcessingStatus.EMBEDDING)
+                    # ERROR can be set from any non-terminal state (already handled by terminal check above)
+
+            # fast in-place update with state validation
             stmt = (
                 update(FileMetadataModel)
-                .where(
-                    FileMetadataModel.id == file_id,
-                    FileMetadataModel.organization_id == actor.organization_id,
-                )
+                .where(*where_conditions)
                 .values(**values)
+                .returning(FileMetadataModel.id)  # return id if update succeeded
             )
-            await session.execute(stmt)
+            result = await session.execute(stmt)
+            updated_id = result.scalar()
+
+            if not updated_id:
+                # update was blocked
+                await session.commit()
+
+                if enforce_state_transitions:
+                    # update was blocked by state transition rules - raise error
+                    # fetch current state to provide informative error
+                    current_file = await FileMetadataModel.read_async(
+                        db_session=session,
+                        identifier=file_id,
+                        actor=actor,
+                    )
+                    current_status = current_file.processing_status
+
+                    # build informative error message
+                    if processing_status is not None:
+                        if current_status in [FileProcessingStatus.ERROR, FileProcessingStatus.COMPLETED]:
+                            raise ValueError(
+                                f"Cannot update file {file_id} status from terminal state {current_status} to {processing_status}"
+                            )
+                        else:
+                            raise ValueError(f"Invalid state transition for file {file_id}: {current_status} -> {processing_status}")
+                    else:
+                        raise ValueError(f"Cannot update file {file_id} in terminal state {current_status}")
+                else:
+                    # validation was bypassed but update still failed (e.g., file doesn't exist)
+                    return None
+
             await session.commit()
 
             # invalidate cache for this file
             await self._invalidate_file_caches(file_id, actor)
 
-            # Reload via normal accessor so we return a fully-attached object
+            # reload via normal accessor so we return a fully-attached object
             file_orm = await FileMetadataModel.read_async(
                 db_session=session,
                 identifier=file_id,
@@ -317,12 +392,12 @@ class FileManager:
 
     @enforce_types
     @trace_method
-    @async_redis_cache(
-        key_func=lambda self, original_filename, source_id, actor: f"{original_filename}:{source_id}:{actor.organization_id}",
-        prefix="file_by_name",
-        ttl_s=3600,
-        model_class=PydanticFileMetadata,
-    )
+    # @async_redis_cache(
+    #     key_func=lambda self, original_filename, source_id, actor: f"{original_filename}:{source_id}:{actor.organization_id}",
+    #     prefix="file_by_name",
+    #     ttl_s=3600,
+    #     model_class=PydanticFileMetadata,
+    # )
     async def get_file_by_original_name_and_source(
         self, original_filename: str, source_id: str, actor: PydanticUser
     ) -> Optional[PydanticFileMetadata]:

letta/services/file_processor/embedder/pinecone_embedder.py

@@ -1,8 +1,9 @@
-from typing import List
+from typing import List, Optional
 
 from letta.helpers.pinecone_utils import upsert_file_records_to_pinecone_index
 from letta.log import get_logger
 from letta.otel.tracing import log_event, trace_method
+from letta.schemas.embedding_config import EmbeddingConfig
 from letta.schemas.passage import Passage
 from letta.schemas.user import User
 from letta.services.file_processor.embedder.base_embedder import BaseEmbedder
@@ -18,10 +19,15 @@ logger = get_logger(__name__)
 class PineconeEmbedder(BaseEmbedder):
     """Pinecone-based embedding generation"""
 
-    def __init__(self):
+    def __init__(self, embedding_config: Optional[EmbeddingConfig] = None):
         if not PINECONE_AVAILABLE:
             raise ImportError("Pinecone package is not installed. Install it with: pip install pinecone")
 
+        # set default embedding config if not provided
+        if embedding_config is None:
+            embedding_config = EmbeddingConfig.default_config(provider="pinecone")
+
+        self.embedding_config = embedding_config
         super().__init__()
 
     @trace_method

letta/services/file_processor/file_processor.py

@@ -10,12 +10,12 @@ from letta.schemas.enums import FileProcessingStatus
 from letta.schemas.file import FileMetadata
 from letta.schemas.passage import Passage
 from letta.schemas.user import User
-from letta.server.server import SyncServer
+from letta.services.agent_manager import AgentManager
 from letta.services.file_manager import FileManager
 from letta.services.file_processor.chunker.line_chunker import LineChunker
 from letta.services.file_processor.chunker.llama_index_chunker import LlamaIndexChunker
 from letta.services.file_processor.embedder.base_embedder import BaseEmbedder
-from letta.services.file_processor.parser.mistral_parser import MistralFileParser
+from letta.services.file_processor.parser.base_parser import FileParser
 from letta.services.job_manager import JobManager
 from letta.services.passage_manager import PassageManager
 from letta.services.source_manager import SourceManager
@@ -28,7 +28,7 @@ class FileProcessor:
 
     def __init__(
         self,
-        file_parser: MistralFileParser,
+        file_parser: FileParser,
         embedder: BaseEmbedder,
         actor: User,
         using_pinecone: bool,
@@ -42,6 +42,7 @@ class FileProcessor:
         self.source_manager = SourceManager()
         self.passage_manager = PassageManager()
         self.job_manager = JobManager()
+        self.agent_manager = AgentManager()
         self.actor = actor
         self.using_pinecone = using_pinecone
 
@@ -50,7 +51,7 @@ class FileProcessor:
         filename = file_metadata.file_name
 
         # Create file-type-specific chunker
-        text_chunker = LlamaIndexChunker(file_type=file_metadata.file_type)
+        text_chunker = LlamaIndexChunker(file_type=file_metadata.file_type, chunk_size=self.embedder.embedding_config.embedding_chunk_size)
 
         # First attempt with file-specific chunker
         try:
@@ -58,18 +59,30 @@ class FileProcessor:
             for page in ocr_response.pages:
                 chunks = text_chunker.chunk_text(page)
                 if not chunks:
-                    log_event("file_processor.chunking_failed", {"filename": filename, "page_index": ocr_response.pages.index(page)})
+                    log_event(
+                        "file_processor.chunking_failed",
+                        {
+                            "filename": filename,
+                            "page_index": ocr_response.pages.index(page),
+                        },
+                    )
                     raise ValueError("No chunks created from text")
                 all_chunks.extend(chunks)
 
             all_passages = await self.embedder.generate_embedded_passages(
-                file_id=file_metadata.id, source_id=source_id, chunks=all_chunks, actor=self.actor
+                file_id=file_metadata.id,
+                source_id=source_id,
+                chunks=all_chunks,
+                actor=self.actor,
             )
             return all_passages
 
         except Exception as e:
             logger.warning(f"Failed to chunk/embed with file-specific chunker for {filename}: {str(e)}. Retrying with default chunker.")
-            log_event("file_processor.embedding_failed_retrying", {"filename": filename, "error": str(e), "error_type": type(e).__name__})
+            log_event(
+                "file_processor.embedding_failed_retrying",
+                {"filename": filename, "error": str(e), "error_type": type(e).__name__},
+            )
 
             # Retry with default chunker
             try:
@@ -80,31 +93,49 @@ class FileProcessor:
                     chunks = text_chunker.default_chunk_text(page)
                     if not chunks:
                         log_event(
-                            "file_processor.default_chunking_failed", {"filename": filename, "page_index": ocr_response.pages.index(page)}
+                            "file_processor.default_chunking_failed",
+                            {
+                                "filename": filename,
+                                "page_index": ocr_response.pages.index(page),
+                            },
                         )
                         raise ValueError("No chunks created from text with default chunker")
                     all_chunks.extend(chunks)
 
                 all_passages = await self.embedder.generate_embedded_passages(
-                    file_id=file_metadata.id, source_id=source_id, chunks=all_chunks, actor=self.actor
+                    file_id=file_metadata.id,
+                    source_id=source_id,
+                    chunks=all_chunks,
+                    actor=self.actor,
                 )
                 logger.info(f"Successfully generated passages with default chunker for {filename}")
-                log_event("file_processor.default_chunking_success", {"filename": filename, "total_chunks": len(all_chunks)})
+                log_event(
+                    "file_processor.default_chunking_success",
+                    {"filename": filename, "total_chunks": len(all_chunks)},
+                )
                 return all_passages
 
             except Exception as fallback_error:
                 logger.error("Default chunking also failed for %s: %s", filename, fallback_error)
                 log_event(
                     "file_processor.default_chunking_also_failed",
-                    {"filename": filename, "fallback_error": str(fallback_error), "fallback_error_type": type(fallback_error).__name__},
+                    {
+                        "filename": filename,
+                        "fallback_error": str(fallback_error),
+                        "fallback_error_type": type(fallback_error).__name__,
+                    },
                 )
                 raise fallback_error
 
     # TODO: Factor this function out of SyncServer
     @trace_method
     async def process(
-        self, server: SyncServer, agent_states: List[AgentState], source_id: str, content: bytes, file_metadata: FileMetadata
-    ) -> List[Passage]:
+        self,
+        agent_states: list[AgentState],
+        source_id: str,
+        content: bytes,
+        file_metadata: FileMetadata,
+    ) -> list[Passage]:
         filename = file_metadata.file_name
 
         # Create file as early as possible with no content
@@ -151,7 +182,7 @@ class FileProcessor:
             )
             file_metadata = await self.file_manager.upsert_file_content(file_id=file_metadata.id, text=raw_markdown_text, actor=self.actor)
 
-            await server.insert_file_into_context_windows(
+            await self.agent_manager.insert_file_into_context_windows(
                 source_id=source_id,
                 file_metadata_with_content=file_metadata,
                 actor=self.actor,
@@ -170,18 +201,28 @@ class FileProcessor:
                 raise ValueError("No text extracted from PDF")
 
             logger.info("Chunking extracted text")
-            log_event("file_processor.chunking_started", {"filename": filename, "pages_to_process": len(ocr_response.pages)})
+            log_event(
+                "file_processor.chunking_started",
+                {"filename": filename, "pages_to_process": len(ocr_response.pages)},
+            )
 
             # Chunk and embed with fallback logic
             all_passages = await self._chunk_and_embed_with_fallback(
-                file_metadata=file_metadata, ocr_response=ocr_response, source_id=source_id
+                file_metadata=file_metadata,
+                ocr_response=ocr_response,
+                source_id=source_id,
             )
 
             if not self.using_pinecone:
                 all_passages = await self.passage_manager.create_many_source_passages_async(
-                    passages=all_passages, file_metadata=file_metadata, actor=self.actor
+                    passages=all_passages,
+                    file_metadata=file_metadata,
+                    actor=self.actor,
+                )
+                log_event(
+                    "file_processor.passages_created",
+                    {"filename": filename, "total_passages": len(all_passages)},
                 )
-                log_event("file_processor.passages_created", {"filename": filename, "total_passages": len(all_passages)})
 
             logger.info(f"Successfully processed {filename}: {len(all_passages)} passages")
             log_event(
@@ -197,17 +238,22 @@ class FileProcessor:
             # update job status
             if not self.using_pinecone:
                 await self.file_manager.update_file_status(
-                    file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.COMPLETED
+                    file_id=file_metadata.id,
+                    actor=self.actor,
+                    processing_status=FileProcessingStatus.COMPLETED,
                 )
             else:
                 await self.file_manager.update_file_status(
-                    file_id=file_metadata.id, actor=self.actor, total_chunks=len(all_passages), chunks_embedded=0
+                    file_id=file_metadata.id,
+                    actor=self.actor,
+                    total_chunks=len(all_passages),
+                    chunks_embedded=0,
                 )
 
             return all_passages
 
         except Exception as e:
-            logger.error("File processing failed for %s: %s", filename, e)
+            logger.exception("File processing failed for %s: %s", filename, e)
             log_event(
                 "file_processor.processing_failed",
                 {
@@ -254,7 +300,7 @@ class FileProcessor:
             # Create OCR response from existing content
             ocr_response = self._create_ocr_response_from_content(content)
 
-            # Update file status to embedding
+            # Update file status to embedding (valid transition from PARSING)
             file_metadata = await self.file_manager.update_file_status(
                 file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.EMBEDDING
             )
@@ -274,12 +320,14 @@ class FileProcessor:
                 )
                 log_event("file_processor.import_passages_created", {"filename": filename, "total_passages": len(all_passages)})
 
-            # Update file status to completed
+            # Update file status to completed (valid transition from EMBEDDING)
             if not self.using_pinecone:
                 await self.file_manager.update_file_status(
                     file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.COMPLETED
                 )
             else:
+                # For Pinecone, update chunk counts but keep status at EMBEDDING
+                # The status will be updated to COMPLETED later when chunks are confirmed embedded
                 await self.file_manager.update_file_status(
                     file_id=file_metadata.id, actor=self.actor, total_chunks=len(all_passages), chunks_embedded=0
                 )
@@ -310,7 +358,10 @@ class FileProcessor:
                 },
             )
             await self.file_manager.update_file_status(
-                file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.ERROR, error_message=str(e)
+                file_id=file_metadata.id,
+                actor=self.actor,
+                processing_status=FileProcessingStatus.ERROR,
+                error_message=str(e),
             )
 
             return []