letta-nightly 0.11.4.dev20250826104242__py3-none-any.whl → 0.11.6.dev20250827050912__py3-none-any.whl

letta/services/agent_serialization_manager.py

@@ -1,8 +1,16 @@
+import asyncio
+import uuid
 from datetime import datetime, timezone
 from typing import Any, Dict, List, Optional
 
 from letta.constants import MCP_TOOL_TAG_NAME_PREFIX
-from letta.errors import AgentExportIdMappingError, AgentExportProcessingError, AgentFileImportError, AgentNotFoundForExportError
+from letta.errors import (
+    AgentExportIdMappingError,
+    AgentExportProcessingError,
+    AgentFileExportError,
+    AgentFileImportError,
+    AgentNotFoundForExportError,
+)
 from letta.helpers.pinecone_utils import should_use_pinecone
 from letta.log import get_logger
 from letta.schemas.agent import AgentState, CreateAgent
@@ -420,6 +428,8 @@ class AgentSerializationManager:
         self,
         schema: AgentFileSchema,
         actor: User,
+        append_copy_suffix: bool = False,
+        override_existing_tools: bool = True,
         dry_run: bool = False,
         env_vars: Optional[Dict[str, Any]] = None,
     ) -> ImportResult:
@@ -481,7 +491,9 @@ class AgentSerializationManager:
                     pydantic_tools.append(Tool(**tool_schema.model_dump(exclude={"id"})))
 
                 # bulk upsert all tools at once
-                created_tools = await self.tool_manager.bulk_upsert_tools_async(pydantic_tools, actor)
+                created_tools = await self.tool_manager.bulk_upsert_tools_async(
+                    pydantic_tools, actor, override_existing_tools=override_existing_tools
+                )
 
                 # map file ids to database ids
                 # note: tools are matched by name during upsert, so we need to match by name here too
@@ -513,8 +525,20 @@ class AgentSerializationManager:
             if schema.sources:
                 # convert source schemas to pydantic sources
                 pydantic_sources = []
+
+                # First, do a fast batch check for existing source names to avoid conflicts
+                source_names_to_check = [s.name for s in schema.sources]
+                existing_source_names = await self.source_manager.get_existing_source_names(source_names_to_check, actor)
+
                 for source_schema in schema.sources:
                     source_data = source_schema.model_dump(exclude={"id", "embedding", "embedding_chunk_size"})
+
+                    # Check if source name already exists, if so add unique suffix
+                    original_name = source_data["name"]
+                    if original_name in existing_source_names:
+                        unique_suffix = uuid.uuid4().hex[:8]
+                        source_data["name"] = f"{original_name}_{unique_suffix}"
+
                     pydantic_sources.append(Source(**source_data))
 
                 # bulk upsert all sources at once
@@ -523,13 +547,15 @@ class AgentSerializationManager:
                 # map file ids to database ids
                 # note: sources are matched by name during upsert, so we need to match by name here too
                 created_sources_by_name = {source.name: source for source in created_sources}
-                for source_schema in schema.sources:
-                    created_source = created_sources_by_name.get(source_schema.name)
+                for i, source_schema in enumerate(schema.sources):
+                    # Use the pydantic source name (which may have been modified for uniqueness)
+                    source_name = pydantic_sources[i].name
+                    created_source = created_sources_by_name.get(source_name)
                     if created_source:
                         file_to_db_ids[source_schema.id] = created_source.id
                         imported_count += 1
                     else:
-                        logger.warning(f"Source {source_schema.name} was not created during bulk upsert")
+                        logger.warning(f"Source {source_name} was not created during bulk upsert")
 
             # 4. Create files (depends on sources)
             for file_schema in schema.files:
@@ -548,38 +574,49 @@ class AgentSerializationManager:
                 imported_count += 1
 
             # 5. Process files for chunking/embedding (depends on files and sources)
-            if should_use_pinecone():
-                embedder = PineconeEmbedder(embedding_config=schema.agents[0].embedding_config)
-            else:
-                embedder = OpenAIEmbedder(embedding_config=schema.agents[0].embedding_config)
-            file_processor = FileProcessor(
-                file_parser=self.file_parser,
-                embedder=embedder,
-                actor=actor,
-                using_pinecone=self.using_pinecone,
-            )
+            # Start background tasks for file processing
+            background_tasks = []
+            if schema.files and any(f.content for f in schema.files):
+                if should_use_pinecone():
+                    embedder = PineconeEmbedder(embedding_config=schema.agents[0].embedding_config)
+                else:
+                    embedder = OpenAIEmbedder(embedding_config=schema.agents[0].embedding_config)
+                file_processor = FileProcessor(
+                    file_parser=self.file_parser,
+                    embedder=embedder,
+                    actor=actor,
+                    using_pinecone=self.using_pinecone,
+                )
 
-            for file_schema in schema.files:
-                if file_schema.content:  # Only process files with content
-                    file_db_id = file_to_db_ids[file_schema.id]
-                    source_db_id = file_to_db_ids[file_schema.source_id]
+                for file_schema in schema.files:
+                    if file_schema.content:  # Only process files with content
+                        file_db_id = file_to_db_ids[file_schema.id]
+                        source_db_id = file_to_db_ids[file_schema.source_id]
 
-                    # Get the created file metadata (with caching)
-                    if file_db_id not in file_metadata_cache:
-                        file_metadata_cache[file_db_id] = await self.file_manager.get_file_by_id(file_db_id, actor)
-                    file_metadata = file_metadata_cache[file_db_id]
+                        # Get the created file metadata (with caching)
+                        if file_db_id not in file_metadata_cache:
+                            file_metadata_cache[file_db_id] = await self.file_manager.get_file_by_id(file_db_id, actor)
+                        file_metadata = file_metadata_cache[file_db_id]
 
-                    # Save the db call of fetching content again
-                    file_metadata.content = file_schema.content
+                        # Save the db call of fetching content again
+                        file_metadata.content = file_schema.content
 
-                    # Process the file for chunking/embedding
-                    passages = await file_processor.process_imported_file(file_metadata=file_metadata, source_id=source_db_id)
-                    imported_count += len(passages)
+                        # Create background task for file processing
+                        # TODO: This can be moved to celery or RQ or something
+                        task = asyncio.create_task(
+                            self._process_file_async(
+                                file_metadata=file_metadata, source_id=source_db_id, file_processor=file_processor, actor=actor
+                            )
+                        )
+                        background_tasks.append(task)
+                        logger.info(f"Started background processing for file {file_metadata.file_name} (ID: {file_db_id})")
 
             # 6. Create agents with empty message history
             for agent_schema in schema.agents:
                 # Convert AgentSchema back to CreateAgent, remapping tool/block IDs
                 agent_data = agent_schema.model_dump(exclude={"id", "in_context_message_ids", "messages"})
+                if append_copy_suffix:
+                    agent_data["name"] = agent_data.get("name") + "_copy"
 
                 # Remap tool_ids from file IDs to database IDs
                 if agent_data.get("tool_ids"):
@@ -589,6 +626,10 @@ class AgentSerializationManager:
                 if agent_data.get("block_ids"):
                     agent_data["block_ids"] = [file_to_db_ids[file_id] for file_id in agent_data["block_ids"]]
 
+                # Remap source_ids from file IDs to database IDs
+                if agent_data.get("source_ids"):
+                    agent_data["source_ids"] = [file_to_db_ids[file_id] for file_id in agent_data["source_ids"]]
+
                 if env_vars:
                     for var in agent_data["tool_exec_environment_variables"]:
                         var["value"] = env_vars.get(var["key"], "")
@@ -635,14 +676,16 @@ class AgentSerializationManager:
                     for file_agent_schema in agent_schema.files_agents:
                         file_db_id = file_to_db_ids[file_agent_schema.file_id]
 
-                        # Use cached file metadata if available
+                        # Use cached file metadata if available (with content)
                         if file_db_id not in file_metadata_cache:
-                            file_metadata_cache[file_db_id] = await self.file_manager.get_file_by_id(file_db_id, actor)
+                            file_metadata_cache[file_db_id] = await self.file_manager.get_file_by_id(
+                                file_db_id, actor, include_content=True
+                            )
                         file_metadata = file_metadata_cache[file_db_id]
                         files_for_agent.append(file_metadata)
 
                         if file_agent_schema.visible_content:
-                            visible_content_map[file_db_id] = file_agent_schema.visible_content
+                            visible_content_map[file_metadata.file_name] = file_agent_schema.visible_content
 
                     # Bulk attach files to agent
                     await self.file_agent_manager.attach_files_bulk(
@@ -669,9 +712,19 @@ class AgentSerializationManager:
                 file_to_db_ids[group.id] = created_group.id
                 imported_count += 1
 
+            # prepare result message
+            num_background_tasks = len(background_tasks)
+            if num_background_tasks > 0:
+                message = (
+                    f"Import completed successfully. Imported {imported_count} entities. "
+                    f"{num_background_tasks} file(s) are being processed in the background for embeddings."
+                )
+            else:
+                message = f"Import completed successfully. Imported {imported_count} entities."
+
             return ImportResult(
                 success=True,
-                message=f"Import completed successfully. Imported {imported_count} entities.",
+                message=message,
                 imported_count=imported_count,
                 imported_agent_ids=imported_agent_ids,
                 id_mappings=file_to_db_ids,
@@ -849,3 +902,47 @@ class AgentSerializationManager:
         except AttributeError:
             allowed = model_cls.__fields__.keys()  # Pydantic v1
         return {k: v for k, v in data.items() if k in allowed}
+
+    async def _process_file_async(self, file_metadata: FileMetadata, source_id: str, file_processor: FileProcessor, actor: User):
+        """
+        Process a file asynchronously in the background.
+
+        This method handles chunking and embedding of file content without blocking
+        the main import process.
+
+        Args:
+            file_metadata: The file metadata with content
+            source_id: The database ID of the source
+            file_processor: The file processor instance to use
+            actor: The user performing the action
+        """
+        file_id = file_metadata.id
+        file_name = file_metadata.file_name
+
+        try:
+            logger.info(f"Starting background processing for file {file_name} (ID: {file_id})")
+
+            # process the file for chunking/embedding
+            passages = await file_processor.process_imported_file(file_metadata=file_metadata, source_id=source_id)
+
+            logger.info(f"Successfully processed file {file_name} with {len(passages)} passages")
+
+            # file status is automatically updated to COMPLETED by process_imported_file
+            return passages
+
+        except Exception as e:
+            logger.error(f"Failed to process file {file_name} (ID: {file_id}) in background: {e}")
+
+            # update file status to ERROR
+            try:
+                await self.file_manager.update_file_status(
+                    file_id=file_id,
+                    actor=actor,
+                    processing_status=FileProcessingStatus.ERROR,
+                    error_message=str(e) if str(e) else f"Agent serialization failed: {type(e).__name__}",
+                )
+            except Exception as update_error:
+                logger.error(f"Failed to update file status to ERROR for {file_id}: {update_error}")
+
+            # we don't re-raise here since this is a background task
+            # the file will be marked as ERROR and the import can continue

letta/services/file_manager.py

@@ -1,6 +1,6 @@
 import asyncio
 import os
-from datetime import datetime
+from datetime import datetime, timedelta, timezone
 from typing import List, Optional
 
 from sqlalchemy import func, select, update
@@ -9,6 +9,8 @@ from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import selectinload
 
 from letta.constants import MAX_FILENAME_LENGTH
+from letta.helpers.pinecone_utils import list_pinecone_index_for_files, should_use_pinecone
+from letta.log import get_logger
 from letta.orm.errors import NoResultFound
 from letta.orm.file import FileContent as FileContentModel
 from letta.orm.file import FileMetadata as FileMetadataModel
@@ -20,8 +22,11 @@ from letta.schemas.source import Source as PydanticSource
 from letta.schemas.source_metadata import FileStats, OrganizationSourcesStats, SourceStats
 from letta.schemas.user import User as PydanticUser
 from letta.server.db import db_registry
+from letta.settings import settings
 from letta.utils import enforce_types
 
+logger = get_logger(__name__)
+
 
 class DuplicateFileError(Exception):
     """Raised when a duplicate file is encountered and error handling is specified"""
@@ -174,6 +179,10 @@ class FileManager:
         if processing_status is None and error_message is None and total_chunks is None and chunks_embedded is None:
             raise ValueError("Nothing to update")
 
+        # validate that ERROR status must have an error message
+        if processing_status == FileProcessingStatus.ERROR and not error_message:
+            raise ValueError("Error message is required when setting processing status to ERROR")
+
         values: dict[str, object] = {"updated_at": datetime.utcnow()}
         if processing_status is not None:
             values["processing_status"] = processing_status
@@ -273,6 +282,79 @@ class FileManager:
             )
             return await file_orm.to_pydantic_async()
 
+    @enforce_types
+    @trace_method
+    async def check_and_update_file_status(
+        self,
+        file_metadata: PydanticFileMetadata,
+        actor: PydanticUser,
+    ) -> PydanticFileMetadata:
+        """
+        Check and update file status for timeout and embedding completion.
+
+        This method consolidates logic for:
+        1. Checking if a file has timed out during processing
+        2. Checking Pinecone embedding status and updating counts
+
+        Args:
+            file_metadata: The file metadata to check
+            actor: User performing the check
+
+        Returns:
+            Updated file metadata with current status
+        """
+        # check for timeout if status is not terminal
+        if not file_metadata.processing_status.is_terminal_state():
+            if file_metadata.created_at:
+                # handle timezone differences between PostgreSQL (timezone-aware) and SQLite (timezone-naive)
+                if settings.letta_pg_uri_no_default:
+                    # postgresql: both datetimes are timezone-aware
+                    timeout_threshold = datetime.now(timezone.utc) - timedelta(minutes=settings.file_processing_timeout_minutes)
+                    file_created_at = file_metadata.created_at
+                else:
+                    # sqlite: both datetimes should be timezone-naive
+                    timeout_threshold = datetime.utcnow() - timedelta(minutes=settings.file_processing_timeout_minutes)
+                    file_created_at = file_metadata.created_at
+
+                if file_created_at < timeout_threshold:
+                    # move file to error status with timeout message
+                    timeout_message = settings.file_processing_timeout_error_message.format(settings.file_processing_timeout_minutes)
+                    try:
+                        file_metadata = await self.update_file_status(
+                            file_id=file_metadata.id,
+                            actor=actor,
+                            processing_status=FileProcessingStatus.ERROR,
+                            error_message=timeout_message,
+                        )
+                    except ValueError as e:
+                        # state transition was blocked - log it but don't fail
+                        logger.warning(f"Could not update file to timeout error state: {str(e)}")
+                        # continue with existing file_metadata
+
+        # check pinecone embedding status
+        if should_use_pinecone() and file_metadata.processing_status == FileProcessingStatus.EMBEDDING:
+            ids = await list_pinecone_index_for_files(file_id=file_metadata.id, actor=actor)
+            logger.info(
+                f"Embedded chunks {len(ids)}/{file_metadata.total_chunks} for {file_metadata.id} ({file_metadata.file_name}) in organization {actor.organization_id}"
+            )
+
+            if len(ids) != file_metadata.chunks_embedded or len(ids) == file_metadata.total_chunks:
+                if len(ids) != file_metadata.total_chunks:
+                    file_status = file_metadata.processing_status
+                else:
+                    file_status = FileProcessingStatus.COMPLETED
+                try:
+                    file_metadata = await self.update_file_status(
+                        file_id=file_metadata.id, actor=actor, chunks_embedded=len(ids), processing_status=file_status
+                    )
+                except ValueError as e:
+                    # state transition was blocked - this is a race condition
+                    # log it but don't fail since we're just checking status
+                    logger.warning(f"Race condition detected in check_and_update_file_status: {str(e)}")
+                    # return the current file state without updating
+
+        return file_metadata
+
     @enforce_types
     @trace_method
     async def upsert_file_content(
@@ -328,8 +410,22 @@ class FileManager:
         limit: Optional[int] = 50,
         include_content: bool = False,
         strip_directory_prefix: bool = False,
+        check_status_updates: bool = False,
     ) -> List[PydanticFileMetadata]:
-        """List all files with optional pagination."""
+        """List all files with optional pagination and status checking.
+
+        Args:
+            source_id: Source to list files from
+            actor: User performing the request
+            after: Pagination cursor
+            limit: Maximum number of files to return
+            include_content: Whether to include file content
+            strip_directory_prefix: Whether to strip directory prefix from filenames
+            check_status_updates: Whether to check and update status for timeout and embedding completion
+
+        Returns:
+            List of file metadata
+        """
         async with db_registry.async_session() as session:
             options = [selectinload(FileMetadataModel.content)] if include_content else None
 
@@ -341,10 +437,19 @@ class FileManager:
                 source_id=source_id,
                 query_options=options,
             )
-            return [
-                await file.to_pydantic_async(include_content=include_content, strip_directory_prefix=strip_directory_prefix)
-                for file in files
-            ]
+
+            # convert all files to pydantic models
+            file_metadatas = await asyncio.gather(
+                *[file.to_pydantic_async(include_content=include_content, strip_directory_prefix=strip_directory_prefix) for file in files]
+            )
+
+            # if status checking is enabled, check all files concurrently
+            if check_status_updates:
+                file_metadatas = await asyncio.gather(
+                    *[self.check_and_update_file_status(file_metadata, actor) for file_metadata in file_metadatas]
+                )
+
+            return file_metadatas
 
     @enforce_types
     @trace_method

letta/services/file_processor/file_processor.py

@@ -264,7 +264,10 @@ class FileProcessor:
                 },
             )
             await self.file_manager.update_file_status(
-                file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.ERROR, error_message=str(e)
+                file_id=file_metadata.id,
+                actor=self.actor,
+                processing_status=FileProcessingStatus.ERROR,
+                error_message=str(e) if str(e) else f"File processing failed: {type(e).__name__}",
             )
 
             return []
@@ -361,7 +364,7 @@ class FileProcessor:
                 file_id=file_metadata.id,
                 actor=self.actor,
                 processing_status=FileProcessingStatus.ERROR,
-                error_message=str(e),
+                error_message=str(e) if str(e) else f"Import file processing failed: {type(e).__name__}",
             )
 
             return []

letta/services/files_agents_manager.py

@@ -293,6 +293,66 @@ class FileAgentManager:
             else:
                 return [r.to_pydantic() for r in rows]
 
+    @enforce_types
+    @trace_method
+    async def list_files_for_agent_paginated(
+        self,
+        agent_id: str,
+        actor: PydanticUser,
+        cursor: Optional[str] = None,
+        limit: int = 20,
+        is_open: Optional[bool] = None,
+    ) -> tuple[List[PydanticFileAgent], Optional[str], bool]:
+        """
+        Return paginated file associations for an agent.
+
+        Args:
+            agent_id: The agent ID to get files for
+            actor: User performing the action
+            cursor: Pagination cursor (file-agent ID to start after)
+            limit: Maximum number of results to return
+            is_open: Optional filter for open/closed status (None = all, True = open only, False = closed only)
+
+        Returns:
+            Tuple of (file_agents, next_cursor, has_more)
+        """
+        async with db_registry.async_session() as session:
+            conditions = [
+                FileAgentModel.agent_id == agent_id,
+                FileAgentModel.organization_id == actor.organization_id,
+                FileAgentModel.is_deleted == False,
+            ]
+
+            # apply is_open filter if specified
+            if is_open is not None:
+                conditions.append(FileAgentModel.is_open == is_open)
+
+            # apply cursor if provided (get records after this ID)
+            if cursor:
+                conditions.append(FileAgentModel.id > cursor)
+
+            query = select(FileAgentModel).where(and_(*conditions))
+
+            # order by ID for stable pagination
+            query = query.order_by(FileAgentModel.id)
+
+            # fetch limit + 1 to check if there are more results
+            query = query.limit(limit + 1)
+
+            result = await session.execute(query)
+            rows = result.scalars().all()
+
+            # check if we got more records than requested (meaning there are more pages)
+            has_more = len(rows) > limit
+            if has_more:
+                # trim back to the requested limit
+                rows = rows[:limit]
+
+            # get cursor for next page (ID of last item in current page)
+            next_cursor = rows[-1].id if rows else None
+
+            return [r.to_pydantic() for r in rows], next_cursor, has_more
+
     @enforce_types
     @trace_method
     async def list_agents_for_file(