letta-nightly 0.8.0.dev20250606195656__py3-none-any.whl → 0.8.3.dev20250607000559__py3-none-any.whl

letta/services/file_processor/file_processor.py

@@ -5,12 +5,13 @@ from fastapi import UploadFile
 
 from letta.log import get_logger
 from letta.schemas.agent import AgentState
-from letta.schemas.enums import JobStatus
+from letta.schemas.enums import FileProcessingStatus, JobStatus
 from letta.schemas.file import FileMetadata
 from letta.schemas.job import Job, JobUpdate
 from letta.schemas.passage import Passage
 from letta.schemas.user import User
 from letta.server.server import SyncServer
+from letta.services.file_processor.chunker.line_chunker import LineChunker
 from letta.services.file_processor.chunker.llama_index_chunker import LlamaIndexChunker
 from letta.services.file_processor.embedder.openai_embedder import OpenAIEmbedder
 from letta.services.file_processor.parser.mistral_parser import MistralFileParser
@@ -34,6 +35,7 @@ class FileProcessor:
     ):
         self.file_parser = file_parser
         self.text_chunker = text_chunker
+        self.line_chunker = LineChunker()
         self.embedder = embedder
         self.max_file_size = max_file_size
         self.source_manager = SourceManager()
@@ -52,9 +54,12 @@ class FileProcessor:
         job: Optional[Job] = None,
     ) -> List[Passage]:
         file_metadata = self._extract_upload_file_metadata(file, source_id=source_id)
-        file_metadata = await self.source_manager.create_file(file_metadata, self.actor)
         filename = file_metadata.file_name
 
+        # Create file as early as possible with no content
+        file_metadata.processing_status = FileProcessingStatus.PARSING  # Parsing now
+        file_metadata = await self.source_manager.create_file(file_metadata, self.actor)
+
         try:
             # Ensure we're working with bytes
             if isinstance(content, str):
@@ -66,11 +71,35 @@ class FileProcessor:
             logger.info(f"Starting OCR extraction for {filename}")
             ocr_response = await self.file_parser.extract_text(content, mime_type=file_metadata.file_type)
 
+            # update file with raw text
+            raw_markdown_text = "".join([page.markdown for page in ocr_response.pages])
+            file_metadata = await self.source_manager.upsert_file_content(
+                file_id=file_metadata.id, text=raw_markdown_text, actor=self.actor
+            )
+            file_metadata = await self.source_manager.update_file_status(
+                file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.EMBEDDING
+            )
+
+            # Insert to agent context window
+            # TODO: Rethink this line chunking mechanism
+            content_lines = self.line_chunker.chunk_text(text=raw_markdown_text)
+            visible_content = "\n".join(content_lines)
+
+            await server.insert_file_into_context_windows(
+                source_id=source_id,
+                text=visible_content,
+                file_id=file_metadata.id,
+                file_name=file_metadata.file_name,
+                actor=self.actor,
+                agent_states=agent_states,
+            )
+
             if not ocr_response or len(ocr_response.pages) == 0:
                 raise ValueError("No text extracted from PDF")
 
             logger.info("Chunking extracted text")
             all_passages = []
+
             for page in ocr_response.pages:
                 chunks = self.text_chunker.chunk_text(page)
 
@@ -82,28 +111,26 @@ class FileProcessor:
                 )
                 all_passages.extend(passages)
 
-            all_passages = await self.passage_manager.create_many_passages_async(all_passages, self.actor)
+            all_passages = await self.passage_manager.create_many_source_passages_async(
+                passages=all_passages, file_metadata=file_metadata, actor=self.actor
+            )
 
             logger.info(f"Successfully processed {filename}: {len(all_passages)} passages")
 
-            await server.insert_file_into_context_windows(
-                source_id=source_id,
-                text="".join([ocr_response.pages[i].markdown for i in range(min(3, len(ocr_response.pages)))]),
-                file_id=file_metadata.id,
-                actor=self.actor,
-                agent_states=agent_states,
-            )
-
             # update job status
             if job:
                 job.status = JobStatus.completed
                 job.metadata["num_passages"] = len(all_passages)
                 await self.job_manager.update_job_by_id_async(job_id=job.id, job_update=JobUpdate(**job.model_dump()), actor=self.actor)
 
+            await self.source_manager.update_file_status(
+                file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.COMPLETED
+            )
+
             return all_passages
 
         except Exception as e:
-            logger.error(f"PDF processing failed for {filename}: {str(e)}")
+            logger.error(f"File processing failed for {filename}: {str(e)}")
 
             # update job status
             if job:
@@ -111,6 +138,10 @@ class FileProcessor:
                 job.metadata["error"] = str(e)
                 await self.job_manager.update_job_by_id_async(job_id=job.id, job_update=JobUpdate(**job.model_dump()), actor=self.actor)
 
+            await self.source_manager.update_file_status(
+                file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.ERROR, error_message=str(e)
+            )
+
             return []
 
     def _extract_upload_file_metadata(self, file: UploadFile, source_id: str) -> FileMetadata:

letta/services/file_processor/parser/mistral_parser.py

@@ -9,6 +9,16 @@ from letta.settings import settings
 logger = get_logger(__name__)
 
 
+SIMPLE_TEXT_MIME_TYPES = {
+    "text/plain",
+    "text/markdown",
+    "text/x-markdown",
+    "application/json",
+    "application/jsonl",
+    "application/x-jsonlines",
+}
+
+
 class MistralFileParser(FileParser):
     """Mistral-based OCR extraction"""
 
@@ -23,7 +33,7 @@ class MistralFileParser(FileParser):
 
             # TODO: Kind of hacky...we try to exit early here?
             # TODO: Create our internal file parser representation we return instead of OCRResponse
-            if mime_type == "text/plain":
+            if mime_type in SIMPLE_TEXT_MIME_TYPES or mime_type.startswith("text/"):
                 text = content.decode("utf-8", errors="replace")
                 return OCRResponse(
                     model=self.model,

letta/services/files_agents_manager.py

@@ -5,10 +5,11 @@ from sqlalchemy import and_, func, select, update
 
 from letta.orm.errors import NoResultFound
 from letta.orm.files_agents import FileAgent as FileAgentModel
+from letta.otel.tracing import trace_method
+from letta.schemas.block import Block as PydanticBlock
 from letta.schemas.file import FileAgent as PydanticFileAgent
 from letta.schemas.user import User as PydanticUser
 from letta.server.db import db_registry
-from letta.tracing import trace_method
 from letta.utils import enforce_types
 
 
@@ -22,6 +23,7 @@ class FileAgentManager:
         *,
         agent_id: str,
         file_id: str,
+        file_name: str,
         actor: PydanticUser,
         is_open: bool = True,
         visible_content: Optional[str] = None,
@@ -38,6 +40,7 @@ class FileAgentManager:
                 and_(
                     FileAgentModel.agent_id == agent_id,
                     FileAgentModel.file_id == file_id,
+                    FileAgentModel.file_name == file_name,
                     FileAgentModel.organization_id == actor.organization_id,
                 )
             )
@@ -61,6 +64,7 @@ class FileAgentManager:
             assoc = FileAgentModel(
                 agent_id=agent_id,
                 file_id=file_id,
+                file_name=file_name,
                 organization_id=actor.organization_id,
                 is_open=is_open,
                 visible_content=visible_content,
@@ -71,7 +75,7 @@ class FileAgentManager:
 
     @enforce_types
     @trace_method
-    async def update_file_agent(
+    async def update_file_agent_by_id(
         self,
         *,
         agent_id: str,
@@ -82,7 +86,33 @@ class FileAgentManager:
     ) -> PydanticFileAgent:
         """Patch an existing association row."""
         async with db_registry.async_session() as session:
-            assoc = await self._get_association(session, agent_id, file_id, actor)
+            assoc = await self._get_association_by_file_id(session, agent_id, file_id, actor)
+
+            if is_open is not None:
+                assoc.is_open = is_open
+            if visible_content is not None:
+                assoc.visible_content = visible_content
+
+            # touch timestamp
+            assoc.last_accessed_at = datetime.now(timezone.utc)
+
+            await assoc.update_async(session, actor=actor)
+            return assoc.to_pydantic()
+
+    @enforce_types
+    @trace_method
+    async def update_file_agent_by_name(
+        self,
+        *,
+        agent_id: str,
+        file_name: str,
+        actor: PydanticUser,
+        is_open: Optional[bool] = None,
+        visible_content: Optional[str] = None,
+    ) -> PydanticFileAgent:
+        """Patch an existing association row."""
+        async with db_registry.async_session() as session:
+            assoc = await self._get_association_by_file_name(session, agent_id, file_name, actor)
 
             if is_open is not None:
                 assoc.is_open = is_open
@@ -100,15 +130,61 @@ class FileAgentManager:
     async def detach_file(self, *, agent_id: str, file_id: str, actor: PydanticUser) -> None:
         """Hard-delete the association."""
         async with db_registry.async_session() as session:
-            assoc = await self._get_association(session, agent_id, file_id, actor)
+            assoc = await self._get_association_by_file_id(session, agent_id, file_id, actor)
             await assoc.hard_delete_async(session, actor=actor)
 
     @enforce_types
     @trace_method
-    async def get_file_agent(self, *, agent_id: str, file_id: str, actor: PydanticUser) -> Optional[PydanticFileAgent]:
+    async def get_file_agent_by_id(self, *, agent_id: str, file_id: str, actor: PydanticUser) -> Optional[PydanticFileAgent]:
+        async with db_registry.async_session() as session:
+            try:
+                assoc = await self._get_association_by_file_id(session, agent_id, file_id, actor)
+                return assoc.to_pydantic()
+            except NoResultFound:
+                return None
+
+    @enforce_types
+    @trace_method
+    async def get_all_file_blocks_by_name(
+        self,
+        *,
+        file_names: List[str],
+        actor: PydanticUser,
+    ) -> List[PydanticBlock]:
+        """
+        Retrieve multiple FileAgent associations by their IDs in a single query.
+
+        Args:
+            file_names: List of file names to retrieve
+            actor: The user making the request
+
+        Returns:
+            List of PydanticFileAgent objects found (may be fewer than requested if some IDs don't exist)
+        """
+        if not file_names:
+            return []
+
+        async with db_registry.async_session() as session:
+            # Use IN clause for efficient bulk retrieval
+            query = select(FileAgentModel).where(
+                and_(
+                    FileAgentModel.file_name.in_(file_names),
+                    FileAgentModel.organization_id == actor.organization_id,
+                )
+            )
+
+            # Execute query and get all results
+            rows = (await session.execute(query)).scalars().all()
+
+            # Convert to Pydantic models
+            return [row.to_pydantic_block() for row in rows]
+
+    @enforce_types
+    @trace_method
+    async def get_file_agent_by_file_name(self, *, agent_id: str, file_name: str, actor: PydanticUser) -> Optional[PydanticFileAgent]:
         async with db_registry.async_session() as session:
             try:
-                assoc = await self._get_association(session, agent_id, file_id, actor)
+                assoc = await self._get_association_by_file_name(session, agent_id, file_name, actor)
                 return assoc.to_pydantic()
             except NoResultFound:
                 return None
@@ -170,7 +246,7 @@ class FileAgentManager:
             await session.execute(stmt)
             await session.commit()
 
-    async def _get_association(self, session, agent_id: str, file_id: str, actor: PydanticUser) -> FileAgentModel:
+    async def _get_association_by_file_id(self, session, agent_id: str, file_id: str, actor: PydanticUser) -> FileAgentModel:
         q = select(FileAgentModel).where(
             and_(
                 FileAgentModel.agent_id == agent_id,
@@ -182,3 +258,16 @@ class FileAgentManager:
         if not assoc:
             raise NoResultFound(f"FileAgent(agent_id={agent_id}, file_id={file_id}) not found in org {actor.organization_id}")
         return assoc
+
+    async def _get_association_by_file_name(self, session, agent_id: str, file_name: str, actor: PydanticUser) -> FileAgentModel:
+        q = select(FileAgentModel).where(
+            and_(
+                FileAgentModel.agent_id == agent_id,
+                FileAgentModel.file_name == file_name,
+                FileAgentModel.organization_id == actor.organization_id,
+            )
+        )
+        assoc = await session.scalar(q)
+        if not assoc:
+            raise NoResultFound(f"FileAgent(agent_id={agent_id}, file_name={file_name}) not found in org {actor.organization_id}")
+        return assoc

letta/services/group_manager.py

@@ -7,13 +7,13 @@ from letta.orm.agent import Agent as AgentModel
 from letta.orm.errors import NoResultFound
 from letta.orm.group import Group as GroupModel
 from letta.orm.message import Message as MessageModel
+from letta.otel.tracing import trace_method
 from letta.schemas.group import Group as PydanticGroup
 from letta.schemas.group import GroupCreate, GroupUpdate, ManagerType
 from letta.schemas.letta_message import LettaMessage
 from letta.schemas.message import Message as PydanticMessage
 from letta.schemas.user import User as PydanticUser
 from letta.server.db import db_registry
-from letta.tracing import trace_method
 from letta.utils import enforce_types
 
 
@@ -152,9 +152,9 @@ class GroupManager:
 
     @trace_method
     @enforce_types
-    def modify_group(self, group_id: str, group_update: GroupUpdate, actor: PydanticUser) -> PydanticGroup:
-        with db_registry.session() as session:
-            group = GroupModel.read(db_session=session, identifier=group_id, actor=actor)
+    async def modify_group_async(self, group_id: str, group_update: GroupUpdate, actor: PydanticUser) -> PydanticGroup:
+        async with db_registry.async_session() as session:
+            group = await GroupModel.read_async(db_session=session, identifier=group_id, actor=actor)
 
             sleeptime_agent_frequency = None
             max_message_buffer_length = None
@@ -206,11 +206,11 @@ class GroupManager:
             if group_update.description:
                 group.description = group_update.description
             if group_update.agent_ids:
-                self._process_agent_relationship(
+                await self._process_agent_relationship_async(
                     session=session, group=group, agent_ids=group_update.agent_ids, allow_partial=False, replace=True
                 )
 
-            group.update(session, actor=actor)
+            await group.update_async(session, actor=actor)
             return group.to_pydantic()
 
     @trace_method