letta-nightly 0.8.15.dev20250720104313__py3-none-any.whl → 0.8.16.dev20250721104533__py3-none-any.whl

letta/services/file_processor/file_processor.py

@@ -1,5 +1,7 @@
 from typing import List
 
+from mistralai import OCRPageObject, OCRResponse, OCRUsageInfo
+
 from letta.log import get_logger
 from letta.otel.context import get_ctx_attributes
 from letta.otel.tracing import log_event, trace_method
@@ -221,3 +223,94 @@ class FileProcessor:
             )
 
             return []
+
+    def _create_ocr_response_from_content(self, content: str):
+        """Create minimal OCR response from existing content"""
+        return OCRResponse(
+            model="import-skip-ocr",
+            pages=[
+                OCRPageObject(
+                    index=0,
+                    markdown=content,
+                    images=[],
+                    dimensions=None,
+                )
+            ],
+            usage_info=OCRUsageInfo(pages_processed=1),
+            document_annotation=None,
+        )
+
+    @trace_method
+    async def process_imported_file(self, file_metadata: FileMetadata, source_id: str) -> List[Passage]:
+        """Process an imported file that already has content - skip OCR, do chunking/embedding"""
+        filename = file_metadata.file_name
+
+        if not file_metadata.content:
+            logger.warning(f"No content found for imported file {filename}")
+            return []
+
+        content = file_metadata.content
+        try:
+            # Create OCR response from existing content
+            ocr_response = self._create_ocr_response_from_content(content)
+
+            # Update file status to embedding
+            file_metadata = await self.file_manager.update_file_status(
+                file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.EMBEDDING
+            )
+
+            logger.info(f"Chunking imported file content for {filename}")
+            log_event("file_processor.import_chunking_started", {"filename": filename, "content_length": len(content)})
+
+            # Chunk and embed using existing logic
+            all_passages = await self._chunk_and_embed_with_fallback(
+                file_metadata=file_metadata, ocr_response=ocr_response, source_id=source_id
+            )
+
+            # Create passages in database (unless using Pinecone)
+            if not self.using_pinecone:
+                all_passages = await self.passage_manager.create_many_source_passages_async(
+                    passages=all_passages, file_metadata=file_metadata, actor=self.actor
+                )
+                log_event("file_processor.import_passages_created", {"filename": filename, "total_passages": len(all_passages)})
+
+            # Update file status to completed
+            if not self.using_pinecone:
+                await self.file_manager.update_file_status(
+                    file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.COMPLETED
+                )
+            else:
+                await self.file_manager.update_file_status(
+                    file_id=file_metadata.id, actor=self.actor, total_chunks=len(all_passages), chunks_embedded=0
+                )
+
+            logger.info(f"Successfully processed imported file {filename}: {len(all_passages)} passages")
+            log_event(
+                "file_processor.import_processing_completed",
+                {
+                    "filename": filename,
+                    "file_id": str(file_metadata.id),
+                    "total_passages": len(all_passages),
+                    "status": FileProcessingStatus.COMPLETED.value,
+                },
+            )
+
+            return all_passages
+
+        except Exception as e:
+            logger.exception("Import file processing failed for %s: %s", filename, e)
+            log_event(
+                "file_processor.import_processing_failed",
+                {
+                    "filename": filename,
+                    "file_id": str(file_metadata.id),
+                    "error": str(e),
+                    "error_type": type(e).__name__,
+                    "status": FileProcessingStatus.ERROR.value,
+                },
+            )
+            await self.file_manager.update_file_status(
+                file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.ERROR, error_message=str(e)
+            )
+
+            return []

letta/services/files_agents_manager.py

@@ -573,3 +573,31 @@ class FileAgentManager:
         if not assoc:
             raise NoResultFound(f"FileAgent(agent_id={agent_id}, file_name={file_name}) not found in org {actor.organization_id}")
         return assoc
+
+    @enforce_types
+    @trace_method
+    async def get_files_agents_for_agents_async(self, agent_ids: List[str], actor: PydanticUser) -> List[PydanticFileAgent]:
+        """
+        Get all file-agent relationships for multiple agents in a single query.
+
+        Args:
+            agent_ids: List of agent IDs to find file-agent relationships for
+            actor: User performing the action
+
+        Returns:
+            List[PydanticFileAgent]: List of file-agent relationships for these agents
+        """
+        if not agent_ids:
+            return []
+
+        async with db_registry.async_session() as session:
+            query = select(FileAgentModel).where(
+                FileAgentModel.agent_id.in_(agent_ids),
+                FileAgentModel.organization_id == actor.organization_id,
+                FileAgentModel.is_deleted == False,
+            )
+
+            result = await session.execute(query)
+            file_agents_orm = result.scalars().all()
+
+            return [file_agent.to_pydantic() for file_agent in file_agents_orm]

letta/services/group_manager.py

@@ -18,7 +18,6 @@ from letta.utils import enforce_types
 
 
 class GroupManager:
-
     @enforce_types
     @trace_method
     def list_groups(
@@ -164,7 +163,7 @@ class GroupManager:
             manager_agent_id = None
             if group_update.manager_config:
                 if group_update.manager_config.manager_type != group.manager_type:
-                    raise ValueError(f"Cannot change group pattern after creation")
+                    raise ValueError("Cannot change group pattern after creation")
                 match group_update.manager_config.manager_type:
                     case ManagerType.round_robin:
                         max_turns = group_update.manager_config.max_turns
@@ -473,7 +472,7 @@ class GroupManager:
         # 1) require both-or-none
         if (max_value is None) != (min_value is None):
             raise ValueError(
-                f"Both '{max_name}' and '{min_name}' must be provided together " f"(got {max_name}={max_value}, {min_name}={min_value})"
+                f"Both '{max_name}' and '{min_name}' must be provided together (got {max_name}={max_value}, {min_name}={min_value})"
             )
 
         # no further checks if neither is provided
@@ -488,9 +487,9 @@ class GroupManager:
             )
         if max_value <= 4 or min_value <= 4:
             raise ValueError(
-                f"Both '{max_name}' and '{min_name}' must be greater than 4 " f"(got {max_name}={max_value}, {min_name}={min_value})"
+                f"Both '{max_name}' and '{min_name}' must be greater than 4 (got {max_name}={max_value}, {min_name}={min_value})"
             )
 
         # 3) ordering
         if max_value <= min_value:
-            raise ValueError(f"'{max_name}' must be greater than '{min_name}' " f"(got {max_name}={max_value} <= {min_name}={min_value})")
+            raise ValueError(f"'{max_name}' must be greater than '{min_name}' (got {max_name}={max_value} <= {min_name}={min_value})")

letta/services/helpers/agent_manager_helper.py

@@ -4,6 +4,7 @@ from typing import List, Literal, Optional, Set
 
 import numpy as np
 from sqlalchemy import Select, and_, asc, desc, func, literal, nulls_last, or_, select, union_all
+from sqlalchemy.orm import noload
 from sqlalchemy.sql.expression import exists
 
 from letta import system
@@ -37,7 +38,7 @@ from letta.schemas.memory import Memory
 from letta.schemas.message import Message, MessageCreate
 from letta.schemas.tool_rule import ToolRule
 from letta.schemas.user import User
-from letta.settings import settings
+from letta.settings import DatabaseChoice, settings
 from letta.system import get_initial_boot_messages, get_login_event, package_function_response
 
 
@@ -382,7 +383,6 @@ def package_initial_message_sequence(
                     role=message_create.role,
                     content=[TextContent(text=packed_message)],
                     name=message_create.name,
-                    organization_id=actor.organization_id,
                     agent_id=agent_id,
                     model=model,
                 )
@@ -397,7 +397,6 @@ def package_initial_message_sequence(
                     role=message_create.role,
                     content=[TextContent(text=packed_message)],
                     name=message_create.name,
-                    organization_id=actor.organization_id,
                     agent_id=agent_id,
                     model=model,
                 )
@@ -418,7 +417,6 @@ def package_initial_message_sequence(
                     role=MessageRole.assistant,
                     content=None,
                     name=message_create.name,
-                    organization_id=actor.organization_id,
                     agent_id=agent_id,
                     model=model,
                     tool_calls=[
@@ -438,7 +436,6 @@ def package_initial_message_sequence(
                     role=MessageRole.tool,
                     content=[TextContent(text=function_response)],
                     name=message_create.name,
-                    organization_id=actor.organization_id,
                     agent_id=agent_id,
                     model=model,
                     tool_call_id=tool_call_id,
@@ -551,6 +548,9 @@ async def _apply_pagination_async(
         result = (await session.execute(select(sort_column, AgentModel.id).where(AgentModel.id == after))).first()
         if result:
             after_sort_value, after_id = result
+            # SQLite does not support as granular timestamping, so we need to round the timestamp
+            if settings.database_engine is DatabaseChoice.SQLITE and isinstance(after_sort_value, datetime):
+                after_sort_value = after_sort_value.strftime("%Y-%m-%d %H:%M:%S")
             query = query.where(
                 _cursor_filter(sort_column, AgentModel.id, after_sort_value, after_id, forward=ascending, nulls_last=sort_nulls_last)
             )
@@ -559,6 +559,9 @@ async def _apply_pagination_async(
         result = (await session.execute(select(sort_column, AgentModel.id).where(AgentModel.id == before))).first()
         if result:
             before_sort_value, before_id = result
+            # SQLite does not support as granular timestamping, so we need to round the timestamp
+            if settings.database_engine is DatabaseChoice.SQLITE and isinstance(before_sort_value, datetime):
+                before_sort_value = before_sort_value.strftime("%Y-%m-%d %H:%M:%S")
             query = query.where(
                 _cursor_filter(sort_column, AgentModel.id, before_sort_value, before_id, forward=not ascending, nulls_last=sort_nulls_last)
             )
@@ -649,7 +652,12 @@ def _apply_filters(
         query = query.where(AgentModel.name == name)
     # Apply a case-insensitive partial match for the agent's name.
     if query_text:
-        query = query.where(AgentModel.name.ilike(f"%{query_text}%"))
+        if settings.database_engine is DatabaseChoice.POSTGRES:
+            # PostgreSQL: Use ILIKE for case-insensitive search
+            query = query.where(AgentModel.name.ilike(f"%{query_text}%"))
+        else:
+            # SQLite: Use LIKE with LOWER for case-insensitive search
+            query = query.where(func.lower(AgentModel.name).like(func.lower(f"%{query_text}%")))
     # Filter agents by project ID.
     if project_id:
         query = query.where(AgentModel.project_id == project_id)
@@ -662,6 +670,24 @@ def _apply_filters(
     return query
 
 
+def _apply_relationship_filters(query, include_relationships: Optional[List[str]] = None):
+    if include_relationships is None:
+        return query
+
+    if "memory" not in include_relationships:
+        query = query.options(noload(AgentModel.core_memory), noload(AgentModel.file_agents))
+    if "identity_ids" not in include_relationships:
+        query = query.options(noload(AgentModel.identities))
+
+    relationships = ["tool_exec_environment_variables", "tools", "sources", "tags", "multi_agent_group"]
+
+    for rel in relationships:
+        if rel not in include_relationships:
+            query = query.options(noload(getattr(AgentModel, rel)))
+
+    return query
+
+
 def build_passage_query(
     actor: User,
     agent_id: Optional[str] = None,
@@ -790,7 +816,7 @@ def build_passage_query(
 
     # Vector search
     if embedded_text:
-        if settings.letta_pg_uri_no_default:
+        if settings.database_engine is DatabaseChoice.POSTGRES:
             # PostgreSQL with pgvector
             main_query = main_query.order_by(combined_query.c.embedding.cosine_distance(embedded_text).asc())
         else:
@@ -917,7 +943,7 @@ def build_source_passage_query(
 
     # Handle text search or vector search
     if embedded_text:
-        if settings.letta_pg_uri_no_default:
+        if settings.database_engine is DatabaseChoice.POSTGRES:
             # PostgreSQL with pgvector
             query = query.order_by(SourcePassage.embedding.cosine_distance(embedded_text).asc())
         else:
@@ -1004,7 +1030,7 @@ def build_agent_passage_query(
 
     # Handle text search or vector search
     if embedded_text:
-        if settings.letta_pg_uri_no_default:
+        if settings.database_engine is DatabaseChoice.POSTGRES:
             # PostgreSQL with pgvector
             query = query.order_by(AgentPassage.embedding.cosine_distance(embedded_text).asc())
         else:
@@ -1070,3 +1096,25 @@ def calculate_multi_agent_tools() -> Set[str]:
         return set(MULTI_AGENT_TOOLS) - set(LOCAL_ONLY_MULTI_AGENT_TOOLS)
     else:
         return set(MULTI_AGENT_TOOLS)
+
+
+@trace_method
+async def validate_agent_exists_async(session, agent_id: str, actor: User) -> None:
+    """
+    Validate that an agent exists and user has access to it using raw SQL for efficiency.
+
+    Args:
+        session: Database session
+        agent_id: ID of the agent to validate
+        actor: User performing the action
+
+    Raises:
+        NoResultFound: If agent doesn't exist or user doesn't have access
+    """
+    agent_exists_query = select(
+        exists().where(and_(AgentModel.id == agent_id, AgentModel.organization_id == actor.organization_id, AgentModel.is_deleted == False))
+    )
+    result = await session.execute(agent_exists_query)
+
+    if not result.scalar():
+        raise NoResultFound(f"Agent with ID {agent_id} not found")

letta/services/identity_manager.py

@@ -6,12 +6,14 @@ from sqlalchemy.exc import NoResultFound
 
 from letta.orm.agent import Agent as AgentModel
 from letta.orm.block import Block as BlockModel
+from letta.orm.errors import UniqueConstraintViolationError
 from letta.orm.identity import Identity as IdentityModel
 from letta.otel.tracing import trace_method
 from letta.schemas.identity import Identity as PydanticIdentity
 from letta.schemas.identity import IdentityCreate, IdentityProperty, IdentityType, IdentityUpdate, IdentityUpsert
 from letta.schemas.user import User as PydanticUser
 from letta.server.db import db_registry
+from letta.settings import DatabaseChoice, settings
 from letta.utils import enforce_types
 
 
@@ -64,6 +66,26 @@ class IdentityManager:
     async def _create_identity_async(self, db_session, identity: IdentityCreate, actor: PydanticUser) -> PydanticIdentity:
         new_identity = IdentityModel(**identity.model_dump(exclude={"agent_ids", "block_ids"}, exclude_unset=True))
         new_identity.organization_id = actor.organization_id
+
+        # For SQLite compatibility: check for unique constraint violation manually
+        # since SQLite doesn't support postgresql_nulls_not_distinct=True
+        if settings.database_engine is DatabaseChoice.SQLITE:
+            # Check if an identity with the same identifier_key, project_id, and organization_id exists
+            query = select(IdentityModel).where(
+                IdentityModel.identifier_key == new_identity.identifier_key,
+                IdentityModel.project_id == new_identity.project_id,
+                IdentityModel.organization_id == new_identity.organization_id,
+            )
+            result = await db_session.execute(query)
+            existing_identity = result.scalar_one_or_none()
+            if existing_identity is not None:
+                raise UniqueConstraintViolationError(
+                    f"A unique constraint was violated for Identity. "
+                    f"An identity with identifier_key='{new_identity.identifier_key}', "
+                    f"project_id='{new_identity.project_id}', and "
+                    f"organization_id='{new_identity.organization_id}' already exists."
+                )
+
         await self._process_relationship_async(
             db_session=db_session,
             identity=new_identity,