letta-nightly 0.8.17.dev20250722104501__py3-none-any.whl → 0.9.0.dev20250724081419__py3-none-any.whl

letta/services/file_processor/parser/markitdown_parser.py

@@ -0,0 +1,95 @@
+import logging
+import os
+import tempfile
+
+from markitdown import MarkItDown
+from mistralai import OCRPageObject, OCRResponse, OCRUsageInfo
+
+from letta.log import get_logger
+from letta.otel.tracing import trace_method
+from letta.services.file_processor.file_types import is_simple_text_mime_type
+from letta.services.file_processor.parser.base_parser import FileParser
+
+logger = get_logger(__name__)
+
+# Suppress pdfminer warnings that occur during PDF processing
+logging.getLogger("pdfminer.pdffont").setLevel(logging.ERROR)
+logging.getLogger("pdfminer.pdfinterp").setLevel(logging.ERROR)
+logging.getLogger("pdfminer.pdfpage").setLevel(logging.ERROR)
+logging.getLogger("pdfminer.converter").setLevel(logging.ERROR)
+
+
+class MarkitdownFileParser(FileParser):
+    """Markitdown-based file parsing for documents"""
+
+    def __init__(self, model: str = "markitdown"):
+        self.model = model
+
+    @trace_method
+    async def extract_text(self, content: bytes, mime_type: str) -> OCRResponse:
+        """Extract text using markitdown."""
+        try:
+            # Handle simple text files directly
+            if is_simple_text_mime_type(mime_type):
+                logger.info(f"Extracting text directly (no processing needed): {self.model}")
+                text = content.decode("utf-8", errors="replace")
+                return OCRResponse(
+                    model=self.model,
+                    pages=[
+                        OCRPageObject(
+                            index=0,
+                            markdown=text,
+                            images=[],
+                            dimensions=None,
+                        )
+                    ],
+                    usage_info=OCRUsageInfo(pages_processed=1),
+                    document_annotation=None,
+                )
+
+            logger.info(f"Extracting text using markitdown: {self.model}")
+
+            # Create temporary file to pass to markitdown
+            with tempfile.NamedTemporaryFile(delete=False, suffix=self._get_file_extension(mime_type)) as temp_file:
+                temp_file.write(content)
+                temp_file_path = temp_file.name
+
+            try:
+                md = MarkItDown(enable_plugins=False)
+                result = md.convert(temp_file_path)
+
+                return OCRResponse(
+                    model=self.model,
+                    pages=[
+                        OCRPageObject(
+                            index=0,
+                            markdown=result.text_content,
+                            images=[],
+                            dimensions=None,
+                        )
+                    ],
+                    usage_info=OCRUsageInfo(pages_processed=1),
+                    document_annotation=None,
+                )
+            finally:
+                # Clean up temporary file
+                os.unlink(temp_file_path)
+
+        except Exception as e:
+            logger.error(f"Markitdown text extraction failed: {str(e)}")
+            raise
+
+    def _get_file_extension(self, mime_type: str) -> str:
+        """Get file extension based on MIME type for markitdown processing."""
+        mime_to_ext = {
+            "application/pdf": ".pdf",
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
+            "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
+            "application/vnd.ms-excel": ".xls",
+            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
+            "text/csv": ".csv",
+            "application/json": ".json",
+            "text/xml": ".xml",
+            "application/xml": ".xml",
+        }
+        return mime_to_ext.get(mime_type, ".txt")

letta/services/files_agents_manager.py

@@ -1,14 +1,14 @@
 from datetime import datetime, timezone
-from typing import List, Optional
+from typing import List, Optional, Union
 
-from sqlalchemy import and_, func, select, update
+from sqlalchemy import and_, delete, func, or_, select, update
 
-from letta.constants import MAX_FILES_OPEN
 from letta.log import get_logger
 from letta.orm.errors import NoResultFound
 from letta.orm.files_agents import FileAgent as FileAgentModel
 from letta.otel.tracing import trace_method
 from letta.schemas.block import Block as PydanticBlock
+from letta.schemas.block import FileBlock as PydanticFileBlock
 from letta.schemas.file import FileAgent as PydanticFileAgent
 from letta.schemas.file import FileMetadata
 from letta.schemas.user import User as PydanticUser
@@ -31,6 +31,7 @@ class FileAgentManager:
         file_name: str,
         source_id: str,
         actor: PydanticUser,
+        max_files_open: int,
         is_open: bool = True,
         visible_content: Optional[str] = None,
     ) -> tuple[PydanticFileAgent, List[str]]:
@@ -40,7 +41,7 @@ class FileAgentManager:
         • If the row already exists → update `is_open`, `visible_content`
           and always refresh `last_accessed_at`.
         • Otherwise create a brand-new association.
-        • If is_open=True, enforces MAX_FILES_OPEN using LRU eviction.
+        • If is_open=True, enforces max_files_open using LRU eviction.
 
         Returns:
             Tuple of (file_agent, closed_file_names)
@@ -54,6 +55,7 @@ class FileAgentManager:
                 source_id=source_id,
                 actor=actor,
                 visible_content=visible_content or "",
+                max_files_open=max_files_open,
             )
 
             # Get the updated file agent to return
@@ -160,6 +162,36 @@ class FileAgentManager:
             assoc = await self._get_association_by_file_id(session, agent_id, file_id, actor)
             await assoc.hard_delete_async(session, actor=actor)
 
+    @enforce_types
+    @trace_method
+    async def detach_file_bulk(self, *, agent_file_pairs: List, actor: PydanticUser) -> int:  # List of (agent_id, file_id) tuples
+        """
+        Bulk delete multiple agent-file associations in a single query.
+
+        Args:
+            agent_file_pairs: List of (agent_id, file_id) tuples to delete
+            actor: User performing the action
+
+        Returns:
+            Number of rows deleted
+        """
+        if not agent_file_pairs:
+            return 0
+
+        async with db_registry.async_session() as session:
+            # Build compound OR conditions for each agent-file pair
+            conditions = []
+            for agent_id, file_id in agent_file_pairs:
+                conditions.append(and_(FileAgentModel.agent_id == agent_id, FileAgentModel.file_id == file_id))
+
+            # Create delete statement with all conditions
+            stmt = delete(FileAgentModel).where(and_(or_(*conditions), FileAgentModel.organization_id == actor.organization_id))
+
+            result = await session.execute(stmt)
+            await session.commit()
+
+            return result.rowcount
+
     @enforce_types
     @trace_method
     async def get_file_agent_by_id(self, *, agent_id: str, file_id: str, actor: PydanticUser) -> Optional[PydanticFileAgent]:
@@ -177,6 +209,7 @@ class FileAgentManager:
         *,
         file_names: List[str],
         agent_id: str,
+        per_file_view_window_char_limit: int,
         actor: PydanticUser,
     ) -> List[PydanticBlock]:
         """
@@ -185,6 +218,7 @@ class FileAgentManager:
         Args:
             file_names: List of file names to retrieve
             agent_id: ID of the agent to retrieve file blocks for
+            per_file_view_window_char_limit: The per-file view window char limit
             actor: The user making the request
 
         Returns:
@@ -207,7 +241,7 @@ class FileAgentManager:
             rows = (await session.execute(query)).scalars().all()
 
             # Convert to Pydantic models
-            return [row.to_pydantic_block() for row in rows]
+            return [row.to_pydantic_block(per_file_view_window_char_limit=per_file_view_window_char_limit) for row in rows]
 
     @enforce_types
     @trace_method
@@ -222,8 +256,13 @@ class FileAgentManager:
     @enforce_types
     @trace_method
     async def list_files_for_agent(
-        self, agent_id: str, actor: PydanticUser, is_open_only: bool = False, return_as_blocks: bool = False
-    ) -> List[PydanticFileAgent]:
+        self,
+        agent_id: str,
+        per_file_view_window_char_limit: int,
+        actor: PydanticUser,
+        is_open_only: bool = False,
+        return_as_blocks: bool = False,
+    ) -> Union[List[PydanticFileAgent], List[PydanticFileBlock]]:
         """Return associations for *agent_id* (filtering by `is_open` if asked)."""
         async with db_registry.async_session() as session:
             conditions = [
@@ -236,7 +275,7 @@ class FileAgentManager:
             rows = (await session.execute(select(FileAgentModel).where(and_(*conditions)))).scalars().all()
 
             if return_as_blocks:
-                return [r.to_pydantic_block() for r in rows]
+                return [r.to_pydantic_block(per_file_view_window_char_limit=per_file_view_window_char_limit) for r in rows]
             else:
                 return [r.to_pydantic() for r in rows]
 
@@ -334,7 +373,7 @@ class FileAgentManager:
     @enforce_types
     @trace_method
     async def enforce_max_open_files_and_open(
-        self, *, agent_id: str, file_id: str, file_name: str, source_id: str, actor: PydanticUser, visible_content: str
+        self, *, agent_id: str, file_id: str, file_name: str, source_id: str, actor: PydanticUser, visible_content: str, max_files_open: int
     ) -> tuple[List[str], bool]:
         """
         Efficiently handle LRU eviction and file opening in a single transaction.
@@ -343,7 +382,7 @@ class FileAgentManager:
             agent_id: ID of the agent
             file_id: ID of the file to open
             file_name: Name of the file to open
-            source_id: ID of the source (denormalized from files.source_id)
+            source_id: ID of the source
             actor: User performing the action
             visible_content: Content to set for the opened file
 
@@ -386,7 +425,7 @@ class FileAgentManager:
 
             # Calculate how many files need to be closed
             current_other_count = len(other_open_files)
-            target_other_count = MAX_FILES_OPEN - 1  # Reserve 1 slot for file we're opening
+            target_other_count = max_files_open - 1  # Reserve 1 slot for file we're opening
 
             closed_file_names = []
             if current_other_count > target_other_count:
@@ -443,6 +482,7 @@ class FileAgentManager:
         *,
         agent_id: str,
         files_metadata: list[FileMetadata],
+        max_files_open: int,
         visible_content_map: Optional[dict[str, str]] = None,
         actor: PydanticUser,
     ) -> list[str]:
@@ -496,17 +536,17 @@ class FileAgentManager:
             still_open_names = [r.file_name for r in currently_open if r.file_name not in new_names_set]
 
             # decide final open set
-            if len(new_names) >= MAX_FILES_OPEN:
-                final_open = new_names[:MAX_FILES_OPEN]
+            if len(new_names) >= max_files_open:
+                final_open = new_names[:max_files_open]
             else:
-                room_for_old = MAX_FILES_OPEN - len(new_names)
+                room_for_old = max_files_open - len(new_names)
                 final_open = new_names + still_open_names[-room_for_old:]
             final_open_set = set(final_open)
 
             closed_file_names = [r.file_name for r in currently_open if r.file_name not in final_open_set]
-            # Add new files that won't be opened due to MAX_FILES_OPEN limit
-            if len(new_names) >= MAX_FILES_OPEN:
-                closed_file_names.extend(new_names[MAX_FILES_OPEN:])
+            # Add new files that won't be opened due to max_files_open limit
+            if len(new_names) >= max_files_open:
+                closed_file_names.extend(new_names[max_files_open:])
             evicted_ids = [r.file_id for r in currently_open if r.file_name in closed_file_names]
 
             # upsert requested files

letta/services/group_manager.py

@@ -220,6 +220,13 @@ class GroupManager:
             group = GroupModel.read(db_session=session, identifier=group_id, actor=actor)
             group.hard_delete(session)
 
+    @enforce_types
+    @trace_method
+    async def delete_group_async(self, group_id: str, actor: PydanticUser) -> None:
+        async with db_registry.async_session() as session:
+            group = await GroupModel.read_async(db_session=session, identifier=group_id, actor=actor)
+            await group.hard_delete_async(session)
+
     @enforce_types
     @trace_method
     def list_group_messages(

letta/services/helpers/agent_manager_helper.py

@@ -22,11 +22,12 @@ from letta.constants import (
 from letta.embeddings import embedding_model
 from letta.helpers import ToolRulesSolver
 from letta.helpers.datetime_helpers import format_datetime, get_local_time, get_local_time_fast
-from letta.orm import AgentPassage, SourcePassage, SourcesAgents
 from letta.orm.agent import Agent as AgentModel
 from letta.orm.agents_tags import AgentsTags
 from letta.orm.errors import NoResultFound
 from letta.orm.identity import Identity
+from letta.orm.passage import AgentPassage, SourcePassage
+from letta.orm.sources_agents import SourcesAgents
 from letta.orm.sqlite_functions import adapt_array
 from letta.otel.tracing import trace_method
 from letta.prompts import gpt_system
@@ -45,7 +46,7 @@ from letta.system import get_initial_boot_messages, get_login_event, package_fun
 # Static methods
 @trace_method
 def _process_relationship(
-    session, agent: AgentModel, relationship_name: str, model_class, item_ids: List[str], allow_partial=False, replace=True
+    session, agent: "AgentModel", relationship_name: str, model_class, item_ids: List[str], allow_partial=False, replace=True
 ):
     """
     Generalized function to handle relationships like tools, sources, and blocks using item IDs.
@@ -88,7 +89,7 @@ def _process_relationship(
 
 @trace_method
 async def _process_relationship_async(
-    session, agent: AgentModel, relationship_name: str, model_class, item_ids: List[str], allow_partial=False, replace=True
+    session, agent: "AgentModel", relationship_name: str, model_class, item_ids: List[str], allow_partial=False, replace=True
 ):
     """
     Generalized function to handle relationships like tools, sources, and blocks using item IDs.
@@ -130,7 +131,7 @@ async def _process_relationship_async(
         current_relationship.extend(new_items)
 
 
-def _process_tags(agent: AgentModel, tags: List[str], replace=True):
+def _process_tags(agent: "AgentModel", tags: List[str], replace=True):
     """
     Handles tags for an agent.
 
@@ -207,16 +208,21 @@ def compile_memory_metadata_block(
     timestamp_str = format_datetime(memory_edit_timestamp, timezone)
 
     # Create a metadata block of info so the agent knows about the metadata of out-of-context memories
-    memory_metadata_block = "\n".join(
-        [
-            "<memory_metadata>",
-            f"- The current time is: {get_local_time_fast(timezone)}",
-            f"- Memory blocks were last modified: {timestamp_str}",
-            f"- {previous_message_count} previous messages between you and the user are stored in recall memory (use tools to access them)",
-            f"- {archival_memory_size} total memories you created are stored in archival memory (use tools to access them)",
-            "</memory_metadata>",
-        ]
-    )
+    metadata_lines = [
+        "<memory_metadata>",
+        f"- The current time is: {get_local_time_fast(timezone)}",
+        f"- Memory blocks were last modified: {timestamp_str}",
+        f"- {previous_message_count} previous messages between you and the user are stored in recall memory (use tools to access them)",
+    ]
+
+    # Only include archival memory line if there are archival memories
+    if archival_memory_size > 0:
+        metadata_lines.append(
+            f"- {archival_memory_size} total memories you created are stored in archival memory (use tools to access them)"
+        )
+
+    metadata_lines.append("</memory_metadata>")
+    memory_metadata_block = "\n".join(metadata_lines)
     return memory_metadata_block
 
 
@@ -253,6 +259,7 @@ def compile_system_message(
     archival_memory_size: int = 0,
     tool_rules_solver: Optional[ToolRulesSolver] = None,
     sources: Optional[List] = None,
+    max_files_open: Optional[int] = None,
 ) -> str:
     """Prepare the final/full system message that will be fed into the LLM API
 
@@ -285,7 +292,9 @@ def compile_system_message(
             timezone=timezone,
         )
 
-        memory_with_sources = in_context_memory.compile(tool_usage_rules=tool_constraint_block, sources=sources)
+        memory_with_sources = in_context_memory.compile(
+            tool_usage_rules=tool_constraint_block, sources=sources, max_files_open=max_files_open
+        )
         full_memory_string = memory_with_sources + "\n\n" + memory_metadata_string
 
         # Add to the variables list to inject
@@ -337,6 +346,7 @@ def initialize_message_sequence(
         previous_message_count=previous_message_count,
         archival_memory_size=archival_memory_size,
         sources=agent_state.sources,
+        max_files_open=agent_state.max_files_open,
     )
     first_user_message = get_login_event(agent_state.timezone)  # event letting Letta know the user just logged in
 

letta/services/provider_manager.py

@@ -207,7 +207,7 @@ class ProviderManager:
 
     @enforce_types
     @trace_method
-    def check_provider_api_key(self, provider_check: ProviderCheck) -> None:
+    async def check_provider_api_key(self, provider_check: ProviderCheck) -> None:
         provider = PydanticProvider(
             name=provider_check.provider_type.value,
             provider_type=provider_check.provider_type,
@@ -221,4 +221,4 @@ class ProviderManager:
         if not provider.api_key:
             raise ValueError("API key is required")
 
-        provider.check_api_key()
+        await provider.check_api_key()

letta/services/source_manager.py

@@ -1,5 +1,5 @@
 import asyncio
-from typing import List, Optional
+from typing import List, Optional, Union
 
 from sqlalchemy import and_, exists, select
 
@@ -234,37 +234,56 @@ class SourceManager:
 
     @enforce_types
     @trace_method
-    async def list_attached_agents(self, source_id: str, actor: PydanticUser) -> List[PydanticAgentState]:
+    async def list_attached_agents(
+        self, source_id: str, actor: PydanticUser, ids_only: bool = False
+    ) -> Union[List[PydanticAgentState], List[str]]:
         """
         Lists all agents that have the specified source attached.
 
         Args:
             source_id: ID of the source to find attached agents for
             actor: User performing the action
+            ids_only: If True, return only agent IDs instead of full agent states
 
         Returns:
-            List[PydanticAgentState]: List of agents that have this source attached
+            List[PydanticAgentState] | List[str]: List of agents or agent IDs that have this source attached
         """
         async with db_registry.async_session() as session:
             # Verify source exists and user has permission to access it
             await self._validate_source_exists_async(session, source_id, actor)
 
-            # Use junction table query instead of relationship to avoid performance issues
-            query = (
-                select(AgentModel)
-                .join(SourcesAgents, AgentModel.id == SourcesAgents.agent_id)
-                .where(
-                    SourcesAgents.source_id == source_id,
-                    AgentModel.organization_id == actor.organization_id,
-                    AgentModel.is_deleted == False,
+            if ids_only:
+                # Query only agent IDs for performance
+                query = (
+                    select(AgentModel.id)
+                    .join(SourcesAgents, AgentModel.id == SourcesAgents.agent_id)
+                    .where(
+                        SourcesAgents.source_id == source_id,
+                        AgentModel.organization_id == actor.organization_id,
+                        AgentModel.is_deleted == False,
+                    )
+                    .order_by(AgentModel.created_at.desc(), AgentModel.id)
                 )
-                .order_by(AgentModel.created_at.desc(), AgentModel.id)
-            )
 
-            result = await session.execute(query)
-            agents_orm = result.scalars().all()
+                result = await session.execute(query)
+                return list(result.scalars().all())
+            else:
+                # Use junction table query instead of relationship to avoid performance issues
+                query = (
+                    select(AgentModel)
+                    .join(SourcesAgents, AgentModel.id == SourcesAgents.agent_id)
+                    .where(
+                        SourcesAgents.source_id == source_id,
+                        AgentModel.organization_id == actor.organization_id,
+                        AgentModel.is_deleted == False,
+                    )
+                    .order_by(AgentModel.created_at.desc(), AgentModel.id)
+                )
+
+                result = await session.execute(query)
+                agents_orm = result.scalars().all()
 
-            return await asyncio.gather(*[agent.to_pydantic_async() for agent in agents_orm])
+                return await asyncio.gather(*[agent.to_pydantic_async() for agent in agents_orm])
 
     @enforce_types
     @trace_method

letta/services/tool_executor/files_tool_executor.py

@@ -2,7 +2,7 @@ import asyncio
 import re
 from typing import Any, Dict, List, Optional
 
-from letta.constants import MAX_FILES_OPEN, PINECONE_TEXT_FIELD_NAME
+from letta.constants import PINECONE_TEXT_FIELD_NAME
 from letta.functions.types import FileOpenRequest
 from letta.helpers.pinecone_utils import search_pinecone_index, should_use_pinecone
 from letta.log import get_logger
@@ -117,8 +117,10 @@ class LettaFileToolExecutor(ToolExecutor):
         file_requests = parsed_requests
 
         # Validate file count first
-        if len(file_requests) > MAX_FILES_OPEN:
-            raise ValueError(f"Cannot open {len(file_requests)} files: exceeds maximum limit of {MAX_FILES_OPEN} files")
+        if len(file_requests) > agent_state.max_files_open:
+            raise ValueError(
+                f"Cannot open {len(file_requests)} files: exceeds configured maximum limit of {agent_state.max_files_open} files"
+            )
 
         if not file_requests:
             raise ValueError("No file requests provided")
@@ -186,6 +188,7 @@ class LettaFileToolExecutor(ToolExecutor):
                 source_id=file.source_id,
                 actor=self.actor,
                 visible_content=visible_content,
+                max_files_open=agent_state.max_files_open,
             )
 
             opened_files.append(file_name)
@@ -329,7 +332,9 @@ class LettaFileToolExecutor(ToolExecutor):
             include_regex = re.compile(include_pattern, re.IGNORECASE)
 
         # Get all attached files for this agent
-        file_agents = await self.files_agents_manager.list_files_for_agent(agent_id=agent_state.id, actor=self.actor)
+        file_agents = await self.files_agents_manager.list_files_for_agent(
+            agent_id=agent_state.id, per_file_view_window_char_limit=agent_state.per_file_view_window_char_limit, actor=self.actor
+        )
 
         if not file_agents:
             return "No files are currently attached to search"
@@ -509,7 +514,9 @@ class LettaFileToolExecutor(ToolExecutor):
             return f"No valid source IDs found for attached files"
 
         # Get all attached files for this agent
-        file_agents = await self.files_agents_manager.list_files_for_agent(agent_id=agent_state.id, actor=self.actor)
+        file_agents = await self.files_agents_manager.list_files_for_agent(
+            agent_id=agent_state.id, per_file_view_window_char_limit=agent_state.per_file_view_window_char_limit, actor=self.actor
+        )
         if not file_agents:
             return "No files are currently attached to search"
 

letta/services/tool_manager.py

@@ -3,6 +3,8 @@ import os
 import warnings
 from typing import List, Optional, Set, Union
 
+from sqlalchemy import func, select
+
 from letta.constants import (
     BASE_FUNCTION_RETURN_CHAR_LIMIT,
     BASE_MEMORY_TOOLS,
@@ -290,6 +292,16 @@ class ToolManager:
         except NoResultFound:
             return None
 
+    @enforce_types
+    @trace_method
+    async def tool_exists_async(self, tool_id: str, actor: PydanticUser) -> bool:
+        """Check if a tool exists and belongs to the user's organization (lightweight check)."""
+        async with db_registry.async_session() as session:
+            query = select(func.count(ToolModel.id)).where(ToolModel.id == tool_id, ToolModel.organization_id == actor.organization_id)
+            result = await session.execute(query)
+            count = result.scalar()
+            return count > 0
+
     @enforce_types
     @trace_method
     async def list_tools_async(