letta-nightly 0.8.10.dev20250707035305__py3-none-any.whl → 0.8.11.dev20250708000504__py3-none-any.whl

letta/__init__.py

@@ -5,7 +5,7 @@ try:
     __version__ = version("letta")
 except PackageNotFoundError:
     # Fallback for development installations
-    __version__ = "0.8.10"
+    __version__ = "0.8.11"
 
 
 if os.environ.get("LETTA_VERSION"):

letta/constants.py

@@ -371,3 +371,4 @@ PINECONE_TEXT_FIELD_NAME = "chunk_text"
 PINECONE_METRIC = "cosine"
 PINECONE_CLOUD = "aws"
 PINECONE_REGION = "us-east-1"
+PINECONE_MAX_BATCH_SIZE = 96

letta/helpers/pinecone_utils.py

@@ -2,7 +2,14 @@ from typing import Any, Dict, List
 
 from pinecone import PineconeAsyncio
 
-from letta.constants import PINECONE_CLOUD, PINECONE_EMBEDDING_MODEL, PINECONE_METRIC, PINECONE_REGION, PINECONE_TEXT_FIELD_NAME
+from letta.constants import (
+    PINECONE_CLOUD,
+    PINECONE_EMBEDDING_MODEL,
+    PINECONE_MAX_BATCH_SIZE,
+    PINECONE_METRIC,
+    PINECONE_REGION,
+    PINECONE_TEXT_FIELD_NAME,
+)
 from letta.log import get_logger
 from letta.schemas.user import User
 from letta.settings import settings
@@ -90,7 +97,10 @@ async def upsert_records_to_pinecone_index(records: List[dict], actor: User):
     async with PineconeAsyncio(api_key=settings.pinecone_api_key) as pc:
         description = await pc.describe_index(name=settings.pinecone_source_index)
         async with pc.IndexAsyncio(host=description.index.host) as dense_index:
-            await dense_index.upsert_records(actor.organization_id, records)
+            # Process records in batches to avoid exceeding Pinecone limits
+            for i in range(0, len(records), PINECONE_MAX_BATCH_SIZE):
+                batch = records[i : i + PINECONE_MAX_BATCH_SIZE]
+                await dense_index.upsert_records(actor.organization_id, batch)
 
 
 async def search_pinecone_index(query: str, limit: int, filter: Dict[str, Any], actor: User) -> Dict[str, Any]:

letta/jobs/scheduler.py

@@ -4,10 +4,11 @@ from typing import Optional
 
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from apscheduler.triggers.interval import IntervalTrigger
+from sqlalchemy import text
 
 from letta.jobs.llm_batch_job_polling import poll_running_llm_batches
 from letta.log import get_logger
-from letta.server.db import db_context
+from letta.server.db import db_registry
 from letta.server.server import SyncServer
 from letta.settings import settings
 
@@ -16,68 +17,54 @@ scheduler = AsyncIOScheduler()
 logger = get_logger(__name__)
 ADVISORY_LOCK_KEY = 0x12345678ABCDEF00
 
-_advisory_lock_conn = None  # Holds the raw DB connection if leader
-_advisory_lock_cur = None  # Holds the cursor for the lock connection if leader
+_advisory_lock_session = None  # Holds the async session if leader
 _lock_retry_task: Optional[asyncio.Task] = None  # Background task handle for non-leaders
 _is_scheduler_leader = False  # Flag indicating if this instance runs the scheduler
 
 
 async def _try_acquire_lock_and_start_scheduler(server: SyncServer) -> bool:
     """Attempts to acquire lock, starts scheduler if successful."""
-    global _advisory_lock_conn, _advisory_lock_cur, _is_scheduler_leader, scheduler
+    global _advisory_lock_session, _is_scheduler_leader, scheduler
 
     if _is_scheduler_leader:
         return True  # Already leading
 
-    raw_conn = None
-    cur = None
+    lock_session = None
     acquired_lock = False
     try:
-        # Use a temporary connection context for the attempt initially
-        with db_context() as session:
+        async with db_registry.async_session() as session:
             engine = session.get_bind()
             engine_name = engine.name
             logger.info(f"Database engine type: {engine_name}")
-
             if engine_name != "postgresql":
                 logger.warning(f"Advisory locks not supported for {engine_name} database. Starting scheduler without leader election.")
-                acquired_lock = True  # For SQLite, assume we can start the scheduler
+                acquired_lock = True
             else:
-                # Get raw connection - MUST be kept open if lock is acquired
-                raw_conn = engine.raw_connection()
-                cur = raw_conn.cursor()
-
-                cur.execute("SELECT pg_try_advisory_lock(CAST(%s AS bigint))", (ADVISORY_LOCK_KEY,))
-                acquired_lock = cur.fetchone()[0]
+                lock_session = db_registry.get_async_session_factory()()
+                result = await lock_session.execute(
+                    text("SELECT pg_try_advisory_lock(CAST(:lock_key AS bigint))"), {"lock_key": ADVISORY_LOCK_KEY}
+                )
+                acquired_lock = result.scalar()
 
         if not acquired_lock:
-            if cur:
-                cur.close()
-            if raw_conn:
-                raw_conn.close()
+            if lock_session:
+                await lock_session.close()
             logger.info("Scheduler lock held by another instance.")
             return False
 
-        # --- Lock Acquired ---
         if engine_name == "postgresql":
             logger.info("Acquired PostgreSQL advisory lock.")
-            _advisory_lock_conn = raw_conn  # Keep connection for lock duration
-            _advisory_lock_cur = cur  # Keep cursor for lock duration
-            raw_conn = None  # Prevent closing in finally block
-            cur = None  # Prevent closing in finally block
+            _advisory_lock_session = lock_session
+            lock_session = None
         else:
             logger.info("Starting scheduler for non-PostgreSQL database.")
-            # For SQLite, we don't need to keep the connection open
-            if cur:
-                cur.close()
-            if raw_conn:
-                raw_conn.close()
-            raw_conn = None
-            cur = None
+            if lock_session:
+                await lock_session.close()
+            lock_session = None
 
         trigger = IntervalTrigger(
             seconds=settings.poll_running_llm_batches_interval_seconds,
-            jitter=10,  # Jitter for the job execution
+            jitter=10,
         )
         scheduler.add_job(
             poll_running_llm_batches,
@@ -91,7 +78,7 @@ async def _try_acquire_lock_and_start_scheduler(server: SyncServer) -> bool:
 
         if not scheduler.running:
             scheduler.start()
-        elif scheduler.state == 2:  # PAUSED
+        elif scheduler.state == 2:
             scheduler.resume()
 
         _is_scheduler_leader = True
@@ -99,38 +86,27 @@ async def _try_acquire_lock_and_start_scheduler(server: SyncServer) -> bool:
 
     except Exception as e:
         logger.error(f"Error during lock acquisition/scheduler start: {e}", exc_info=True)
-        if acquired_lock:  # If lock was acquired before error, try to release
+        if acquired_lock:
             logger.warning("Attempting to release lock due to error during startup.")
             try:
-                # Use the cursor/connection we were about to store
-                _advisory_lock_cur = cur
-                _advisory_lock_conn = raw_conn
-                await _release_advisory_lock()  # Attempt cleanup
+                _advisory_lock_session = lock_session
+                await _release_advisory_lock()
             except Exception as unlock_err:
                 logger.error(f"Failed to release lock during error handling: {unlock_err}", exc_info=True)
             finally:
-                # Ensure globals are cleared after failed attempt
-                _advisory_lock_cur = None
-                _advisory_lock_conn = None
+                _advisory_lock_session = None
                 _is_scheduler_leader = False
 
-        # Ensure scheduler is stopped if we failed partially
         if scheduler.running:
             try:
                 scheduler.shutdown(wait=False)
             except:
-                pass  # Best effort
+                pass
         return False
     finally:
-        # Clean up temporary resources if lock wasn't acquired or error occurred before storing
-        if cur:
-            try:
-                cur.close()
-            except:
-                pass
-        if raw_conn:
+        if lock_session:
             try:
-                raw_conn.close()
+                await lock_session.close()
             except:
                 pass
 
@@ -141,63 +117,50 @@ async def _background_lock_retry_loop(server: SyncServer):
     logger.info("Starting background task to periodically check for scheduler lock.")
 
     while True:
-        if _is_scheduler_leader:  # Should be cancelled first, but safety check
+        if _is_scheduler_leader:
             break
         try:
             wait_time = settings.poll_lock_retry_interval_seconds
             await asyncio.sleep(wait_time)
 
-            # Re-check state before attempting lock
             if _is_scheduler_leader or _lock_retry_task is None:
-                break  # Stop if became leader or task was cancelled
+                break
 
             acquired = await _try_acquire_lock_and_start_scheduler(server)
             if acquired:
                 logger.info("Background task acquired lock and started scheduler.")
-                _lock_retry_task = None  # Clear self handle
-                break  # Exit loop, we are now the leader
+                _lock_retry_task = None
+                break
 
         except asyncio.CancelledError:
             logger.info("Background lock retry task cancelled.")
             break
         except Exception as e:
             logger.error(f"Error in background lock retry loop: {e}", exc_info=True)
-            # Avoid tight loop on persistent errors
             await asyncio.sleep(settings.poll_lock_retry_interval_seconds)
 
 
 async def _release_advisory_lock():
-    """Releases the advisory lock using the stored connection."""
-    global _advisory_lock_conn, _advisory_lock_cur
+    """Releases the advisory lock using the stored session."""
+    global _advisory_lock_session
 
-    lock_cur = _advisory_lock_cur
-    lock_conn = _advisory_lock_conn
-    _advisory_lock_cur = None  # Clear global immediately
-    _advisory_lock_conn = None  # Clear global immediately
+    lock_session = _advisory_lock_session
+    _advisory_lock_session = None
 
-    if lock_cur is not None and lock_conn is not None:
+    if lock_session is not None:
         logger.info(f"Attempting to release PostgreSQL advisory lock {ADVISORY_LOCK_KEY}")
         try:
-            # Try to execute unlock - connection/cursor validity is checked by attempting the operation
-            lock_cur.execute("SELECT pg_advisory_unlock(CAST(%s AS bigint))", (ADVISORY_LOCK_KEY,))
-            lock_cur.fetchone()  # Consume result
-            lock_conn.commit()
+            await lock_session.execute(text("SELECT pg_advisory_unlock(CAST(:lock_key AS bigint))"), {"lock_key": ADVISORY_LOCK_KEY})
             logger.info(f"Executed pg_advisory_unlock for lock {ADVISORY_LOCK_KEY}")
         except Exception as e:
             logger.error(f"Error executing pg_advisory_unlock: {e}", exc_info=True)
         finally:
-            # Ensure resources are closed regardless of unlock success
-            try:
-                if lock_cur:
-                    lock_cur.close()
-            except Exception as e:
-                logger.error(f"Error closing advisory lock cursor: {e}", exc_info=True)
             try:
-                if lock_conn:
-                    lock_conn.close()
-                logger.info("Closed database connection that held advisory lock.")
+                if lock_session:
+                    await lock_session.close()
+                logger.info("Closed database session that held advisory lock.")
             except Exception as e:
-                logger.error(f"Error closing advisory lock connection: {e}", exc_info=True)
+                logger.error(f"Error closing advisory lock session: {e}", exc_info=True)
     else:
         logger.info("No PostgreSQL advisory lock to release (likely using SQLite or non-PostgreSQL database).")
 
@@ -220,7 +183,6 @@ async def start_scheduler_with_leader_election(server: SyncServer):
     acquired_immediately = await _try_acquire_lock_and_start_scheduler(server)
 
     if not acquired_immediately and _lock_retry_task is None:
-        # Failed initial attempt, start background retry task
         loop = asyncio.get_running_loop()
         _lock_retry_task = loop.create_task(_background_lock_retry_loop(server))
 
@@ -232,48 +194,40 @@ async def shutdown_scheduler_and_release_lock():
     """
     global _is_scheduler_leader, _lock_retry_task, scheduler
 
-    # 1. Cancel retry task if running (for non-leaders)
     if _lock_retry_task is not None:
         logger.info("Shutting down: Cancelling background lock retry task.")
         current_task = _lock_retry_task
-        _lock_retry_task = None  # Clear handle first
+        _lock_retry_task = None
         current_task.cancel()
         try:
-            await current_task  # Wait for cancellation
+            await current_task
         except asyncio.CancelledError:
             logger.info("Background lock retry task successfully cancelled.")
         except Exception as e:
             logger.warning(f"Exception waiting for cancelled retry task: {e}", exc_info=True)
 
-    # 2. Shutdown scheduler and release lock if we were the leader
     if _is_scheduler_leader:
         logger.info("Shutting down: Leader instance stopping scheduler and releasing lock.")
         if scheduler.running:
             try:
-                # Force synchronous shutdown to prevent callback scheduling
                 scheduler.shutdown(wait=True)
 
-                # wait for any internal cleanup to complete
                 await asyncio.sleep(0.1)
 
                 logger.info("APScheduler shut down.")
             except Exception as e:
-                # Handle SchedulerNotRunningError and other shutdown exceptions
                 logger.warning(f"Exception during APScheduler shutdown: {e}")
                 if "not running" not in str(e).lower():
                     logger.error(f"Unexpected error shutting down APScheduler: {e}", exc_info=True)
 
         await _release_advisory_lock()
-        _is_scheduler_leader = False  # Update state after cleanup
+        _is_scheduler_leader = False
     else:
         logger.info("Shutting down: Non-leader instance.")
 
-    # Final cleanup check for scheduler state (belt and suspenders)
-    # This should rarely be needed if shutdown logic above worked correctly
     try:
         if scheduler.running:
             logger.warning("Scheduler still running after shutdown logic completed? Forcing shutdown.")
             scheduler.shutdown(wait=False)
     except Exception as e:
-        # Catch SchedulerNotRunningError and other shutdown exceptions
         logger.debug(f"Expected exception during final scheduler cleanup: {e}")

letta/schemas/source_metadata.py

@@ -0,0 +1,32 @@
+from typing import List, Optional
+
+from pydantic import Field
+
+from letta.schemas.letta_base import LettaBase
+
+
+class FileStats(LettaBase):
+    """File statistics for metadata endpoint"""
+
+    file_id: str = Field(..., description="Unique identifier of the file")
+    file_name: str = Field(..., description="Name of the file")
+    file_size: Optional[int] = Field(None, description="Size of the file in bytes")
+
+
+class SourceStats(LettaBase):
+    """Aggregated metadata for a source"""
+
+    source_id: str = Field(..., description="Unique identifier of the source")
+    source_name: str = Field(..., description="Name of the source")
+    file_count: int = Field(0, description="Number of files in the source")
+    total_size: int = Field(0, description="Total size of all files in bytes")
+    files: List[FileStats] = Field(default_factory=list, description="List of file statistics")
+
+
+class OrganizationSourcesStats(LettaBase):
+    """Complete metadata response for organization sources"""
+
+    total_sources: int = Field(0, description="Total number of sources")
+    total_files: int = Field(0, description="Total number of files across all sources")
+    total_size: int = Field(0, description="Total size of all files in bytes")
+    sources: List[SourceStats] = Field(default_factory=list, description="List of source metadata")

letta/server/db.py

@@ -226,6 +226,32 @@ class DatabaseRegistry:
     @contextmanager
     def session(self, name: str = "default") -> Generator[Any, None, None]:
         """Context manager for database sessions."""
+        caller_info = "unknown caller"
+        try:
+            import inspect
+
+            frame = inspect.currentframe()
+            stack = inspect.getouterframes(frame)
+
+            for i, frame_info in enumerate(stack):
+                module = inspect.getmodule(frame_info.frame)
+                module_name = module.__name__ if module else "unknown"
+
+                if module_name != "contextlib" and "db.py" not in frame_info.filename:
+                    caller_module = module_name
+                    caller_function = frame_info.function
+                    caller_lineno = frame_info.lineno
+                    caller_file = frame_info.filename.split("/")[-1]
+
+                    caller_info = f"{caller_module}.{caller_function}:{caller_lineno} ({caller_file})"
+                    break
+        except:
+            pass
+        finally:
+            del frame
+
+        self.session_caller_trace(caller_info)
+
         session_factory = self.get_session_factory(name)
         if not session_factory:
             raise ValueError(f"No session factory found for '{name}'")
@@ -250,6 +276,11 @@ class DatabaseRegistry:
         finally:
             await session.close()
 
+    @trace_method
+    def session_caller_trace(self, caller_info: str):
+        """Trace sync db caller information for debugging purposes."""
+        pass  # wrapper used for otel tracing only
+
 
 # Create a singleton instance
 db_registry = DatabaseRegistry()

letta/server/rest_api/routers/v1/sources.py

@@ -23,10 +23,10 @@ from letta.schemas.enums import FileProcessingStatus
 from letta.schemas.file import FileMetadata
 from letta.schemas.passage import Passage
 from letta.schemas.source import Source, SourceCreate, SourceUpdate
+from letta.schemas.source_metadata import OrganizationSourcesStats
 from letta.schemas.user import User
 from letta.server.rest_api.utils import get_letta_server
 from letta.server.server import SyncServer
-from letta.services.file_processor.chunker.llama_index_chunker import LlamaIndexChunker
 from letta.services.file_processor.embedder.openai_embedder import OpenAIEmbedder
 from letta.services.file_processor.embedder.pinecone_embedder import PineconeEmbedder
 from letta.services.file_processor.file_processor import FileProcessor
@@ -95,6 +95,24 @@ async def get_source_id_by_name(
     return source.id
 
 
+@router.get("/metadata", response_model=OrganizationSourcesStats, operation_id="get_sources_metadata")
+async def get_sources_metadata(
+    server: "SyncServer" = Depends(get_letta_server),
+    actor_id: Optional[str] = Header(None, alias="user_id"),
+):
+    """
+    Get aggregated metadata for all sources in an organization.
+
+    Returns structured metadata including:
+    - Total number of sources
+    - Total number of files across all sources
+    - Total size of all files
+    - Per-source breakdown with file details (file_name, file_size per file)
+    """
+    actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)
+    return await server.file_manager.get_organization_sources_metadata(actor=actor)
+
+
 @router.get("/", response_model=List[Source], operation_id="list_sources")
 async def list_sources(
     server: "SyncServer" = Depends(get_letta_server),
@@ -344,7 +362,9 @@ async def get_file_metadata(
 
     if should_use_pinecone() and not file_metadata.is_processing_terminal():
         ids = await list_pinecone_index_for_files(file_id=file_id, actor=actor, limit=file_metadata.total_chunks)
-        logger.info(f"Embedded chunks {len(ids)}/{file_metadata.total_chunks} for {file_id} in organization {actor.organization_id}")
+        logger.info(
+            f"Embedded chunks {len(ids)}/{file_metadata.total_chunks} for {file_id} ({file_metadata.file_name}) in organization {actor.organization_id}"
+        )
 
         if len(ids) != file_metadata.chunks_embedded or len(ids) == file_metadata.total_chunks:
             if len(ids) != file_metadata.total_chunks:
@@ -424,15 +444,12 @@ async def load_file_to_source_cloud(
     file_metadata: FileMetadata,
 ):
     file_processor = MistralFileParser()
-    text_chunker = LlamaIndexChunker(chunk_size=embedding_config.embedding_chunk_size)
     using_pinecone = should_use_pinecone()
     if using_pinecone:
         embedder = PineconeEmbedder()
     else:
         embedder = OpenAIEmbedder(embedding_config=embedding_config)
-    file_processor = FileProcessor(
-        file_parser=file_processor, text_chunker=text_chunker, embedder=embedder, actor=actor, using_pinecone=using_pinecone
-    )
+    file_processor = FileProcessor(file_parser=file_processor, embedder=embedder, actor=actor, using_pinecone=using_pinecone)
     await file_processor.process(
         server=server, agent_states=agent_states, source_id=source_id, content=content, file_metadata=file_metadata
     )

letta/services/file_manager.py

@@ -16,6 +16,7 @@ from letta.otel.tracing import trace_method
 from letta.schemas.enums import FileProcessingStatus
 from letta.schemas.file import FileMetadata as PydanticFileMetadata
 from letta.schemas.source import Source as PydanticSource
+from letta.schemas.source_metadata import FileStats, OrganizationSourcesStats, SourceStats
 from letta.schemas.user import User as PydanticUser
 from letta.server.db import db_registry
 from letta.utils import enforce_types
@@ -272,3 +273,72 @@ class FileManager:
             else:
                 # Add numeric suffix
                 return f"{source.name}/{base}_({count}){ext}"
+
+    @enforce_types
+    @trace_method
+    async def get_organization_sources_metadata(self, actor: PydanticUser) -> OrganizationSourcesStats:
+        """
+        Get aggregated metadata for all sources in an organization with optimized queries.
+
+        Returns structured metadata including:
+        - Total number of sources
+        - Total number of files across all sources
+        - Total size of all files
+        - Per-source breakdown with file details
+        """
+        async with db_registry.async_session() as session:
+            # Import here to avoid circular imports
+            from letta.orm.source import Source as SourceModel
+
+            # Single optimized query to get all sources with their file aggregations
+            query = (
+                select(
+                    SourceModel.id,
+                    SourceModel.name,
+                    func.count(FileMetadataModel.id).label("file_count"),
+                    func.coalesce(func.sum(FileMetadataModel.file_size), 0).label("total_size"),
+                )
+                .outerjoin(FileMetadataModel, (FileMetadataModel.source_id == SourceModel.id) & (FileMetadataModel.is_deleted == False))
+                .where(SourceModel.organization_id == actor.organization_id)
+                .where(SourceModel.is_deleted == False)
+                .group_by(SourceModel.id, SourceModel.name)
+                .order_by(SourceModel.name)
+            )
+
+            result = await session.execute(query)
+            source_aggregations = result.fetchall()
+
+            # Build response
+            metadata = OrganizationSourcesStats()
+
+            for row in source_aggregations:
+                source_id, source_name, file_count, total_size = row
+
+                # Get individual file details for this source
+                files_query = (
+                    select(FileMetadataModel.id, FileMetadataModel.file_name, FileMetadataModel.file_size)
+                    .where(
+                        FileMetadataModel.source_id == source_id,
+                        FileMetadataModel.organization_id == actor.organization_id,
+                        FileMetadataModel.is_deleted == False,
+                    )
+                    .order_by(FileMetadataModel.file_name)
+                )
+
+                files_result = await session.execute(files_query)
+                files_rows = files_result.fetchall()
+
+                # Build file stats
+                files = [FileStats(file_id=file_row[0], file_name=file_row[1], file_size=file_row[2]) for file_row in files_rows]
+
+                # Build source metadata
+                source_metadata = SourceStats(
+                    source_id=source_id, source_name=source_name, file_count=file_count, total_size=total_size, files=files
+                )
+
+                metadata.sources.append(source_metadata)
+                metadata.total_files += file_count
+                metadata.total_size += total_size
+
+            metadata.total_sources = len(metadata.sources)
+            return metadata

letta/services/file_processor/chunker/line_chunker.py

@@ -99,7 +99,12 @@ class LineChunker:
         return [line for line in lines if line.strip()]
 
     def chunk_text(
-        self, file_metadata: FileMetadata, start: Optional[int] = None, end: Optional[int] = None, add_metadata: bool = True
+        self,
+        file_metadata: FileMetadata,
+        start: Optional[int] = None,
+        end: Optional[int] = None,
+        add_metadata: bool = True,
+        validate_range: bool = False,
     ) -> List[str]:
         """Content-aware text chunking based on file type"""
         strategy = self._determine_chunking_strategy(file_metadata)
@@ -116,11 +121,31 @@ class LineChunker:
             content_lines = self._chunk_by_lines(text, preserve_indentation=False)
 
         total_chunks = len(content_lines)
+        chunk_type = (
+            "sentences" if strategy == ChunkingStrategy.DOCUMENTATION else "chunks" if strategy == ChunkingStrategy.PROSE else "lines"
+        )
+
+        # Validate range if requested
+        if validate_range and (start is not None or end is not None):
+            if start is not None and start >= total_chunks:
+                # Convert to 1-indexed for user-friendly error message
+                start_display = start + 1
+                raise ValueError(
+                    f"File {file_metadata.file_name} has only {total_chunks} lines, but requested offset {start_display} is out of range"
+                )
+
+            if start is not None and end is not None and end > total_chunks:
+                # Convert to 1-indexed for user-friendly error message
+                start_display = start + 1
+                end_display = end
+                raise ValueError(
+                    f"File {file_metadata.file_name} has only {total_chunks} lines, but requested range {start_display} to {end_display} extends beyond file bounds"
+                )
 
         # Handle start/end slicing
-        if start is not None and end is not None:
+        if start is not None or end is not None:
             content_lines = content_lines[start:end]
-            line_offset = start
+            line_offset = start if start is not None else 0
         else:
             line_offset = 0
 
@@ -129,14 +154,15 @@ class LineChunker:
 
         # Add metadata about total chunks
         if add_metadata:
-            chunk_type = (
-                "sentences" if strategy == ChunkingStrategy.DOCUMENTATION else "chunks" if strategy == ChunkingStrategy.PROSE else "lines"
-            )
             if start is not None and end is not None:
                 # Display 1-indexed ranges for users
                 start_display = start + 1
                 end_display = end
                 content_lines.insert(0, f"[Viewing {chunk_type} {start_display} to {end_display} (out of {total_chunks} {chunk_type})]")
+            elif start is not None:
+                # Only start specified - viewing from start to end
+                start_display = start + 1
+                content_lines.insert(0, f"[Viewing {chunk_type} {start_display} to end (out of {total_chunks} {chunk_type})]")
             else:
                 content_lines.insert(0, f"[Viewing file start (out of {total_chunks} {chunk_type})]")
 

letta/services/file_processor/chunker/llama_index_chunker.py

@@ -1,119 +1,169 @@
-from typing import List, Tuple
+from typing import List, Optional, Union
 
 from mistralai import OCRPageObject
 
 from letta.log import get_logger
 from letta.otel.tracing import trace_method
+from letta.services.file_processor.file_types import ChunkingStrategy, file_type_registry
 
 logger = get_logger(__name__)
 
 
 class LlamaIndexChunker:
-    """LlamaIndex-based text chunking"""
+    """LlamaIndex-based text chunking with automatic splitter selection"""
 
-    def __init__(self, chunk_size: int = 512, chunk_overlap: int = 50):
+    # Conservative default chunk sizes for fallback scenarios
+    DEFAULT_CONSERVATIVE_CHUNK_SIZE = 384
+    DEFAULT_CONSERVATIVE_CHUNK_OVERLAP = 25
+
+    def __init__(self, chunk_size: int = 512, chunk_overlap: int = 50, file_type: Optional[str] = None):
         self.chunk_size = chunk_size
         self.chunk_overlap = chunk_overlap
+        self.file_type = file_type
 
-        from llama_index.core.node_parser import SentenceSplitter
+        # Create appropriate parser based on file type
+        self.parser = self._create_parser_for_file_type(file_type, chunk_size, chunk_overlap)
 
-        self.parser = SentenceSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        # Log which parser was selected
+        parser_name = type(self.parser).__name__
+        logger.info(f"LlamaIndexChunker initialized with {parser_name} for file type: {file_type}")
+
+    def _create_parser_for_file_type(self, file_type: Optional[str], chunk_size: int, chunk_overlap: int):
+        """Create appropriate parser based on file type"""
+        if not file_type:
+            # Default fallback
+            from llama_index.core.node_parser import SentenceSplitter
+
+            return SentenceSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
 
-    # TODO: Make this more general beyond Mistral
-    @trace_method
-    def chunk_text(self, page: OCRPageObject) -> List[str]:
-        """Chunk text using LlamaIndex splitter"""
         try:
-            return self.parser.split_text(page.markdown)
+            # Get chunking strategy from file type registry
+            chunking_strategy = file_type_registry.get_chunking_strategy_by_mime_type(file_type)
+            logger.debug(f"Chunking strategy for {file_type}: {chunking_strategy}")
 
-        except Exception as e:
-            logger.error(f"Chunking failed: {str(e)}")
-            raise
+            if chunking_strategy == ChunkingStrategy.CODE:
+                from llama_index.core.node_parser import CodeSplitter
 
+                return CodeSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
 
-class MarkdownChunker:
-    """Markdown-specific chunker that preserves line numbers for citation purposes"""
+            elif chunking_strategy == ChunkingStrategy.DOCUMENTATION:
+                if file_type in ["text/markdown", "text/x-markdown"]:
+                    from llama_index.core.node_parser import MarkdownNodeParser
 
-    def __init__(self, chunk_size: int = 2048):
-        self.chunk_size = chunk_size
-        # No overlap for line-based citations to avoid ambiguity
+                    return MarkdownNodeParser()
+                elif file_type in ["text/html"]:
+                    from llama_index.core.node_parser import HTMLNodeParser
+
+                    return HTMLNodeParser()
+                else:
+                    # Fall back to sentence splitter for other documentation
+                    from llama_index.core.node_parser import SentenceSplitter
+
+                    return SentenceSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+
+            elif chunking_strategy == ChunkingStrategy.STRUCTURED_DATA:
+                if file_type in ["application/json", "application/jsonl"]:
+                    from llama_index.core.node_parser import JSONNodeParser
 
-        from llama_index.core.node_parser import MarkdownNodeParser
+                    return JSONNodeParser()
+                else:
+                    # Fall back to sentence splitter for other structured data
+                    from llama_index.core.node_parser import SentenceSplitter
 
-        self.parser = MarkdownNodeParser()
+                    return SentenceSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
 
-    def chunk_markdown_with_line_numbers(self, markdown_content: str) -> List[Tuple[str, int, int]]:
-        """
-        Chunk markdown content while preserving line number mappings.
+            else:
+                # Default to sentence splitter for PROSE and LINE_BASED
+                from llama_index.core.node_parser import SentenceSplitter
 
-        Returns:
-            List of tuples: (chunk_text, start_line, end_line)
-        """
+                return SentenceSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+
+        except Exception as e:
+            logger.warning(f"Failed to create specialized parser for {file_type}: {str(e)}. Using default SentenceSplitter.")
+            from llama_index.core.node_parser import SentenceSplitter
+
+            return SentenceSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+
+    @trace_method
+    def chunk_text(self, content: Union[OCRPageObject, str]) -> List[str]:
+        """Chunk text using LlamaIndex splitter"""
         try:
-            # Split content into lines for line number tracking
-            lines = markdown_content.split("\n")
+            # Handle different input types
+            if isinstance(content, OCRPageObject):
+                # Extract markdown from OCR page object
+                text_content = content.markdown
+            else:
+                # Assume it's a string
+                text_content = content
+
+            # Use the selected parser
+            if hasattr(self.parser, "split_text"):
+                # Most parsers have split_text method
+                return self.parser.split_text(text_content)
+            elif hasattr(self.parser, "get_nodes_from_documents"):
+                # Some parsers need Document objects
+                from llama_index.core import Document
+                from llama_index.core.node_parser import SentenceSplitter
+
+                document = Document(text=text_content)
+                nodes = self.parser.get_nodes_from_documents([document])
+
+                # Further split nodes that exceed chunk_size using SentenceSplitter
+                final_chunks = []
+                sentence_splitter = SentenceSplitter(chunk_size=self.chunk_size, chunk_overlap=self.chunk_overlap)
+
+                for node in nodes:
+                    if len(node.text) > self.chunk_size:
+                        # Split oversized nodes with sentence splitter
+                        sub_chunks = sentence_splitter.split_text(node.text)
+                        final_chunks.extend(sub_chunks)
+                    else:
+                        final_chunks.append(node.text)
+
+                return final_chunks
+            else:
+                # Fallback - try to call the parser directly
+                return self.parser(text_content)
 
-            # Create nodes using MarkdownNodeParser
-            from llama_index.core import Document
+        except Exception as e:
+            logger.error(f"Chunking failed with {type(self.parser).__name__}: {str(e)}")
+            # Try fallback with SentenceSplitter
+            try:
+                logger.info("Attempting fallback to SentenceSplitter")
+                from llama_index.core.node_parser import SentenceSplitter
 
-            document = Document(text=markdown_content)
-            nodes = self.parser.get_nodes_from_documents([document])
+                fallback_parser = SentenceSplitter(chunk_size=self.chunk_size, chunk_overlap=self.chunk_overlap)
 
-            chunks_with_line_numbers = []
+                # Extract text content if needed
+                if isinstance(content, OCRPageObject):
+                    text_content = content.markdown
+                else:
+                    text_content = content
 
-            for node in nodes:
-                chunk_text = node.text
+                return fallback_parser.split_text(text_content)
+            except Exception as fallback_error:
+                logger.error(f"Fallback chunking also failed: {str(fallback_error)}")
+                raise e  # Raise the original error
 
-                # Find the line numbers for this chunk
-                start_line, end_line = self._find_line_numbers(chunk_text, lines)
+    @trace_method
+    def default_chunk_text(self, content: Union[OCRPageObject, str], chunk_size: int = None, chunk_overlap: int = None) -> List[str]:
+        """Chunk text using default SentenceSplitter regardless of file type with conservative defaults"""
+        try:
+            from llama_index.core.node_parser import SentenceSplitter
 
-                chunks_with_line_numbers.append((chunk_text, start_line, end_line))
+            # Use provided defaults or fallback to conservative values
+            chunk_size = chunk_size if chunk_size is not None else self.DEFAULT_CONSERVATIVE_CHUNK_SIZE
+            chunk_overlap = chunk_overlap if chunk_overlap is not None else self.DEFAULT_CONSERVATIVE_CHUNK_OVERLAP
+            default_parser = SentenceSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
 
-            return chunks_with_line_numbers
+            # Handle different input types
+            if isinstance(content, OCRPageObject):
+                text_content = content.markdown
+            else:
+                text_content = content
+
+            return default_parser.split_text(text_content)
 
         except Exception as e:
-            logger.error(f"Markdown chunking failed: {str(e)}")
-            # Fallback to simple line-based chunking
-            return self._fallback_line_chunking(markdown_content)
-
-    def _find_line_numbers(self, chunk_text: str, lines: List[str]) -> Tuple[int, int]:
-        """Find the start and end line numbers for a given chunk of text."""
-        chunk_lines = chunk_text.split("\n")
-
-        # Find the first line of the chunk in the original document
-        start_line = 1
-        for i, line in enumerate(lines):
-            if chunk_lines[0].strip() in line.strip() and len(chunk_lines[0].strip()) > 10:  # Avoid matching short lines
-                start_line = i + 1
-                break
-
-        # Calculate end line
-        end_line = start_line + len(chunk_lines) - 1
-
-        return start_line, min(end_line, len(lines))
-
-    def _fallback_line_chunking(self, markdown_content: str) -> List[Tuple[str, int, int]]:
-        """Fallback chunking method that simply splits by lines with no overlap."""
-        lines = markdown_content.split("\n")
-        chunks = []
-
-        i = 0
-        while i < len(lines):
-            chunk_lines = []
-            start_line = i + 1
-            char_count = 0
-
-            # Build chunk until we hit size limit
-            while i < len(lines) and char_count < self.chunk_size:
-                line = lines[i]
-                chunk_lines.append(line)
-                char_count += len(line) + 1  # +1 for newline
-                i += 1
-
-            end_line = i
-            chunk_text = "\n".join(chunk_lines)
-            chunks.append((chunk_text, start_line, end_line))
-
-            # No overlap - continue from where we left off
-
-        return chunks
+            logger.error(f"Default chunking failed: {str(e)}")
+            raise

letta/services/file_processor/embedder/openai_embedder.py

@@ -91,7 +91,7 @@ class OpenAIEmbedder(BaseEmbedder):
             try:
                 return await self._embed_batch(batch, indices)
             except Exception as e:
-                logger.error(f"Failed to embed batch of size {len(batch)}: {str(e)}")
+                logger.error("Failed to embed batch of size %s: %s", len(batch), e)
                 log_event("embedder.batch_failed", {"batch_size": len(batch), "error": str(e), "error_type": type(e).__name__})
                 raise
 

letta/services/file_processor/file_processor.py

@@ -26,14 +26,12 @@ class FileProcessor:
     def __init__(
         self,
         file_parser: MistralFileParser,
-        text_chunker: LlamaIndexChunker,
         embedder: BaseEmbedder,
         actor: User,
         using_pinecone: bool,
         max_file_size: int = 50 * 1024 * 1024,  # 50MB default
     ):
         self.file_parser = file_parser
-        self.text_chunker = text_chunker
         self.line_chunker = LineChunker()
         self.embedder = embedder
         self.max_file_size = max_file_size
@@ -44,6 +42,61 @@ class FileProcessor:
         self.actor = actor
         self.using_pinecone = using_pinecone
 
+    async def _chunk_and_embed_with_fallback(self, file_metadata: FileMetadata, ocr_response, source_id: str) -> List:
+        """Chunk text and generate embeddings with fallback to default chunker if needed"""
+        filename = file_metadata.file_name
+
+        # Create file-type-specific chunker
+        text_chunker = LlamaIndexChunker(file_type=file_metadata.file_type)
+
+        # First attempt with file-specific chunker
+        try:
+            all_chunks = []
+            for page in ocr_response.pages:
+                chunks = text_chunker.chunk_text(page)
+                if not chunks:
+                    log_event("file_processor.chunking_failed", {"filename": filename, "page_index": ocr_response.pages.index(page)})
+                    raise ValueError("No chunks created from text")
+                all_chunks.extend(chunks)
+
+            all_passages = await self.embedder.generate_embedded_passages(
+                file_id=file_metadata.id, source_id=source_id, chunks=all_chunks, actor=self.actor
+            )
+            return all_passages
+
+        except Exception as e:
+            logger.warning(f"Failed to chunk/embed with file-specific chunker for {filename}: {str(e)}. Retrying with default chunker.")
+            log_event("file_processor.embedding_failed_retrying", {"filename": filename, "error": str(e), "error_type": type(e).__name__})
+
+            # Retry with default chunker
+            try:
+                logger.info(f"Retrying chunking with default SentenceSplitter for {filename}")
+                all_chunks = []
+
+                for page in ocr_response.pages:
+                    chunks = text_chunker.default_chunk_text(page)
+                    if not chunks:
+                        log_event(
+                            "file_processor.default_chunking_failed", {"filename": filename, "page_index": ocr_response.pages.index(page)}
+                        )
+                        raise ValueError("No chunks created from text with default chunker")
+                    all_chunks.extend(chunks)
+
+                all_passages = await self.embedder.generate_embedded_passages(
+                    file_id=file_metadata.id, source_id=source_id, chunks=all_chunks, actor=self.actor
+                )
+                logger.info(f"Successfully generated passages with default chunker for {filename}")
+                log_event("file_processor.default_chunking_success", {"filename": filename, "total_chunks": len(all_chunks)})
+                return all_passages
+
+            except Exception as fallback_error:
+                logger.error("Default chunking also failed for %s: %s", filename, fallback_error)
+                log_event(
+                    "file_processor.default_chunking_also_failed",
+                    {"filename": filename, "fallback_error": str(fallback_error), "fallback_error_type": type(fallback_error).__name__},
+                )
+                raise fallback_error
+
     # TODO: Factor this function out of SyncServer
     @trace_method
     async def process(
@@ -111,19 +164,10 @@ class FileProcessor:
 
             logger.info("Chunking extracted text")
             log_event("file_processor.chunking_started", {"filename": filename, "pages_to_process": len(ocr_response.pages)})
-            all_chunks = []
-
-            for page in ocr_response.pages:
-                chunks = self.text_chunker.chunk_text(page)
 
-                if not chunks:
-                    log_event("file_processor.chunking_failed", {"filename": filename, "page_index": ocr_response.pages.index(page)})
-                    raise ValueError("No chunks created from text")
-
-                all_chunks.extend(self.text_chunker.chunk_text(page))
-
-            all_passages = await self.embedder.generate_embedded_passages(
-                file_id=file_metadata.id, source_id=source_id, chunks=all_chunks, actor=self.actor
+            # Chunk and embed with fallback logic
+            all_passages = await self._chunk_and_embed_with_fallback(
+                file_metadata=file_metadata, ocr_response=ocr_response, source_id=source_id
             )
 
             if not self.using_pinecone:
@@ -156,7 +200,7 @@ class FileProcessor:
             return all_passages
 
         except Exception as e:
-            logger.error(f"File processing failed for {filename}: {str(e)}")
+            logger.error("File processing failed for %s: %s", filename, e)
             log_event(
                 "file_processor.processing_failed",
                 {

letta/services/tool_executor/files_tool_executor.py

@@ -175,7 +175,7 @@ class LettaFileToolExecutor(ToolExecutor):
             file = await self.file_manager.get_file_by_id(file_id=file_id, actor=self.actor, include_content=True)
 
             # Process file content
-            content_lines = LineChunker().chunk_text(file_metadata=file, start=start, end=end)
+            content_lines = LineChunker().chunk_text(file_metadata=file, start=start, end=end, validate_range=True)
             visible_content = "\n".join(content_lines)
 
             # Handle LRU eviction and file opening

letta/settings.py

@@ -247,7 +247,7 @@ class Settings(BaseSettings):
     # cron job parameters
     enable_batch_job_polling: bool = False
     poll_running_llm_batches_interval_seconds: int = 5 * 60
-    poll_lock_retry_interval_seconds: int = 5 * 60
+    poll_lock_retry_interval_seconds: int = 8 * 60
     batch_job_polling_lookback_weeks: int = 2
     batch_job_polling_batch_size: Optional[int] = None
 

{letta_nightly-0.8.10.dev20250707035305.dist-info → letta_nightly-0.8.11.dev20250708000504.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: letta-nightly
-Version: 0.8.10.dev20250707035305
+Version: 0.8.11.dev20250708000504
 Summary: Create LLM agents with long-term memory and custom tools
 License: Apache License
 Author: Letta Team

{letta_nightly-0.8.10.dev20250707035305.dist-info → letta_nightly-0.8.11.dev20250708000504.dist-info}/RECORD

@@ -1,4 +1,4 @@
-letta/__init__.py,sha256=Bp20c-Yi_EunL5Bz76_a8uG3RnCyxVwt9lpnslSkfkg,1223
+letta/__init__.py,sha256=MY4Eqs94gdtwy8X6KBPrk0zKk7LUqwyQWvQU-ssrpes,1223
 letta/agent.py,sha256=esW2W5hBzO7aPr7ghEDb_fLnUxgYqBYDq_VWtQDrB0c,89153
 letta/agents/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 letta/agents/base_agent.py,sha256=Z1jgCTMFRTLnaLRcfdo8TmsP8tuCYqNcOM8ov9kviMA,6869
@@ -17,7 +17,7 @@ letta/client/client.py,sha256=l_yKUUzl1-qfxFkDHsOMHxSwyzOBbx-2mi0GfI3WlJE,84906
 letta/client/streaming.py,sha256=UsDS_tDTsA3HgYryIDvGGmx_dWfnfQwtmEwLi4Z89Ik,4701
 letta/client/utils.py,sha256=VCGV-op5ZSmurd4yw7Vhf93XDQ0BkyBT8qsuV7EqfiU,2859
 letta/config.py,sha256=JFGY4TWW0Wm5fTbZamOwWqk5G8Nn-TXyhgByGoAqy2c,12375
-letta/constants.py,sha256=oxe2dtsOhBcfRJSEf5lTvPDCOYhBhALWEisTl2cKhJg,14704
+letta/constants.py,sha256=eGXwXbvIZyvxiX2Trg28i8a7ABfhWqtqb36D3ciyr_8,14733
 letta/data_sources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 letta/data_sources/connectors.py,sha256=V8mUgE3V6CX-CcOyvkPSQ_ZWP2VtuqgTEXkCN1j0p68,7920
 letta/data_sources/connectors_helper.py,sha256=oQpVlc-BjSz9sTZ7sp4PsJSXJbBKpZPi3Dam03CURTQ,3376
@@ -59,7 +59,7 @@ letta/helpers/datetime_helpers.py,sha256=8AwZInX-NX_XQiqej2arozYqfC2ysnWpCJ9ETv8
 letta/helpers/decorators.py,sha256=jyywXMxO5XPDSe93ybVXIOjTWkGX514S9BMcy_gP0j8,5891
 letta/helpers/json_helpers.py,sha256=PWZ5HhSqGXO4e563dM_8M72q7ScirjXQ4Rv1ckohaV8,396
 letta/helpers/message_helper.py,sha256=Xzf_VCMAXT0Ys8LVUh1ySVtgJwabSQYksOdPr7P4EJU,3549
-letta/helpers/pinecone_utils.py,sha256=wey0quuu6WTwW5dlcMwkXa1GqoeUqzhKcDI76Svhalg,6244
+letta/helpers/pinecone_utils.py,sha256=LMfrynzenK_IuVGEsZmULz4AAtZ58Wof02ENj-NwtLQ,6511
 letta/helpers/singleton.py,sha256=Y4dG_ZBCcrogvl9iZ69bSLq-QltrdP8wHqKkhef8OBI,370
 letta/helpers/tool_execution_helper.py,sha256=BgBgVLZzbc-JTdOGwyU9miV_-zM3A30jkMpwH1otxaU,7599
 letta/helpers/tool_rule_solver.py,sha256=avRMQzqxE2r6gRvw7oTImYmkSvuoMHlADPND0__feBw,11620
@@ -75,7 +75,7 @@ letta/interfaces/utils.py,sha256=c6jvO0dBYHh8DQnlN-B0qeNC64d3CSunhfqlFA4pJTY,278
 letta/jobs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 letta/jobs/helpers.py,sha256=kO4aj954xsQ1RAmkjY6LQQ7JEIGuhaxB1e9pzrYKHAY,914
 letta/jobs/llm_batch_job_polling.py,sha256=r_6D5RcqEJQgrdh-rnN7vdLD0GAQl-GGmIfCnV0naHQ,10299
-letta/jobs/scheduler.py,sha256=P_M_x7NtF3foHGRVCaDMXSILiHur78r4lXYe4rq4Z-M,11491
+letta/jobs/scheduler.py,sha256=bnwvgT_72ULlKmSFG-26T6tfmiEzxSrm9ARl1MuOEvA,8818
 letta/jobs/types.py,sha256=K8GKEnqEgAT6Kq4F2hUrBC4ZAFM9OkfOjVMStzxKuXQ,742
 letta/llm_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 letta/llm_api/anthropic.py,sha256=tbMy4483TySrEmbXD3juM6TpPRrV9_M3Fgp59sDBcqE,47935
@@ -265,6 +265,7 @@ letta/schemas/response_format.py,sha256=pXNsjbtpA3Tf8HsDyIa40CSmoUbVR_7n2WOfQaX4
 letta/schemas/run.py,sha256=1lVOWlHVbk9MYIOiIrE1gCoQvBhErKo7UMSeWyMExbw,2089
 letta/schemas/sandbox_config.py,sha256=thI4p7R4nnW1W-F_PBNkpmyHXpSH_lorlQX8YxDXSe0,5252
 letta/schemas/source.py,sha256=ZDeTjkNp1rKamG7xZzoUHeCptjpW9WNLzAcJ9QQRxlM,3444
+letta/schemas/source_metadata.py,sha256=_dGjuXhGcVMlc53ja9yuk16Uj64ggEzilRDgmkqYfNs,1334
 letta/schemas/step.py,sha256=QudHSpLMcNVC-oI26Uy48lsp3FOrcd3JYAp5ubDEIHY,2651
 letta/schemas/tool.py,sha256=C2HdnmwrjAWoBjB8H2lpO8oIys3HlkRrRWtR8uMcUfc,14375
 letta/schemas/tool_execution_result.py,sha256=4P77llsUsZBnRd0PtPiC4VzGjx7i_-fUNgXQfCpMS9U,896
@@ -283,7 +284,7 @@ letta/serialize_schemas/marshmallow_tool.py,sha256=jwU69BDCakPlYPSk-ta21kuvsURKO
 letta/serialize_schemas/pydantic_agent_schema.py,sha256=CqGqSFzArYE2CzFsIU8LXVmH1A1jYFQpFy7Sj62n_4A,3171
 letta/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 letta/server/constants.py,sha256=yAdGbLkzlOU_dLTx0lKDmAnj0ZgRXCEaIcPJWO69eaE,92
-letta/server/db.py,sha256=WYuTpqzJxi98WownUX5VZFPzsc99Pp91xPSNYiU5_KU,10631
+letta/server/db.py,sha256=q5wCpTxpcbutk2HuF2ZpyhlPGCdUuLpRFZ0QE5VkZ38,11751
 letta/server/generate_openapi_schema.sh,sha256=0OtBhkC1g6CobVmNEd_m2B6sTdppjbJLXaM95icejvE,371
 letta/server/rest_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 letta/server/rest_api/app.py,sha256=JEJlOKBPKfnkQeM6BAd5Olf4EM7O2TKD4_MHObFuci8,18084
@@ -310,7 +311,7 @@ letta/server/rest_api/routers/v1/organizations.py,sha256=5NEjTOdGKWrfN584jfPpJhA
 letta/server/rest_api/routers/v1/providers.py,sha256=8SJ_RsSk7L4nh1f_uFE31JOxefmGhOfN-fMJ0Sp6SJo,4353
 letta/server/rest_api/routers/v1/runs.py,sha256=vieUp7uTvRTdAte0Nw1bqX2APMATZhKTr2R1HVNJT74,8879
 letta/server/rest_api/routers/v1/sandbox_configs.py,sha256=pKuy88GD3atrBkKa7VVfKTjg8Y07e1vVtdw4TtxkQBk,8910
-letta/server/rest_api/routers/v1/sources.py,sha256=5OkHUlBUVeE3vI69F8xtM5lmYnxxjDuP-0183926rG4,18952
+letta/server/rest_api/routers/v1/sources.py,sha256=RfCp7XCFpEIF6eTHfbKXvE7mAdHFUrpDU_AXbwvJj3o,19584
 letta/server/rest_api/routers/v1/steps.py,sha256=N863b0Oyzz64rKHqpyQnXEQBw0SCQ8kAxWaZ7huV1Rk,4925
 letta/server/rest_api/routers/v1/tags.py,sha256=ef94QitUSJ3NQVffWF1ZqANUZ2b2jRyGHp_I3UUjhno,912
 letta/server/rest_api/routers/v1/telemetry.py,sha256=z53BW3Pefi3eWy47FPJyGhFWbZicX9jPJUi5LC5c3sk,790
@@ -339,16 +340,16 @@ letta/services/block_manager.py,sha256=YwDGdy6f6MNXVXVOxIMOOP6IEWT8h-k5uQlveof0p
 letta/services/context_window_calculator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 letta/services/context_window_calculator/context_window_calculator.py,sha256=H0-Ello1DHV28MnzMseWrg--jarDc6YwCcgwPlWjtZk,6527
 letta/services/context_window_calculator/token_counter.py,sha256=Ai9-aPkNvhhMTj9zlvdiQAdVqroTzIyAn0TrHpHNQZY,2954
-letta/services/file_manager.py,sha256=d3T0836BimRAu6yqrPt6WQGGrCu6TAplk96V-bSH0rU,11154
+letta/services/file_manager.py,sha256=Zm0wK4pkKz_rkPtANZCaxp2mXgi6rWg9moJ-nic6Bms,14277
 letta/services/file_processor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 letta/services/file_processor/chunker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-letta/services/file_processor/chunker/line_chunker.py,sha256=6VGwthka1Xg9z2r84YbaSG6G_UmrQnbe7aELgkmp2d4,5752
-letta/services/file_processor/chunker/llama_index_chunker.py,sha256=BFNfkcof-9yhrSI9oa7eutb8qNdOdUvfKOV0UPtkAVQ,4044
+letta/services/file_processor/chunker/line_chunker.py,sha256=m02molsKXU_RUEebbHhMA6LNxg3JmFlCTOuX6kZcz3E,7024
+letta/services/file_processor/chunker/llama_index_chunker.py,sha256=zHjwQUE4QTJonxHpG09sd_0fgt4KTUyjRJawUcGDAyI,7615
 letta/services/file_processor/embedder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 letta/services/file_processor/embedder/base_embedder.py,sha256=cuHF2kAlBFL9Hr63Q5vJQYYrfyDNtm31vYvW5boUQ58,518
-letta/services/file_processor/embedder/openai_embedder.py,sha256=03FwtBL-bz6rZN9JlilfX-Yei3IOHk7SS-J0Kb6nJAE,5717
+letta/services/file_processor/embedder/openai_embedder.py,sha256=qafYDdbbBDCv5Mg-gdZozc5qFCdraaG8B8OCLd8_3vY,5715
 letta/services/file_processor/embedder/pinecone_embedder.py,sha256=O33NGvDyOG07Iz-tEhZDu_PKq7NfWIaBzjJuLi8hDiU,2841
-letta/services/file_processor/file_processor.py,sha256=nED4LNm_ztJoQ9KFMTq9Wg1LIfyvAxhRquYzxWT2-FA,7587
+letta/services/file_processor/file_processor.py,sha256=E2lEoootYA8mAgNKGq2u2KCVqJqg8QCAO3-3pVaV60U,10047
 letta/services/file_processor/file_types.py,sha256=9k3Lt_bquQjJ7T6L12fPS9IS5wldhJ2puSkH6rhfCaE,13128
 letta/services/file_processor/parser/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 letta/services/file_processor/parser/base_parser.py,sha256=WfnXP6fL-xQz4eIHEWa6-ZNEAARbF_alowqH4BAUzJo,238
@@ -385,7 +386,7 @@ letta/services/tool_executor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
 letta/services/tool_executor/builtin_tool_executor.py,sha256=4hVeFnExExxuZHPIOD3LK9fPlFUt0s-b_TpKuvyxLwk,17384
 letta/services/tool_executor/composio_tool_executor.py,sha256=ia2AA_WDOseR8Ylam-HEayR7OiyfNSb1sSUrjwqlmFM,2308
 letta/services/tool_executor/core_tool_executor.py,sha256=YfUTxo78FNFPFNc6714RR_ztrf9I1GATs4cZhSbiEag,20503
-letta/services/tool_executor/files_tool_executor.py,sha256=p_MeCjSTGGBsEacULy0d7IBzc9WDmtGAAj3umERJ4Ew,29024
+letta/services/tool_executor/files_tool_executor.py,sha256=QZM0Q9zTsTSOnXq3gV5doy9HB6Mey_QdbAXWHpYlPDU,29045
 letta/services/tool_executor/mcp_tool_executor.py,sha256=x8V8J4Xi1ZVbwfaR_IwnUGRrD9w5wgV4G54sjraVBw4,1676
 letta/services/tool_executor/multi_agent_tool_executor.py,sha256=dfaZeldEnzJDg2jGHlGy3YXKjsJpokJW1tvVeoCCDrk,5496
 letta/services/tool_executor/tool_execution_manager.py,sha256=flCbTmtxZvYNcTGNC2MrYWkdPIatqFPTWnI8oJUWTIY,6399
@@ -398,7 +399,7 @@ letta/services/tool_sandbox/base.py,sha256=Vt4CnxuY5otUD6Kv8PpJNrAtl9eI8tjfcwkOd
 letta/services/tool_sandbox/e2b_sandbox.py,sha256=TrWWav56H1AsnaKgNZuq0RI-FeWHOZvOubtUywPH72s,11125
 letta/services/tool_sandbox/local_sandbox.py,sha256=RQ3iSZqP1nndSdU8pN8GoNDbQr8PRuRIK2-BinIYtK4,11810
 letta/services/user_manager.py,sha256=Neik-mxXgf9jc9jBiiBIlK38UukJonUy9NRS2soFR98,10405
-letta/settings.py,sha256=VWR9Ey1JI11BvZ4olZHPTOF4G8b-iORwp8xkHLH_bcI,11304
+letta/settings.py,sha256=lWaLL1t06s9pp4VK5ojQvTiI7D85VcJPjaVKak5LFxs,11304
 letta/streaming_interface.py,sha256=c-T7zoMTXGXFwDWJJXrv7UypeMPXwPOmNHeuuh0b9zk,16398
 letta/streaming_utils.py,sha256=jLqFTVhUL76FeOuYk8TaRQHmPTf3HSRc2EoJwxJNK6U,11946
 letta/system.py,sha256=-cfh9Xpl2Ef_H7N3oZQtNuJqb1EEskdDXNa-VwKsF0A,8977
@@ -409,8 +410,8 @@ letta/templates/summary_request_text.j2,sha256=ZttQwXonW2lk4pJLYzLK0pmo4EO4EtUUI
 letta/templates/template_helper.py,sha256=uHWO1PukgMoIIvgqQdPyHq3o3CQ6mcjUjTGvx9VLGkk,409
 letta/types/__init__.py,sha256=hokKjCVFGEfR7SLMrtZsRsBfsC7yTIbgKPLdGg4K1eY,147
 letta/utils.py,sha256=4segcFYPNsPrzMpiouYoV6Qzj4TIHuqtCyzVwAMildM,36172
-letta_nightly-0.8.10.dev20250707035305.dist-info/LICENSE,sha256=mExtuZ_GYJgDEI38GWdiEYZizZS4KkVt2SF1g_GPNhI,10759
-letta_nightly-0.8.10.dev20250707035305.dist-info/METADATA,sha256=6FN18pUoS6OGBhQ7Txas9Q4ZbFWPHiUHn-Lld91Mejo,22892
-letta_nightly-0.8.10.dev20250707035305.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-letta_nightly-0.8.10.dev20250707035305.dist-info/entry_points.txt,sha256=2zdiyGNEZGV5oYBuS-y2nAAgjDgcC9yM_mHJBFSRt5U,40
-letta_nightly-0.8.10.dev20250707035305.dist-info/RECORD,,
+letta_nightly-0.8.11.dev20250708000504.dist-info/LICENSE,sha256=mExtuZ_GYJgDEI38GWdiEYZizZS4KkVt2SF1g_GPNhI,10759
+letta_nightly-0.8.11.dev20250708000504.dist-info/METADATA,sha256=8fhj7YWfX0Qt65Shn8EfP5GjgmIU8bs86OMIfRo4pjg,22892
+letta_nightly-0.8.11.dev20250708000504.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+letta_nightly-0.8.11.dev20250708000504.dist-info/entry_points.txt,sha256=2zdiyGNEZGV5oYBuS-y2nAAgjDgcC9yM_mHJBFSRt5U,40
+letta_nightly-0.8.11.dev20250708000504.dist-info/RECORD,,