letta-nightly 0.8.13.dev20250714104447__tar.gz → 0.8.14.dev20250714180504__tar.gz

{letta_nightly-0.8.13.dev20250714104447 → letta_nightly-0.8.14.dev20250714180504}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: letta-nightly
-Version: 0.8.13.dev20250714104447
+Version: 0.8.14.dev20250714180504
 Summary: Create LLM agents with long-term memory and custom tools
 License: Apache License
 Author: Letta Team

{letta_nightly-0.8.13.dev20250714104447 → letta_nightly-0.8.14.dev20250714180504}/letta/__init__.py

@@ -5,7 +5,7 @@ try:
     __version__ = version("letta")
 except PackageNotFoundError:
     # Fallback for development installations
-    __version__ = "0.8.13"
+    __version__ = "0.8.14"
 
 if os.environ.get("LETTA_VERSION"):
     __version__ = os.environ["LETTA_VERSION"]

{letta_nightly-0.8.13.dev20250714104447 → letta_nightly-0.8.14.dev20250714180504}/letta/constants.py

@@ -372,3 +372,9 @@ PINECONE_METRIC = "cosine"
 PINECONE_CLOUD = "aws"
 PINECONE_REGION = "us-east-1"
 PINECONE_MAX_BATCH_SIZE = 96
+
+# retry configuration
+PINECONE_MAX_RETRY_ATTEMPTS = 3
+PINECONE_RETRY_BASE_DELAY = 1.0  # seconds
+PINECONE_RETRY_MAX_DELAY = 60.0  # seconds
+PINECONE_RETRY_BACKOFF_FACTOR = 2.0

{letta_nightly-0.8.13.dev20250714104447 → letta_nightly-0.8.14.dev20250714180504}/letta/functions/function_sets/base.py

@@ -135,7 +135,7 @@ def core_memory_append(agent_state: "AgentState", label: str, content: str) -> O
     Append to the contents of core memory.
 
     Args:
-        label (str): Section of the memory to be edited (persona or human).
+        label (str): Section of the memory to be edited.
         content (str): Content to write to the memory. All unicode (including emojis) are supported.
 
     Returns:
@@ -152,7 +152,7 @@ def core_memory_replace(agent_state: "AgentState", label: str, old_content: str,
     Replace the contents of core memory. To delete memories, use an empty string for new_content.
 
     Args:
-        label (str): Section of the memory to be edited (persona or human).
+        label (str): Section of the memory to be edited.
         old_content (str): String to replace. Must be an exact match.
         new_content (str): Content to write to the memory. All unicode (including emojis) are supported.
 

letta_nightly-0.8.14.dev20250714180504/letta/helpers/pinecone_utils.py

@@ -0,0 +1,333 @@
+import asyncio
+import random
+import time
+from functools import wraps
+from typing import Any, Dict, List
+
+from letta.otel.tracing import trace_method
+
+try:
+    from pinecone import IndexEmbed, PineconeAsyncio
+    from pinecone.exceptions.exceptions import (
+        ForbiddenException,
+        NotFoundException,
+        PineconeApiException,
+        ServiceException,
+        UnauthorizedException,
+    )
+
+    PINECONE_AVAILABLE = True
+except ImportError:
+    PINECONE_AVAILABLE = False
+
+from letta.constants import (
+    PINECONE_CLOUD,
+    PINECONE_EMBEDDING_MODEL,
+    PINECONE_MAX_BATCH_SIZE,
+    PINECONE_MAX_RETRY_ATTEMPTS,
+    PINECONE_METRIC,
+    PINECONE_REGION,
+    PINECONE_RETRY_BACKOFF_FACTOR,
+    PINECONE_RETRY_BASE_DELAY,
+    PINECONE_RETRY_MAX_DELAY,
+    PINECONE_TEXT_FIELD_NAME,
+)
+from letta.log import get_logger
+from letta.schemas.user import User
+from letta.settings import settings
+
+logger = get_logger(__name__)
+
+
+def pinecone_retry(
+    max_attempts: int = PINECONE_MAX_RETRY_ATTEMPTS,
+    base_delay: float = PINECONE_RETRY_BASE_DELAY,
+    max_delay: float = PINECONE_RETRY_MAX_DELAY,
+    backoff_factor: float = PINECONE_RETRY_BACKOFF_FACTOR,
+):
+    """
+    Decorator to retry Pinecone operations with exponential backoff.
+
+    Args:
+        max_attempts: Maximum number of retry attempts
+        base_delay: Base delay in seconds for the first retry
+        max_delay: Maximum delay in seconds between retries
+        backoff_factor: Factor to increase delay after each failed attempt
+    """
+
+    def decorator(func):
+        @wraps(func)
+        async def wrapper(*args, **kwargs):
+            operation_name = func.__name__
+            start_time = time.time()
+
+            for attempt in range(max_attempts):
+                try:
+                    logger.debug(f"[Pinecone] Starting {operation_name} (attempt {attempt + 1}/{max_attempts})")
+                    result = await func(*args, **kwargs)
+
+                    execution_time = time.time() - start_time
+                    logger.info(f"[Pinecone] {operation_name} completed successfully in {execution_time:.2f}s")
+                    return result
+
+                except (ServiceException, PineconeApiException) as e:
+                    # retryable server errors
+                    if attempt == max_attempts - 1:
+                        execution_time = time.time() - start_time
+                        logger.error(f"[Pinecone] {operation_name} failed after {max_attempts} attempts in {execution_time:.2f}s: {str(e)}")
+                        raise
+
+                    # calculate delay with exponential backoff and jitter
+                    delay = min(base_delay * (backoff_factor**attempt), max_delay)
+                    jitter = random.uniform(0, delay * 0.1)  # add up to 10% jitter
+                    total_delay = delay + jitter
+
+                    logger.warning(
+                        f"[Pinecone] {operation_name} failed (attempt {attempt + 1}/{max_attempts}): {str(e)}. Retrying in {total_delay:.2f}s"
+                    )
+                    await asyncio.sleep(total_delay)
+
+                except (UnauthorizedException, ForbiddenException) as e:
+                    # non-retryable auth errors
+                    execution_time = time.time() - start_time
+                    logger.error(f"[Pinecone] {operation_name} failed with auth error in {execution_time:.2f}s: {str(e)}")
+                    raise
+
+                except NotFoundException as e:
+                    # non-retryable not found errors
+                    execution_time = time.time() - start_time
+                    logger.warning(f"[Pinecone] {operation_name} failed with not found error in {execution_time:.2f}s: {str(e)}")
+                    raise
+
+                except Exception as e:
+                    # other unexpected errors - retry once then fail
+                    if attempt == max_attempts - 1:
+                        execution_time = time.time() - start_time
+                        logger.error(f"[Pinecone] {operation_name} failed after {max_attempts} attempts in {execution_time:.2f}s: {str(e)}")
+                        raise
+
+                    delay = min(base_delay * (backoff_factor**attempt), max_delay)
+                    jitter = random.uniform(0, delay * 0.1)
+                    total_delay = delay + jitter
+
+                    logger.warning(
+                        f"[Pinecone] {operation_name} failed with unexpected error (attempt {attempt + 1}/{max_attempts}): {str(e)}. Retrying in {total_delay:.2f}s"
+                    )
+                    await asyncio.sleep(total_delay)
+
+        return wrapper
+
+    return decorator
+
+
+def should_use_pinecone(verbose: bool = False):
+    if verbose:
+        logger.info(
+            "Pinecone check: enable_pinecone=%s, api_key=%s, agent_index=%s, source_index=%s",
+            settings.enable_pinecone,
+            bool(settings.pinecone_api_key),
+            bool(settings.pinecone_agent_index),
+            bool(settings.pinecone_source_index),
+        )
+
+    return all(
+        (
+            PINECONE_AVAILABLE,
+            settings.enable_pinecone,
+            settings.pinecone_api_key,
+            settings.pinecone_agent_index,
+            settings.pinecone_source_index,
+        )
+    )
+
+
+@pinecone_retry()
+@trace_method
+async def upsert_pinecone_indices():
+    if not PINECONE_AVAILABLE:
+        raise ImportError("Pinecone is not available. Please install pinecone to use this feature.")
+
+    indices = get_pinecone_indices()
+    logger.info(f"[Pinecone] Upserting {len(indices)} indices: {indices}")
+
+    for index_name in indices:
+        async with PineconeAsyncio(api_key=settings.pinecone_api_key) as pc:
+            if not await pc.has_index(index_name):
+                logger.info(f"[Pinecone] Creating index {index_name} with model {PINECONE_EMBEDDING_MODEL}")
+                await pc.create_index_for_model(
+                    name=index_name,
+                    cloud=PINECONE_CLOUD,
+                    region=PINECONE_REGION,
+                    embed=IndexEmbed(model=PINECONE_EMBEDDING_MODEL, field_map={"text": PINECONE_TEXT_FIELD_NAME}, metric=PINECONE_METRIC),
+                )
+                logger.info(f"[Pinecone] Successfully created index {index_name}")
+            else:
+                logger.debug(f"[Pinecone] Index {index_name} already exists")
+
+
+def get_pinecone_indices() -> List[str]:
+    return [settings.pinecone_agent_index, settings.pinecone_source_index]
+
+
+@pinecone_retry()
+@trace_method
+async def upsert_file_records_to_pinecone_index(file_id: str, source_id: str, chunks: List[str], actor: User):
+    if not PINECONE_AVAILABLE:
+        raise ImportError("Pinecone is not available. Please install pinecone to use this feature.")
+
+    logger.info(f"[Pinecone] Preparing to upsert {len(chunks)} chunks for file {file_id} source {source_id}")
+
+    records = []
+    for i, chunk in enumerate(chunks):
+        record = {
+            "_id": f"{file_id}_{i}",
+            PINECONE_TEXT_FIELD_NAME: chunk,
+            "file_id": file_id,
+            "source_id": source_id,
+        }
+        records.append(record)
+
+    logger.debug(f"[Pinecone] Created {len(records)} records for file {file_id}")
+    return await upsert_records_to_pinecone_index(records, actor)
+
+
+@pinecone_retry()
+@trace_method
+async def delete_file_records_from_pinecone_index(file_id: str, actor: User):
+    if not PINECONE_AVAILABLE:
+        raise ImportError("Pinecone is not available. Please install pinecone to use this feature.")
+
+    namespace = actor.organization_id
+    logger.info(f"[Pinecone] Deleting records for file {file_id} from index {settings.pinecone_source_index} namespace {namespace}")
+
+    try:
+        async with PineconeAsyncio(api_key=settings.pinecone_api_key) as pc:
+            description = await pc.describe_index(name=settings.pinecone_source_index)
+            async with pc.IndexAsyncio(host=description.index.host) as dense_index:
+                await dense_index.delete(
+                    filter={
+                        "file_id": {"$eq": file_id},
+                    },
+                    namespace=namespace,
+                )
+                logger.info(f"[Pinecone] Successfully deleted records for file {file_id}")
+    except NotFoundException:
+        logger.warning(f"[Pinecone] Namespace {namespace} not found for file {file_id} and org {actor.organization_id}")
+
+
+@pinecone_retry()
+@trace_method
+async def delete_source_records_from_pinecone_index(source_id: str, actor: User):
+    if not PINECONE_AVAILABLE:
+        raise ImportError("Pinecone is not available. Please install pinecone to use this feature.")
+
+    namespace = actor.organization_id
+    logger.info(f"[Pinecone] Deleting records for source {source_id} from index {settings.pinecone_source_index} namespace {namespace}")
+
+    try:
+        async with PineconeAsyncio(api_key=settings.pinecone_api_key) as pc:
+            description = await pc.describe_index(name=settings.pinecone_source_index)
+            async with pc.IndexAsyncio(host=description.index.host) as dense_index:
+                await dense_index.delete(filter={"source_id": {"$eq": source_id}}, namespace=namespace)
+                logger.info(f"[Pinecone] Successfully deleted records for source {source_id}")
+    except NotFoundException:
+        logger.warning(f"[Pinecone] Namespace {namespace} not found for source {source_id} and org {actor.organization_id}")
+
+
+@pinecone_retry()
+@trace_method
+async def upsert_records_to_pinecone_index(records: List[dict], actor: User):
+    if not PINECONE_AVAILABLE:
+        raise ImportError("Pinecone is not available. Please install pinecone to use this feature.")
+
+    logger.info(f"[Pinecone] Upserting {len(records)} records to index {settings.pinecone_source_index} for org {actor.organization_id}")
+
+    async with PineconeAsyncio(api_key=settings.pinecone_api_key) as pc:
+        description = await pc.describe_index(name=settings.pinecone_source_index)
+        async with pc.IndexAsyncio(host=description.index.host) as dense_index:
+            # process records in batches to avoid exceeding pinecone limits
+            total_batches = (len(records) + PINECONE_MAX_BATCH_SIZE - 1) // PINECONE_MAX_BATCH_SIZE
+            logger.debug(f"[Pinecone] Processing {total_batches} batches of max {PINECONE_MAX_BATCH_SIZE} records each")
+
+            for i in range(0, len(records), PINECONE_MAX_BATCH_SIZE):
+                batch = records[i : i + PINECONE_MAX_BATCH_SIZE]
+                batch_num = (i // PINECONE_MAX_BATCH_SIZE) + 1
+
+                logger.debug(f"[Pinecone] Upserting batch {batch_num}/{total_batches} with {len(batch)} records")
+                await dense_index.upsert_records(actor.organization_id, batch)
+
+            logger.info(f"[Pinecone] Successfully upserted all {len(records)} records in {total_batches} batches")
+
+
+@pinecone_retry()
+@trace_method
+async def search_pinecone_index(query: str, limit: int, filter: Dict[str, Any], actor: User) -> Dict[str, Any]:
+    if not PINECONE_AVAILABLE:
+        raise ImportError("Pinecone is not available. Please install pinecone to use this feature.")
+
+    namespace = actor.organization_id
+    logger.info(
+        f"[Pinecone] Searching index {settings.pinecone_source_index} namespace {namespace} with query length {len(query)} chars, limit {limit}"
+    )
+    logger.debug(f"[Pinecone] Search filter: {filter}")
+
+    async with PineconeAsyncio(api_key=settings.pinecone_api_key) as pc:
+        description = await pc.describe_index(name=settings.pinecone_source_index)
+        async with pc.IndexAsyncio(host=description.index.host) as dense_index:
+            try:
+                # search the dense index with reranking
+                search_results = await dense_index.search(
+                    namespace=namespace,
+                    query={
+                        "top_k": limit,
+                        "inputs": {"text": query},
+                        "filter": filter,
+                    },
+                    rerank={"model": "bge-reranker-v2-m3", "top_n": limit, "rank_fields": [PINECONE_TEXT_FIELD_NAME]},
+                )
+
+                result_count = len(search_results.get("matches", []))
+                logger.info(f"[Pinecone] Search completed, found {result_count} matches")
+                return search_results
+
+            except Exception as e:
+                logger.warning(f"[Pinecone] Failed to search namespace {namespace}: {str(e)}")
+                raise e
+
+
+@pinecone_retry()
+@trace_method
+async def list_pinecone_index_for_files(file_id: str, actor: User, limit: int = None, pagination_token: str = None) -> List[str]:
+    if not PINECONE_AVAILABLE:
+        raise ImportError("Pinecone is not available. Please install pinecone to use this feature.")
+
+    namespace = actor.organization_id
+    logger.info(f"[Pinecone] Listing records for file {file_id} from index {settings.pinecone_source_index} namespace {namespace}")
+    logger.debug(f"[Pinecone] List params - limit: {limit}, pagination_token: {pagination_token}")
+
+    try:
+        async with PineconeAsyncio(api_key=settings.pinecone_api_key) as pc:
+            description = await pc.describe_index(name=settings.pinecone_source_index)
+            async with pc.IndexAsyncio(host=description.index.host) as dense_index:
+
+                kwargs = {"namespace": namespace, "prefix": file_id}
+                if limit is not None:
+                    kwargs["limit"] = limit
+                if pagination_token is not None:
+                    kwargs["pagination_token"] = pagination_token
+
+                try:
+                    result = []
+                    async for ids in dense_index.list(**kwargs):
+                        result.extend(ids)
+
+                    logger.info(f"[Pinecone] Successfully listed {len(result)} records for file {file_id}")
+                    return result
+
+                except Exception as e:
+                    logger.warning(f"[Pinecone] Failed to list records for file {file_id} in namespace {namespace}: {str(e)}")
+                    raise e
+
+    except NotFoundException:
+        logger.warning(f"[Pinecone] Namespace {namespace} not found for file {file_id} and org {actor.organization_id}")
+        return []

{letta_nightly-0.8.13.dev20250714104447 → letta_nightly-0.8.14.dev20250714180504}/letta/orm/file.py

@@ -1,5 +1,5 @@
 import uuid
-from typing import TYPE_CHECKING, List, Optional
+from typing import TYPE_CHECKING, Optional
 
 from sqlalchemy import ForeignKey, Index, Integer, String, Text, UniqueConstraint, desc
 from sqlalchemy.ext.asyncio import AsyncAttrs
@@ -11,10 +11,7 @@ from letta.schemas.enums import FileProcessingStatus
 from letta.schemas.file import FileMetadata as PydanticFileMetadata
 
 if TYPE_CHECKING:
-    from letta.orm.files_agents import FileAgent
-    from letta.orm.organization import Organization
-    from letta.orm.passage import SourcePassage
-    from letta.orm.source import Source
+    pass
 
 
 # TODO: Note that this is NOT organization scoped, this is potentially dangerous if we misuse this
@@ -64,18 +61,6 @@ class FileMetadata(SqlalchemyBase, OrganizationMixin, SourceMixin, AsyncAttrs):
     chunks_embedded: Mapped[Optional[int]] = mapped_column(Integer, nullable=True, doc="Number of chunks that have been embedded.")
 
     # relationships
-    organization: Mapped["Organization"] = relationship("Organization", back_populates="files", lazy="selectin")
-    source: Mapped["Source"] = relationship("Source", back_populates="files", lazy="selectin")
-    source_passages: Mapped[List["SourcePassage"]] = relationship(
-        "SourcePassage", back_populates="file", lazy="selectin", cascade="all, delete-orphan"
-    )
-    file_agents: Mapped[List["FileAgent"]] = relationship(
-        "FileAgent",
-        back_populates="file",
-        lazy="selectin",
-        cascade="all, delete-orphan",
-        passive_deletes=True,  # ← add this
-    )
     content: Mapped[Optional["FileContent"]] = relationship(
         "FileContent",
         uselist=False,

{letta_nightly-0.8.13.dev20250714104447 → letta_nightly-0.8.14.dev20250714180504}/letta/orm/files_agents.py

@@ -12,7 +12,7 @@ from letta.schemas.block import Block as PydanticBlock
 from letta.schemas.file import FileAgent as PydanticFileAgent
 
 if TYPE_CHECKING:
-    from letta.orm.file import FileMetadata
+    pass
 
 
 class FileAgent(SqlalchemyBase, OrganizationMixin):
@@ -55,6 +55,12 @@ class FileAgent(SqlalchemyBase, OrganizationMixin):
         nullable=False,
         doc="ID of the agent",
     )
+    source_id: Mapped[str] = mapped_column(
+        String,
+        ForeignKey("sources.id", ondelete="CASCADE"),
+        nullable=False,
+        doc="ID of the source (denormalized from files.source_id)",
+    )
 
     file_name: Mapped[str] = mapped_column(
         String,
@@ -78,13 +84,6 @@ class FileAgent(SqlalchemyBase, OrganizationMixin):
         back_populates="file_agents",
         lazy="selectin",
     )
-    file: Mapped["FileMetadata"] = relationship(
-        "FileMetadata",
-        foreign_keys=[file_id],
-        lazy="selectin",
-        back_populates="file_agents",
-        passive_deletes=True,  # ← add this
-    )
 
     # TODO: This is temporary as we figure out if we want FileBlock as a first class citizen
     def to_pydantic_block(self) -> PydanticBlock:
@@ -99,8 +98,8 @@ class FileAgent(SqlalchemyBase, OrganizationMixin):
         return PydanticBlock(
             organization_id=self.organization_id,
             value=visible_content,
-            label=self.file.file_name,
+            label=self.file_name,  # use denormalized file_name instead of self.file.file_name
             read_only=True,
-            metadata={"source_id": self.file.source_id},
+            metadata={"source_id": self.source_id},  # use denormalized source_id
             limit=CORE_MEMORY_SOURCE_CHAR_LIMIT,
         )

{letta_nightly-0.8.13.dev20250714104447 → letta_nightly-0.8.14.dev20250714180504}/letta/orm/organization.py

@@ -9,7 +9,6 @@ if TYPE_CHECKING:
     from letta.orm.agent import Agent
     from letta.orm.agent_passage import AgentPassage
     from letta.orm.block import Block
-    from letta.orm.file import FileMetadata
     from letta.orm.group import Group
     from letta.orm.identity import Identity
     from letta.orm.llm_batch_item import LLMBatchItem
@@ -18,7 +17,6 @@ if TYPE_CHECKING:
     from letta.orm.provider import Provider
     from letta.orm.sandbox_config import AgentEnvironmentVariable, SandboxConfig
     from letta.orm.sandbox_environment_variable import SandboxEnvironmentVariable
-    from letta.orm.source import Source
     from letta.orm.source_passage import SourcePassage
     from letta.orm.tool import Tool
     from letta.orm.user import User
@@ -38,8 +36,6 @@ class Organization(SqlalchemyBase):
     tools: Mapped[List["Tool"]] = relationship("Tool", back_populates="organization", cascade="all, delete-orphan")
     # mcp_servers: Mapped[List["MCPServer"]] = relationship("MCPServer", back_populates="organization", cascade="all, delete-orphan")
     blocks: Mapped[List["Block"]] = relationship("Block", back_populates="organization", cascade="all, delete-orphan")
-    sources: Mapped[List["Source"]] = relationship("Source", back_populates="organization", cascade="all, delete-orphan")
-    files: Mapped[List["FileMetadata"]] = relationship("FileMetadata", back_populates="organization", cascade="all, delete-orphan")
     sandbox_configs: Mapped[List["SandboxConfig"]] = relationship(
         "SandboxConfig", back_populates="organization", cascade="all, delete-orphan"
     )

{letta_nightly-0.8.13.dev20250714104447 → letta_nightly-0.8.14.dev20250714180504}/letta/orm/passage.py

@@ -49,11 +49,6 @@ class SourcePassage(BasePassage, FileMixin, SourceMixin):
 
     file_name: Mapped[str] = mapped_column(doc="The name of the file that this passage was derived from")
 
-    @declared_attr
-    def file(cls) -> Mapped["FileMetadata"]:
-        """Relationship to file"""
-        return relationship("FileMetadata", back_populates="source_passages", lazy="selectin")
-
     @declared_attr
     def organization(cls) -> Mapped["Organization"]:
         return relationship("Organization", back_populates="source_passages", lazy="selectin")
@@ -74,11 +69,6 @@ class SourcePassage(BasePassage, FileMixin, SourceMixin):
             {"extend_existing": True},
         )
 
-    @declared_attr
-    def source(cls) -> Mapped["Source"]:
-        """Relationship to source"""
-        return relationship("Source", back_populates="passages", lazy="selectin", passive_deletes=True)
-
 
 class AgentPassage(BasePassage, AgentMixin):
     """Passages created by agents as archival memories"""

{letta_nightly-0.8.13.dev20250714104447 → letta_nightly-0.8.14.dev20250714180504}/letta/orm/source.py

@@ -1,9 +1,8 @@
-from typing import TYPE_CHECKING, List, Optional
+from typing import TYPE_CHECKING, Optional
 
 from sqlalchemy import JSON, Index, UniqueConstraint
-from sqlalchemy.orm import Mapped, mapped_column, relationship
+from sqlalchemy.orm import Mapped, mapped_column
 
-from letta.orm import FileMetadata
 from letta.orm.custom_columns import EmbeddingConfigColumn
 from letta.orm.mixins import OrganizationMixin
 from letta.orm.sqlalchemy_base import SqlalchemyBase
@@ -11,10 +10,7 @@ from letta.schemas.embedding_config import EmbeddingConfig
 from letta.schemas.source import Source as PydanticSource
 
 if TYPE_CHECKING:
-    from letta.orm.agent import Agent
-    from letta.orm.file import FileMetadata
-    from letta.orm.organization import Organization
-    from letta.orm.passage import SourcePassage
+    pass
 
 
 class Source(SqlalchemyBase, OrganizationMixin):
@@ -34,16 +30,3 @@ class Source(SqlalchemyBase, OrganizationMixin):
     instructions: Mapped[str] = mapped_column(nullable=True, doc="instructions for how to use the source")
     embedding_config: Mapped[EmbeddingConfig] = mapped_column(EmbeddingConfigColumn, doc="Configuration settings for embedding.")
     metadata_: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True, doc="metadata for the source.")
-
-    # relationships
-    organization: Mapped["Organization"] = relationship("Organization", back_populates="sources")
-    files: Mapped[List["FileMetadata"]] = relationship("FileMetadata", back_populates="source", cascade="all, delete-orphan")
-    passages: Mapped[List["SourcePassage"]] = relationship("SourcePassage", back_populates="source", cascade="all, delete-orphan")
-    agents: Mapped[List["Agent"]] = relationship(
-        "Agent",
-        secondary="sources_agents",
-        back_populates="sources",
-        lazy="selectin",
-        cascade="save-update",  # Only propagate save and update operations
-        passive_deletes=True,  # Let the database handle deletions
-    )

{letta_nightly-0.8.13.dev20250714104447 → letta_nightly-0.8.14.dev20250714180504}/letta/schemas/file.py

@@ -85,6 +85,7 @@ class FileAgent(FileAgentBase):
     )
     agent_id: str = Field(..., description="Unique identifier of the agent.")
     file_id: str = Field(..., description="Unique identifier of the file.")
+    source_id: str = Field(..., description="Unique identifier of the source (denormalized from files.source_id).")
     file_name: str = Field(..., description="Name of the file.")
     is_open: bool = Field(True, description="True if the agent currently has the file open.")
     visible_content: Optional[str] = Field(

{letta_nightly-0.8.13.dev20250714104447 → letta_nightly-0.8.14.dev20250714180504}/letta/schemas/memory.py

@@ -210,7 +210,7 @@ class BasicBlockMemory(Memory):
         Append to the contents of core memory.
 
         Args:
-            label (str): Section of the memory to be edited (persona or human).
+            label (str): Section of the memory to be edited.
             content (str): Content to write to the memory. All unicode (including emojis) are supported.
 
         Returns:
@@ -226,7 +226,7 @@ class BasicBlockMemory(Memory):
         Replace the contents of core memory. To delete memories, use an empty string for new_content.
 
         Args:
-            label (str): Section of the memory to be edited (persona or human).
+            label (str): Section of the memory to be edited.
             old_content (str): String to replace. Must be an exact match.
             new_content (str): Content to write to the memory. All unicode (including emojis) are supported.
 

{letta_nightly-0.8.13.dev20250714104447 → letta_nightly-0.8.14.dev20250714180504}/letta/server/rest_api/routers/v1/agents.py

@@ -272,14 +272,14 @@ async def modify_agent(
 
 
 @router.get("/{agent_id}/tools", response_model=list[Tool], operation_id="list_agent_tools")
-def list_agent_tools(
+async def list_agent_tools(
     agent_id: str,
     server: "SyncServer" = Depends(get_letta_server),
     actor_id: str | None = Header(None, alias="user_id"),  # Extract user_id from header, default to None if not present
 ):
     """Get tools from an existing agent"""
-    actor = server.user_manager.get_user_or_default(user_id=actor_id)
-    return server.agent_manager.list_attached_tools(agent_id=agent_id, actor=actor)
+    actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)
+    return await server.agent_manager.list_attached_tools_async(agent_id=agent_id, actor=actor)
 
 
 @router.patch("/{agent_id}/tools/attach/{tool_id}", response_model=AgentState, operation_id="attach_tool")
@@ -1072,7 +1072,7 @@ async def _process_message_background(
             completed_at=datetime.now(timezone.utc),
             metadata={"error": str(e)},
         )
-        await server.job_manager.update_job_by_id_async(job_id=job_id, job_update=job_update, actor=actor)
+        await server.job_manager.update_job_by_id_async(job_id=run_id, job_update=job_update, actor=actor)
 
 
 @router.post(

{letta_nightly-0.8.13.dev20250714104447 → letta_nightly-0.8.14.dev20250714180504}/letta/server/rest_api/routers/v1/messages.py

@@ -1,6 +1,6 @@
 from typing import List, Optional
 
-from fastapi import APIRouter, Body, Depends, Header, Query, status
+from fastapi import APIRouter, Body, Depends, Header, Query
 from fastapi.exceptions import HTTPException
 from starlette.requests import Request
 
@@ -45,12 +45,8 @@ async def create_messages_batch(
         if length > max_bytes:
             raise HTTPException(status_code=413, detail=f"Request too large ({length} bytes). Max is {max_bytes} bytes.")
 
-    # Reject request if env var is not set
     if not settings.enable_batch_job_polling:
-        raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"Server misconfiguration: LETTA_ENABLE_BATCH_JOB_POLLING is set to False.",
-        )
+        logger.warning("Batch job polling is disabled. Enable batch processing by setting LETTA_ENABLE_BATCH_JOB_POLLING to True.")
 
     actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)
     batch_job = BatchJob(

{letta_nightly-0.8.13.dev20250714104447 → letta_nightly-0.8.14.dev20250714180504}/letta/server/rest_api/routers/v1/sources.py

@@ -391,8 +391,8 @@ async def get_file_metadata(
     if file_metadata.source_id != source_id:
         raise HTTPException(status_code=404, detail=f"File with id={file_id} not found in source {source_id}.")
 
-    if should_use_pinecone() and not file_metadata.is_processing_terminal():
-        ids = await list_pinecone_index_for_files(file_id=file_id, actor=actor, limit=file_metadata.total_chunks)
+    if should_use_pinecone() and file_metadata.processing_status == FileProcessingStatus.EMBEDDING:
+        ids = await list_pinecone_index_for_files(file_id=file_id, actor=actor)
         logger.info(
             f"Embedded chunks {len(ids)}/{file_metadata.total_chunks} for {file_id} ({file_metadata.file_name}) in organization {actor.organization_id}"
         )
@@ -402,7 +402,7 @@ async def get_file_metadata(
                 file_status = file_metadata.processing_status
             else:
                 file_status = FileProcessingStatus.COMPLETED
-            await server.file_manager.update_file_status(
+            file_metadata = await server.file_manager.update_file_status(
                 file_id=file_metadata.id, actor=actor, chunks_embedded=len(ids), processing_status=file_status
             )
 

{letta_nightly-0.8.13.dev20250714104447 → letta_nightly-0.8.14.dev20250714180504}/letta/server/server.py

@@ -1342,9 +1342,6 @@ class SyncServer(Server):
             new_passage_size = await self.agent_manager.passage_size_async(actor=actor, agent_id=agent_id)
             assert new_passage_size >= curr_passage_size  # in case empty files are added
 
-            # rebuild system prompt and force
-            agent_state = await self.agent_manager.rebuild_system_prompt_async(agent_id=agent_id, actor=actor, force=True)
-
         # update job status
         job.status = JobStatus.completed
         job.metadata["num_passages"] = num_passages