letta-nightly 0.7.30.dev20250603104343__py3-none-any.whl → 0.8.0.dev20250604104349__py3-none-any.whl

letta/orm/organization.py

@@ -6,12 +6,20 @@ from letta.orm.sqlalchemy_base import SqlalchemyBase
 from letta.schemas.organization import Organization as PydanticOrganization
 
 if TYPE_CHECKING:
-
     from letta.orm.agent import Agent
+    from letta.orm.agent_passage import AgentPassage
+    from letta.orm.block import Block
     from letta.orm.file import FileMetadata
+    from letta.orm.group import Group
     from letta.orm.identity import Identity
+    from letta.orm.llm_batch_item import LLMBatchItem
+    from letta.orm.llm_batch_job import LLMBatchJob
+    from letta.orm.message import Message
     from letta.orm.provider import Provider
-    from letta.orm.sandbox_config import AgentEnvironmentVariable
+    from letta.orm.sandbox_config import AgentEnvironmentVariable, SandboxConfig
+    from letta.orm.sandbox_environment_variable import SandboxEnvironmentVariable
+    from letta.orm.source import Source
+    from letta.orm.source_passage import SourcePassage
     from letta.orm.tool import Tool
     from letta.orm.user import User
 
@@ -28,6 +36,7 @@ class Organization(SqlalchemyBase):
     # relationships
     users: Mapped[List["User"]] = relationship("User", back_populates="organization", cascade="all, delete-orphan")
     tools: Mapped[List["Tool"]] = relationship("Tool", back_populates="organization", cascade="all, delete-orphan")
+    # mcp_servers: Mapped[List["MCPServer"]] = relationship("MCPServer", back_populates="organization", cascade="all, delete-orphan")
     blocks: Mapped[List["Block"]] = relationship("Block", back_populates="organization", cascade="all, delete-orphan")
     sources: Mapped[List["Source"]] = relationship("Source", back_populates="organization", cascade="all, delete-orphan")
     files: Mapped[List["FileMetadata"]] = relationship("FileMetadata", back_populates="organization", cascade="all, delete-orphan")

letta/orm/passage.py

@@ -41,16 +41,6 @@ class BasePassage(SqlalchemyBase, OrganizationMixin):
         """Relationship to organization"""
         return relationship("Organization", back_populates="passages", lazy="selectin")
 
-    @declared_attr
-    def __table_args__(cls):
-        if settings.letta_pg_uri_no_default:
-            return (
-                Index(f"{cls.__tablename__}_org_idx", "organization_id"),
-                Index(f"{cls.__tablename__}_created_at_id_idx", "created_at", "id"),
-                {"extend_existing": True},
-            )
-        return (Index(f"{cls.__tablename__}_created_at_id_idx", "created_at", "id"), {"extend_existing": True})
-
 
 class SourcePassage(BasePassage, FileMixin, SourceMixin):
     """Passages derived from external files/sources"""
@@ -66,6 +56,16 @@ class SourcePassage(BasePassage, FileMixin, SourceMixin):
     def organization(cls) -> Mapped["Organization"]:
         return relationship("Organization", back_populates="source_passages", lazy="selectin")
 
+    @declared_attr
+    def __table_args__(cls):
+        if settings.letta_pg_uri_no_default:
+            return (
+                Index("source_passages_org_idx", "organization_id"),
+                Index("source_passages_created_at_id_idx", "created_at", "id"),
+                {"extend_existing": True},
+            )
+        return (Index("source_passages_created_at_id_idx", "created_at", "id"), {"extend_existing": True})
+
     @declared_attr
     def source(cls) -> Mapped["Source"]:
         """Relationship to source"""
@@ -80,3 +80,18 @@ class AgentPassage(BasePassage, AgentMixin):
     @declared_attr
     def organization(cls) -> Mapped["Organization"]:
         return relationship("Organization", back_populates="agent_passages", lazy="selectin")
+
+    @declared_attr
+    def __table_args__(cls):
+        if settings.letta_pg_uri_no_default:
+            return (
+                Index("agent_passages_org_idx", "organization_id"),
+                Index("ix_agent_passages_org_agent", "organization_id", "agent_id"),
+                Index("agent_passages_created_at_id_idx", "created_at", "id"),
+                {"extend_existing": True},
+            )
+        return (
+            Index("ix_agent_passages_org_agent", "organization_id", "agent_id"),
+            Index("agent_passages_created_at_id_idx", "created_at", "id"),
+            {"extend_existing": True},
+        )

letta/orm/sandbox_config.py

@@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Dict, List, Optional
 
 from sqlalchemy import JSON
 from sqlalchemy import Enum as SqlEnum
-from sqlalchemy import String, UniqueConstraint
+from sqlalchemy import Index, String, UniqueConstraint
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from letta.orm.mixins import AgentMixin, OrganizationMixin, SandboxConfigMixin
@@ -61,7 +61,10 @@ class AgentEnvironmentVariable(SqlalchemyBase, OrganizationMixin, AgentMixin):
 
     __tablename__ = "agent_environment_variables"
     # We cannot have duplicate key names for the same agent, the env var would get overwritten
-    __table_args__ = (UniqueConstraint("key", "agent_id", name="uix_key_agent"),)
+    __table_args__ = (
+        UniqueConstraint("key", "agent_id", name="uix_key_agent"),
+        Index("idx_agent_environment_variables_agent_id", "agent_id"),
+    )
 
     # agent_env_var generates its own id
     # TODO: We want to migrate all the ORM models to do this, so we will need to move this to the SqlalchemyBase

letta/orm/sqlalchemy_base.py

@@ -4,7 +4,7 @@ from functools import wraps
 from pprint import pformat
 from typing import TYPE_CHECKING, List, Literal, Optional, Tuple, Union
 
-from sqlalchemy import String, and_, func, or_, select
+from sqlalchemy import String, and_, delete, func, or_, select, text
 from sqlalchemy.exc import DBAPIError, IntegrityError, TimeoutError
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import Mapped, Session, mapped_column
@@ -188,56 +188,55 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
 
         logger.debug(f"Listing {cls.__name__} with kwarg filters {kwargs}")
 
-        async with db_session as session:
-            # Get the reference objects for pagination
-            before_obj = None
-            after_obj = None
-
-            if before:
-                before_obj = await session.get(cls, before)
-                if not before_obj:
-                    raise NoResultFound(f"No {cls.__name__} found with id {before}")
-
-            if after:
-                after_obj = await session.get(cls, after)
-                if not after_obj:
-                    raise NoResultFound(f"No {cls.__name__} found with id {after}")
-
-            # Validate that before comes after the after object if both are provided
-            if before_obj and after_obj and before_obj.created_at < after_obj.created_at:
-                raise ValueError("'before' reference must be later than 'after' reference")
-
-            query = cls._list_preprocess(
-                before_obj=before_obj,
-                after_obj=after_obj,
-                start_date=start_date,
-                end_date=end_date,
-                limit=limit,
-                query_text=query_text,
-                query_embedding=query_embedding,
-                ascending=ascending,
-                actor=actor,
-                access=access,
-                access_type=access_type,
-                join_model=join_model,
-                join_conditions=join_conditions,
-                identifier_keys=identifier_keys,
-                identity_id=identity_id,
-                **kwargs,
-            )
+        # Get the reference objects for pagination
+        before_obj = None
+        after_obj = None
+
+        if before:
+            before_obj = await db_session.get(cls, before)
+            if not before_obj:
+                raise NoResultFound(f"No {cls.__name__} found with id {before}")
+
+        if after:
+            after_obj = await db_session.get(cls, after)
+            if not after_obj:
+                raise NoResultFound(f"No {cls.__name__} found with id {after}")
+
+        # Validate that before comes after the after object if both are provided
+        if before_obj and after_obj and before_obj.created_at < after_obj.created_at:
+            raise ValueError("'before' reference must be later than 'after' reference")
+
+        query = cls._list_preprocess(
+            before_obj=before_obj,
+            after_obj=after_obj,
+            start_date=start_date,
+            end_date=end_date,
+            limit=limit,
+            query_text=query_text,
+            query_embedding=query_embedding,
+            ascending=ascending,
+            actor=actor,
+            access=access,
+            access_type=access_type,
+            join_model=join_model,
+            join_conditions=join_conditions,
+            identifier_keys=identifier_keys,
+            identity_id=identity_id,
+            **kwargs,
+        )
+
+        # Execute the query
+        results = await db_session.execute(query)
 
-            # Execute the query
-            results = await session.execute(query)
+        results = list(results.scalars())
+        results = cls._list_postprocess(
+            before=before,
+            after=after,
+            limit=limit,
+            results=results,
+        )
 
-            results = list(results.scalars())
-            results = cls._list_postprocess(
-                before=before,
-                after=after,
-                limit=limit,
-                results=results,
-            )
-
-            return results
+        return results
 
     @classmethod
     def _list_preprocess(
@@ -258,6 +257,7 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
         join_conditions: Optional[Union[Tuple, List]] = None,
         identifier_keys: Optional[List[str]] = None,
         identity_id: Optional[str] = None,
+        check_is_deleted: bool = False,
         **kwargs,
     ):
         """
@@ -280,15 +280,28 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
             query = query.join(cls.identities).filter(cls.identities.property.mapper.class_.id == identity_id)
 
         # Apply filtering logic from kwargs
+        # 1 part: <column> // 2 parts: <table>.<column> OR <column>.<json_key> // 3 parts: <table>.<column>.<json_key>
+        # TODO (cliandy): can make this more robust down the line
         for key, value in kwargs.items():
-            if "." in key:
-                # Handle joined table columns
-                table_name, column_name = key.split(".")
+            parts = key.split(".")
+            if len(parts) == 1:
+                column = getattr(cls, key)
+            elif len(parts) == 2:
+                if locals().get(parts[0]) or globals().get(parts[0]):
+                    # It's a joined table column
+                    joined_table = locals().get(parts[0]) or globals().get(parts[0])
+                    column = getattr(joined_table, parts[1])
+                else:
+                    # It's a JSON field on the main table
+                    column = getattr(cls, parts[0])
+                    column = column.op("->>")(parts[1])
+            elif len(parts) == 3:
+                table_name, column_name, json_key = parts
                 joined_table = locals().get(table_name) or globals().get(table_name)
                 column = getattr(joined_table, column_name)
+                column = column.op("->>")(json_key)
             else:
-                # Handle columns from main table
-                column = getattr(cls, key)
+                raise ValueError(f"Unhandled column name {key}")
 
             if isinstance(value, (list, tuple, set)):
                 query = query.where(column.in_(value))
@@ -361,7 +374,7 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
                 is_ordered = True
 
         # Handle soft deletes
-        if hasattr(cls, "is_deleted"):
+        if check_is_deleted and hasattr(cls, "is_deleted"):
             query = query.where(cls.is_deleted == False)
 
         # Apply ordering
@@ -405,6 +418,7 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
         actor: Optional["User"] = None,
         access: Optional[List[Literal["read", "write", "admin"]]] = ["read"],
         access_type: AccessType = AccessType.ORGANIZATION,
+        check_is_deleted: bool = False,
         **kwargs,
     ) -> "SqlalchemyBase":
         """The primary accessor for an ORM record.
@@ -421,7 +435,7 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
         """
         # this is ok because read_multiple will check if the
         identifiers = [] if identifier is None else [identifier]
-        found = cls.read_multiple(db_session, identifiers, actor, access, access_type, **kwargs)
+        found = cls.read_multiple(db_session, identifiers, actor, access, access_type, check_is_deleted, **kwargs)
         if len(found) == 0:
             # for backwards compatibility.
             conditions = []
@@ -429,7 +443,7 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
                 conditions.append(f"id={identifier}")
             if actor:
                 conditions.append(f"access level in {access} for {actor}")
-            if hasattr(cls, "is_deleted"):
+            if check_is_deleted and hasattr(cls, "is_deleted"):
                 conditions.append("is_deleted=False")
             raise NoResultFound(f"{cls.__name__} not found with {', '.join(conditions if conditions else ['no conditions'])}")
         return found[0]
@@ -443,6 +457,7 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
         actor: Optional["User"] = None,
         access: Optional[List[Literal["read", "write", "admin"]]] = ["read"],
         access_type: AccessType = AccessType.ORGANIZATION,
+        check_is_deleted: bool = False,
         **kwargs,
     ) -> "SqlalchemyBase":
         """The primary accessor for an ORM record. Async version of read method.
@@ -457,20 +472,18 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
         Raises:
             NoResultFound: if the object is not found
         """
-        # this is ok because read_multiple will check if the
         identifiers = [] if identifier is None else [identifier]
-        found = await cls.read_multiple_async(db_session, identifiers, actor, access, access_type, **kwargs)
-        if len(found) == 0:
-            # for backwards compatibility.
-            conditions = []
-            if identifier:
-                conditions.append(f"id={identifier}")
-            if actor:
-                conditions.append(f"access level in {access} for {actor}")
-            if hasattr(cls, "is_deleted"):
-                conditions.append("is_deleted=False")
-            raise NoResultFound(f"{cls.__name__} not found with {', '.join(conditions if conditions else ['no conditions'])}")
-        return found[0]
+        query, query_conditions = cls._read_multiple_preprocess(identifiers, actor, access, access_type, check_is_deleted, **kwargs)
+        await db_session.execute(text("SET LOCAL enable_seqscan = OFF"))
+        try:
+            result = await db_session.execute(query)
+            item = result.scalar_one_or_none()
+        finally:
+            await db_session.execute(text("SET LOCAL enable_seqscan = ON"))
+
+        if item is None:
+            raise NoResultFound(f"{cls.__name__} not found with {', '.join(query_conditions if query_conditions else ['no conditions'])}")
+        return item
 
     @classmethod
     @handle_db_timeout
@@ -481,6 +494,7 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
         actor: Optional["User"] = None,
         access: Optional[List[Literal["read", "write", "admin"]]] = ["read"],
         access_type: AccessType = AccessType.ORGANIZATION,
+        check_is_deleted: bool = False,
         **kwargs,
     ) -> List["SqlalchemyBase"]:
         """The primary accessor for ORM record(s)
@@ -495,7 +509,7 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
         Raises:
             NoResultFound: if the object is not found
         """
-        query, query_conditions = cls._read_multiple_preprocess(identifiers, actor, access, access_type, **kwargs)
+        query, query_conditions = cls._read_multiple_preprocess(identifiers, actor, access, access_type, check_is_deleted, **kwargs)
         results = db_session.execute(query).scalars().all()
         return cls._read_multiple_postprocess(results, identifiers, query_conditions)
 
@@ -508,13 +522,14 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
         actor: Optional["User"] = None,
         access: Optional[List[Literal["read", "write", "admin"]]] = ["read"],
         access_type: AccessType = AccessType.ORGANIZATION,
+        check_is_deleted: bool = False,
         **kwargs,
     ) -> List["SqlalchemyBase"]:
         """
         Async version of read_multiple(...)
         The primary accessor for ORM record(s)
         """
-        query, query_conditions = cls._read_multiple_preprocess(identifiers, actor, access, access_type, **kwargs)
+        query, query_conditions = cls._read_multiple_preprocess(identifiers, actor, access, access_type, check_is_deleted, **kwargs)
         results = await db_session.execute(query)
         return cls._read_multiple_postprocess(results.scalars().all(), identifiers, query_conditions)
 
@@ -525,6 +540,7 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
         actor: Optional["User"],
         access: Optional[List[Literal["read", "write", "admin"]]],
         access_type: AccessType,
+        check_is_deleted: bool,
         **kwargs,
     ):
         logger.debug(f"Reading {cls.__name__} with ID(s): {identifiers} with actor={actor}")
@@ -535,8 +551,11 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
         query_conditions = []
 
         # If an identifier is provided, add it to the query conditions
-        if len(identifiers) > 0:
-            query = query.where(cls.id.in_(identifiers))
+        if identifiers:
+            if len(identifiers) == 1:
+                query = query.where(cls.id == identifiers[0])
+            else:
+                query = query.where(cls.id.in_(identifiers))
             query_conditions.append(f"id='{identifiers}'")
         elif not kwargs:
             logger.debug(f"No identifiers provided for {cls.__name__}, returning empty list")
@@ -550,7 +569,7 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
             query = cls.apply_access_predicate(query, actor, access, access_type)
             query_conditions.append(f"access level in {access} for actor='{actor}'")
 
-        if hasattr(cls, "is_deleted"):
+        if check_is_deleted and hasattr(cls, "is_deleted"):
             query = query.where(cls.is_deleted == False)
             query_conditions.append("is_deleted=False")
 
@@ -679,22 +698,21 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
                 item._set_created_and_updated_by_fields(actor.id)
 
         try:
-            async with db_session as session:
-                session.add_all(items)
-                await session.flush()  # Flush to generate IDs but don't commit yet
+            db_session.add_all(items)
+            await db_session.flush()  # Flush to generate IDs but don't commit yet
 
-                # Collect IDs to fetch the complete objects after commit
-                item_ids = [item.id for item in items]
+            # Collect IDs to fetch the complete objects after commit
+            item_ids = [item.id for item in items]
 
-                await session.commit()
+            await db_session.commit()
 
-                # Re-query the objects to get them with relationships loaded
-                query = select(cls).where(cls.id.in_(item_ids))
-                if hasattr(cls, "created_at"):
-                    query = query.order_by(cls.created_at)
+            # Re-query the objects to get them with relationships loaded
+            query = select(cls).where(cls.id.in_(item_ids))
+            if hasattr(cls, "created_at"):
+                query = query.order_by(cls.created_at)
 
-                result = await session.execute(query)
-                return list(result.scalars())
+            result = await db_session.execute(query)
+            return list(result.scalars())
 
         except (DBAPIError, IntegrityError) as e:
             cls._handle_dbapi_error(e)
@@ -741,14 +759,42 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
         """Permanently removes the record from the database asynchronously."""
         logger.debug(f"Hard deleting {self.__class__.__name__} with ID: {self.id} with actor={actor} (async)")
 
-        async with db_session as session:
-            try:
-                await session.delete(self)
-                await session.commit()
-            except Exception as e:
-                await session.rollback()
-                logger.exception(f"Failed to hard delete {self.__class__.__name__} with ID {self.id}")
-                raise ValueError(f"Failed to hard delete {self.__class__.__name__} with ID {self.id}: {e}")
+        try:
+            await db_session.delete(self)
+            await db_session.commit()
+        except Exception as e:
+            await db_session.rollback()
+            logger.exception(f"Failed to hard delete {self.__class__.__name__} with ID {self.id}")
+            raise ValueError(f"Failed to hard delete {self.__class__.__name__} with ID {self.id}: {e}")
+
+    @classmethod
+    @handle_db_timeout
+    async def bulk_hard_delete_async(
+        cls,
+        db_session: "AsyncSession",
+        identifiers: List[str],
+        actor: Optional["User"],
+        access: Optional[List[Literal["read", "write", "admin"]]] = ["write"],
+        access_type: AccessType = AccessType.ORGANIZATION,
+    ) -> None:
+        """Permanently removes the record from the database asynchronously."""
+        logger.debug(f"Hard deleting {cls.__name__} with IDs: {identifiers} with actor={actor} (async)")
+
+        if len(identifiers) == 0:
+            logger.debug(f"No identifiers provided for {cls.__name__}, nothing to delete")
+            return
+
+        query = delete(cls)
+        query = query.where(cls.id.in_(identifiers))
+        query = cls.apply_access_predicate(query, actor, access, access_type)
+        try:
+            result = await db_session.execute(query)
+            await db_session.commit()
+            logger.debug(f"Successfully deleted {result.rowcount} {cls.__name__} records")
+        except Exception as e:
+            await db_session.rollback()
+            logger.exception(f"Failed to hard delete {cls.__name__} with identifiers {identifiers}")
+            raise ValueError(f"Failed to hard delete {cls.__name__} with identifiers {identifiers}: {e}")
 
     @handle_db_timeout
     def update(self, db_session: Session, actor: Optional["User"] = None, no_commit: bool = False) -> "SqlalchemyBase":
@@ -790,10 +836,11 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
         actor: Optional["User"] = None,
         access: Optional[List[Literal["read", "write", "admin"]]] = ["read"],
         access_type: AccessType = AccessType.ORGANIZATION,
+        check_is_deleted: bool = False,
         **kwargs,
     ):
         logger.debug(f"Calculating size for {cls.__name__} with filters {kwargs}")
-        query = select(func.count()).select_from(cls)
+        query = select(func.count(1)).select_from(cls)
 
         if actor:
             query = cls.apply_access_predicate(query, actor, access, access_type)
@@ -809,8 +856,7 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
                 else:  # Single value for equality filtering
                     query = query.where(column == value)
 
-        # Handle soft deletes if the class has the 'is_deleted' attribute
-        if hasattr(cls, "is_deleted"):
+        if check_is_deleted and hasattr(cls, "is_deleted"):
             query = query.where(cls.is_deleted == False)
 
         return query
@@ -824,6 +870,7 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
         actor: Optional["User"] = None,
         access: Optional[List[Literal["read", "write", "admin"]]] = ["read"],
         access_type: AccessType = AccessType.ORGANIZATION,
+        check_is_deleted: bool = False,
         **kwargs,
     ) -> int:
         """
@@ -840,7 +887,14 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
             DBAPIError: If a database error occurs
         """
         with db_session as session:
-            query = cls._size_preprocess(db_session=session, actor=actor, access=access, access_type=access_type, **kwargs)
+            query = cls._size_preprocess(
+                db_session=session,
+                actor=actor,
+                access=access,
+                access_type=access_type,
+                check_is_deleted=check_is_deleted,
+                **kwargs,
+            )
 
             try:
                 count = session.execute(query).scalar()
@@ -858,6 +912,7 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
         actor: Optional["User"] = None,
         access: Optional[List[Literal["read", "write", "admin"]]] = ["read"],
         access_type: AccessType = AccessType.ORGANIZATION,
+        check_is_deleted: bool = False,
         **kwargs,
     ) -> int:
         """
@@ -870,16 +925,22 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
         Raises:
             DBAPIError: If a database error occurs
         """
-        async with db_session as session:
-            query = cls._size_preprocess(db_session=session, actor=actor, access=access, access_type=access_type, **kwargs)
+        query = cls._size_preprocess(
+            db_session=db_session,
+            actor=actor,
+            access=access,
+            access_type=access_type,
+            check_is_deleted=check_is_deleted,
+            **kwargs,
+        )
 
-            try:
-                result = await session.execute(query)
-                count = result.scalar()
-                return count if count else 0
-            except DBAPIError as e:
-                logger.exception(f"Failed to calculate size for {cls.__name__}")
-                raise e
+        try:
+            result = await db_session.execute(query)
+            count = result.scalar()
+            return count if count else 0
+        except DBAPIError as e:
+            logger.exception(f"Failed to calculate size for {cls.__name__}")
+            raise e
 
     @classmethod
     def apply_access_predicate(
@@ -905,12 +966,12 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):
             org_id = getattr(actor, "organization_id", None)
             if not org_id:
                 raise ValueError(f"object {actor} has no organization accessor")
-            return query.where(cls.organization_id == org_id, cls.is_deleted == False)
+            return query.where(cls.organization_id == org_id)
         elif access_type == AccessType.USER:
             user_id = getattr(actor, "id", None)
             if not user_id:
                 raise ValueError(f"object {actor} has no user accessor")
-            return query.where(cls.user_id == user_id, cls.is_deleted == False)
+            return query.where(cls.user_id == user_id)
         else:
             raise ValueError(f"unknown access_type: {access_type}")
 

letta/prompts/system/memgpt_base.txt

@@ -41,9 +41,14 @@ You can edit your core memory using the 'core_memory_append' and 'core_memory_re
 
 Archival memory (infinite size):
 Your archival memory is infinite size, but is held outside of your immediate context, so you must explicitly run a retrieval/search operation to see data inside it.
-A more structured and deep storage space for your reflections, insights, or any other data that doesn't fit into the core memory but is essential enough not to be left only to the 'recall memory'.
+A more structured and deep storage space for your reflections, insights, or any memories that arise from interacting with the user doesn't fit into the core memory but is essential enough not to be left only to the 'recall memory'.
 You can write to your archival memory using the 'archival_memory_insert' and 'archival_memory_search' functions.
 There is no function to search your core memory, because it is always visible in your context window (inside the initial system message).
 
+Data sources:
+You may be given access to external sources of data, relevant to the user's interaction. For example, code, style guides, and documentation relevant
+to the current interaction with the user. Your core memory will contain information about the contents of these data sources. You will have access
+to functions to open and close the files as a filesystem and maintain only the files that are relevant to the user's interaction.
+
 Base instructions finished.
 From now on, you are going to act as your persona.