letta-nightly 0.6.2.dev20241210104242__py3-none-any.whl → 0.6.3.dev20241211050151__py3-none-any.whl

letta/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.6.2"
+__version__ = "0.6.3"
 
 # import clients
 from letta.client.client import LocalClient, RESTClient, create_client

letta/agent.py

@@ -28,7 +28,7 @@ from letta.interface import AgentInterface
 from letta.llm_api.helpers import is_context_overflow_error
 from letta.llm_api.llm_api_tools import create
 from letta.local_llm.utils import num_tokens_from_functions, num_tokens_from_messages
-from letta.memory import ArchivalMemory, EmbeddingArchivalMemory, summarize_messages
+from letta.memory import summarize_messages
 from letta.metadata import MetadataStore
 from letta.orm import User
 from letta.schemas.agent import AgentState, AgentStepResponse
@@ -52,6 +52,7 @@ from letta.schemas.usage import LettaUsageStatistics
 from letta.schemas.user import User as PydanticUser
 from letta.services.block_manager import BlockManager
 from letta.services.message_manager import MessageManager
+from letta.services.passage_manager import PassageManager
 from letta.services.source_manager import SourceManager
 from letta.services.tool_execution_sandbox import ToolExecutionSandbox
 from letta.services.user_manager import UserManager
@@ -85,7 +86,7 @@ def compile_memory_metadata_block(
     actor: PydanticUser,
     agent_id: str,
     memory_edit_timestamp: datetime.datetime,
-    archival_memory: Optional[ArchivalMemory] = None,
+    passage_manager: Optional[PassageManager] = None,
     message_manager: Optional[MessageManager] = None,
 ) -> str:
     # Put the timestamp in the local timezone (mimicking get_local_time())
@@ -96,7 +97,7 @@ def compile_memory_metadata_block(
         [
             f"### Memory [last modified: {timestamp_str}]",
             f"{message_manager.size(actor=actor, agent_id=agent_id) if message_manager else 0} previous messages between you and the user are stored in recall memory (use functions to access them)",
-            f"{archival_memory.count() if archival_memory else 0} total memories you created are stored in archival memory (use functions to access them)",
+            f"{passage_manager.size(actor=actor, agent_id=agent_id) if passage_manager else 0} total memories you created are stored in archival memory (use functions to access them)",
             "\nCore memory shown below (limited in size, additional information stored in archival / recall memory):",
         ]
     )
@@ -109,7 +110,7 @@ def compile_system_message(
     in_context_memory: Memory,
     in_context_memory_last_edit: datetime.datetime,  # TODO move this inside of BaseMemory?
     actor: PydanticUser,
-    archival_memory: Optional[ArchivalMemory] = None,
+    passage_manager: Optional[PassageManager] = None,
     message_manager: Optional[MessageManager] = None,
     user_defined_variables: Optional[dict] = None,
     append_icm_if_missing: bool = True,
@@ -138,7 +139,7 @@ def compile_system_message(
             actor=actor,
             agent_id=agent_id,
             memory_edit_timestamp=in_context_memory_last_edit,
-            archival_memory=archival_memory,
+            passage_manager=passage_manager,
             message_manager=message_manager,
         )
         full_memory_string = memory_metadata_string + "\n" + in_context_memory.compile()
@@ -175,7 +176,7 @@ def initialize_message_sequence(
     agent_id: str,
     memory: Memory,
     actor: PydanticUser,
-    archival_memory: Optional[ArchivalMemory] = None,
+    passage_manager: Optional[PassageManager] = None,
     message_manager: Optional[MessageManager] = None,
     memory_edit_timestamp: Optional[datetime.datetime] = None,
     include_initial_boot_message: bool = True,
@@ -184,7 +185,7 @@ def initialize_message_sequence(
         memory_edit_timestamp = get_local_time()
 
     # full_system_message = construct_system_with_memory(
-    # system, memory, memory_edit_timestamp, archival_memory=archival_memory, recall_memory=recall_memory
+    # system, memory, memory_edit_timestamp, passage_manager=passage_manager, recall_memory=recall_memory
     # )
     full_system_message = compile_system_message(
         agent_id=agent_id,
@@ -192,7 +193,7 @@ def initialize_message_sequence(
         in_context_memory=memory,
         in_context_memory_last_edit=memory_edit_timestamp,
         actor=actor,
-        archival_memory=archival_memory,
+        passage_manager=passage_manager,
         message_manager=message_manager,
         user_defined_variables=None,
         append_icm_if_missing=True,
@@ -294,7 +295,7 @@ class Agent(BaseAgent):
         self.interface = interface
 
         # Create the persistence manager object based on the AgentState info
-        self.archival_memory = EmbeddingArchivalMemory(agent_state)
+        self.passage_manager = PassageManager()
         self.message_manager = MessageManager()
 
         # State needed for heartbeat pausing
@@ -325,7 +326,7 @@ class Agent(BaseAgent):
                 agent_id=self.agent_state.id,
                 memory=self.agent_state.memory,
                 actor=self.user,
-                archival_memory=None,
+                passage_manager=None,
                 message_manager=None,
                 memory_edit_timestamp=get_utc_time(),
                 include_initial_boot_message=True,
@@ -350,7 +351,7 @@ class Agent(BaseAgent):
                     memory=self.agent_state.memory,
                     agent_id=self.agent_state.id,
                     actor=self.user,
-                    archival_memory=None,
+                    passage_manager=None,
                     message_manager=None,
                     memory_edit_timestamp=get_utc_time(),
                     include_initial_boot_message=True,
@@ -1306,7 +1307,7 @@ class Agent(BaseAgent):
             in_context_memory=self.agent_state.memory,
             in_context_memory_last_edit=memory_edit_timestamp,
             actor=self.user,
-            archival_memory=self.archival_memory,
+            passage_manager=self.passage_manager,
             message_manager=self.message_manager,
             user_defined_variables=None,
             append_icm_if_missing=True,
@@ -1371,45 +1372,33 @@ class Agent(BaseAgent):
         # TODO: recall memory
         raise NotImplementedError()
 
-    def attach_source(self, source_id: str, source_connector: StorageConnector, ms: MetadataStore):
+    def attach_source(self, user: PydanticUser, source_id: str, source_manager: SourceManager, ms: MetadataStore):
         """Attach data with name `source_name` to the agent from source_connector."""
         # TODO: eventually, adding a data source should just give access to the retriever the source table, rather than modifying archival memory
-        user = UserManager().get_user_by_id(self.agent_state.user_id)
-        filters = {"user_id": self.agent_state.user_id, "source_id": source_id}
-        size = source_connector.size(filters)
         page_size = 100
-        generator = source_connector.get_all_paginated(filters=filters, page_size=page_size)  # yields List[Passage]
-        all_passages = []
-        for i in tqdm(range(0, size, page_size)):
-            passages = next(generator)
+        passages = self.passage_manager.list_passages(actor=user, source_id=source_id, limit=page_size)
 
-            # need to associated passage with agent (for filtering)
-            for passage in passages:
-                assert isinstance(passage, Passage), f"Generate yielded bad non-Passage type: {type(passage)}"
-                passage.agent_id = self.agent_state.id
+        for passage in passages:
+            assert isinstance(passage, Passage), f"Generate yielded bad non-Passage type: {type(passage)}"
+            passage.agent_id = self.agent_state.id
+            self.passage_manager.update_passage_by_id(passage_id=passage.id, passage=passage, actor=user)
 
-                # regenerate passage ID (avoid duplicates)
-                # TODO: need to find another solution to the text duplication issue
-                # passage.id = create_uuid_from_string(f"{source_id}_{str(passage.agent_id)}_{passage.text}")
-
-            # insert into agent archival memory
-            self.archival_memory.storage.insert_many(passages)
-            all_passages += passages
-
-        assert size == len(all_passages), f"Expected {size} passages, but only got {len(all_passages)}"
-
-        # save destination storage
-        self.archival_memory.storage.save()
+        agents_passages = self.passage_manager.list_passages(actor=user, agent_id=self.agent_state.id, source_id=source_id, limit=page_size)
+        passage_size = self.passage_manager.size(actor=user, agent_id=self.agent_state.id, source_id=source_id)
+        assert all([p.agent_id == self.agent_state.id for p in agents_passages])
+        assert len(agents_passages) == passage_size # sanity check
+        assert passage_size == len(passages), f"Expected {len(passages)} passages, got {passage_size}"
 
         # attach to agent
-        source = SourceManager().get_source_by_id(source_id=source_id, actor=user)
+        source = source_manager.get_source_by_id(source_id=source_id, actor=user)
         assert source is not None, f"Source {source_id} not found in metadata store"
-        ms.attach_source(agent_id=self.agent_state.id, source_id=source_id, user_id=self.agent_state.user_id)
 
-        total_agent_passages = self.archival_memory.storage.size()
+        # NOTE: need this redundant line here because we haven't migrated agent to ORM yet
+        # TODO: delete @matt and remove
+        ms.attach_source(agent_id=self.agent_state.id, source_id=source_id, user_id=self.agent_state.user_id)
 
         printd(
-            f"Attached data source {source.name} to agent {self.agent_state.name}, consisting of {len(all_passages)}. Agent now has {total_agent_passages} embeddings in archival memory.",
+            f"Attached data source {source.name} to agent {self.agent_state.name}, consisting of {len(passages)}. Agent now has {passage_size} embeddings in archival memory.",
         )
 
     def update_message(self, message_id: str, request: MessageUpdate) -> Message:
@@ -1565,13 +1554,13 @@ class Agent(BaseAgent):
                 num_tokens_from_messages(messages=messages_openai_format[1:], model=self.model) if len(messages_openai_format) > 1 else 0
             )
 
-        num_archival_memory = self.archival_memory.storage.size()
+        passage_manager_size = self.passage_manager.size(actor=self.user, agent_id=self.agent_state.id)
         message_manager_size = self.message_manager.size(actor=self.user, agent_id=self.agent_state.id)
         external_memory_summary = compile_memory_metadata_block(
             actor=self.user,
             agent_id=self.agent_state.id,
             memory_edit_timestamp=get_utc_time(),  # dummy timestamp
-            archival_memory=self.archival_memory,
+            passage_manager=self.passage_manager,
             message_manager=self.message_manager,
         )
         num_tokens_external_memory_summary = count_tokens(external_memory_summary)
@@ -1597,7 +1586,7 @@ class Agent(BaseAgent):
         return ContextWindowOverview(
             # context window breakdown (in messages)
             num_messages=len(self._messages),
-            num_archival_memory=num_archival_memory,
+            num_archival_memory=passage_manager_size,
             num_recall_memory=message_manager_size,
             num_tokens_external_memory_summary=num_tokens_external_memory_summary,
             # top-level information

letta/agent_store/db.py

@@ -1,4 +1,5 @@
 import base64
+import json
 import os
 from datetime import datetime
 from typing import Dict, List, Optional
@@ -32,7 +33,7 @@ from letta.orm.base import Base
 from letta.orm.file import FileMetadata as FileMetadataModel
 
 # from letta.schemas.message import Message, Passage, Record, RecordType, ToolCall
-from letta.schemas.passage import Passage
+from letta.orm.passage import Passage as PassageModel
 from letta.settings import settings
 
 config = LettaConfig()
@@ -66,56 +67,6 @@ class CommonVector(TypeDecorator):
         # For PostgreSQL, value is already in bytes
         return np.frombuffer(value, dtype=np.float32)
 
-
-class PassageModel(Base):
-    """Defines data model for storing Passages (consisting of text, embedding)"""
-
-    __tablename__ = "passages"
-    __table_args__ = {"extend_existing": True}
-
-    # Assuming passage_id is the primary key
-    id = Column(String, primary_key=True)
-    user_id = Column(String, nullable=False)
-    text = Column(String)
-    file_id = Column(String)
-    agent_id = Column(String)
-    source_id = Column(String)
-
-    # vector storage
-    if settings.letta_pg_uri_no_default:
-        from pgvector.sqlalchemy import Vector
-
-        embedding = mapped_column(Vector(MAX_EMBEDDING_DIM))
-    elif config.archival_storage_type == "sqlite" or config.archival_storage_type == "chroma":
-        embedding = Column(CommonVector)
-    else:
-        raise ValueError(f"Unsupported archival_storage_type: {config.archival_storage_type}")
-    embedding_config = Column(EmbeddingConfigColumn)
-    metadata_ = Column(MutableJson)
-
-    # Add a datetime column, with default value as the current time
-    created_at = Column(DateTime(timezone=True))
-
-    Index("passage_idx_user", user_id, agent_id, file_id),
-
-    def __repr__(self):
-        return f"<Passage(passage_id='{self.id}', text='{self.text}', embedding='{self.embedding})>"
-
-    def to_record(self):
-        return Passage(
-            text=self.text,
-            embedding=self.embedding,
-            embedding_config=self.embedding_config,
-            file_id=self.file_id,
-            user_id=self.user_id,
-            id=self.id,
-            source_id=self.source_id,
-            agent_id=self.agent_id,
-            metadata_=self.metadata_,
-            created_at=self.created_at,
-        )
-
-
 class SQLStorageConnector(StorageConnector):
     def __init__(self, table_type: str, config: LettaConfig, user_id, agent_id=None):
         super().__init__(table_type=table_type, config=config, user_id=user_id, agent_id=agent_id)
@@ -320,8 +271,9 @@ class PostgresStorageConnector(SQLStorageConnector):
         self.session_maker = db_context
 
         # TODO: move to DB init
-        with self.session_maker() as session:
-            session.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))  # Enables the vector extension
+        if settings.pg_uri:
+            with self.session_maker() as session:
+                session.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))  # Enables the vector extension
 
     def query(self, query: str, query_vec: List[float], top_k: int = 10, filters: Optional[Dict] = {}):
         filters = self.get_filters(filters)
@@ -419,7 +371,13 @@ class SQLLiteStorageConnector(SQLStorageConnector):
 
         # get storage URI
         if table_type == TableType.ARCHIVAL_MEMORY or table_type == TableType.PASSAGES:
-            raise ValueError(f"Table type {table_type} not implemented")
+            self.db_model = PassageModel
+            if settings.letta_pg_uri_no_default:
+                self.uri = settings.letta_pg_uri_no_default
+            else:
+                # For SQLite, use the archival storage path
+                self.path = config.archival_storage_path
+                self.uri = f"sqlite:///{os.path.join(config.archival_storage_path, 'letta.db')}"
         elif table_type == TableType.FILES:
             self.path = self.config.metadata_storage_path
             if self.path is None:

letta/agent_store/storage.py

@@ -45,11 +45,13 @@ class StorageConnector:
         self,
         table_type: Union[TableType.ARCHIVAL_MEMORY, TableType.RECALL_MEMORY, TableType.PASSAGES, TableType.FILES],
         config: LettaConfig,
-        user_id,
-        agent_id=None,
+        user_id: str,
+        agent_id: Optional[str] = None,
+        organization_id: Optional[str] = None,
     ):
         self.user_id = user_id
         self.agent_id = agent_id
+        self.organization_id = organization_id
         self.table_type = table_type
 
         # get object type
@@ -74,10 +76,12 @@ class StorageConnector:
             # agent-specific table
             assert agent_id is not None, "Agent ID must be provided for agent-specific tables"
             self.filters = {"user_id": self.user_id, "agent_id": self.agent_id}
-        elif self.table_type == TableType.PASSAGES or self.table_type == TableType.FILES:
+        elif self.table_type == TableType.FILES:
             # setup base filters for user-specific tables
             assert agent_id is None, "Agent ID must not be provided for user-specific tables"
             self.filters = {"user_id": self.user_id}
+        elif self.table_type == TableType.PASSAGES:
+            self.filters = {"organization_id": self.organization_id}
         else:
             raise ValueError(f"Table type {table_type} not implemented")
 
@@ -85,8 +89,9 @@ class StorageConnector:
     def get_storage_connector(
         table_type: Union[TableType.ARCHIVAL_MEMORY, TableType.RECALL_MEMORY, TableType.PASSAGES, TableType.FILES],
         config: LettaConfig,
-        user_id,
-        agent_id=None,
+        user_id: str,
+        organization_id: Optional[str] = None,
+        agent_id: Optional[str] = None,
     ):
         if table_type == TableType.ARCHIVAL_MEMORY or table_type == TableType.PASSAGES:
             storage_type = config.archival_storage_type
@@ -101,10 +106,6 @@ class StorageConnector:
             from letta.agent_store.db import PostgresStorageConnector
 
             return PostgresStorageConnector(table_type, config, user_id, agent_id)
-        elif storage_type == "chroma":
-            from letta.agent_store.chroma import ChromaStorageConnector
-
-            return ChromaStorageConnector(table_type, config, user_id, agent_id)
 
         elif storage_type == "qdrant":
             from letta.agent_store.qdrant import QdrantStorageConnector

letta/cli/cli.py

@@ -53,6 +53,7 @@ def server(
     debug: Annotated[bool, typer.Option(help="Turn debugging output on")] = False,
     ade: Annotated[bool, typer.Option(help="Allows remote access")] = False,  # NOTE: deprecated
     secure: Annotated[bool, typer.Option(help="Adds simple security access")] = False,
+    localhttps: Annotated[bool, typer.Option(help="Setup local https")] = False,
 ):
     """Launch a Letta server process"""
     if type == ServerChoice.rest_api:

letta/client/client.py

@@ -742,7 +742,8 @@ class RESTClient(AbstractClient):
         agents = [AgentState(**agent) for agent in response.json()]
         if len(agents) == 0:
             return None
-        assert len(agents) == 1, f"Multiple agents with the same name: {agents}"
+        agents = [agents[0]] # TODO: @matt monkeypatched
+        assert len(agents) == 1, f"Multiple agents with the same name: {[(agents.name, agents.id) for agents in agents]}"
         return agents[0].id
 
     # memory
@@ -3107,7 +3108,7 @@ class LocalClient(AbstractClient):
             passages (List[Passage]): List of passages
         """
 
-        return self.server.get_agent_archival_cursor(user_id=self.user_id, agent_id=agent_id, before=before, after=after, limit=limit)
+        return self.server.get_agent_archival_cursor(user_id=self.user_id, agent_id=agent_id, limit=limit)
 
     # recall memory
 

letta/config.py

@@ -62,8 +62,8 @@ class LettaConfig:
     # @norton120 these are the metdadatastore
 
     # database configs: archival
-    archival_storage_type: str = "chroma"  # local, db
-    archival_storage_path: str = os.path.join(LETTA_DIR, "chroma")
+    archival_storage_type: str = "sqlite"  # local, db
+    archival_storage_path: str = LETTA_DIR
     archival_storage_uri: str = None  # TODO: eventually allow external vector DB
 
     # database configs: recall

letta/data_sources/connectors.py

@@ -1,4 +1,4 @@
-from typing import Dict, Iterator, List, Tuple
+from typing import Dict, Iterator, List, Tuple, Optional
 
 import typer
 
@@ -42,7 +42,7 @@ class DataConnector:
         """
 
 
-def load_data(connector: DataConnector, source: Source, passage_store: StorageConnector, source_manager: SourceManager, actor: "User"):
+def load_data(connector: DataConnector, source: Source, passage_store: StorageConnector, source_manager: SourceManager, actor: "User", agent_id: Optional[str] = None):
     """Load data from a connector (generates file and passages) into a specified source_id, associated with a user_id."""
     embedding_config = source.embedding_config
 
@@ -82,9 +82,10 @@ def load_data(connector: DataConnector, source: Source, passage_store: StorageCo
                 id=create_uuid_from_string(f"{str(source.id)}_{passage_text}"),
                 text=passage_text,
                 file_id=file_metadata.id,
+                agent_id=agent_id,
                 source_id=source.id,
                 metadata_=passage_metadata,
-                user_id=source.created_by_id,
+                organization_id=source.organization_id,
                 embedding_config=source.embedding_config,
                 embedding=embedding,
             )

letta/embeddings.py

@@ -1,4 +1,3 @@
-import os
 import uuid
 from typing import Any, List, Optional
 
@@ -141,14 +140,35 @@ class AzureOpenAIEmbedding:
         return embeddings
 
 
-def default_embedding_model():
-    # default to hugging face model running local
-    # warning: this is a terrible model
-    from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+class OllamaEmbeddings:
 
-    os.environ["TOKENIZERS_PARALLELISM"] = "False"
-    model = "BAAI/bge-small-en-v1.5"
-    return HuggingFaceEmbedding(model_name=model)
+    # Format:
+    # curl http://localhost:11434/api/embeddings -d '{
+    #   "model": "mxbai-embed-large",
+    #   "prompt": "Llamas are members of the camelid family"
+    # }'
+
+    def __init__(self, model: str, base_url: str, ollama_additional_kwargs: dict):
+        self.model = model
+        self.base_url = base_url
+        self.ollama_additional_kwargs = ollama_additional_kwargs
+
+    def get_text_embedding(self, text: str):
+        import httpx
+
+        headers = {"Content-Type": "application/json"}
+        json_data = {"model": self.model, "prompt": text}
+        json_data.update(self.ollama_additional_kwargs)
+
+        with httpx.Client() as client:
+            response = client.post(
+                f"{self.base_url}/api/embeddings",
+                headers=headers,
+                json=json_data,
+            )
+
+        response_json = response.json()
+        return response_json["embedding"]
 
 
 def query_embedding(embedding_model, query_text: str):
@@ -228,4 +248,4 @@ def embedding_model(config: EmbeddingConfig, user_id: Optional[uuid.UUID] = None
         return model
 
     else:
-        return default_embedding_model()
+        raise ValueError(f"Unknown endpoint type {endpoint_type}")

letta/functions/function_sets/base.py

@@ -164,17 +164,23 @@ def archival_memory_insert(self: "Agent", content: str) -> Optional[str]:
     Returns:
         Optional[str]: None is always returned as this function does not produce a response.
     """
-    self.archival_memory.insert(content)
+    self.passage_manager.insert_passage(
+        agent_state=self.agent_state,
+        agent_id=self.agent_state.id,
+        text=content,
+        actor=self.user,
+    )
     return None
 
 
-def archival_memory_search(self: "Agent", query: str, page: Optional[int] = 0) -> Optional[str]:
+def archival_memory_search(self: "Agent", query: str, page: Optional[int] = 0, start: Optional[int] = 0) -> Optional[str]:
     """
     Search archival memory using semantic (embedding-based) search.
 
     Args:
         query (str): String to search for.
         page (Optional[int]): Allows you to page through results. Only use on a follow-up query. Defaults to 0 (first page).
+        start (Optional[int]): Starting index for the search results. Defaults to 0.
 
     Returns:
         str: Query result string
@@ -191,15 +197,34 @@ def archival_memory_search(self: "Agent", query: str, page: Optional[int] = 0) -
     except:
         raise ValueError(f"'page' argument must be an integer")
     count = RETRIEVAL_QUERY_DEFAULT_PAGE_SIZE
-    results, total = self.archival_memory.search(query, count=count, start=page * count)
-    num_pages = math.ceil(total / count) - 1  # 0 index
-    if len(results) == 0:
-        results_str = f"No results found."
-    else:
-        results_pref = f"Showing {len(results)} of {total} results (page {page}/{num_pages}):"
-        results_formatted = [f"timestamp: {d['timestamp']}, memory: {d['content']}" for d in results]
-        results_str = f"{results_pref} {json_dumps(results_formatted)}"
-    return results_str
+    
+    try:
+        # Get results using passage manager
+        all_results = self.passage_manager.list_passages(
+            actor=self.user,
+            query_text=query,
+            limit=count + start,  # Request enough results to handle offset
+            embedding_config=self.agent_state.embedding_config,
+            embed_query=True
+        )
+
+        # Apply pagination
+        end = min(count + start, len(all_results))
+        paged_results = all_results[start:end]
+
+        # Format results to match previous implementation
+        formatted_results = [
+            {
+                "timestamp": str(result.created_at),
+                "content": result.text
+            }
+            for result in paged_results
+        ]
+
+        return formatted_results, len(formatted_results)
+
+    except Exception as e:
+        raise e
 
 
 def core_memory_append(agent_state: "AgentState", label: str, content: str) -> Optional[str]:  # type: ignore

letta/metadata.py

@@ -363,8 +363,19 @@ class MetadataStore:
         with self.session_maker() as session:
             # TODO: remove this (is a hack)
             mapping_id = f"{user_id}-{agent_id}-{source_id}"
-            session.add(AgentSourceMappingModel(id=mapping_id, user_id=user_id, agent_id=agent_id, source_id=source_id))
-            session.commit()
+            existing = session.query(AgentSourceMappingModel).filter(
+                AgentSourceMappingModel.id == mapping_id
+            ).first()
+
+            if existing is None:
+                # Only create if it doesn't exist
+                session.add(AgentSourceMappingModel(
+                    id=mapping_id, 
+                    user_id=user_id, 
+                    agent_id=agent_id, 
+                    source_id=source_id
+                ))
+                session.commit()
 
     @enforce_types
     def list_attached_source_ids(self, agent_id: str) -> List[str]:

letta/o1_agent.py

@@ -67,9 +67,8 @@ class O1Agent(Agent):
         total_usage = UsageStatistics()
         step_count = 0
         while step_count < self.max_thinking_steps:
-            # This is hacky but we need to do this for now
-            for m in next_input_message:
-                m.id = m._generate_id()
+            if counter > 0:
+                next_input_message = []
 
             kwargs["ms"] = ms
             kwargs["first_message"] = False

letta/offline_memory_agent.py

@@ -130,8 +130,9 @@ class OfflineMemoryAgent(Agent):
         # extras
         first_message_verify_mono: bool = False,
         max_memory_rethinks: int = 10,
+        initial_message_sequence: Optional[List[Message]] = None,  
     ):
-        super().__init__(interface, agent_state, user)
+        super().__init__(interface, agent_state, user, initial_message_sequence=initial_message_sequence)
         self.first_message_verify_mono = first_message_verify_mono
         self.max_memory_rethinks = max_memory_rethinks
 

letta/orm/__init__.py

@@ -6,6 +6,7 @@ from letta.orm.file import FileMetadata
 from letta.orm.job import Job
 from letta.orm.message import Message
 from letta.orm.organization import Organization
+from letta.orm.passage import Passage
 from letta.orm.sandbox_config import SandboxConfig, SandboxEnvironmentVariable
 from letta.orm.source import Source
 from letta.orm.tool import Tool

letta/orm/file.py

@@ -27,3 +27,4 @@ class FileMetadata(SqlalchemyBase, OrganizationMixin, SourceMixin):
     # relationships
     organization: Mapped["Organization"] = relationship("Organization", back_populates="files", lazy="selectin")
     source: Mapped["Source"] = relationship("Source", back_populates="files", lazy="selectin")
+    passages: Mapped[List["Passage"]] = relationship("Passage", back_populates="file", lazy="selectin", cascade="all, delete-orphan")

letta/orm/mixins.py

@@ -1,3 +1,4 @@
+from typing import Optional
 from uuid import UUID
 
 from sqlalchemy import ForeignKey, String
@@ -30,6 +31,12 @@ class UserMixin(Base):
 
     user_id: Mapped[str] = mapped_column(String, ForeignKey("users.id"))
 
+class FileMixin(Base):
+    """Mixin for models that belong to a file."""
+
+    __abstract__ = True
+
+    file_id: Mapped[str] = mapped_column(String, ForeignKey("files.id"))
 
 class AgentMixin(Base):
     """Mixin for models that belong to an agent."""
@@ -38,13 +45,16 @@ class AgentMixin(Base):
 
     agent_id: Mapped[str] = mapped_column(String, ForeignKey("agents.id"))
 
-
 class FileMixin(Base):
     """Mixin for models that belong to a file."""
 
     __abstract__ = True
 
-    file_id: Mapped[str] = mapped_column(String, ForeignKey("files.id"))
+    file_id: Mapped[Optional[str]] = mapped_column(
+        String, 
+        ForeignKey("files.id", ondelete="CASCADE"),
+        nullable=True
+    )
 
 
 class SourceMixin(Base):