letta-nightly 0.5.2.dev20241119104253__py3-none-any.whl → 0.5.3.dev20241120010849__py3-none-any.whl

letta/schemas/block.py

@@ -14,36 +14,30 @@ class BaseBlock(LettaBase, validate_assignment=True):
     __id_prefix__ = "block"
 
     # data value
-    value: Optional[str] = Field(None, description="Value of the block.")
+    value: str = Field(..., description="Value of the block.")
     limit: int = Field(2000, description="Character limit of the block.")
 
     # template data (optional)
     template_name: Optional[str] = Field(None, description="Name of the block if it is a template.", alias="name")
-    template: bool = Field(False, description="Whether the block is a template (e.g. saved human/persona options).")
+    is_template: bool = Field(False, description="Whether the block is a template (e.g. saved human/persona options).")
 
     # context window label
-    label: str = Field(None, description="Label of the block (e.g. 'human', 'persona') in the context window.")
+    label: Optional[str] = Field(None, description="Label of the block (e.g. 'human', 'persona') in the context window.")
 
     # metadata
     description: Optional[str] = Field(None, description="Description of the block.")
     metadata_: Optional[dict] = Field({}, description="Metadata of the block.")
 
-    # associated user/agent
-    user_id: Optional[str] = Field(None, description="The unique identifier of the user associated with the block.")
-
     @model_validator(mode="after")
     def verify_char_limit(self) -> Self:
-        try:
-            assert len(self) <= self.limit
-        except AssertionError:
-            error_msg = f"Edit failed: Exceeds {self.limit} character limit (requested {len(self)}) - {str(self)}."
+        if len(self.value) > self.limit:
+            error_msg = f"Edit failed: Exceeds {self.limit} character limit (requested {len(self.value)}) - {str(self)}."
             raise ValueError(error_msg)
-        except Exception as e:
-            raise e
+
         return self
 
-    def __len__(self):
-        return len(self.value)
+    # def __len__(self):
+    #     return len(self.value)
 
     def __setattr__(self, name, value):
         """Run validation if self.value is updated"""
@@ -52,6 +46,9 @@ class BaseBlock(LettaBase, validate_assignment=True):
             # run validation
             self.__class__.model_validate(self.model_dump(exclude_unset=True))
 
+    class Config:
+        extra = "ignore"  # Ignores extra fields
+
 
 class Block(BaseBlock):
     """
@@ -61,15 +58,22 @@ class Block(BaseBlock):
         label (str): The label of the block (e.g. 'human', 'persona'). This defines a category for the block.
         value (str): The value of the block. This is the string that is represented in the context window.
         limit (int): The character limit of the block.
+        is_template (bool): Whether the block is a template (e.g. saved human/persona options). Non-template blocks are not stored in the database and are ephemeral, while templated blocks are stored in the database.
+        label (str): The label of the block (e.g. 'human', 'persona'). This defines a category for the block.
         template_name (str): The name of the block template (if it is a template).
-        template (bool): Whether the block is a template (e.g. saved human/persona options). Non-template blocks are not stored in the database and are ephemeral, while templated blocks are stored in the database.
         description (str): Description of the block.
         metadata_ (Dict): Metadata of the block.
         user_id (str): The unique identifier of the user associated with the block.
     """
 
     id: str = BaseBlock.generate_id_field()
-    value: str = Field(..., description="Value of the block.")
+
+    # associated user/agent
+    organization_id: Optional[str] = Field(None, description="The unique identifier of the organization associated with the block.")
+
+    # default orm fields
+    created_by_id: Optional[str] = Field(None, description="The id of the user that made this Block.")
+    last_updated_by_id: Optional[str] = Field(None, description="The id of the user that last updated this Block.")
 
 
 class Human(Block):
@@ -84,41 +88,42 @@ class Persona(Block):
     label: str = "persona"
 
 
-class CreateBlock(BaseBlock):
+class BlockCreate(BaseBlock):
     """Create a block"""
 
-    template: bool = True
+    is_template: bool = True
     label: str = Field(..., description="Label of the block.")
 
 
-class CreatePersona(BaseBlock):
+class CreatePersona(BlockCreate):
     """Create a persona block"""
 
-    template: bool = True
     label: str = "persona"
 
 
-class CreateHuman(BaseBlock):
+class CreateHuman(BlockCreate):
     """Create a human block"""
 
-    template: bool = True
     label: str = "human"
 
 
-class UpdateBlock(BaseBlock):
+class BlockUpdate(BaseBlock):
     """Update a block"""
 
-    id: str = Field(..., description="The unique identifier of the block.")
     limit: Optional[int] = Field(2000, description="Character limit of the block.")
+    value: Optional[str] = Field(None, description="Value of the block.")
+
+    class Config:
+        extra = "ignore"  # Ignores extra fields
 
 
-class UpdatePersona(UpdateBlock):
+class UpdatePersona(BlockUpdate):
     """Update a persona block"""
 
     label: str = "persona"
 
 
-class UpdateHuman(UpdateBlock):
+class UpdateHuman(BlockUpdate):
     """Update a human block"""
 
     label: str = "human"

letta/schemas/letta_base.py

@@ -77,6 +77,6 @@ class LettaBase(BaseModel):
         """
         _ = values  # for SCA
         if isinstance(v, UUID):
-            logger.warning("Bare UUIDs are deprecated, please use the full prefixed id!")
+            logger.warning(f"Bare UUIDs are deprecated, please use the full prefixed id ({cls.__id_prefix__})!")
             return f"{cls.__id_prefix__}-{v}"
         return v

letta/server/rest_api/routers/v1/blocks.py

@@ -2,7 +2,8 @@ from typing import TYPE_CHECKING, List, Optional
 
 from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query
 
-from letta.schemas.block import Block, CreateBlock, UpdateBlock
+from letta.orm.errors import NoResultFound
+from letta.schemas.block import Block, BlockCreate, BlockUpdate
 from letta.server.rest_api.utils import get_letta_server
 from letta.server.server import SyncServer
 
@@ -22,54 +23,49 @@ def list_blocks(
     user_id: Optional[str] = Header(None, alias="user_id"),  # Extract user_id from header, default to None if not present
 ):
     actor = server.get_user_or_default(user_id=user_id)
-
-    blocks = server.get_blocks(user_id=actor.id, label=label, template=templates_only, name=name)
-    if blocks is None:
-        return []
-    return blocks
+    return server.block_manager.get_blocks(actor=actor, label=label, is_template=templates_only, template_name=name)
 
 
 @router.post("/", response_model=Block, operation_id="create_memory_block")
 def create_block(
-    create_block: CreateBlock = Body(...),
+    create_block: BlockCreate = Body(...),
     server: SyncServer = Depends(get_letta_server),
     user_id: Optional[str] = Header(None, alias="user_id"),  # Extract user_id from header, default to None if not present
 ):
     actor = server.get_user_or_default(user_id=user_id)
-
-    create_block.user_id = actor.id
-    return server.create_block(user_id=actor.id, request=create_block)
+    block = Block(**create_block.model_dump())
+    return server.block_manager.create_or_update_block(actor=actor, block=block)
 
 
 @router.patch("/{block_id}", response_model=Block, operation_id="update_memory_block")
 def update_block(
     block_id: str,
-    updated_block: UpdateBlock = Body(...),
+    updated_block: BlockUpdate = Body(...),
     server: SyncServer = Depends(get_letta_server),
+    user_id: Optional[str] = Header(None, alias="user_id"),
 ):
-    # actor = server.get_current_user()
-
-    updated_block.id = block_id
-    return server.update_block(request=updated_block)
+    actor = server.get_user_or_default(user_id=user_id)
+    return server.block_manager.update_block(block_id=block_id, block_update=updated_block, actor=actor)
 
 
-# TODO: delete should not return anything
 @router.delete("/{block_id}", response_model=Block, operation_id="delete_memory_block")
 def delete_block(
     block_id: str,
     server: SyncServer = Depends(get_letta_server),
+    user_id: Optional[str] = Header(None, alias="user_id"),
 ):
-
-    return server.delete_block(block_id=block_id)
+    actor = server.get_user_or_default(user_id=user_id)
+    return server.block_manager.delete_block(block_id=block_id, actor=actor)
 
 
 @router.get("/{block_id}", response_model=Block, operation_id="get_memory_block")
 def get_block(
     block_id: str,
     server: SyncServer = Depends(get_letta_server),
+    user_id: Optional[str] = Header(None, alias="user_id"),
 ):
-
-    block = server.get_block(block_id=block_id)
-    if block is None:
+    actor = server.get_user_or_default(user_id=user_id)
+    try:
+        return server.block_manager.get_block_by_id(block_id=block_id, actor=actor)
+    except NoResultFound:
         raise HTTPException(status_code=404, detail="Block not found")
-    return block

letta/server/rest_api/routers/v1/sources.py

@@ -18,6 +18,7 @@ from letta.schemas.passage import Passage
 from letta.schemas.source import Source, SourceCreate, SourceUpdate
 from letta.server.rest_api.utils import get_letta_server
 from letta.server.server import SyncServer
+from letta.utils import sanitize_filename
 
 # These can be forward refs, but because Fastapi needs them at runtime the must be imported normally
 
@@ -170,7 +171,7 @@ def upload_file_to_source(
     server.ms.create_job(job)
 
     # create background task
-    background_tasks.add_task(load_file_to_source_async, server, source_id=source.id, job_id=job.id, file=file, bytes=bytes)
+    background_tasks.add_task(load_file_to_source_async, server, source_id=source.id, file=file, job_id=job.id, bytes=bytes)
 
     # return job information
     job = server.ms.get_job(job_id=job_id)
@@ -227,10 +228,15 @@ def delete_file_from_source(
 
 
 def load_file_to_source_async(server: SyncServer, source_id: str, job_id: str, file: UploadFile, bytes: bytes):
-    # write the file to a temporary directory (deleted after the context manager exits)
+    # Create a temporary directory (deleted after the context manager exits)
     with tempfile.TemporaryDirectory() as tmpdirname:
-        file_path = os.path.join(str(tmpdirname), str(file.filename))
+        # Sanitize the filename
+        sanitized_filename = sanitize_filename(file.filename)
+        file_path = os.path.join(tmpdirname, sanitized_filename)
+
+        # Write the file to the sanitized path
         with open(file_path, "wb") as buffer:
             buffer.write(bytes)
 
+        # Pass the file to load_file_to_source
         server.load_file_to_source(source_id, file_path, job_id)

letta/server/server.py

@@ -55,13 +55,6 @@ from letta.providers import (
 )
 from letta.schemas.agent import AgentState, AgentType, CreateAgent, UpdateAgentState
 from letta.schemas.api_key import APIKey, APIKeyCreate
-from letta.schemas.block import (
-    Block,
-    CreateBlock,
-    CreateHuman,
-    CreatePersona,
-    UpdateBlock,
-)
 from letta.schemas.embedding_config import EmbeddingConfig
 
 # openai schemas
@@ -83,6 +76,7 @@ from letta.schemas.tool import Tool, ToolCreate
 from letta.schemas.usage import LettaUsageStatistics
 from letta.schemas.user import User
 from letta.services.agents_tags_manager import AgentsTagsManager
+from letta.services.block_manager import BlockManager
 from letta.services.organization_manager import OrganizationManager
 from letta.services.source_manager import SourceManager
 from letta.services.tool_manager import ToolManager
@@ -250,6 +244,7 @@ class SyncServer(Server):
         self.organization_manager = OrganizationManager()
         self.user_manager = UserManager()
         self.tool_manager = ToolManager()
+        self.block_manager = BlockManager()
         self.source_manager = SourceManager()
         self.agents_tags_manager = AgentsTagsManager()
 
@@ -257,7 +252,7 @@ class SyncServer(Server):
         if init_with_default_org_and_user:
             self.default_org = self.organization_manager.create_default_organization()
             self.default_user = self.user_manager.create_default_user()
-            self.add_default_blocks(self.default_user.id)
+            self.block_manager.add_default_blocks(actor=self.default_user)
             self.tool_manager.add_base_tools(actor=self.default_user)
 
             # If there is a default org/user
@@ -333,15 +328,6 @@ class SyncServer(Server):
                 )
             )
 
-    def save_agents(self):
-        """Saves all the agents that are in the in-memory object store"""
-        for agent_d in self.active_agents:
-            try:
-                save_agent(agent_d["agent"], self.ms)
-                logger.debug(f"Saved agent {agent_d['agent_id']}")
-            except Exception as e:
-                logger.exception(f"Error occurred while trying to save agent {agent_d['agent_id']}:\n{e}")
-
     def _get_agent(self, user_id: str, agent_id: str) -> Union[Agent, None]:
         """Get the agent object from the in-memory object store"""
         for d in self.active_agents:
@@ -399,9 +385,9 @@ class SyncServer(Server):
             assert isinstance(agent_state.memory, Memory)
 
             if agent_state.agent_type == AgentType.memgpt_agent:
-                letta_agent = Agent(agent_state=agent_state, interface=interface, tools=tool_objs)
+                letta_agent = Agent(agent_state=agent_state, interface=interface, tools=tool_objs, user=actor)
             elif agent_state.agent_type == AgentType.o1_agent:
-                letta_agent = O1Agent(agent_state=agent_state, interface=interface, tools=tool_objs)
+                letta_agent = O1Agent(agent_state=agent_state, interface=interface, tools=tool_objs, user=actor)
             else:
                 raise NotImplementedError("Not a supported agent type")
 
@@ -884,6 +870,7 @@ class SyncServer(Server):
                     first_message_verify_mono=(
                         True if (llm_config and llm_config.model is not None and "gpt-4" in llm_config.model) else False
                     ),
+                    user=actor,
                     initial_message_sequence=request.initial_message_sequence,
                 )
             elif request.agent_type == AgentType.o1_agent:
@@ -895,6 +882,7 @@ class SyncServer(Server):
                     first_message_verify_mono=(
                         True if (llm_config and llm_config.model is not None and "gpt-4" in llm_config.model) else False
                     ),
+                    user=actor,
                 )
             # rebuilding agent memory on agent create in case shared memory blocks
             # were specified in the new agent's memory config. we're doing this for two reasons:
@@ -1130,56 +1118,6 @@ class SyncServer(Server):
 
             return [self.get_agent_state(user_id=user.id, agent_id=agent_id) for agent_id in agent_ids]
 
-    def get_blocks(
-        self,
-        user_id: Optional[str] = None,
-        label: Optional[str] = None,
-        template: Optional[bool] = None,
-        name: Optional[str] = None,
-        id: Optional[str] = None,
-    ) -> Optional[List[Block]]:
-
-        return self.ms.get_blocks(user_id=user_id, label=label, template=template, template_name=name, id=id)
-
-    def get_block(self, block_id: str):
-
-        blocks = self.get_blocks(id=block_id)
-        if blocks is None or len(blocks) == 0:
-            raise ValueError("Block does not exist")
-        if len(blocks) > 1:
-            raise ValueError("Multiple blocks with the same id")
-        return blocks[0]
-
-    def create_block(self, request: CreateBlock, user_id: str, update: bool = False) -> Block:
-        existing_blocks = self.ms.get_blocks(
-            template_name=request.template_name, user_id=user_id, template=request.template, label=request.label
-        )
-
-        # for templates, update existing block template if exists
-        if existing_blocks is not None and request.template:
-            existing_block = existing_blocks[0]
-            assert len(existing_blocks) == 1
-            if update:
-                return self.update_block(UpdateBlock(id=existing_block.id, **vars(request)))
-            else:
-                raise ValueError(f"Block with name {request.template_name} already exists")
-        block = Block(**vars(request))
-        self.ms.create_block(block)
-        return block
-
-    def update_block(self, request: UpdateBlock) -> Block:
-        block = self.get_block(request.id)
-        block.limit = request.limit if request.limit is not None else block.limit
-        block.value = request.value if request.value is not None else block.value
-        block.template_name = request.template_name if request.template_name is not None else block.template_name
-        self.ms.update_block(block=block)
-        return self.ms.get_block(block_id=request.id)
-
-    def delete_block(self, block_id: str):
-        block = self.get_block(block_id)
-        self.ms.delete_block(block_id)
-        return block
-
     # convert name->id
 
     def get_agent_id(self, name: str, user_id: str):
@@ -1790,21 +1728,6 @@ class SyncServer(Server):
 
         return success
 
-    def add_default_blocks(self, user_id: str):
-        from letta.utils import list_human_files, list_persona_files
-
-        assert user_id is not None, "User ID must be provided"
-
-        for persona_file in list_persona_files():
-            text = open(persona_file, "r", encoding="utf-8").read()
-            name = os.path.basename(persona_file).replace(".txt", "")
-            self.create_block(CreatePersona(user_id=user_id, template_name=name, value=text, template=True), user_id=user_id, update=True)
-
-        for human_file in list_human_files():
-            text = open(human_file, "r", encoding="utf-8").read()
-            name = os.path.basename(human_file).replace(".txt", "")
-            self.create_block(CreateHuman(user_id=user_id, template_name=name, value=text, template=True), user_id=user_id, update=True)
-
     def get_agent_message(self, agent_id: str, message_id: str) -> Optional[Message]:
         """Get a single message from the agent's memory"""
         # Get the agent object (loaded in memory)

letta/services/block_manager.py

@@ -0,0 +1,103 @@
+import os
+from typing import List, Optional
+
+from letta.orm.block import Block as BlockModel
+from letta.orm.errors import NoResultFound
+from letta.schemas.block import Block
+from letta.schemas.block import Block as PydanticBlock
+from letta.schemas.block import BlockUpdate, Human, Persona
+from letta.schemas.user import User as PydanticUser
+from letta.utils import enforce_types, list_human_files, list_persona_files
+
+
+class BlockManager:
+    """Manager class to handle business logic related to Blocks."""
+
+    def __init__(self):
+        # Fetching the db_context similarly as in ToolManager
+        from letta.server.server import db_context
+
+        self.session_maker = db_context
+
+    @enforce_types
+    def create_or_update_block(self, block: Block, actor: PydanticUser) -> PydanticBlock:
+        """Create a new block based on the Block schema."""
+        db_block = self.get_block_by_id(block.id, actor)
+        if db_block:
+            update_data = BlockUpdate(**block.model_dump(exclude_none=True))
+            self.update_block(block.id, update_data, actor)
+        else:
+            with self.session_maker() as session:
+                data = block.model_dump(exclude_none=True)
+                block = BlockModel(**data, organization_id=actor.organization_id)
+                block.create(session, actor=actor)
+            return block.to_pydantic()
+
+    @enforce_types
+    def update_block(self, block_id: str, block_update: BlockUpdate, actor: PydanticUser) -> PydanticBlock:
+        """Update a block by its ID with the given BlockUpdate object."""
+        with self.session_maker() as session:
+            block = BlockModel.read(db_session=session, identifier=block_id, actor=actor)
+            update_data = block_update.model_dump(exclude_unset=True, exclude_none=True)
+            for key, value in update_data.items():
+                setattr(block, key, value)
+            block.update(db_session=session, actor=actor)
+            return block.to_pydantic()
+
+    @enforce_types
+    def delete_block(self, block_id: str, actor: PydanticUser) -> PydanticBlock:
+        """Delete a block by its ID."""
+        with self.session_maker() as session:
+            block = BlockModel.read(db_session=session, identifier=block_id)
+            block.hard_delete(db_session=session, actor=actor)
+            return block.to_pydantic()
+
+    @enforce_types
+    def get_blocks(
+        self,
+        actor: PydanticUser,
+        label: Optional[str] = None,
+        is_template: Optional[bool] = None,
+        template_name: Optional[str] = None,
+        id: Optional[str] = None,
+        cursor: Optional[str] = None,
+        limit: Optional[int] = 50,
+    ) -> List[PydanticBlock]:
+        """Retrieve blocks based on various optional filters."""
+        with self.session_maker() as session:
+            # Prepare filters
+            filters = {"organization_id": actor.organization_id}
+            if label:
+                filters["label"] = label
+            if is_template is not None:
+                filters["is_template"] = is_template
+            if template_name:
+                filters["template_name"] = template_name
+            if id:
+                filters["id"] = id
+
+            blocks = BlockModel.list(db_session=session, cursor=cursor, limit=limit, **filters)
+
+            return [block.to_pydantic() for block in blocks]
+
+    @enforce_types
+    def get_block_by_id(self, block_id, actor: PydanticUser) -> Optional[PydanticBlock]:
+        """Retrieve a block by its name."""
+        with self.session_maker() as session:
+            try:
+                block = BlockModel.read(db_session=session, identifier=block_id, actor=actor)
+                return block.to_pydantic()
+            except NoResultFound:
+                return None
+
+    @enforce_types
+    def add_default_blocks(self, actor: PydanticUser):
+        for persona_file in list_persona_files():
+            text = open(persona_file, "r", encoding="utf-8").read()
+            name = os.path.basename(persona_file).replace(".txt", "")
+            self.create_or_update_block(Persona(template_name=name, value=text, is_template=True), actor=actor)
+
+        for human_file in list_human_files():
+            text = open(human_file, "r", encoding="utf-8").read()
+            name = os.path.basename(human_file).replace(".txt", "")
+            self.create_or_update_block(Human(template_name=name, value=text, is_template=True), actor=actor)

letta/utils.py

@@ -21,6 +21,7 @@ from urllib.parse import urljoin, urlparse
 import demjson3 as demjson
 import pytz
 import tiktoken
+from pathvalidate import sanitize_filename as pathvalidate_sanitize_filename
 
 import letta
 from letta.constants import (
@@ -29,6 +30,7 @@ from letta.constants import (
     CORE_MEMORY_PERSONA_CHAR_LIMIT,
     FUNCTION_RETURN_CHAR_LIMIT,
     LETTA_DIR,
+    MAX_FILENAME_LENGTH,
     TOOL_CALL_ID_MAX_LEN,
 )
 from letta.schemas.openai.chat_completion_response import ChatCompletionResponse
@@ -1071,3 +1073,40 @@ def json_dumps(data, indent=2):
 
 def json_loads(data):
     return json.loads(data, strict=False)
+
+
+def sanitize_filename(filename: str) -> str:
+    """
+    Sanitize the given filename to prevent directory traversal, invalid characters,
+    and reserved names while ensuring it fits within the maximum length allowed by the filesystem.
+
+    Parameters:
+        filename (str): The user-provided filename.
+
+    Returns:
+        str: A sanitized filename that is unique and safe for use.
+    """
+    # Extract the base filename to avoid directory components
+    filename = os.path.basename(filename)
+
+    # Split the base and extension
+    base, ext = os.path.splitext(filename)
+
+    # External sanitization library
+    base = pathvalidate_sanitize_filename(base)
+
+    # Cannot start with a period
+    if base.startswith("."):
+        raise ValueError(f"Invalid filename - derived file name {base} cannot start with '.'")
+
+    # Truncate the base name to fit within the maximum allowed length
+    max_base_length = MAX_FILENAME_LENGTH - len(ext) - 33  # 32 for UUID + 1 for `_`
+    if len(base) > max_base_length:
+        base = base[:max_base_length]
+
+    # Append a unique UUID suffix for uniqueness
+    unique_suffix = uuid.uuid4().hex
+    sanitized_filename = f"{base}_{unique_suffix}{ext}"
+
+    # Return the sanitized filename
+    return sanitized_filename

{letta_nightly-0.5.2.dev20241119104253.dist-info → letta_nightly-0.5.3.dev20241120010849.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: letta-nightly
-Version: 0.5.2.dev20241119104253
+Version: 0.5.3.dev20241120010849
 Summary: Create LLM agents with long-term memory and custom tools
 License: Apache License
 Author: Letta Team
@@ -47,6 +47,7 @@ Requires-Dist: llama-index-embeddings-openai (>=0.2.5,<0.3.0)
 Requires-Dist: locust (>=2.31.5,<3.0.0)
 Requires-Dist: nltk (>=3.8.1,<4.0.0)
 Requires-Dist: numpy (>=1.26.2,<2.0.0)
+Requires-Dist: pathvalidate (>=3.2.1,<4.0.0)
 Requires-Dist: pexpect (>=4.9.0,<5.0.0) ; extra == "dev"
 Requires-Dist: pg8000 (>=1.30.3,<2.0.0) ; extra == "postgres"
 Requires-Dist: pgvector (>=0.2.3,<0.3.0) ; extra == "postgres"