letta-nightly 0.4.1.dev20241014104152__py3-none-any.whl → 0.5.0.dev20241015014828__py3-none-any.whl

letta/data_sources/connectors_helper.py

@@ -0,0 +1,97 @@
+import mimetypes
+import os
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional
+
+
+def extract_file_metadata(file_path) -> dict:
+    """Extracts metadata from a single file."""
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(file_path)
+
+    file_metadata = {
+        "file_name": os.path.basename(file_path),
+        "file_path": file_path,
+        "file_type": mimetypes.guess_type(file_path)[0] or "unknown",
+        "file_size": os.path.getsize(file_path),
+        "file_creation_date": datetime.fromtimestamp(os.path.getctime(file_path)).strftime("%Y-%m-%d"),
+        "file_last_modified_date": datetime.fromtimestamp(os.path.getmtime(file_path)).strftime("%Y-%m-%d"),
+    }
+    return file_metadata
+
+
+def extract_metadata_from_files(file_list):
+    """Extracts metadata for a list of files."""
+    metadata = []
+    for file_path in file_list:
+        file_metadata = extract_file_metadata(file_path)
+        if file_metadata:
+            metadata.append(file_metadata)
+    return metadata
+
+
+def get_filenames_in_dir(
+    input_dir: str, recursive: bool = True, required_exts: Optional[List[str]] = None, exclude: Optional[List[str]] = None
+):
+    """
+    Recursively reads files from the directory, applying required_exts and exclude filters.
+    Ensures that required_exts and exclude do not overlap.
+
+    Args:
+        input_dir (str): The directory to scan for files.
+        recursive (bool): Whether to scan directories recursively.
+        required_exts (list): List of file extensions to include (e.g., ['pdf', 'txt']).
+                             If None or empty, matches any file extension.
+        exclude (list): List of file patterns to exclude (e.g., ['*png', '*jpg']).
+
+    Returns:
+        list: A list of matching file paths.
+    """
+    required_exts = required_exts or []
+    exclude = exclude or []
+
+    # Ensure required_exts and exclude do not overlap
+    ext_set = set(required_exts)
+    exclude_set = set(exclude)
+    overlap = ext_set & exclude_set
+    if overlap:
+        raise ValueError(f"Extensions in required_exts and exclude overlap: {overlap}")
+
+    def is_excluded(file_name):
+        """Check if a file matches any pattern in the exclude list."""
+        for pattern in exclude:
+            if Path(file_name).match(pattern):
+                return True
+        return False
+
+    files = []
+    search_pattern = "**/*" if recursive else "*"
+
+    for file_path in Path(input_dir).glob(search_pattern):
+        if file_path.is_file() and not is_excluded(file_path.name):
+            ext = file_path.suffix.lstrip(".")
+            # If required_exts is empty, match any file
+            if not required_exts or ext in required_exts:
+                files.append(file_path)
+
+    return files
+
+
+def assert_all_files_exist_locally(file_paths: List[str]) -> bool:
+    """
+    Checks if all file paths in the provided list exist locally.
+    Raises a FileNotFoundError with a list of missing files if any do not exist.
+
+    Args:
+        file_paths (List[str]): List of file paths to check.
+
+    Returns:
+        bool: True if all files exist, raises FileNotFoundError if any file is missing.
+    """
+    missing_files = [file_path for file_path in file_paths if not Path(file_path).exists()]
+
+    if missing_files:
+        raise FileNotFoundError(missing_files)
+
+    return True

letta/llm_api/mistral.py

@@ -0,0 +1,47 @@
+import requests
+
+from letta.utils import printd, smart_urljoin
+
+
+def mistral_get_model_list(url: str, api_key: str) -> dict:
+    url = smart_urljoin(url, "models")
+
+    headers = {"Content-Type": "application/json"}
+    if api_key is not None:
+        headers["Authorization"] = f"Bearer {api_key}"
+
+    printd(f"Sending request to {url}")
+    response = None
+    try:
+        # TODO add query param "tool" to be true
+        response = requests.get(url, headers=headers)
+        response.raise_for_status()  # Raises HTTPError for 4XX/5XX status
+        response_json = response.json()  # convert to dict from string
+        return response_json
+    except requests.exceptions.HTTPError as http_err:
+        # Handle HTTP errors (e.g., response 4XX, 5XX)
+        try:
+            if response:
+                response = response.json()
+        except:
+            pass
+        printd(f"Got HTTPError, exception={http_err}, response={response}")
+        raise http_err
+    except requests.exceptions.RequestException as req_err:
+        # Handle other requests-related errors (e.g., connection error)
+        try:
+            if response:
+                response = response.json()
+        except:
+            pass
+        printd(f"Got RequestException, exception={req_err}, response={response}")
+        raise req_err
+    except Exception as e:
+        # Handle other potential errors
+        try:
+            if response:
+                response = response.json()
+        except:
+            pass
+        printd(f"Got unknown Exception, exception={e}, response={response}")
+        raise e

letta/metadata.py

@@ -11,6 +11,7 @@ from sqlalchemy import (
     Column,
     DateTime,
     Index,
+    Integer,
     String,
     TypeDecorator,
     desc,
@@ -24,6 +25,7 @@ from letta.schemas.api_key import APIKey
 from letta.schemas.block import Block, Human, Persona
 from letta.schemas.embedding_config import EmbeddingConfig
 from letta.schemas.enums import JobStatus
+from letta.schemas.file import FileMetadata
 from letta.schemas.job import Job
 from letta.schemas.llm_config import LLMConfig
 from letta.schemas.memory import Memory
@@ -38,6 +40,41 @@ from letta.settings import settings
 from letta.utils import enforce_types, get_utc_time, printd
 
 
+class FileMetadataModel(Base):
+    __tablename__ = "files"
+    __table_args__ = {"extend_existing": True}
+
+    id = Column(String, primary_key=True, nullable=False)
+    user_id = Column(String, nullable=False)
+    # TODO: Investigate why this breaks during table creation due to FK
+    # source_id = Column(String, ForeignKey("sources.id"), nullable=False)
+    source_id = Column(String, nullable=False)
+    file_name = Column(String, nullable=True)
+    file_path = Column(String, nullable=True)
+    file_type = Column(String, nullable=True)
+    file_size = Column(Integer, nullable=True)
+    file_creation_date = Column(String, nullable=True)
+    file_last_modified_date = Column(String, nullable=True)
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+
+    def __repr__(self):
+        return f"<FileMetadata(id='{self.id}', source_id='{self.source_id}', file_name='{self.file_name}')>"
+
+    def to_record(self):
+        return FileMetadata(
+            id=self.id,
+            user_id=self.user_id,
+            source_id=self.source_id,
+            file_name=self.file_name,
+            file_path=self.file_path,
+            file_type=self.file_type,
+            file_size=self.file_size,
+            file_creation_date=self.file_creation_date,
+            file_last_modified_date=self.file_last_modified_date,
+            created_at=self.created_at,
+        )
+
+
 class LLMConfigColumn(TypeDecorator):
     """Custom type for storing LLMConfig as JSON"""
 
@@ -865,6 +902,27 @@ class MetadataStore:
             session.add(JobModel(**vars(job)))
             session.commit()
 
+    @enforce_types
+    def list_files_from_source(self, source_id: str, limit: int, cursor: Optional[str]):
+        with self.session_maker() as session:
+            # Start with the basic query filtered by source_id
+            query = session.query(FileMetadataModel).filter(FileMetadataModel.source_id == source_id)
+
+            if cursor:
+                # Assuming cursor is the ID of the last file in the previous page
+                query = query.filter(FileMetadataModel.id > cursor)
+
+            # Order by ID or other ordering criteria to ensure correct pagination
+            query = query.order_by(FileMetadataModel.id)
+
+            # Limit the number of results returned
+            results = query.limit(limit).all()
+
+            # Convert the results to the required FileMetadata objects
+            files = [r.to_record() for r in results]
+
+            return files
+
     def delete_job(self, job_id: str):
         with self.session_maker() as session:
             session.query(JobModel).filter(JobModel.id == job_id).delete()

letta/providers.py

@@ -139,6 +139,50 @@ class AnthropicProvider(Provider):
         return []
 
 
+class MistralProvider(Provider):
+    name: str = "mistral"
+    api_key: str = Field(..., description="API key for the Mistral API.")
+    base_url: str = "https://api.mistral.ai/v1"
+
+    def list_llm_models(self) -> List[LLMConfig]:
+        from letta.llm_api.mistral import mistral_get_model_list
+
+        # Some hardcoded support for OpenRouter (so that we only get models with tool calling support)...
+        # See: https://openrouter.ai/docs/requests
+        response = mistral_get_model_list(self.base_url, api_key=self.api_key)
+
+        assert "data" in response, f"Mistral model query response missing 'data' field: {response}"
+
+        configs = []
+        for model in response["data"]:
+            # If model has chat completions and function calling enabled
+            if model["capabilities"]["completion_chat"] and model["capabilities"]["function_calling"]:
+                configs.append(
+                    LLMConfig(
+                        model=model["id"],
+                        model_endpoint_type="openai",
+                        model_endpoint=self.base_url,
+                        context_window=model["max_context_length"],
+                    )
+                )
+
+        return configs
+
+    def list_embedding_models(self) -> List[EmbeddingConfig]:
+        # Not supported for mistral
+        return []
+
+    def get_model_context_window(self, model_name: str) -> Optional[int]:
+        # Redoing this is fine because it's a pretty lightweight call
+        models = self.list_llm_models()
+
+        for m in models:
+            if model_name in m["id"]:
+                return int(m["max_context_length"])
+
+        return None
+
+
 class OllamaProvider(OpenAIProvider):
     """Ollama provider that uses the native /api/generate endpoint
 

letta/schemas/file.py

@@ -0,0 +1,31 @@
+from datetime import datetime
+from typing import Optional
+
+from pydantic import Field
+
+from letta.schemas.letta_base import LettaBase
+from letta.utils import get_utc_time
+
+
+class FileMetadataBase(LettaBase):
+    """Base class for FileMetadata schemas"""
+
+    __id_prefix__ = "file"
+
+
+class FileMetadata(FileMetadataBase):
+    """Representation of a single FileMetadata"""
+
+    id: str = FileMetadataBase.generate_id_field()
+    user_id: str = Field(description="The unique identifier of the user associated with the document.")
+    source_id: str = Field(..., description="The unique identifier of the source associated with the document.")
+    file_name: Optional[str] = Field(None, description="The name of the file.")
+    file_path: Optional[str] = Field(None, description="The path to the file.")
+    file_type: Optional[str] = Field(None, description="The type of the file (MIME type).")
+    file_size: Optional[int] = Field(None, description="The size of the file in bytes.")
+    file_creation_date: Optional[str] = Field(None, description="The creation date of the file.")
+    file_last_modified_date: Optional[str] = Field(None, description="The last modified date of the file.")
+    created_at: datetime = Field(default_factory=get_utc_time, description="The creation date of this file metadata object.")
+
+    class Config:
+        extra = "allow"

letta/schemas/job.py

@@ -15,7 +15,7 @@ class JobBase(LettaBase):
 
 class Job(JobBase):
     """
-    Representation of offline jobs, used for tracking status of data loading tasks (involving parsing and embedding documents).
+    Representation of offline jobs, used for tracking status of data loading tasks (involving parsing and embedding files).
 
     Parameters:
         id (str): The unique identifier of the job.

letta/schemas/letta_request.py

@@ -1,13 +1,13 @@
-from typing import List
+from typing import List, Union
 
 from pydantic import BaseModel, Field
 
 from letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG
-from letta.schemas.message import MessageCreate
+from letta.schemas.message import Message, MessageCreate
 
 
 class LettaRequest(BaseModel):
-    messages: List[MessageCreate] = Field(..., description="The messages to be sent to the agent.")
+    messages: Union[List[MessageCreate], List[Message]] = Field(..., description="The messages to be sent to the agent.")
     run_async: bool = Field(default=False, description="Whether to asynchronously send the messages to the agent.")  # TODO: implement
 
     stream_steps: bool = Field(

letta/schemas/llm_config.py

@@ -33,6 +33,7 @@ class LLMConfig(BaseModel):
         "koboldcpp",
         "vllm",
         "hugging-face",
+        "mistral",
     ] = Field(..., description="The endpoint type for the model.")
     model_endpoint: Optional[str] = Field(None, description="The endpoint for the model.")
     model_wrapper: Optional[str] = Field(None, description="The wrapper for the model.")

letta/schemas/message.py

@@ -2,7 +2,7 @@ import copy
 import json
 import warnings
 from datetime import datetime, timezone
-from typing import List, Optional
+from typing import List, Literal, Optional
 
 from pydantic import Field, field_validator
 
@@ -57,7 +57,11 @@ class BaseMessage(LettaBase):
 class MessageCreate(BaseMessage):
     """Request to create a message"""
 
-    role: MessageRole = Field(..., description="The role of the participant.")
+    # In the simplified format, only allow simple roles
+    role: Literal[
+        MessageRole.user,
+        MessageRole.system,
+    ] = Field(..., description="The role of the participant.")
     text: str = Field(..., description="The text of the message.")
     name: Optional[str] = Field(None, description="The name of the participant.")
 

letta/schemas/passage.py

@@ -19,8 +19,8 @@ class PassageBase(LettaBase):
     # origin data source
     source_id: Optional[str] = Field(None, description="The data source of the passage.")
 
-    # document association
-    doc_id: Optional[str] = Field(None, description="The unique identifier of the document associated with the passage.")
+    # file association
+    file_id: Optional[str] = Field(None, description="The unique identifier of the file associated with the passage.")
     metadata_: Optional[Dict] = Field({}, description="The metadata of the passage.")
 
 
@@ -36,7 +36,7 @@ class Passage(PassageBase):
         user_id (str): The unique identifier of the user associated with the passage.
         agent_id (str): The unique identifier of the agent associated with the passage.
         source_id (str): The data source of the passage.
-        doc_id (str): The unique identifier of the document associated with the passage.
+        file_id (str): The unique identifier of the file associated with the passage.
     """
 
     id: str = PassageBase.generate_id_field()

letta/schemas/source.py

@@ -28,7 +28,7 @@ class SourceCreate(BaseSource):
 
 class Source(BaseSource):
     """
-    Representation of a source, which is a collection of documents and passages.
+    Representation of a source, which is a collection of files and passages.
 
     Parameters:
         id (str): The ID of the source
@@ -59,4 +59,4 @@ class UploadFileToSourceRequest(BaseModel):
 class UploadFileToSourceResponse(BaseModel):
     source: Source = Field(..., description="The source the file was uploaded to.")
     added_passages: int = Field(..., description="The number of passages added to the source.")
-    added_documents: int = Field(..., description="The number of documents added to the source.")
+    added_documents: int = Field(..., description="The number of files added to the source.")

letta/server/rest_api/routers/v1/agents.py

@@ -8,7 +8,7 @@ from starlette.responses import StreamingResponse
 
 from letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG
 from letta.schemas.agent import AgentState, CreateAgent, UpdateAgentState
-from letta.schemas.enums import MessageRole, MessageStreamStatus
+from letta.schemas.enums import MessageStreamStatus
 from letta.schemas.letta_message import (
     LegacyLettaMessage,
     LettaMessage,
@@ -23,7 +23,7 @@ from letta.schemas.memory import (
     Memory,
     RecallMemorySummary,
 )
-from letta.schemas.message import Message, UpdateMessage
+from letta.schemas.message import Message, MessageCreate, UpdateMessage
 from letta.schemas.passage import Passage
 from letta.schemas.source import Source
 from letta.server.rest_api.interface import StreamingServerInterface
@@ -326,14 +326,15 @@ async def send_message(
 
     # TODO(charles): support sending multiple messages
     assert len(request.messages) == 1, f"Multiple messages not supported: {request.messages}"
-    message = request.messages[0]
+    request.messages[0]
 
     return await send_message_to_agent(
         server=server,
         agent_id=agent_id,
         user_id=actor.id,
-        role=message.role,
-        message=message.text,
+        # role=message.role,
+        # message=message.text,
+        messages=request.messages,
         stream_steps=request.stream_steps,
         stream_tokens=request.stream_tokens,
         return_message_object=request.return_message_object,
@@ -349,8 +350,8 @@ async def send_message_to_agent(
     server: SyncServer,
     agent_id: str,
     user_id: str,
-    role: MessageRole,
-    message: str,
+    # role: MessageRole,
+    messages: Union[List[Message], List[MessageCreate]],
     stream_steps: bool,
     stream_tokens: bool,
     # related to whether or not we return `LettaMessage`s or `Message`s
@@ -367,14 +368,6 @@ async def send_message_to_agent(
     # TODO: @charles is this the correct way to handle?
     include_final_message = True
 
-    # determine role
-    if role == MessageRole.user:
-        message_func = server.user_message
-    elif role == MessageRole.system:
-        message_func = server.system_message
-    else:
-        raise HTTPException(status_code=500, detail=f"Bad role {role}")
-
     if not stream_steps and stream_tokens:
         raise HTTPException(status_code=400, detail="stream_steps must be 'true' if stream_tokens is 'true'")
 
@@ -413,7 +406,8 @@ async def send_message_to_agent(
         # Offload the synchronous message_func to a separate thread
         streaming_interface.stream_start()
         task = asyncio.create_task(
-            asyncio.to_thread(message_func, user_id=user_id, agent_id=agent_id, message=message, timestamp=timestamp)
+            # asyncio.to_thread(message_func, user_id=user_id, agent_id=agent_id, message=message, timestamp=timestamp)
+            asyncio.to_thread(server.send_messages, user_id=user_id, agent_id=agent_id, messages=messages)
         )
 
         if stream_steps:

letta/server/rest_api/routers/v1/jobs.py

@@ -1,6 +1,6 @@
 from typing import List, Optional
 
-from fastapi import APIRouter, Depends, Header, Query
+from fastapi import APIRouter, Depends, Header, HTTPException, Query
 
 from letta.schemas.job import Job
 from letta.server.rest_api.utils import get_letta_server
@@ -54,3 +54,19 @@ def get_job(
     """
 
     return server.get_job(job_id=job_id)
+
+
+@router.delete("/{job_id}", response_model=Job, operation_id="delete_job")
+def delete_job(
+    job_id: str,
+    server: "SyncServer" = Depends(get_letta_server),
+):
+    """
+    Delete a job by its job_id.
+    """
+    job = server.get_job(job_id=job_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Job not found")
+
+    server.delete_job(job_id=job_id)
+    return job

letta/server/rest_api/routers/v1/sources.py

@@ -4,7 +4,7 @@ from typing import List, Optional
 
 from fastapi import APIRouter, BackgroundTasks, Depends, Header, Query, UploadFile
 
-from letta.schemas.document import Document
+from letta.schemas.file import FileMetadata
 from letta.schemas.job import Job
 from letta.schemas.passage import Passage
 from letta.schemas.source import Source, SourceCreate, SourceUpdate
@@ -186,19 +186,17 @@ def list_passages(
     return passages
 
 
-@router.get("/{source_id}/documents", response_model=List[Document], operation_id="list_source_documents")
-def list_documents(
+@router.get("/{source_id}/files", response_model=List[FileMetadata], operation_id="list_files_from_source")
+def list_files_from_source(
     source_id: str,
+    limit: int = Query(1000, description="Number of files to return"),
+    cursor: Optional[str] = Query(None, description="Pagination cursor to fetch the next set of results"),
     server: "SyncServer" = Depends(get_letta_server),
-    user_id: Optional[str] = Header(None, alias="user_id"),  # Extract user_id from header, default to None if not present
 ):
     """
-    List all documents associated with a data source.
+    List paginated files associated with a data source.
     """
-    actor = server.get_user_or_default(user_id=user_id)
-
-    documents = server.list_data_source_documents(user_id=actor.id, source_id=source_id)
-    return documents
+    return server.list_files_from_source(source_id=source_id, limit=limit, cursor=cursor)
 
 
 def load_file_to_source_async(server: SyncServer, source_id: str, job_id: str, file: UploadFile, bytes: bytes):