letta-nightly 0.4.1.dev20241013104006__py3-none-any.whl → 0.5.0.dev20241015014828__py3-none-any.whl

letta/data_sources/connectors.py

@@ -1,11 +1,15 @@
-from typing import Dict, Iterator, List, Optional, Tuple
+from typing import Dict, Iterator, List, Tuple
 
 import typer
-from llama_index.core import Document as LlamaIndexDocument
 
 from letta.agent_store.storage import StorageConnector
+from letta.data_sources.connectors_helper import (
+    assert_all_files_exist_locally,
+    extract_metadata_from_files,
+    get_filenames_in_dir,
+)
 from letta.embeddings import embedding_model
-from letta.schemas.document import Document
+from letta.schemas.file import FileMetadata
 from letta.schemas.passage import Passage
 from letta.schemas.source import Source
 from letta.utils import create_uuid_from_string
@@ -13,23 +17,23 @@ from letta.utils import create_uuid_from_string
 
 class DataConnector:
     """
-    Base class for data connectors that can be extended to generate documents and passages from a custom data source.
+    Base class for data connectors that can be extended to generate files and passages from a custom data source.
     """
 
-    def generate_documents(self) -> Iterator[Tuple[str, Dict]]:  # -> Iterator[Document]:
+    def find_files(self, source: Source) -> Iterator[FileMetadata]:
         """
-        Generate document text and metadata from a data source.
+        Generate file metadata from a data source.
 
         Returns:
-            documents (Iterator[Tuple[str, Dict]]): Generate a tuple of string text and metadata dictionary for each document.
+            files (Iterator[FileMetadata]): Generate file metadata for each file found.
         """
 
-    def generate_passages(self, documents: List[Document], chunk_size: int = 1024) -> Iterator[Tuple[str, Dict]]:  # -> Iterator[Passage]:
+    def generate_passages(self, file: FileMetadata, chunk_size: int = 1024) -> Iterator[Tuple[str, Dict]]:  # -> Iterator[Passage]:
         """
-        Generate passage text and metadata from a list of documents.
+        Generate passage text and metadata from a list of files.
 
         Args:
-            documents (List[Document]): List of documents to generate passages from.
+            file (FileMetadata): The document to generate passages from.
             chunk_size (int, optional): Chunk size for splitting passages. Defaults to 1024.
 
         Returns:
@@ -41,33 +45,25 @@ def load_data(
     connector: DataConnector,
     source: Source,
     passage_store: StorageConnector,
-    document_store: Optional[StorageConnector] = None,
+    file_metadata_store: StorageConnector,
 ):
-    """Load data from a connector (generates documents and passages) into a specified source_id, associatedw with a user_id."""
+    """Load data from a connector (generates file and passages) into a specified source_id, associatedw with a user_id."""
     embedding_config = source.embedding_config
 
     # embedding model
     embed_model = embedding_model(embedding_config)
 
-    # insert passages/documents
+    # insert passages/file
     passages = []
     embedding_to_document_name = {}
     passage_count = 0
-    document_count = 0
-    for document_text, document_metadata in connector.generate_documents():
-        # insert document into storage
-        document = Document(
-            text=document_text,
-            metadata_=document_metadata,
-            source_id=source.id,
-            user_id=source.user_id,
-        )
-        document_count += 1
-        if document_store:
-            document_store.insert(document)
+    file_count = 0
+    for file_metadata in connector.find_files(source):
+        file_count += 1
+        file_metadata_store.insert(file_metadata)
 
         # generate passages
-        for passage_text, passage_metadata in connector.generate_passages([document], chunk_size=embedding_config.embedding_chunk_size):
+        for passage_text, passage_metadata in connector.generate_passages(file_metadata, chunk_size=embedding_config.embedding_chunk_size):
             # for some reason, llama index parsers sometimes return empty strings
             if len(passage_text) == 0:
                 typer.secho(
@@ -89,7 +85,7 @@ def load_data(
             passage = Passage(
                 id=create_uuid_from_string(f"{str(source.id)}_{passage_text}"),
                 text=passage_text,
-                doc_id=document.id,
+                file_id=file_metadata.id,
                 source_id=source.id,
                 metadata_=passage_metadata,
                 user_id=source.user_id,
@@ -98,16 +94,16 @@ def load_data(
             )
 
             hashable_embedding = tuple(passage.embedding)
-            document_name = document.metadata_.get("file_path", document.id)
+            file_name = file_metadata.file_name
             if hashable_embedding in embedding_to_document_name:
                 typer.secho(
-                    f"Warning: Duplicate embedding found for passage in {document_name} (already exists in {embedding_to_document_name[hashable_embedding]}), skipping insert into VectorDB.",
+                    f"Warning: Duplicate embedding found for passage in {file_name} (already exists in {embedding_to_document_name[hashable_embedding]}), skipping insert into VectorDB.",
                     fg=typer.colors.YELLOW,
                 )
                 continue
 
             passages.append(passage)
-            embedding_to_document_name[hashable_embedding] = document_name
+            embedding_to_document_name[hashable_embedding] = file_name
             if len(passages) >= 100:
                 # insert passages into passage store
                 passage_store.insert_many(passages)
@@ -120,7 +116,7 @@ def load_data(
         passage_store.insert_many(passages)
         passage_count += len(passages)
 
-    return passage_count, document_count
+    return passage_count, file_count
 
 
 class DirectoryConnector(DataConnector):
@@ -143,105 +139,109 @@ class DirectoryConnector(DataConnector):
         if self.recursive == True:
             assert self.input_directory is not None, "Must provide input directory if recursive is True."
 
-    def generate_documents(self) -> Iterator[Tuple[str, Dict]]:  # -> Iterator[Document]:
-        from llama_index.core import SimpleDirectoryReader
-
+    def find_files(self, source: Source) -> Iterator[FileMetadata]:
         if self.input_directory is not None:
-            reader = SimpleDirectoryReader(
+            files = get_filenames_in_dir(
                 input_dir=self.input_directory,
                 recursive=self.recursive,
                 required_exts=[ext.strip() for ext in str(self.extensions).split(",")],
+                exclude=["*png", "*jpg", "*jpeg"],
             )
         else:
-            assert self.input_files is not None, "Must provide input files if input_dir is None"
-            reader = SimpleDirectoryReader(input_files=[str(f) for f in self.input_files])
-
-        llama_index_docs = reader.load_data(show_progress=True)
-        for llama_index_doc in llama_index_docs:
-            # TODO: add additional metadata?
-            # doc = Document(text=llama_index_doc.text, metadata=llama_index_doc.metadata)
-            # docs.append(doc)
-            yield llama_index_doc.text, llama_index_doc.metadata
-
-    def generate_passages(self, documents: List[Document], chunk_size: int = 1024) -> Iterator[Tuple[str, Dict]]:  # -> Iterator[Passage]:
-        # use llama index to run embeddings code
-        # from llama_index.core.node_parser import SentenceSplitter
+            files = self.input_files
+
+        # Check that file paths are valid
+        assert_all_files_exist_locally(files)
+
+        for metadata in extract_metadata_from_files(files):
+            yield FileMetadata(
+                user_id=source.user_id,
+                source_id=source.id,
+                file_name=metadata.get("file_name"),
+                file_path=metadata.get("file_path"),
+                file_type=metadata.get("file_type"),
+                file_size=metadata.get("file_size"),
+                file_creation_date=metadata.get("file_creation_date"),
+                file_last_modified_date=metadata.get("file_last_modified_date"),
+            )
+
+    def generate_passages(self, file: FileMetadata, chunk_size: int = 1024) -> Iterator[Tuple[str, Dict]]:
+        from llama_index.core import SimpleDirectoryReader
         from llama_index.core.node_parser import TokenTextSplitter
 
         parser = TokenTextSplitter(chunk_size=chunk_size)
-        for document in documents:
-            llama_index_docs = [LlamaIndexDocument(text=document.text, metadata=document.metadata_)]
-            nodes = parser.get_nodes_from_documents(llama_index_docs)
-            for node in nodes:
-                # passage = Passage(
-                #    text=node.text,
-                #    doc_id=document.id,
-                # )
-                yield node.text, None
-
-
-class WebConnector(DirectoryConnector):
-    def __init__(self, urls: List[str] = None, html_to_text: bool = True):
-        self.urls = urls
-        self.html_to_text = html_to_text
-
-    def generate_documents(self) -> Iterator[Tuple[str, Dict]]:  # -> Iterator[Document]:
-        from llama_index.readers.web import SimpleWebPageReader
-
-        documents = SimpleWebPageReader(html_to_text=self.html_to_text).load_data(self.urls)
-        for document in documents:
-            yield document.text, {"url": document.id_}
-
-
-class VectorDBConnector(DataConnector):
-    # NOTE: this class has not been properly tested, so is unlikely to work
-    # TODO: allow loading multiple tables (1:1 mapping between Document and Table)
-
-    def __init__(
-        self,
-        name: str,
-        uri: str,
-        table_name: str,
-        text_column: str,
-        embedding_column: str,
-        embedding_dim: int,
-    ):
-        self.name = name
-        self.uri = uri
-        self.table_name = table_name
-        self.text_column = text_column
-        self.embedding_column = embedding_column
-        self.embedding_dim = embedding_dim
-
-        # connect to db table
-        from sqlalchemy import create_engine
-
-        self.engine = create_engine(uri)
-
-    def generate_documents(self) -> Iterator[Tuple[str, Dict]]:  # -> Iterator[Document]:
-        yield self.table_name, None
-
-    def generate_passages(self, documents: List[Document], chunk_size: int = 1024) -> Iterator[Tuple[str, Dict]]:  # -> Iterator[Passage]:
-        from pgvector.sqlalchemy import Vector
-        from sqlalchemy import Inspector, MetaData, Table, select
-
-        metadata = MetaData()
-        # Create an inspector to inspect the database
-        inspector = Inspector.from_engine(self.engine)
-        table_names = inspector.get_table_names()
-        assert self.table_name in table_names, f"Table {self.table_name} not found in database: tables that exist {table_names}."
-
-        table = Table(self.table_name, metadata, autoload_with=self.engine)
-
-        # Prepare a select statement
-        select_statement = select(table.c[self.text_column], table.c[self.embedding_column].cast(Vector(self.embedding_dim)))
-
-        # Execute the query and fetch the results
-        # TODO: paginate results
-        with self.engine.connect() as connection:
-            result = connection.execute(select_statement).fetchall()
-
-        for text, embedding in result:
-            # assume that embeddings are the same model as in config
-            # TODO: don't re-compute embedding
-            yield text, {"embedding": embedding}
+        documents = SimpleDirectoryReader(input_files=[file.file_path]).load_data()
+        nodes = parser.get_nodes_from_documents(documents)
+        for node in nodes:
+            yield node.text, None
+
+
+"""
+The below isn't used anywhere, it isn't tested, and pretty much should be deleted.
+- Matt
+"""
+# class WebConnector(DirectoryConnector):
+#     def __init__(self, urls: List[str] = None, html_to_text: bool = True):
+#         self.urls = urls
+#         self.html_to_text = html_to_text
+#
+#     def generate_files(self) -> Iterator[Tuple[str, Dict]]:  # -> Iterator[Document]:
+#         from llama_index.readers.web import SimpleWebPageReader
+#
+#         files = SimpleWebPageReader(html_to_text=self.html_to_text).load_data(self.urls)
+#         for document in files:
+#             yield document.text, {"url": document.id_}
+#
+#
+# class VectorDBConnector(DataConnector):
+#     # NOTE: this class has not been properly tested, so is unlikely to work
+#     # TODO: allow loading multiple tables (1:1 mapping between FileMetadata and Table)
+#
+#     def __init__(
+#         self,
+#         name: str,
+#         uri: str,
+#         table_name: str,
+#         text_column: str,
+#         embedding_column: str,
+#         embedding_dim: int,
+#     ):
+#         self.name = name
+#         self.uri = uri
+#         self.table_name = table_name
+#         self.text_column = text_column
+#         self.embedding_column = embedding_column
+#         self.embedding_dim = embedding_dim
+#
+#         # connect to db table
+#         from sqlalchemy import create_engine
+#
+#         self.engine = create_engine(uri)
+#
+#     def generate_files(self) -> Iterator[Tuple[str, Dict]]:  # -> Iterator[Document]:
+#         yield self.table_name, None
+#
+#     def generate_passages(self, file_text: str, file: FileMetadata, chunk_size: int = 1024) -> Iterator[Tuple[str, Dict]]:  # -> Iterator[Passage]:
+#         from pgvector.sqlalchemy import Vector
+#         from sqlalchemy import Inspector, MetaData, Table, select
+#
+#         metadata = MetaData()
+#         # Create an inspector to inspect the database
+#         inspector = Inspector.from_engine(self.engine)
+#         table_names = inspector.get_table_names()
+#         assert self.table_name in table_names, f"Table {self.table_name} not found in database: tables that exist {table_names}."
+#
+#         table = Table(self.table_name, metadata, autoload_with=self.engine)
+#
+#         # Prepare a select statement
+#         select_statement = select(table.c[self.text_column], table.c[self.embedding_column].cast(Vector(self.embedding_dim)))
+#
+#         # Execute the query and fetch the results
+#         # TODO: paginate results
+#         with self.engine.connect() as connection:
+#             result = connection.execute(select_statement).fetchall()
+#
+#         for text, embedding in result:
+#             # assume that embeddings are the same model as in config
+#             # TODO: don't re-compute embedding
+#             yield text, {"embedding": embedding}

letta/data_sources/connectors_helper.py

@@ -0,0 +1,97 @@
+import mimetypes
+import os
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional
+
+
+def extract_file_metadata(file_path) -> dict:
+    """Extracts metadata from a single file."""
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(file_path)
+
+    file_metadata = {
+        "file_name": os.path.basename(file_path),
+        "file_path": file_path,
+        "file_type": mimetypes.guess_type(file_path)[0] or "unknown",
+        "file_size": os.path.getsize(file_path),
+        "file_creation_date": datetime.fromtimestamp(os.path.getctime(file_path)).strftime("%Y-%m-%d"),
+        "file_last_modified_date": datetime.fromtimestamp(os.path.getmtime(file_path)).strftime("%Y-%m-%d"),
+    }
+    return file_metadata
+
+
+def extract_metadata_from_files(file_list):
+    """Extracts metadata for a list of files."""
+    metadata = []
+    for file_path in file_list:
+        file_metadata = extract_file_metadata(file_path)
+        if file_metadata:
+            metadata.append(file_metadata)
+    return metadata
+
+
+def get_filenames_in_dir(
+    input_dir: str, recursive: bool = True, required_exts: Optional[List[str]] = None, exclude: Optional[List[str]] = None
+):
+    """
+    Recursively reads files from the directory, applying required_exts and exclude filters.
+    Ensures that required_exts and exclude do not overlap.
+
+    Args:
+        input_dir (str): The directory to scan for files.
+        recursive (bool): Whether to scan directories recursively.
+        required_exts (list): List of file extensions to include (e.g., ['pdf', 'txt']).
+                             If None or empty, matches any file extension.
+        exclude (list): List of file patterns to exclude (e.g., ['*png', '*jpg']).
+
+    Returns:
+        list: A list of matching file paths.
+    """
+    required_exts = required_exts or []
+    exclude = exclude or []
+
+    # Ensure required_exts and exclude do not overlap
+    ext_set = set(required_exts)
+    exclude_set = set(exclude)
+    overlap = ext_set & exclude_set
+    if overlap:
+        raise ValueError(f"Extensions in required_exts and exclude overlap: {overlap}")
+
+    def is_excluded(file_name):
+        """Check if a file matches any pattern in the exclude list."""
+        for pattern in exclude:
+            if Path(file_name).match(pattern):
+                return True
+        return False
+
+    files = []
+    search_pattern = "**/*" if recursive else "*"
+
+    for file_path in Path(input_dir).glob(search_pattern):
+        if file_path.is_file() and not is_excluded(file_path.name):
+            ext = file_path.suffix.lstrip(".")
+            # If required_exts is empty, match any file
+            if not required_exts or ext in required_exts:
+                files.append(file_path)
+
+    return files
+
+
+def assert_all_files_exist_locally(file_paths: List[str]) -> bool:
+    """
+    Checks if all file paths in the provided list exist locally.
+    Raises a FileNotFoundError with a list of missing files if any do not exist.
+
+    Args:
+        file_paths (List[str]): List of file paths to check.
+
+    Returns:
+        bool: True if all files exist, raises FileNotFoundError if any file is missing.
+    """
+    missing_files = [file_path for file_path in file_paths if not Path(file_path).exists()]
+
+    if missing_files:
+        raise FileNotFoundError(missing_files)
+
+    return True

letta/llm_api/mistral.py

@@ -0,0 +1,47 @@
+import requests
+
+from letta.utils import printd, smart_urljoin
+
+
+def mistral_get_model_list(url: str, api_key: str) -> dict:
+    url = smart_urljoin(url, "models")
+
+    headers = {"Content-Type": "application/json"}
+    if api_key is not None:
+        headers["Authorization"] = f"Bearer {api_key}"
+
+    printd(f"Sending request to {url}")
+    response = None
+    try:
+        # TODO add query param "tool" to be true
+        response = requests.get(url, headers=headers)
+        response.raise_for_status()  # Raises HTTPError for 4XX/5XX status
+        response_json = response.json()  # convert to dict from string
+        return response_json
+    except requests.exceptions.HTTPError as http_err:
+        # Handle HTTP errors (e.g., response 4XX, 5XX)
+        try:
+            if response:
+                response = response.json()
+        except:
+            pass
+        printd(f"Got HTTPError, exception={http_err}, response={response}")
+        raise http_err
+    except requests.exceptions.RequestException as req_err:
+        # Handle other requests-related errors (e.g., connection error)
+        try:
+            if response:
+                response = response.json()
+        except:
+            pass
+        printd(f"Got RequestException, exception={req_err}, response={response}")
+        raise req_err
+    except Exception as e:
+        # Handle other potential errors
+        try:
+            if response:
+                response = response.json()
+        except:
+            pass
+        printd(f"Got unknown Exception, exception={e}, response={response}")
+        raise e

letta/main.py

@@ -356,19 +356,29 @@ def run_agent_loop(
             else:
                 # If message did not begin with command prefix, pass inputs to Letta
                 # Handle user message and append to messages
-                user_message = system.package_user_message(user_input)
+                user_message = str(user_input)
 
         skip_next_user_input = False
 
         def process_agent_step(user_message, no_verify):
-            step_response = letta_agent.step(
-                user_message,
-                first_message=False,
-                skip_verify=no_verify,
-                stream=stream,
-                inner_thoughts_in_kwargs_option=inner_thoughts_in_kwargs,
-                ms=ms,
-            )
+            if user_message is None:
+                step_response = letta_agent.step(
+                    messages=[],
+                    first_message=False,
+                    skip_verify=no_verify,
+                    stream=stream,
+                    inner_thoughts_in_kwargs_option=inner_thoughts_in_kwargs,
+                    ms=ms,
+                )
+            else:
+                step_response = letta_agent.step_user_message(
+                    user_message_str=user_message,
+                    first_message=False,
+                    skip_verify=no_verify,
+                    stream=stream,
+                    inner_thoughts_in_kwargs_option=inner_thoughts_in_kwargs,
+                    ms=ms,
+                )
             new_messages = step_response.messages
             heartbeat_request = step_response.heartbeat_request
             function_failed = step_response.function_failed

letta/metadata.py

@@ -11,6 +11,7 @@ from sqlalchemy import (
     Column,
     DateTime,
     Index,
+    Integer,
     String,
     TypeDecorator,
     desc,
@@ -24,6 +25,7 @@ from letta.schemas.api_key import APIKey
 from letta.schemas.block import Block, Human, Persona
 from letta.schemas.embedding_config import EmbeddingConfig
 from letta.schemas.enums import JobStatus
+from letta.schemas.file import FileMetadata
 from letta.schemas.job import Job
 from letta.schemas.llm_config import LLMConfig
 from letta.schemas.memory import Memory
@@ -38,6 +40,41 @@ from letta.settings import settings
 from letta.utils import enforce_types, get_utc_time, printd
 
 
+class FileMetadataModel(Base):
+    __tablename__ = "files"
+    __table_args__ = {"extend_existing": True}
+
+    id = Column(String, primary_key=True, nullable=False)
+    user_id = Column(String, nullable=False)
+    # TODO: Investigate why this breaks during table creation due to FK
+    # source_id = Column(String, ForeignKey("sources.id"), nullable=False)
+    source_id = Column(String, nullable=False)
+    file_name = Column(String, nullable=True)
+    file_path = Column(String, nullable=True)
+    file_type = Column(String, nullable=True)
+    file_size = Column(Integer, nullable=True)
+    file_creation_date = Column(String, nullable=True)
+    file_last_modified_date = Column(String, nullable=True)
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+
+    def __repr__(self):
+        return f"<FileMetadata(id='{self.id}', source_id='{self.source_id}', file_name='{self.file_name}')>"
+
+    def to_record(self):
+        return FileMetadata(
+            id=self.id,
+            user_id=self.user_id,
+            source_id=self.source_id,
+            file_name=self.file_name,
+            file_path=self.file_path,
+            file_type=self.file_type,
+            file_size=self.file_size,
+            file_creation_date=self.file_creation_date,
+            file_last_modified_date=self.file_last_modified_date,
+            created_at=self.created_at,
+        )
+
+
 class LLMConfigColumn(TypeDecorator):
     """Custom type for storing LLMConfig as JSON"""
 
@@ -865,6 +902,27 @@ class MetadataStore:
             session.add(JobModel(**vars(job)))
             session.commit()
 
+    @enforce_types
+    def list_files_from_source(self, source_id: str, limit: int, cursor: Optional[str]):
+        with self.session_maker() as session:
+            # Start with the basic query filtered by source_id
+            query = session.query(FileMetadataModel).filter(FileMetadataModel.source_id == source_id)
+
+            if cursor:
+                # Assuming cursor is the ID of the last file in the previous page
+                query = query.filter(FileMetadataModel.id > cursor)
+
+            # Order by ID or other ordering criteria to ensure correct pagination
+            query = query.order_by(FileMetadataModel.id)
+
+            # Limit the number of results returned
+            results = query.limit(limit).all()
+
+            # Convert the results to the required FileMetadata objects
+            files = [r.to_record() for r in results]
+
+            return files
+
     def delete_job(self, job_id: str):
         with self.session_maker() as session:
             session.query(JobModel).filter(JobModel.id == job_id).delete()

letta/providers.py

@@ -139,6 +139,50 @@ class AnthropicProvider(Provider):
         return []
 
 
+class MistralProvider(Provider):
+    name: str = "mistral"
+    api_key: str = Field(..., description="API key for the Mistral API.")
+    base_url: str = "https://api.mistral.ai/v1"
+
+    def list_llm_models(self) -> List[LLMConfig]:
+        from letta.llm_api.mistral import mistral_get_model_list
+
+        # Some hardcoded support for OpenRouter (so that we only get models with tool calling support)...
+        # See: https://openrouter.ai/docs/requests
+        response = mistral_get_model_list(self.base_url, api_key=self.api_key)
+
+        assert "data" in response, f"Mistral model query response missing 'data' field: {response}"
+
+        configs = []
+        for model in response["data"]:
+            # If model has chat completions and function calling enabled
+            if model["capabilities"]["completion_chat"] and model["capabilities"]["function_calling"]:
+                configs.append(
+                    LLMConfig(
+                        model=model["id"],
+                        model_endpoint_type="openai",
+                        model_endpoint=self.base_url,
+                        context_window=model["max_context_length"],
+                    )
+                )
+
+        return configs
+
+    def list_embedding_models(self) -> List[EmbeddingConfig]:
+        # Not supported for mistral
+        return []
+
+    def get_model_context_window(self, model_name: str) -> Optional[int]:
+        # Redoing this is fine because it's a pretty lightweight call
+        models = self.list_llm_models()
+
+        for m in models:
+            if model_name in m["id"]:
+                return int(m["max_context_length"])
+
+        return None
+
+
 class OllamaProvider(OpenAIProvider):
     """Ollama provider that uses the native /api/generate endpoint