letta-nightly 0.4.1.dev20241014104152__tar.gz → 0.5.0.dev20241015014828__tar.gz

{letta_nightly-0.4.1.dev20241014104152 → letta_nightly-0.5.0.dev20241015014828}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: letta-nightly
-Version: 0.4.1.dev20241014104152
+Version: 0.5.0.dev20241015014828
 Summary: Create LLM agents with long-term memory and custom tools
 License: Apache License
 Author: Letta Team

{letta_nightly-0.4.1.dev20241014104152 → letta_nightly-0.5.0.dev20241015014828}/letta/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.4.1"
+__version__ = "0.5.0"
 
 # import clients
 from letta.client.admin import Admin
@@ -7,9 +7,9 @@ from letta.client.client import LocalClient, RESTClient, create_client
 # imports for easier access
 from letta.schemas.agent import AgentState
 from letta.schemas.block import Block
-from letta.schemas.document import Document
 from letta.schemas.embedding_config import EmbeddingConfig
 from letta.schemas.enums import JobStatus
+from letta.schemas.file import FileMetadata
 from letta.schemas.job import Job
 from letta.schemas.letta_message import LettaMessage
 from letta.schemas.llm_config import LLMConfig

{letta_nightly-0.4.1.dev20241014104152 → letta_nightly-0.5.0.dev20241015014828}/letta/agent_store/db.py

@@ -28,7 +28,7 @@ from letta.agent_store.storage import StorageConnector, TableType
 from letta.base import Base
 from letta.config import LettaConfig
 from letta.constants import MAX_EMBEDDING_DIM
-from letta.metadata import EmbeddingConfigColumn, ToolCallColumn
+from letta.metadata import EmbeddingConfigColumn, FileMetadataModel, ToolCallColumn
 
 # from letta.schemas.message import Message, Passage, Record, RecordType, ToolCall
 from letta.schemas.message import Message
@@ -141,7 +141,7 @@ class PassageModel(Base):
     id = Column(String, primary_key=True)
     user_id = Column(String, nullable=False)
     text = Column(String)
-    doc_id = Column(String)
+    file_id = Column(String)
     agent_id = Column(String)
     source_id = Column(String)
 
@@ -160,7 +160,7 @@ class PassageModel(Base):
     # Add a datetime column, with default value as the current time
     created_at = Column(DateTime(timezone=True))
 
-    Index("passage_idx_user", user_id, agent_id, doc_id),
+    Index("passage_idx_user", user_id, agent_id, file_id),
 
     def __repr__(self):
         return f"<Passage(passage_id='{self.id}', text='{self.text}', embedding='{self.embedding})>"
@@ -170,7 +170,7 @@ class PassageModel(Base):
             text=self.text,
             embedding=self.embedding,
             embedding_config=self.embedding_config,
-            doc_id=self.doc_id,
+            file_id=self.file_id,
             user_id=self.user_id,
             id=self.id,
             source_id=self.source_id,
@@ -365,12 +365,17 @@ class PostgresStorageConnector(SQLStorageConnector):
                 self.uri = self.config.archival_storage_uri
                 self.db_model = PassageModel
                 if self.config.archival_storage_uri is None:
-                    raise ValueError(f"Must specifiy archival_storage_uri in config {self.config.config_path}")
+                    raise ValueError(f"Must specify archival_storage_uri in config {self.config.config_path}")
             elif table_type == TableType.RECALL_MEMORY:
                 self.uri = self.config.recall_storage_uri
                 self.db_model = MessageModel
                 if self.config.recall_storage_uri is None:
-                    raise ValueError(f"Must specifiy recall_storage_uri in config {self.config.config_path}")
+                    raise ValueError(f"Must specify recall_storage_uri in config {self.config.config_path}")
+            elif table_type == TableType.FILES:
+                self.uri = self.config.metadata_storage_uri
+                self.db_model = FileMetadataModel
+                if self.config.metadata_storage_uri is None:
+                    raise ValueError(f"Must specify metadata_storage_uri in config {self.config.config_path}")
             else:
                 raise ValueError(f"Table type {table_type} not implemented")
 
@@ -487,8 +492,14 @@ class SQLLiteStorageConnector(SQLStorageConnector):
             # TODO: eventually implement URI option
             self.path = self.config.recall_storage_path
             if self.path is None:
-                raise ValueError(f"Must specifiy recall_storage_path in config {self.config.recall_storage_path}")
+                raise ValueError(f"Must specify recall_storage_path in config.")
             self.db_model = MessageModel
+        elif table_type == TableType.FILES:
+            self.path = self.config.metadata_storage_path
+            if self.path is None:
+                raise ValueError(f"Must specify metadata_storage_path in config.")
+            self.db_model = FileMetadataModel
+
         else:
             raise ValueError(f"Table type {table_type} not implemented")
 

{letta_nightly-0.4.1.dev20241014104152 → letta_nightly-0.5.0.dev20241015014828}/letta/agent_store/lancedb.py

@@ -24,7 +24,7 @@ def get_db_model(table_name: str, table_type: TableType):
             id: uuid.UUID
             user_id: str
             text: str
-            doc_id: str
+            file_id: str
             agent_id: str
             data_source: str
             embedding: Vector(config.default_embedding_config.embedding_dim)
@@ -37,7 +37,7 @@ def get_db_model(table_name: str, table_type: TableType):
                 return Passage(
                     text=self.text,
                     embedding=self.embedding,
-                    doc_id=self.doc_id,
+                    file_id=self.file_id,
                     user_id=self.user_id,
                     id=self.id,
                     data_source=self.data_source,

{letta_nightly-0.4.1.dev20241014104152 → letta_nightly-0.5.0.dev20241015014828}/letta/agent_store/milvus.py

@@ -26,7 +26,7 @@ class MilvusStorageConnector(StorageConnector):
             raise ValueError("Please set `archival_storage_uri` in the config file when using Milvus.")
 
         # need to be converted to strings
-        self.uuid_fields = ["id", "user_id", "agent_id", "source_id", "doc_id"]
+        self.uuid_fields = ["id", "user_id", "agent_id", "source_id", "file_id"]
 
     def _create_collection(self):
         schema = MilvusClient.create_schema(

{letta_nightly-0.4.1.dev20241014104152 → letta_nightly-0.5.0.dev20241015014828}/letta/agent_store/qdrant.py

@@ -38,7 +38,7 @@ class QdrantStorageConnector(StorageConnector):
                     distance=models.Distance.COSINE,
                 ),
             )
-        self.uuid_fields = ["id", "user_id", "agent_id", "source_id", "doc_id"]
+        self.uuid_fields = ["id", "user_id", "agent_id", "source_id", "file_id"]
 
     def get_all_paginated(self, filters: Optional[Dict] = {}, page_size: int = 10) -> Iterator[List[RecordType]]:
         from qdrant_client import grpc

{letta_nightly-0.4.1.dev20241014104152 → letta_nightly-0.5.0.dev20241015014828}/letta/agent_store/storage.py

@@ -10,7 +10,7 @@ from typing import Dict, List, Optional, Tuple, Type, Union
 from pydantic import BaseModel
 
 from letta.config import LettaConfig
-from letta.schemas.document import Document
+from letta.schemas.file import FileMetadata
 from letta.schemas.message import Message
 from letta.schemas.passage import Passage
 from letta.utils import printd
@@ -22,7 +22,7 @@ class TableType:
     ARCHIVAL_MEMORY = "archival_memory"  # recall memory table: letta_agent_{agent_id}
     RECALL_MEMORY = "recall_memory"  # archival memory table: letta_agent_recall_{agent_id}
     PASSAGES = "passages"  # TODO
-    DOCUMENTS = "documents"  # TODO
+    FILES = "files"
 
 
 # table names used by Letta
@@ -33,17 +33,17 @@ ARCHIVAL_TABLE_NAME = "letta_archival_memory_agent"  # agent memory
 
 # external data source tables
 PASSAGE_TABLE_NAME = "letta_passages"  # chunked/embedded passages (from source)
-DOCUMENT_TABLE_NAME = "letta_documents"  # original documents (from source)
+FILE_TABLE_NAME = "letta_files"  # original files (from source)
 
 
 class StorageConnector:
-    """Defines a DB connection that is user-specific to access data: Documents, Passages, Archival/Recall Memory"""
+    """Defines a DB connection that is user-specific to access data: files, Passages, Archival/Recall Memory"""
 
     type: Type[BaseModel]
 
     def __init__(
         self,
-        table_type: Union[TableType.ARCHIVAL_MEMORY, TableType.RECALL_MEMORY, TableType.PASSAGES, TableType.DOCUMENTS],
+        table_type: Union[TableType.ARCHIVAL_MEMORY, TableType.RECALL_MEMORY, TableType.PASSAGES, TableType.FILES],
         config: LettaConfig,
         user_id,
         agent_id=None,
@@ -59,9 +59,9 @@ class StorageConnector:
         elif table_type == TableType.RECALL_MEMORY:
             self.type = Message
             self.table_name = RECALL_TABLE_NAME
-        elif table_type == TableType.DOCUMENTS:
-            self.type = Document
-            self.table_name == DOCUMENT_TABLE_NAME
+        elif table_type == TableType.FILES:
+            self.type = FileMetadata
+            self.table_name = FILE_TABLE_NAME
         elif table_type == TableType.PASSAGES:
             self.type = Passage
             self.table_name = PASSAGE_TABLE_NAME
@@ -74,7 +74,7 @@ class StorageConnector:
             # agent-specific table
             assert agent_id is not None, "Agent ID must be provided for agent-specific tables"
             self.filters = {"user_id": self.user_id, "agent_id": self.agent_id}
-        elif self.table_type == TableType.PASSAGES or self.table_type == TableType.DOCUMENTS:
+        elif self.table_type == TableType.PASSAGES or self.table_type == TableType.FILES:
             # setup base filters for user-specific tables
             assert agent_id is None, "Agent ID must not be provided for user-specific tables"
             self.filters = {"user_id": self.user_id}
@@ -83,7 +83,7 @@ class StorageConnector:
 
     @staticmethod
     def get_storage_connector(
-        table_type: Union[TableType.ARCHIVAL_MEMORY, TableType.RECALL_MEMORY, TableType.PASSAGES, TableType.DOCUMENTS],
+        table_type: Union[TableType.ARCHIVAL_MEMORY, TableType.RECALL_MEMORY, TableType.PASSAGES, TableType.FILES],
         config: LettaConfig,
         user_id,
         agent_id=None,
@@ -92,6 +92,8 @@ class StorageConnector:
             storage_type = config.archival_storage_type
         elif table_type == TableType.RECALL_MEMORY:
             storage_type = config.recall_storage_type
+        elif table_type == TableType.FILES:
+            storage_type = config.metadata_storage_type
         else:
             raise ValueError(f"Table type {table_type} not implemented")
 

{letta_nightly-0.4.1.dev20241014104152 → letta_nightly-0.5.0.dev20241015014828}/letta/cli/cli_load.py

@@ -106,7 +106,7 @@ def load_vector_database(
     #            document_store=None,
     #            passage_store=passage_storage,
     #        )
-    #        print(f"Loaded {num_passages} passages and {num_documents} documents from {name}")
+    #        print(f"Loaded {num_passages} passages and {num_documents} files from {name}")
     #    except Exception as e:
     #        typer.secho(f"Failed to load data from provided information.\n{e}", fg=typer.colors.RED)
     #        ms.delete_source(source_id=source.id)

{letta_nightly-0.4.1.dev20241014104152 → letta_nightly-0.5.0.dev20241015014828}/letta/client/client.py

@@ -25,6 +25,7 @@ from letta.schemas.embedding_config import EmbeddingConfig
 
 # new schemas
 from letta.schemas.enums import JobStatus, MessageRole
+from letta.schemas.file import FileMetadata
 from letta.schemas.job import Job
 from letta.schemas.letta_request import LettaRequest
 from letta.schemas.letta_response import LettaResponse, LettaStreamingResponse
@@ -232,6 +233,9 @@ class AbstractClient(object):
     def list_attached_sources(self, agent_id: str) -> List[Source]:
         raise NotImplementedError
 
+    def list_files_from_source(self, source_id: str, limit: int = 1000, cursor: Optional[str] = None) -> List[FileMetadata]:
+        raise NotImplementedError
+
     def update_source(self, source_id: str, name: Optional[str] = None) -> Source:
         raise NotImplementedError
 
@@ -1016,6 +1020,12 @@ class RESTClient(AbstractClient):
             raise ValueError(f"Failed to get job: {response.text}")
         return Job(**response.json())
 
+    def delete_job(self, job_id: str) -> Job:
+        response = requests.delete(f"{self.base_url}/{self.api_prefix}/jobs/{job_id}", headers=self.headers)
+        if response.status_code != 200:
+            raise ValueError(f"Failed to delete job: {response.text}")
+        return Job(**response.json())
+
     def list_jobs(self):
         response = requests.get(f"{self.base_url}/{self.api_prefix}/jobs", headers=self.headers)
         return [Job(**job) for job in response.json()]
@@ -1088,6 +1098,30 @@ class RESTClient(AbstractClient):
             raise ValueError(f"Failed to list attached sources: {response.text}")
         return [Source(**source) for source in response.json()]
 
+    def list_files_from_source(self, source_id: str, limit: int = 1000, cursor: Optional[str] = None) -> List[FileMetadata]:
+        """
+        List files from source with pagination support.
+
+        Args:
+            source_id (str): ID of the source
+            limit (int): Number of files to return
+            cursor (Optional[str]): Pagination cursor for fetching the next page
+
+        Returns:
+            List[FileMetadata]: List of files
+        """
+        # Prepare query parameters for pagination
+        params = {"limit": limit, "cursor": cursor}
+
+        # Make the request to the FastAPI endpoint
+        response = requests.get(f"{self.base_url}/{self.api_prefix}/sources/{source_id}/files", headers=self.headers, params=params)
+
+        if response.status_code != 200:
+            raise ValueError(f"Failed to list files with source id {source_id}: [{response.status_code}] {response.text}")
+
+        # Parse the JSON response
+        return [FileMetadata(**metadata) for metadata in response.json()]
+
     def update_source(self, source_id: str, name: Optional[str] = None) -> Source:
         """
         Update a source
@@ -2162,6 +2196,9 @@ class LocalClient(AbstractClient):
     def get_job(self, job_id: str):
         return self.server.get_job(job_id=job_id)
 
+    def delete_job(self, job_id: str):
+        return self.server.delete_job(job_id)
+
     def list_jobs(self):
         return self.server.list_jobs(user_id=self.user_id)
 
@@ -2261,6 +2298,20 @@ class LocalClient(AbstractClient):
         """
         return self.server.list_attached_sources(agent_id=agent_id)
 
+    def list_files_from_source(self, source_id: str, limit: int = 1000, cursor: Optional[str] = None) -> List[FileMetadata]:
+        """
+        List files from source.
+
+        Args:
+            source_id (str): ID of the source
+            limit (int): The # of items to return
+            cursor (str): The cursor for fetching the next page
+
+        Returns:
+            files (List[FileMetadata]): List of files
+        """
+        return self.server.list_files_from_source(source_id=source_id, limit=limit, cursor=cursor)
+
     def update_source(self, source_id: str, name: Optional[str] = None) -> Source:
         """
         Update a source

letta_nightly-0.5.0.dev20241015014828/letta/data_sources/connectors.py

@@ -0,0 +1,247 @@
+from typing import Dict, Iterator, List, Tuple
+
+import typer
+
+from letta.agent_store.storage import StorageConnector
+from letta.data_sources.connectors_helper import (
+    assert_all_files_exist_locally,
+    extract_metadata_from_files,
+    get_filenames_in_dir,
+)
+from letta.embeddings import embedding_model
+from letta.schemas.file import FileMetadata
+from letta.schemas.passage import Passage
+from letta.schemas.source import Source
+from letta.utils import create_uuid_from_string
+
+
+class DataConnector:
+    """
+    Base class for data connectors that can be extended to generate files and passages from a custom data source.
+    """
+
+    def find_files(self, source: Source) -> Iterator[FileMetadata]:
+        """
+        Generate file metadata from a data source.
+
+        Returns:
+            files (Iterator[FileMetadata]): Generate file metadata for each file found.
+        """
+
+    def generate_passages(self, file: FileMetadata, chunk_size: int = 1024) -> Iterator[Tuple[str, Dict]]:  # -> Iterator[Passage]:
+        """
+        Generate passage text and metadata from a list of files.
+
+        Args:
+            file (FileMetadata): The document to generate passages from.
+            chunk_size (int, optional): Chunk size for splitting passages. Defaults to 1024.
+
+        Returns:
+            passages (Iterator[Tuple[str, Dict]]): Generate a tuple of string text and metadata dictionary for each passage.
+        """
+
+
+def load_data(
+    connector: DataConnector,
+    source: Source,
+    passage_store: StorageConnector,
+    file_metadata_store: StorageConnector,
+):
+    """Load data from a connector (generates file and passages) into a specified source_id, associatedw with a user_id."""
+    embedding_config = source.embedding_config
+
+    # embedding model
+    embed_model = embedding_model(embedding_config)
+
+    # insert passages/file
+    passages = []
+    embedding_to_document_name = {}
+    passage_count = 0
+    file_count = 0
+    for file_metadata in connector.find_files(source):
+        file_count += 1
+        file_metadata_store.insert(file_metadata)
+
+        # generate passages
+        for passage_text, passage_metadata in connector.generate_passages(file_metadata, chunk_size=embedding_config.embedding_chunk_size):
+            # for some reason, llama index parsers sometimes return empty strings
+            if len(passage_text) == 0:
+                typer.secho(
+                    f"Warning: Llama index parser returned empty string, skipping insert of passage with metadata '{passage_metadata}' into VectorDB. You can usually ignore this warning.",
+                    fg=typer.colors.YELLOW,
+                )
+                continue
+
+            # get embedding
+            try:
+                embedding = embed_model.get_text_embedding(passage_text)
+            except Exception as e:
+                typer.secho(
+                    f"Warning: Failed to get embedding for {passage_text} (error: {str(e)}), skipping insert into VectorDB.",
+                    fg=typer.colors.YELLOW,
+                )
+                continue
+
+            passage = Passage(
+                id=create_uuid_from_string(f"{str(source.id)}_{passage_text}"),
+                text=passage_text,
+                file_id=file_metadata.id,
+                source_id=source.id,
+                metadata_=passage_metadata,
+                user_id=source.user_id,
+                embedding_config=source.embedding_config,
+                embedding=embedding,
+            )
+
+            hashable_embedding = tuple(passage.embedding)
+            file_name = file_metadata.file_name
+            if hashable_embedding in embedding_to_document_name:
+                typer.secho(
+                    f"Warning: Duplicate embedding found for passage in {file_name} (already exists in {embedding_to_document_name[hashable_embedding]}), skipping insert into VectorDB.",
+                    fg=typer.colors.YELLOW,
+                )
+                continue
+
+            passages.append(passage)
+            embedding_to_document_name[hashable_embedding] = file_name
+            if len(passages) >= 100:
+                # insert passages into passage store
+                passage_store.insert_many(passages)
+
+                passage_count += len(passages)
+                passages = []
+
+    if len(passages) > 0:
+        # insert passages into passage store
+        passage_store.insert_many(passages)
+        passage_count += len(passages)
+
+    return passage_count, file_count
+
+
+class DirectoryConnector(DataConnector):
+    def __init__(self, input_files: List[str] = None, input_directory: str = None, recursive: bool = False, extensions: List[str] = None):
+        """
+        Connector for reading text data from a directory of files.
+
+        Args:
+            input_files (List[str], optional): List of file paths to read. Defaults to None.
+            input_directory (str, optional): Directory to read files from. Defaults to None.
+            recursive (bool, optional): Whether to read files recursively from the input directory. Defaults to False.
+            extensions (List[str], optional): List of file extensions to read. Defaults to None.
+        """
+        self.connector_type = "directory"
+        self.input_files = input_files
+        self.input_directory = input_directory
+        self.recursive = recursive
+        self.extensions = extensions
+
+        if self.recursive == True:
+            assert self.input_directory is not None, "Must provide input directory if recursive is True."
+
+    def find_files(self, source: Source) -> Iterator[FileMetadata]:
+        if self.input_directory is not None:
+            files = get_filenames_in_dir(
+                input_dir=self.input_directory,
+                recursive=self.recursive,
+                required_exts=[ext.strip() for ext in str(self.extensions).split(",")],
+                exclude=["*png", "*jpg", "*jpeg"],
+            )
+        else:
+            files = self.input_files
+
+        # Check that file paths are valid
+        assert_all_files_exist_locally(files)
+
+        for metadata in extract_metadata_from_files(files):
+            yield FileMetadata(
+                user_id=source.user_id,
+                source_id=source.id,
+                file_name=metadata.get("file_name"),
+                file_path=metadata.get("file_path"),
+                file_type=metadata.get("file_type"),
+                file_size=metadata.get("file_size"),
+                file_creation_date=metadata.get("file_creation_date"),
+                file_last_modified_date=metadata.get("file_last_modified_date"),
+            )
+
+    def generate_passages(self, file: FileMetadata, chunk_size: int = 1024) -> Iterator[Tuple[str, Dict]]:
+        from llama_index.core import SimpleDirectoryReader
+        from llama_index.core.node_parser import TokenTextSplitter
+
+        parser = TokenTextSplitter(chunk_size=chunk_size)
+        documents = SimpleDirectoryReader(input_files=[file.file_path]).load_data()
+        nodes = parser.get_nodes_from_documents(documents)
+        for node in nodes:
+            yield node.text, None
+
+
+"""
+The below isn't used anywhere, it isn't tested, and pretty much should be deleted.
+- Matt
+"""
+# class WebConnector(DirectoryConnector):
+#     def __init__(self, urls: List[str] = None, html_to_text: bool = True):
+#         self.urls = urls
+#         self.html_to_text = html_to_text
+#
+#     def generate_files(self) -> Iterator[Tuple[str, Dict]]:  # -> Iterator[Document]:
+#         from llama_index.readers.web import SimpleWebPageReader
+#
+#         files = SimpleWebPageReader(html_to_text=self.html_to_text).load_data(self.urls)
+#         for document in files:
+#             yield document.text, {"url": document.id_}
+#
+#
+# class VectorDBConnector(DataConnector):
+#     # NOTE: this class has not been properly tested, so is unlikely to work
+#     # TODO: allow loading multiple tables (1:1 mapping between FileMetadata and Table)
+#
+#     def __init__(
+#         self,
+#         name: str,
+#         uri: str,
+#         table_name: str,
+#         text_column: str,
+#         embedding_column: str,
+#         embedding_dim: int,
+#     ):
+#         self.name = name
+#         self.uri = uri
+#         self.table_name = table_name
+#         self.text_column = text_column
+#         self.embedding_column = embedding_column
+#         self.embedding_dim = embedding_dim
+#
+#         # connect to db table
+#         from sqlalchemy import create_engine
+#
+#         self.engine = create_engine(uri)
+#
+#     def generate_files(self) -> Iterator[Tuple[str, Dict]]:  # -> Iterator[Document]:
+#         yield self.table_name, None
+#
+#     def generate_passages(self, file_text: str, file: FileMetadata, chunk_size: int = 1024) -> Iterator[Tuple[str, Dict]]:  # -> Iterator[Passage]:
+#         from pgvector.sqlalchemy import Vector
+#         from sqlalchemy import Inspector, MetaData, Table, select
+#
+#         metadata = MetaData()
+#         # Create an inspector to inspect the database
+#         inspector = Inspector.from_engine(self.engine)
+#         table_names = inspector.get_table_names()
+#         assert self.table_name in table_names, f"Table {self.table_name} not found in database: tables that exist {table_names}."
+#
+#         table = Table(self.table_name, metadata, autoload_with=self.engine)
+#
+#         # Prepare a select statement
+#         select_statement = select(table.c[self.text_column], table.c[self.embedding_column].cast(Vector(self.embedding_dim)))
+#
+#         # Execute the query and fetch the results
+#         # TODO: paginate results
+#         with self.engine.connect() as connection:
+#             result = connection.execute(select_statement).fetchall()
+#
+#         for text, embedding in result:
+#             # assume that embeddings are the same model as in config
+#             # TODO: don't re-compute embedding
+#             yield text, {"embedding": embedding}

letta_nightly-0.5.0.dev20241015014828/letta/data_sources/connectors_helper.py

@@ -0,0 +1,97 @@
+import mimetypes
+import os
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional
+
+
+def extract_file_metadata(file_path) -> dict:
+    """Extracts metadata from a single file."""
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(file_path)
+
+    file_metadata = {
+        "file_name": os.path.basename(file_path),
+        "file_path": file_path,
+        "file_type": mimetypes.guess_type(file_path)[0] or "unknown",
+        "file_size": os.path.getsize(file_path),
+        "file_creation_date": datetime.fromtimestamp(os.path.getctime(file_path)).strftime("%Y-%m-%d"),
+        "file_last_modified_date": datetime.fromtimestamp(os.path.getmtime(file_path)).strftime("%Y-%m-%d"),
+    }
+    return file_metadata
+
+
+def extract_metadata_from_files(file_list):
+    """Extracts metadata for a list of files."""
+    metadata = []
+    for file_path in file_list:
+        file_metadata = extract_file_metadata(file_path)
+        if file_metadata:
+            metadata.append(file_metadata)
+    return metadata
+
+
+def get_filenames_in_dir(
+    input_dir: str, recursive: bool = True, required_exts: Optional[List[str]] = None, exclude: Optional[List[str]] = None
+):
+    """
+    Recursively reads files from the directory, applying required_exts and exclude filters.
+    Ensures that required_exts and exclude do not overlap.
+
+    Args:
+        input_dir (str): The directory to scan for files.
+        recursive (bool): Whether to scan directories recursively.
+        required_exts (list): List of file extensions to include (e.g., ['pdf', 'txt']).
+                             If None or empty, matches any file extension.
+        exclude (list): List of file patterns to exclude (e.g., ['*png', '*jpg']).
+
+    Returns:
+        list: A list of matching file paths.
+    """
+    required_exts = required_exts or []
+    exclude = exclude or []
+
+    # Ensure required_exts and exclude do not overlap
+    ext_set = set(required_exts)
+    exclude_set = set(exclude)
+    overlap = ext_set & exclude_set
+    if overlap:
+        raise ValueError(f"Extensions in required_exts and exclude overlap: {overlap}")
+
+    def is_excluded(file_name):
+        """Check if a file matches any pattern in the exclude list."""
+        for pattern in exclude:
+            if Path(file_name).match(pattern):
+                return True
+        return False
+
+    files = []
+    search_pattern = "**/*" if recursive else "*"
+
+    for file_path in Path(input_dir).glob(search_pattern):
+        if file_path.is_file() and not is_excluded(file_path.name):
+            ext = file_path.suffix.lstrip(".")
+            # If required_exts is empty, match any file
+            if not required_exts or ext in required_exts:
+                files.append(file_path)
+
+    return files
+
+
+def assert_all_files_exist_locally(file_paths: List[str]) -> bool:
+    """
+    Checks if all file paths in the provided list exist locally.
+    Raises a FileNotFoundError with a list of missing files if any do not exist.
+
+    Args:
+        file_paths (List[str]): List of file paths to check.
+
+    Returns:
+        bool: True if all files exist, raises FileNotFoundError if any file is missing.
+    """
+    missing_files = [file_path for file_path in file_paths if not Path(file_path).exists()]
+
+    if missing_files:
+        raise FileNotFoundError(missing_files)
+
+    return True