unstructured-ingest 0.0.0__py3-none-any.whl → 0.0.2__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/milvus.py

@@ -0,0 +1,200 @@
+import json
+import multiprocessing as mp
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Optional, Union
+
+import pandas as pd
+from dateutil import parser
+
+from unstructured_ingest.enhanced_dataclass import enhanced_field
+from unstructured_ingest.error import WriteError
+from unstructured_ingest.utils.data_prep import flatten_dict
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    FileData,
+    UploadContent,
+    Uploader,
+    UploaderConfig,
+    UploadStager,
+    UploadStagerConfig,
+)
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+)
+
+if TYPE_CHECKING:
+    from pymilvus import MilvusClient
+
+CONNECTOR_TYPE = "milvus"
+
+
+@dataclass
+class MilvusAccessConfig(AccessConfig):
+    password: Optional[str] = None
+    token: Optional[str] = None
+
+
+@dataclass
+class MilvusConnectionConfig(ConnectionConfig):
+    access_config: MilvusAccessConfig = enhanced_field(
+        sensitive=True, default_factory=lambda: MilvusAccessConfig()
+    )
+    uri: Optional[str] = None
+    user: Optional[str] = None
+    db_name: Optional[str] = None
+
+    def get_connection_kwargs(self) -> dict[str, Any]:
+        access_config_dict = self.access_config.to_dict()
+        connection_config_dict = self.to_dict()
+        connection_config_dict.pop("access_config", None)
+        connection_config_dict.update(access_config_dict)
+        # Drop any that were not set explicitly
+        connection_config_dict = {k: v for k, v in connection_config_dict.items() if v is not None}
+        return connection_config_dict
+
+    @requires_dependencies(["pymilvus"], extras="milvus")
+    def get_client(self) -> "MilvusClient":
+        from pymilvus import MilvusClient
+
+        return MilvusClient(**self.get_connection_kwargs())
+
+
+@dataclass
+class MilvusUploadStagerConfig(UploadStagerConfig):
+    pass
+
+
+@dataclass
+class MilvusUploadStager(UploadStager):
+    upload_stager_config: MilvusUploadStagerConfig = field(
+        default_factory=lambda: MilvusUploadStagerConfig()
+    )
+
+    @staticmethod
+    def parse_date_string(date_string: str) -> float:
+        try:
+            timestamp = float(date_string)
+            return timestamp
+        except ValueError:
+            pass
+        return parser.parse(date_string).timestamp()
+
+    @classmethod
+    def conform_dict(cls, data: dict) -> None:
+        datetime_columns = [
+            "data_source_date_created",
+            "data_source_date_modified",
+            "data_source_date_processed",
+            "last_modified",
+        ]
+
+        json_dumps_fields = ["languages", "data_source_permissions_data"]
+
+        # TODO: milvus sdk doesn't seem to support defaults via the schema yet,
+        #  remove once that gets updated
+        defaults = {"is_continuation": False}
+
+        if metadata := data.pop("metadata", None):
+            data.update(flatten_dict(metadata, keys_to_omit=["data_source_record_locator"]))
+        for datetime_column in datetime_columns:
+            if datetime_column in data:
+                data[datetime_column] = cls.parse_date_string(data[datetime_column])
+        for json_dumps_field in json_dumps_fields:
+            if json_dumps_field in data:
+                data[json_dumps_field] = json.dumps(data[json_dumps_field])
+        for default in defaults:
+            if default not in data:
+                data[default] = defaults[default]
+
+    def run(
+        self,
+        elements_filepath: Path,
+        file_data: FileData,
+        output_dir: Path,
+        output_filename: str,
+        **kwargs: Any,
+    ) -> Path:
+        with open(elements_filepath) as elements_file:
+            elements_contents: list[dict[str, Any]] = json.load(elements_file)
+        for element in elements_contents:
+            self.conform_dict(data=element)
+
+        output_path = Path(output_dir) / Path(f"{output_filename}.json")
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        with output_path.open("w") as output_file:
+            json.dump(elements_contents, output_file, indent=2)
+        return output_path
+
+
+@dataclass
+class MilvusUploaderConfig(UploaderConfig):
+    collection_name: str
+    num_of_processes: int = 4
+
+
+@dataclass
+class MilvusUploader(Uploader):
+    connection_config: MilvusConnectionConfig
+    upload_config: MilvusUploaderConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def upload(self, content: UploadContent) -> None:
+        file_extension = content.path.suffix
+        if file_extension == ".json":
+            self.upload_json(content=content)
+        elif file_extension == ".csv":
+            self.upload_csv(content=content)
+        else:
+            raise ValueError(f"Unsupported file extension: {file_extension}")
+
+    @requires_dependencies(["pymilvus"], extras="milvus")
+    def insert_results(self, data: Union[dict, list[dict]]):
+        from pymilvus import MilvusException
+
+        logger.debug(
+            f"uploading {len(data)} entries to {self.connection_config.db_name} "
+            f"db in collection {self.upload_config.collection_name}"
+        )
+        client = self.connection_config.get_client()
+
+        try:
+            res = client.insert(collection_name=self.upload_config.collection_name, data=data)
+        except MilvusException as milvus_exception:
+            raise WriteError("failed to upload records to milvus") from milvus_exception
+        if "err_count" in res and isinstance(res["err_count"], int) and res["err_count"] > 0:
+            err_count = res["err_count"]
+            raise WriteError(f"failed to upload {err_count} docs")
+
+    def upload_csv(self, content: UploadContent) -> None:
+        df = pd.read_csv(content.path)
+        data = df.to_dict(orient="records")
+        self.insert_results(data=data)
+
+    def upload_json(self, content: UploadContent) -> None:
+        with content.path.open("r") as file:
+            data: list[dict] = json.load(file)
+        self.insert_results(data=data)
+
+    def run(self, contents: list[UploadContent], **kwargs: Any) -> None:
+        if self.upload_config.num_of_processes == 1:
+            for content in contents:
+                self.upload(content=content)
+
+        else:
+            with mp.Pool(
+                processes=self.upload_config.num_of_processes,
+            ) as pool:
+                pool.map(self.upload, contents)
+
+
+milvus_destination_entry = DestinationRegistryEntry(
+    connection_config=MilvusConnectionConfig,
+    uploader=MilvusUploader,
+    uploader_config=MilvusUploaderConfig,
+    upload_stager=MilvusUploadStager,
+    upload_stager_config=MilvusUploadStagerConfig,
+)

unstructured_ingest/v2/processes/connectors/mongodb.py

@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any, Optional
 from unstructured.__version__ import __version__ as unstructured_version
 
 from unstructured_ingest.enhanced_dataclass import enhanced_field
+from unstructured_ingest.error import DestinationConnectionError
 from unstructured_ingest.utils.data_prep import batch_generator
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
@@ -85,11 +86,15 @@ class MongoDBUploaderConfig(UploaderConfig):
 class MongoDBUploader(Uploader):
     upload_config: MongoDBUploaderConfig
     connection_config: MongoDBConnectionConfig
-    client: Optional["MongoClient"] = field(init=False)
     connector_type: str = CONNECTOR_TYPE
 
-    def __post_init__(self):
-        self.client = self.create_client()
+    def precheck(self) -> None:
+        try:
+            client = self.create_client()
+            client.admin.command("ping")
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise DestinationConnectionError(f"failed to validate connection: {e}")
 
     @requires_dependencies(["pymongo"], extras="mongodb")
     def create_client(self) -> "MongoClient":
@@ -123,7 +128,8 @@ class MongoDBUploader(Uploader):
             f"collection {self.connection_config.collection} "
             f"at {self.connection_config.host}",
         )
-        db = self.client[self.connection_config.database]
+        client = self.create_client()
+        db = client[self.connection_config.database]
         collection = db[self.connection_config.collection]
         for chunk in batch_generator(elements_dict, self.upload_config.batch_size):
             collection.insert_many(chunk)

unstructured_ingest/v2/processes/connectors/onedrive.py

@@ -5,7 +5,6 @@ from time import time
 from typing import TYPE_CHECKING, Any, Generator, Optional
 
 from dateutil import parser
-from unstructured.documents.elements import DataSourceMetadata
 
 from unstructured_ingest.enhanced_dataclass import enhanced_field
 from unstructured_ingest.error import SourceConnectionError, SourceConnectionNetworkError
@@ -17,6 +16,7 @@ from unstructured_ingest.v2.interfaces import (
     DownloaderConfig,
     DownloadResponse,
     FileData,
+    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
     SourceIdentifiers,
@@ -87,6 +87,18 @@ class OnedriveIndexer(Indexer):
     connection_config: OnedriveConnectionConfig
     index_config: OnedriveIndexerConfig
 
+    def precheck(self) -> None:
+        try:
+            token_resp: dict = self.connection_config.get_token()
+            if error := token_resp.get("error"):
+                raise SourceConnectionError(
+                    "{} ({})".format(error, token_resp.get("error_description"))
+                )
+            self.connection_config.get_client()
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise SourceConnectionError(f"failed to validate connection: {e}")
+
     def list_objects(self, folder, recursive) -> list["DriveItem"]:
         drive_items = folder.children.get().execute_query()
         files = [d for d in drive_items if d.is_file]
@@ -136,7 +148,7 @@ class OnedriveIndexer(Indexer):
             source_identifiers=SourceIdentifiers(
                 fullpath=server_path, filename=drive_item.name, rel_path=rel_path
             ),
-            metadata=DataSourceMetadata(
+            metadata=FileDataSourceMetadata(
                 url=drive_item.parent_reference.path + "/" + drive_item.name,
                 version=drive_item.etag,
                 date_modified=str(date_modified_dt.timestamp()) if date_modified_dt else None,

unstructured_ingest/v2/processes/connectors/pinecone.py

@@ -5,10 +5,6 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional
 
-from unstructured.ingest.v2.logger import logger
-from unstructured.ingest.v2.processes.connector_registry import (
-    DestinationRegistryEntry,
-)
 from unstructured.staging.base import flatten_dict
 from unstructured.utils import requires_dependencies
 
@@ -24,6 +20,10 @@ from unstructured_ingest.v2.interfaces import (
     UploadStager,
     UploadStagerConfig,
 )
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+)
 
 if TYPE_CHECKING:
     from pinecone import Index as PineconeIndex
@@ -123,9 +123,12 @@ class PineconeUploader(Uploader):
     connection_config: PineconeConnectionConfig
     connector_type: str = CONNECTOR_TYPE
 
-    @DestinationConnectionError.wrap
-    def check_connection(self):
-        _ = self.connection_config.get_index()
+    def precheck(self):
+        try:
+            self.connection_config.get_index()
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise DestinationConnectionError(f"failed to validate connection: {e}")
 
     @requires_dependencies(["pinecone"], extras="pinecone")
     def upsert_batch(self, batch):

unstructured_ingest/v2/processes/connectors/salesforce.py

@@ -18,10 +18,9 @@ from textwrap import dedent
 from typing import TYPE_CHECKING, Any, Generator, Type
 
 from dateutil import parser
-from unstructured.documents.elements import DataSourceMetadata
 
 from unstructured_ingest.enhanced_dataclass import enhanced_field
-from unstructured_ingest.error import SourceConnectionNetworkError
+from unstructured_ingest.error import SourceConnectionError, SourceConnectionNetworkError
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
@@ -30,6 +29,7 @@ from unstructured_ingest.v2.interfaces import (
     DownloaderConfig,
     DownloadResponse,
     FileData,
+    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
     SourceIdentifiers,
@@ -132,6 +132,13 @@ class SalesforceIndexer(Indexer):
             if record_type not in ACCEPTED_CATEGORIES:
                 raise ValueError(f"{record_type} not currently an accepted Salesforce category")
 
+    def precheck(self) -> None:
+        try:
+            self.connection_config.get_client()
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise SourceConnectionError(f"failed to validate connection: {e}")
+
     def get_file_extension(self, record_type) -> str:
         if record_type == "EmailMessage":
             extension = ".eml"
@@ -172,7 +179,7 @@ class SalesforceIndexer(Indexer):
                                 filename=record_with_extension,
                                 fullpath=f"{record['attributes']['type']}/{record_with_extension}",
                             ),
-                            metadata=DataSourceMetadata(
+                            metadata=FileDataSourceMetadata(
                                 url=record["attributes"]["url"],
                                 version=str(parser.parse(record["SystemModstamp"]).timestamp()),
                                 date_created=str(parser.parse(record["CreatedDate"]).timestamp()),
@@ -207,11 +214,6 @@ class SalesforceDownloader(Downloader):
     )
     connector_type: str = CONNECTOR_TYPE
 
-    def get_download_path(self, file_data: FileData) -> Path:
-        rel_path = file_data.source_identifiers.relative_path
-        rel_path = rel_path[1:] if rel_path.startswith("/") else rel_path
-        return self.download_dir / Path(rel_path)
-
     def _xml_for_record(self, record: OrderedDict) -> str:
         """Creates partitionable xml file from a record"""
         import xml.etree.ElementTree as ET

unstructured_ingest/v2/processes/connectors/sharepoint.py

@@ -6,10 +6,8 @@ from time import time
 from typing import TYPE_CHECKING, Any, Generator, Optional
 from urllib.parse import quote
 
-from unstructured.documents.elements import DataSourceMetadata
-
 from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin, enhanced_field
-from unstructured_ingest.error import SourceConnectionNetworkError
+from unstructured_ingest.error import SourceConnectionError, SourceConnectionNetworkError
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
@@ -18,6 +16,7 @@ from unstructured_ingest.v2.interfaces import (
     DownloaderConfig,
     DownloadResponse,
     FileData,
+    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
     SourceIdentifiers,
@@ -134,6 +133,14 @@ class SharepointIndexer(Indexer):
     connection_config: SharepointConnectionConfig
     index_config: SharepointIndexerConfig = field(default_factory=lambda: SharepointIndexerConfig())
 
+    def precheck(self) -> None:
+        try:
+            site_client = self.connection_config.get_client()
+            site_client.site_pages.pages.get().execute_query()
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise SourceConnectionError(f"failed to validate connection: {e}")
+
     def list_files(self, folder: "Folder", recursive: bool = False) -> list["File"]:
         if not recursive:
             folder.expand(["Files"]).get().execute_query()
@@ -187,7 +194,7 @@ class SharepointIndexer(Indexer):
                 fullpath=file_path,
                 rel_path=file_path.replace(self.index_config.path, ""),
             ),
-            metadata=DataSourceMetadata(
+            metadata=FileDataSourceMetadata(
                 url=url,
                 version=version,
                 date_modified=str(date_modified_dt.timestamp()) if date_modified_dt else None,
@@ -222,7 +229,7 @@ class SharepointIndexer(Indexer):
                 fullpath=fullpath,
                 rel_path=rel_path,
             ),
-            metadata=DataSourceMetadata(
+            metadata=FileDataSourceMetadata(
                 url=absolute_url,
                 version=f"{file.major_version}.{file.minor_version}",
                 date_modified=str(date_modified_dt.timestamp()) if date_modified_dt else None,
@@ -340,10 +347,9 @@ class SharepointDownloader(Downloader):
     connector_type: str = CONNECTOR_TYPE
 
     def get_download_path(self, file_data: FileData) -> Path:
+        download_path = super().get_download_path(file_data=file_data)
+
         content_type = file_data.additional_metadata.get("sharepoint_content_type")
-        rel_path = file_data.source_identifiers.fullpath
-        rel_path = rel_path[1:] if rel_path.startswith("/") else rel_path
-        download_path = self.download_dir / Path(rel_path)
         if content_type == SharepointContentType.SITEPAGE.value:
             # Update output extension to html if site page
             download_path = download_path.with_suffix(".html")

unstructured_ingest/v2/processes/connectors/sql.py

@@ -4,13 +4,14 @@ import uuid
 from dataclasses import dataclass, field
 from datetime import date, datetime
 from pathlib import Path
-from typing import Any, Optional, Union
+from typing import TYPE_CHECKING, Any, Callable, Optional, Union
 
 import numpy as np
 import pandas as pd
 from dateutil import parser
 
 from unstructured_ingest.enhanced_dataclass import enhanced_field
+from unstructured_ingest.error import DestinationConnectionError
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
@@ -25,6 +26,11 @@ from unstructured_ingest.v2.interfaces import (
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
 
+if TYPE_CHECKING:
+    from sqlite3 import Connection as SqliteConnection
+
+    from psycopg2.extensions import connection as PostgresConnection
+
 CONNECTOR_TYPE = "sql"
 ELEMENTS_TABLE_NAME = "elements"
 
@@ -41,7 +47,7 @@ class DatabaseType(str, enum.Enum):
 
 
 @dataclass
-class SimpleSqlConfig(ConnectionConfig):
+class SQLConnectionConfig(ConnectionConfig):
     db_type: DatabaseType = (
         # required default value here because of parent class
         DatabaseType.SQLITE
@@ -134,7 +140,7 @@ class SQLUploadStager(UploadStager):
         **kwargs: Any,
     ) -> Path:
         with open(elements_filepath) as elements_file:
-            elements_contents = json.load(elements_file)
+            elements_contents: list[dict] = json.load(elements_file)
         output_path = Path(output_dir) / Path(f"{output_filename}.json")
         output_path.parent.mkdir(parents=True, exist_ok=True)
 
@@ -151,7 +157,7 @@ class SQLUploadStager(UploadStager):
             data["id"] = str(uuid.uuid4())
 
             # remove extraneous, not supported columns
-            [data.pop(column) for column in data if column not in _COLUMNS]
+            data = {k: v for k, v in data.items() if k in _COLUMNS}
 
             output.append(data)
 
@@ -185,23 +191,32 @@ class SQLUploaderConfig(UploaderConfig):
 class SQLUploader(Uploader):
     connector_type: str = CONNECTOR_TYPE
     upload_config: SQLUploaderConfig
-    connection_config: SimpleSqlConfig
+    connection_config: SQLConnectionConfig
+
+    def precheck(self) -> None:
+        try:
+            cursor = self.connection().cursor()
+            cursor.execute("SELECT 1;")
+            cursor.close()
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise DestinationConnectionError(f"failed to validate connection: {e}")
 
     @property
-    def connection(self):
+    def connection(self) -> Callable[[], Union["SqliteConnection", "PostgresConnection"]]:
         if self.connection_config.db_type == DatabaseType.POSTGRESQL:
             return self._make_psycopg_connection
         elif self.connection_config.db_type == DatabaseType.SQLITE:
             return self._make_sqlite_connection
         raise ValueError(f"Unsupported database {self.connection_config.db_type} connection.")
 
-    def _make_sqlite_connection(self):
+    def _make_sqlite_connection(self) -> "SqliteConnection":
         from sqlite3 import connect
 
         return connect(database=self.connection_config.database)
 
     @requires_dependencies(["psycopg2"], extras="postgres")
-    def _make_psycopg_connection(self):
+    def _make_psycopg_connection(self) -> "PostgresConnection":
         from psycopg2 import connect
 
         return connect(
@@ -261,7 +276,7 @@ class SQLUploader(Uploader):
 
 
 sql_destination_entry = DestinationRegistryEntry(
-    connection_config=SimpleSqlConfig,
+    connection_config=SQLConnectionConfig,
     uploader=SQLUploader,
     uploader_config=SQLUploaderConfig,
     upload_stager=SQLUploadStager,

unstructured_ingest/v2/processes/connectors/weaviate.py

@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Optional
 from dateutil import parser
 
 from unstructured_ingest.enhanced_dataclass import enhanced_field
+from unstructured_ingest.error import DestinationConnectionError
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
@@ -156,15 +157,21 @@ class WeaviateUploaderConfig(UploaderConfig):
 class WeaviateUploader(Uploader):
     upload_config: WeaviateUploaderConfig
     connection_config: WeaviateConnectionConfig
-    client: Optional["Client"] = field(init=False)
     connector_type: str = CONNECTOR_TYPE
 
     @requires_dependencies(["weaviate"], extras="weaviate")
-    def __post_init__(self):
+    def get_client(self) -> "Client":
         from weaviate import Client
 
         auth = self._resolve_auth_method()
-        self.client = Client(url=self.connection_config.host_url, auth_client_secret=auth)
+        return Client(url=self.connection_config.host_url, auth_client_secret=auth)
+
+    def precheck(self) -> None:
+        try:
+            self.get_client()
+        except Exception as e:
+            logger.error(f"Failed to validate connection {e}", exc_info=True)
+            raise DestinationConnectionError(f"failed to validate connection: {e}")
 
     @requires_dependencies(["weaviate"], extras="weaviate")
     def _resolve_auth_method(self):
@@ -215,8 +222,9 @@ class WeaviateUploader(Uploader):
             f"at {self.connection_config.host_url}",
         )
 
-        self.client.batch.configure(batch_size=self.upload_config.batch_size)
-        with self.client.batch as b:
+        client = self.get_client()
+        client.batch.configure(batch_size=self.upload_config.batch_size)
+        with client.batch as b:
             for e in elements_dict:
                 vector = e.pop("embeddings", None)
                 b.add_data_object(

unstructured_ingest/v2/processes/filter.py

@@ -0,0 +1,54 @@
+import fnmatch
+from abc import ABC
+from dataclasses import dataclass, field
+from typing import Any, Callable, Optional
+
+from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from unstructured_ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.interfaces.process import BaseProcess
+from unstructured_ingest.v2.logger import logger
+
+
+@dataclass
+class FiltererConfig(EnhancedDataClassJsonMixin):
+    file_glob: Optional[list[str]] = None
+    max_file_size: Optional[int] = None
+
+
+@dataclass
+class Filterer(BaseProcess, ABC):
+    config: FiltererConfig = field(default_factory=lambda: FiltererConfig())
+    filters: list[Callable[[FileData], bool]] = field(init=False, default_factory=list)
+
+    def __post_init__(self):
+        # Populate the filters based on values in config
+        if self.config.file_glob is not None:
+            self.filters.append(self.glob_filter)
+        if self.config.max_file_size:
+            self.filters.append(self.file_size_filter)
+
+    def is_async(self) -> bool:
+        return False
+
+    def file_size_filter(self, file_data: FileData) -> bool:
+        if filesize_bytes := file_data.metadata.filesize_bytes:
+            return filesize_bytes <= self.config.max_file_size
+        return True
+
+    def glob_filter(self, file_data: FileData) -> bool:
+        patterns = self.config.file_glob
+        path = file_data.source_identifiers.fullpath
+        for pattern in patterns:
+            if fnmatch.filter([path], pattern):
+                return True
+        logger.debug(f"The file {path!r} is discarded as it does not match any given glob.")
+        return False
+
+    def run(self, file_data: FileData, **kwargs: Any) -> Optional[FileData]:
+        for filter in self.filters:
+            if not filter(file_data):
+                logger.debug(
+                    f"filtered out file data due to {filter.__name__}: {file_data.identifier}"
+                )
+                return None
+        return file_data