unstructured-ingest 0.0.2__py3-none-any.whl → 0.0.2.dev0__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/sql.py

@@ -4,14 +4,13 @@ import uuid
 from dataclasses import dataclass, field
 from datetime import date, datetime
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, Optional, Union
+from typing import Any, Optional, Union
 
 import numpy as np
 import pandas as pd
 from dateutil import parser
 
 from unstructured_ingest.enhanced_dataclass import enhanced_field
-from unstructured_ingest.error import DestinationConnectionError
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
@@ -26,11 +25,6 @@ from unstructured_ingest.v2.interfaces import (
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
 
-if TYPE_CHECKING:
-    from sqlite3 import Connection as SqliteConnection
-
-    from psycopg2.extensions import connection as PostgresConnection
-
 CONNECTOR_TYPE = "sql"
 ELEMENTS_TABLE_NAME = "elements"
 
@@ -47,7 +41,7 @@ class DatabaseType(str, enum.Enum):
 
 
 @dataclass
-class SQLConnectionConfig(ConnectionConfig):
+class SimpleSqlConfig(ConnectionConfig):
     db_type: DatabaseType = (
         # required default value here because of parent class
         DatabaseType.SQLITE
@@ -140,7 +134,7 @@ class SQLUploadStager(UploadStager):
         **kwargs: Any,
     ) -> Path:
         with open(elements_filepath) as elements_file:
-            elements_contents: list[dict] = json.load(elements_file)
+            elements_contents = json.load(elements_file)
         output_path = Path(output_dir) / Path(f"{output_filename}.json")
         output_path.parent.mkdir(parents=True, exist_ok=True)
 
@@ -157,7 +151,7 @@ class SQLUploadStager(UploadStager):
             data["id"] = str(uuid.uuid4())
 
             # remove extraneous, not supported columns
-            data = {k: v for k, v in data.items() if k in _COLUMNS}
+            [data.pop(column) for column in data if column not in _COLUMNS]
 
             output.append(data)
 
@@ -191,32 +185,23 @@ class SQLUploaderConfig(UploaderConfig):
 class SQLUploader(Uploader):
     connector_type: str = CONNECTOR_TYPE
     upload_config: SQLUploaderConfig
-    connection_config: SQLConnectionConfig
-
-    def precheck(self) -> None:
-        try:
-            cursor = self.connection().cursor()
-            cursor.execute("SELECT 1;")
-            cursor.close()
-        except Exception as e:
-            logger.error(f"failed to validate connection: {e}", exc_info=True)
-            raise DestinationConnectionError(f"failed to validate connection: {e}")
+    connection_config: SimpleSqlConfig
 
     @property
-    def connection(self) -> Callable[[], Union["SqliteConnection", "PostgresConnection"]]:
+    def connection(self):
         if self.connection_config.db_type == DatabaseType.POSTGRESQL:
             return self._make_psycopg_connection
         elif self.connection_config.db_type == DatabaseType.SQLITE:
             return self._make_sqlite_connection
         raise ValueError(f"Unsupported database {self.connection_config.db_type} connection.")
 
-    def _make_sqlite_connection(self) -> "SqliteConnection":
+    def _make_sqlite_connection(self):
         from sqlite3 import connect
 
         return connect(database=self.connection_config.database)
 
     @requires_dependencies(["psycopg2"], extras="postgres")
-    def _make_psycopg_connection(self) -> "PostgresConnection":
+    def _make_psycopg_connection(self):
         from psycopg2 import connect
 
         return connect(
@@ -276,7 +261,7 @@ class SQLUploader(Uploader):
 
 
 sql_destination_entry = DestinationRegistryEntry(
-    connection_config=SQLConnectionConfig,
+    connection_config=SimpleSqlConfig,
     uploader=SQLUploader,
     uploader_config=SQLUploaderConfig,
     upload_stager=SQLUploadStager,

unstructured_ingest/v2/processes/connectors/weaviate.py

@@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Any, Optional
 from dateutil import parser
 
 from unstructured_ingest.enhanced_dataclass import enhanced_field
-from unstructured_ingest.error import DestinationConnectionError
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
@@ -157,21 +156,15 @@ class WeaviateUploaderConfig(UploaderConfig):
 class WeaviateUploader(Uploader):
     upload_config: WeaviateUploaderConfig
     connection_config: WeaviateConnectionConfig
+    client: Optional["Client"] = field(init=False)
     connector_type: str = CONNECTOR_TYPE
 
     @requires_dependencies(["weaviate"], extras="weaviate")
-    def get_client(self) -> "Client":
+    def __post_init__(self):
         from weaviate import Client
 
         auth = self._resolve_auth_method()
-        return Client(url=self.connection_config.host_url, auth_client_secret=auth)
-
-    def precheck(self) -> None:
-        try:
-            self.get_client()
-        except Exception as e:
-            logger.error(f"Failed to validate connection {e}", exc_info=True)
-            raise DestinationConnectionError(f"failed to validate connection: {e}")
+        self.client = Client(url=self.connection_config.host_url, auth_client_secret=auth)
 
     @requires_dependencies(["weaviate"], extras="weaviate")
     def _resolve_auth_method(self):
@@ -222,9 +215,8 @@ class WeaviateUploader(Uploader):
             f"at {self.connection_config.host_url}",
         )
 
-        client = self.get_client()
-        client.batch.configure(batch_size=self.upload_config.batch_size)
-        with client.batch as b:
+        self.client.batch.configure(batch_size=self.upload_config.batch_size)
+        with self.client.batch as b:
             for e in elements_dict:
                 vector = e.pop("embeddings", None)
                 b.add_data_object(

{unstructured_ingest-0.0.2.dist-info → unstructured_ingest-0.0.2.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.0.2
+Version: 0.0.2.dev0
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -29,8 +29,8 @@ Requires-Dist: pyairtable ; extra == 'airtable'
 Provides-Extra: astra
 Requires-Dist: astrapy ; extra == 'astra'
 Provides-Extra: azure
-Requires-Dist: fsspec ; extra == 'azure'
 Requires-Dist: adlfs ; extra == 'azure'
+Requires-Dist: fsspec ; extra == 'azure'
 Provides-Extra: azure-cognitive-search
 Requires-Dist: azure-search-documents ; extra == 'azure-cognitive-search'
 Provides-Extra: bedrock
@@ -42,9 +42,9 @@ Provides-Extra: box
 Requires-Dist: boxfs ; extra == 'box'
 Requires-Dist: fsspec ; extra == 'box'
 Provides-Extra: chroma
-Requires-Dist: importlib-metadata >=7.1.0 ; extra == 'chroma'
 Requires-Dist: typer <=0.9.0 ; extra == 'chroma'
 Requires-Dist: chromadb ; extra == 'chroma'
+Requires-Dist: importlib-metadata >=7.1.0 ; extra == 'chroma'
 Provides-Extra: clarifai
 Requires-Dist: clarifai ; extra == 'clarifai'
 Provides-Extra: confluence
@@ -54,8 +54,8 @@ Requires-Dist: unstructured[tsv] ; extra == 'csv'
 Provides-Extra: databricks-volumes
 Requires-Dist: databricks-sdk ; extra == 'databricks-volumes'
 Provides-Extra: delta-table
-Requires-Dist: deltalake ; extra == 'delta-table'
 Requires-Dist: fsspec ; extra == 'delta-table'
+Requires-Dist: deltalake ; extra == 'delta-table'
 Provides-Extra: discord
 Requires-Dist: discord-py ; extra == 'discord'
 Provides-Extra: doc
@@ -63,21 +63,21 @@ Requires-Dist: unstructured[docx] ; extra == 'doc'
 Provides-Extra: docx
 Requires-Dist: unstructured[docx] ; extra == 'docx'
 Provides-Extra: dropbox
-Requires-Dist: dropboxdrivefs ; extra == 'dropbox'
 Requires-Dist: fsspec ; extra == 'dropbox'
+Requires-Dist: dropboxdrivefs ; extra == 'dropbox'
 Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async] ; extra == 'elasticsearch'
 Provides-Extra: embed-huggingface
+Requires-Dist: langchain-community ; extra == 'embed-huggingface'
 Requires-Dist: huggingface ; extra == 'embed-huggingface'
 Requires-Dist: sentence-transformers ; extra == 'embed-huggingface'
-Requires-Dist: langchain-community ; extra == 'embed-huggingface'
 Provides-Extra: embed-octoai
 Requires-Dist: tiktoken ; extra == 'embed-octoai'
 Requires-Dist: openai ; extra == 'embed-octoai'
 Provides-Extra: embed-vertexai
-Requires-Dist: langchain-community ; extra == 'embed-vertexai'
 Requires-Dist: langchain ; extra == 'embed-vertexai'
 Requires-Dist: langchain-google-vertexai ; extra == 'embed-vertexai'
+Requires-Dist: langchain-community ; extra == 'embed-vertexai'
 Provides-Extra: embed-voyageai
 Requires-Dist: langchain ; extra == 'embed-voyageai'
 Requires-Dist: langchain-voyageai ; extra == 'embed-voyageai'
@@ -85,8 +85,8 @@ Provides-Extra: epub
 Requires-Dist: unstructured[epub] ; extra == 'epub'
 Provides-Extra: gcs
 Requires-Dist: bs4 ; extra == 'gcs'
-Requires-Dist: gcsfs ; extra == 'gcs'
 Requires-Dist: fsspec ; extra == 'gcs'
+Requires-Dist: gcsfs ; extra == 'gcs'
 Provides-Extra: github
 Requires-Dist: pygithub >1.58.0 ; extra == 'github'
 Provides-Extra: gitlab
@@ -109,25 +109,25 @@ Requires-Dist: pymongo ; extra == 'mongodb'
 Provides-Extra: msg
 Requires-Dist: unstructured[msg] ; extra == 'msg'
 Provides-Extra: notion
-Requires-Dist: htmlBuilder ; extra == 'notion'
 Requires-Dist: notion-client ; extra == 'notion'
+Requires-Dist: htmlBuilder ; extra == 'notion'
 Provides-Extra: odt
 Requires-Dist: unstructured[odt] ; extra == 'odt'
 Provides-Extra: onedrive
+Requires-Dist: Office365-REST-Python-Client ; extra == 'onedrive'
 Requires-Dist: msal ; extra == 'onedrive'
 Requires-Dist: bs4 ; extra == 'onedrive'
-Requires-Dist: Office365-REST-Python-Client ; extra == 'onedrive'
 Provides-Extra: openai
-Requires-Dist: tiktoken ; extra == 'openai'
-Requires-Dist: openai ; extra == 'openai'
 Requires-Dist: langchain-community ; extra == 'openai'
+Requires-Dist: openai ; extra == 'openai'
+Requires-Dist: tiktoken ; extra == 'openai'
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py ; extra == 'opensearch'
 Provides-Extra: org
 Requires-Dist: unstructured[org] ; extra == 'org'
 Provides-Extra: outlook
-Requires-Dist: msal ; extra == 'outlook'
 Requires-Dist: Office365-REST-Python-Client ; extra == 'outlook'
+Requires-Dist: msal ; extra == 'outlook'
 Provides-Extra: pdf
 Requires-Dist: unstructured[pdf] ; extra == 'pdf'
 Provides-Extra: pinecone
@@ -152,11 +152,11 @@ Requires-Dist: fsspec ; extra == 's3'
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce ; extra == 'salesforce'
 Provides-Extra: sftp
-Requires-Dist: paramiko ; extra == 'sftp'
 Requires-Dist: fsspec ; extra == 'sftp'
+Requires-Dist: paramiko ; extra == 'sftp'
 Provides-Extra: sharepoint
-Requires-Dist: msal ; extra == 'sharepoint'
 Requires-Dist: Office365-REST-Python-Client ; extra == 'sharepoint'
+Requires-Dist: msal ; extra == 'sharepoint'
 Provides-Extra: singlestore
 Requires-Dist: singlestoredb ; extra == 'singlestore'
 Provides-Extra: slack

{unstructured_ingest-0.0.2.dist-info → unstructured_ingest-0.0.2.dev0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=t0CFzEk7qlIWbgyEWA53ytTKmHbZ9ow2lAyjeP1bFqw,42
+unstructured_ingest/__version__.py,sha256=neOBPct_gjgXqs6YN8HnzdaRiPiugEbJpwI6SDZ7qac,47
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/evaluate.py,sha256=R-mKLFXbVX1xQ1tjGsLHjdP-TbSSV-925IHzggW_bIg,9793
 unstructured_ingest/interfaces.py,sha256=uS8L5mS0mXD8I4XTfVlKZxAwqnpJ4yrRqn4vxWVRhQI,31107
@@ -260,10 +260,10 @@ unstructured_ingest/v2/cli/cli.py,sha256=qHXIs-PcvMgDZhP1AR9iDMxh8FXBMJCEDksPBfi
 unstructured_ingest/v2/cli/interfaces.py,sha256=4Bbow6QHks2a1H56tmVQ4vG3sZy-577ZbwrPmDfizmE,829
 unstructured_ingest/v2/cli/utils.py,sha256=QK-ee6FzxPf-IbaNXXWlH-GhvqeNnjK2m8ljBD1SusU,9075
 unstructured_ingest/v2/cli/base/__init__.py,sha256=zXCa7F4FMqItmzxfUIVmyI-CeGh8X85yF8lRxwX_OYQ,83
-unstructured_ingest/v2/cli/base/cmd.py,sha256=qi9N5rcyyE2nmswFaoKWbs1PonhHsMC5llqND9-rQso,9790
+unstructured_ingest/v2/cli/base/cmd.py,sha256=qVHmquVsVDoYyPByKdUTVCwAFfILMYBw5w6eTTVku-E,9308
 unstructured_ingest/v2/cli/base/dest.py,sha256=YMbVIHmYDqvOtxZeEY93stmF2p2ImjuJts7-u-NznYw,2887
 unstructured_ingest/v2/cli/base/importer.py,sha256=nRt0QQ3qpi264-n_mR0l55C2ddM8nowTNzT1jsWaam8,1128
-unstructured_ingest/v2/cli/base/src.py,sha256=oUPO9GPEbkYm1udfD4YQBYTfaefbhpoIN1HPnD672SQ,2460
+unstructured_ingest/v2/cli/base/src.py,sha256=7LnZh9FgUX9rerBH6cizVtTWmM6R2sRkxatnGsxYHG0,2410
 unstructured_ingest/v2/cli/cmds/__init__.py,sha256=DWPMD6Wqus22sSoIEyTSiOJAm97aNjvdpdrXgsL4uQ0,2647
 unstructured_ingest/v2/cli/cmds/astra.py,sha256=L-GR2KSP_cFQkQm0aVcdiXmgYMJZCVKIAH794y8qT1M,2590
 unstructured_ingest/v2/cli/cmds/azure_cognitive_search.py,sha256=VTCSUYeIYKnP60lC7DeBYqoqAJnWuBZrwevCXbeIEzw,2248
@@ -271,7 +271,7 @@ unstructured_ingest/v2/cli/cmds/chroma.py,sha256=RinNOPripk2zRYx1Rt-u-jywXbwh7Js
 unstructured_ingest/v2/cli/cmds/databricks_volumes.py,sha256=53d9A7UunJLYZFwwwHEraVshFc3gSzUbmKjMOiv7hn4,5920
 unstructured_ingest/v2/cli/cmds/elasticsearch.py,sha256=joUfnV992fAwEDCtFVJaABwgpyQiWeDl1ZCBEudRtnk,5258
 unstructured_ingest/v2/cli/cmds/google_drive.py,sha256=mXozabpi8kjRFb0S7kw-xMGtEuFoVUxnvefwL5ZIPHc,2334
-unstructured_ingest/v2/cli/cmds/local.py,sha256=UOTYjSdNqCFxhZfN6bdxm8jRp6Ijun2K-WpQq1X83OQ,1544
+unstructured_ingest/v2/cli/cmds/local.py,sha256=lGBFOVDRlrcCtPFjyk0IAYHLRWg95Kunu1Kli7t0ZK4,1899
 unstructured_ingest/v2/cli/cmds/milvus.py,sha256=PB1ib1rFGGH_-KDi1bSIO3BIiVcqSJEHCBFFrzQrnmI,1998
 unstructured_ingest/v2/cli/cmds/mongodb.py,sha256=oyV6tacuuxm3dN-AXQgbxvYJiDYo2OOWQKRSBCUGj0E,1823
 unstructured_ingest/v2/cli/cmds/onedrive.py,sha256=DKqhQyyF-swZxs3C9G5W8ECleq8sWpDbpTuiAHXukXQ,2781
@@ -286,34 +286,32 @@ unstructured_ingest/v2/cli/cmds/fsspec/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JC
 unstructured_ingest/v2/cli/cmds/fsspec/azure.py,sha256=ZHfchzSpGkZ99Fq1050JvHP0-aG1pZsBZxxozcFfxwI,2784
 unstructured_ingest/v2/cli/cmds/fsspec/box.py,sha256=kslkI-0-GyGSJOU7bKgrZeQRXh8HFexDq87ew8kT8kE,1338
 unstructured_ingest/v2/cli/cmds/fsspec/dropbox.py,sha256=LtcR3rCQPgzJNbV3S90HlL0LPPbW9lYEfE8BG4F-dSI,1349
-unstructured_ingest/v2/cli/cmds/fsspec/fsspec.py,sha256=BlJLEu6YJUejLLFzlSCVSoZDp2pdjoTsdoFFHVwwkVY,2031
+unstructured_ingest/v2/cli/cmds/fsspec/fsspec.py,sha256=Mgr_nto1FWxnGmbGdVlIfm-xuBGL0HEi8k3FEmQnZng,2414
 unstructured_ingest/v2/cli/cmds/fsspec/gcs.py,sha256=3-0LYnDs0fgNrDqnHpNZKj_6rwNj9wQVaV0lGOhVFPE,2737
 unstructured_ingest/v2/cli/cmds/fsspec/s3.py,sha256=EXQzYkDtkFli2sfcj4cRDRPFac7b7z1DfQqYlGQcE6o,2279
 unstructured_ingest/v2/cli/cmds/fsspec/sftp.py,sha256=YY2xKguawMyLdcG0qDYKUgk7DT0KgyZJlV17MfwIhpo,2036
-unstructured_ingest/v2/cli/configs/__init__.py,sha256=nAJ1gT3yCAzoZbEbYswE2SMfSk7TEPxa_1v3qEUsgIQ,336
+unstructured_ingest/v2/cli/configs/__init__.py,sha256=5NMXm872QQZTvUFZFS06c8c1b6K940K5gxs9lbp8W6M,258
 unstructured_ingest/v2/cli/configs/chunk.py,sha256=KvIhmIRIZxazCumMztAKdWs-4MK7qzOb5h6Ned_2bdU,3547
 unstructured_ingest/v2/cli/configs/embed.py,sha256=q_TwnkxKTKOsMgVYfW6xxbD8FWjU_Uh_X2BQ5-_VLGM,2725
-unstructured_ingest/v2/cli/configs/filter.py,sha256=KYe65_au6m7H4VrjgugC2ain6vsUSWswNSEgcG66VPU,841
 unstructured_ingest/v2/cli/configs/partition.py,sha256=7wdI18V6c4kaXuf50Lh66n9LbtrYHYd8ffEgDQLqvSk,3931
 unstructured_ingest/v2/cli/configs/processor.py,sha256=ZHu2DBIuE8VgL3mEt73yYimw2k_PaOEtdxxFqzHfk84,3350
-unstructured_ingest/v2/interfaces/__init__.py,sha256=Rfa8crx6De7WNOK-EjsWWwFVpsUfCc6gY8B8tQ3ae9I,899
+unstructured_ingest/v2/interfaces/__init__.py,sha256=-CHWUlT4rISd-gSfcGKGYFqqSFhMY9lKsT5wxwmOThM,845
 unstructured_ingest/v2/interfaces/connector.py,sha256=u4hE1DpTPDC04-n_IzYyn9w1gNCiPT81anrUoEh30Z8,855
-unstructured_ingest/v2/interfaces/downloader.py,sha256=zs7cxhzbWVc5L0bV4gdCTexWGMVeXTQ9jJF6PCYSAss,2790
-unstructured_ingest/v2/interfaces/file_data.py,sha256=PZrPJBkNC63lNO_1nwvnAeKRxjM3CsjIY6jSO8T9bVM,1665
+unstructured_ingest/v2/interfaces/downloader.py,sha256=aWlacZZrI6SGw6retnRJtZbqT5voOYq_fb326ynNOhI,2506
+unstructured_ingest/v2/interfaces/file_data.py,sha256=5TCMkblUW-Jvy-rS5FqRT22VzDmJqAiQRIWYarpAi64,1543
 unstructured_ingest/v2/interfaces/indexer.py,sha256=pMw0abNHk_tEuA4BkXX1BdAfIwHdytxj7s6tGxMvYRE,821
-unstructured_ingest/v2/interfaces/process.py,sha256=_l4dyaM0u0XxTqQw1Ghr8k2QMpQJMFapLOLhWqSdTdo,512
+unstructured_ingest/v2/interfaces/process.py,sha256=0ecz7mAjlY_DUi9-HhPc9zXphmGclispYwv37O8gvJ0,466
 unstructured_ingest/v2/interfaces/processor.py,sha256=uHVHeKo5Gt_zFkaEXw7xgaCBDTEl2-Amh-ByA07258o,1620
 unstructured_ingest/v2/interfaces/upload_stager.py,sha256=SylhDl9pK6qa7hvfrhpabCkjwE03yIlI6oM-mQnqtho,1220
 unstructured_ingest/v2/interfaces/uploader.py,sha256=bzfx3Ei4poXKu-hsgjAB4sj4jKij9CoaRSadUM5LtGk,1083
 unstructured_ingest/v2/pipeline/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-unstructured_ingest/v2/pipeline/interfaces.py,sha256=Z50-6XFZNajfmJbLKunLxw3RuYMzCYiUp6F0jhQwERE,6441
-unstructured_ingest/v2/pipeline/pipeline.py,sha256=dqn4_O4il6gZ33mE0DVC1wQKRcXMrD_jll999NoyQgw,14283
+unstructured_ingest/v2/pipeline/interfaces.py,sha256=Zz76fLHNKw6BDsBSYQXiRa6CvyW91ulvZU0yw5vVQ5M,6544
+unstructured_ingest/v2/pipeline/pipeline.py,sha256=r8jRMZI2RF8GQIuTcjIFBDeFtMnqpOJmKhEriy6Vo5Y,11616
 unstructured_ingest/v2/pipeline/utils.py,sha256=oPAitfdnITqh2O8Z0uf6VOHg9BTJhitRzNmKXqTwPxg,422
 unstructured_ingest/v2/pipeline/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/v2/pipeline/steps/chunk.py,sha256=lfCsBo6A9u1cT57YaEjvNI79gc29nW8c-2_WZNjiO5Y,3275
-unstructured_ingest/v2/pipeline/steps/download.py,sha256=qYeuRU5jeICyuTN7E4YUdnbi6X1X2qKoooJMm4Orbdw,7499
+unstructured_ingest/v2/pipeline/steps/download.py,sha256=GA5-zTH4c7Ac8oBoc4AeDA8sQ0dYT-KUOZ4n31K3Jpg,4882
 unstructured_ingest/v2/pipeline/steps/embed.py,sha256=VCdDBUXK6Yx8RTvRBpEFdFE7n0izvkP73w6s8Tv2sgg,3253
-unstructured_ingest/v2/pipeline/steps/filter.py,sha256=mYVccl_zp0CGYFWBrSrPelvSElrXhZahebuymGuirV8,1341
 unstructured_ingest/v2/pipeline/steps/index.py,sha256=i4RcJ1oRqNp-rFdc6rvKVGcSzNhdB7woW7_W364uThQ,2269
 unstructured_ingest/v2/pipeline/steps/partition.py,sha256=q7-rpCj5Vy4BXtd7T72gxGb3xg6lmVyNmTwUfHil7Rg,3199
 unstructured_ingest/v2/pipeline/steps/stage.py,sha256=A8i6VAFY4_xFJR0uBEyBNJlQXmTMGaflXsa6Wa6U1wQ,2274
@@ -323,39 +321,38 @@ unstructured_ingest/v2/processes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
 unstructured_ingest/v2/processes/chunker.py,sha256=U6zQhaUG_dii66zqL9iEEGodHENNxnpn6V3pC-e7MMI,4233
 unstructured_ingest/v2/processes/connector_registry.py,sha256=KOrvJNNRdpBPyqFwmTm42kD1xXuo7fNS_5yXjtqAz-c,2100
 unstructured_ingest/v2/processes/embedder.py,sha256=QjAsiXAjWtZzh6lJ4D5LsTMBD81zuMBkegXNWq-FZt0,3308
-unstructured_ingest/v2/processes/filter.py,sha256=CfQihLV_r4yTJgAc66mmbP4_xo3wcDlro5oR_KR--bM,1986
 unstructured_ingest/v2/processes/partitioner.py,sha256=f6UQoQHVKjl8rmM5J9EcuP30RTFLSLrArGdC6qh-ffE,7645
 unstructured_ingest/v2/processes/uncompress.py,sha256=x-JZYNs1zJOtRS7xNgiMyrYoAbzKM0p18O8NAl7avCA,1631
 unstructured_ingest/v2/processes/connectors/__init__.py,sha256=7QMKd8gtEJTIuK352Ho6XyoFvLLhrWIzgdu0dXwXWOE,3960
-unstructured_ingest/v2/processes/connectors/astra.py,sha256=m6A34wYjnctRfIF-14bnbGIFBwht5Y8UWZ4g8R9x6a8,5241
-unstructured_ingest/v2/processes/connectors/azure_cognitive_search.py,sha256=N_--5t_hxFzFEK4vERzm46gfg-Ghozb71_NmUlEYIMA,8277
-unstructured_ingest/v2/processes/connectors/chroma.py,sha256=W995GLn7D85GoUhSqHQXP5QQ8OglgykA5rcNmg9Ruf4,7158
-unstructured_ingest/v2/processes/connectors/databricks_volumes.py,sha256=8bVht450bxp0K4ub1XdCDmMKEooXVV4DNY5b5GWF0Ig,3636
-unstructured_ingest/v2/processes/connectors/elasticsearch.py,sha256=myY2FRXtlBYhH-kbTSsn7j9UDzh36NYHqFRP-ys8am4,15358
-unstructured_ingest/v2/processes/connectors/google_drive.py,sha256=-iYpwt4xxaICRlHD5Bpap7Ck5HRJcapa6uHl60E1uZ4,12702
-unstructured_ingest/v2/processes/connectors/local.py,sha256=IJ5DjASp-5lPmb6J7Y8NROYjIS3sfdRhlcDAZEEGVAw,6573
+unstructured_ingest/v2/processes/connectors/astra.py,sha256=TSI_3GHnEh3gYAC30RTG4b2eEB07agroEFmJ38GnQY4,4903
+unstructured_ingest/v2/processes/connectors/azure_cognitive_search.py,sha256=PT02ZKiJuHMrmBClxqBsyDS0aXUQYLVg02Ns2qh1hD4,7935
+unstructured_ingest/v2/processes/connectors/chroma.py,sha256=nYzNz-8oq-DN0Z4r7lHQFmlved76IaYeRvm7-EmbGUE,6998
+unstructured_ingest/v2/processes/connectors/databricks_volumes.py,sha256=MTLK7SvQqWU-PXmEbGajM4f-CqGWlmlfeED6a5StEWw,3226
+unstructured_ingest/v2/processes/connectors/elasticsearch.py,sha256=6QBvVzPk3mWj9ZqJZN7NvhcJaOO6nSLqLwU6zggP59A,14864
+unstructured_ingest/v2/processes/connectors/google_drive.py,sha256=IkLVafUu280OOoqYmdfdfMB6zlpiWjs2Z5J31ZzJOj4,12681
+unstructured_ingest/v2/processes/connectors/local.py,sha256=maAXVKpRRXj_jseC6EPLTosMgw6ll-0lnGsDdAFLWAE,6646
 unstructured_ingest/v2/processes/connectors/milvus.py,sha256=FWH4FH-zns7gh8sITg9pLYE9uKm_3GeOXJ4wjY6PMno,6776
-unstructured_ingest/v2/processes/connectors/mongodb.py,sha256=XZCgkF28HCR4DtMmr8jlxb59txXgEvfCabovROUrv6Y,4602
-unstructured_ingest/v2/processes/connectors/onedrive.py,sha256=_TFO-vlyCxIxMk6hv20CEsicrlh87wCrbi4I1chsMUw,8822
+unstructured_ingest/v2/processes/connectors/mongodb.py,sha256=ErZWAD-su3OCRGv1h84X1PpAWleUPVZcFDEIYjtyP4E,4310
+unstructured_ingest/v2/processes/connectors/onedrive.py,sha256=WDDoFEfd8M_QBTpkGNI2zZGZZ_CR1rQiCsBWYOO2JoA,8311
 unstructured_ingest/v2/processes/connectors/opensearch.py,sha256=HNRZVQsWnjLLm0yAGiIyHRbhAsBnGSXBO_VkUfIdwdE,5463
-unstructured_ingest/v2/processes/connectors/pinecone.py,sha256=0rpOh_pi4GormyIQsnEJbKVb7FeizAbLcbljpnjtpeY,5908
-unstructured_ingest/v2/processes/connectors/salesforce.py,sha256=S0dEjT1UxReCC6qE9DlSQBgcSzQbOaIq7SMJqXUpNWQ,10858
-unstructured_ingest/v2/processes/connectors/sharepoint.py,sha256=NRn0lbOuXqIYqZT15IVFeFQCxpCKzZC_M8pVYZeeNfo,17933
+unstructured_ingest/v2/processes/connectors/pinecone.py,sha256=PtAodxemYgiBZESx-g9a8fcL6XagJd9DIDQjrhE8aPk,5746
+unstructured_ingest/v2/processes/connectors/salesforce.py,sha256=Cz4qEtnbsD9-m1DXANxnVRZTHX2ZaUUBPVFPu5wnFRk,10832
+unstructured_ingest/v2/processes/connectors/sharepoint.py,sha256=SNovgGUE5tHdfX_lF5zwM_QRZK7mahHzLZKhnqfk6Tc,17696
 unstructured_ingest/v2/processes/connectors/singlestore.py,sha256=upF2O4hJ2uiBhDRrpQ8CSJUvzmqu2j5H1b_QbReHJpw,5168
-unstructured_ingest/v2/processes/connectors/sql.py,sha256=mbhBI2tcX4q1YJwR3Nr7HGbr-rb8ppUYq2JcN88We3U,9076
+unstructured_ingest/v2/processes/connectors/sql.py,sha256=T0rpCbhEipWlezoJOMiUewcZuk6Had6TkmsDT-PeOL0,8360
 unstructured_ingest/v2/processes/connectors/utils.py,sha256=nmpZZCeX0O7rGrwHSWM_heBgpZK9tKT6EV1Moer-z40,576
-unstructured_ingest/v2/processes/connectors/weaviate.py,sha256=HtJuOUhBs_HA7uOXlEIuYtx0elb0ecsCvP8N822tOMQ,8564
+unstructured_ingest/v2/processes/connectors/weaviate.py,sha256=7H3s44zVKbN3_eR35sbKTKSDOt6ZIIQkX-4t65LuJ6c,8254
 unstructured_ingest/v2/processes/connectors/fsspec/__init__.py,sha256=TtdeImM7Ypl_n6sl7I1JqX6bGSG0t_FqvCqE3Cy24og,1846
 unstructured_ingest/v2/processes/connectors/fsspec/azure.py,sha256=RN7zoifocIWVgoP9aMDMz4TP-Z9KhE-HbCCBq33fY90,4674
 unstructured_ingest/v2/processes/connectors/fsspec/box.py,sha256=UnD-F9g7yOOBStrAqeKq6GuQjEyHdwOA3jYLj8YZIRM,4088
 unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py,sha256=I6mPG9EIso9TcIczCw5Y14Yqd-EhTQ2CLw1MJx1V3dY,4420
-unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py,sha256=zKrwKTVGnhnitD8h_Url5HRFsJZjM66o3jWrzAm-_UA,12153
+unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py,sha256=MgOUhDGtTAUuzmAsLvBwV_3ggyL5DDpMm-sb4KNck88,12689
 unstructured_ingest/v2/processes/connectors/fsspec/gcs.py,sha256=RYZq_8hKF7bRxuB5Gozv5AzB3_nTuuooE4UfRjXwEFU,4443
-unstructured_ingest/v2/processes/connectors/fsspec/s3.py,sha256=PXK9a5O3woDuBWSf4R5XLQI5mzHtap8wAKpHI8Rh5gQ,5462
+unstructured_ingest/v2/processes/connectors/fsspec/s3.py,sha256=7lOm5hjb0LBkbe-OWXnV3wDC-3mM_GWwwmdKW0xzh8c,5333
 unstructured_ingest/v2/processes/connectors/fsspec/sftp.py,sha256=J7Ej-j7dtXAluHunwynUfHlNsYwymb-LsrGUFcljcsA,5700
 unstructured_ingest/v2/processes/connectors/fsspec/utils.py,sha256=jec_Qfe2hbfahBuY-u8FnvHuv933AI5HwPFjOL3kEEY,456
-unstructured_ingest-0.0.2.dist-info/METADATA,sha256=a68Sz8-m1-ZRFz0p4yic64BhgwTuMdIMmCuPECdhWwA,21568
-unstructured_ingest-0.0.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-unstructured_ingest-0.0.2.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-0.0.2.dist-info/top_level.txt,sha256=QaTxTcjfM5Hr9sZJ6weOJvSe5ESQc0F8AWkhHInTCf8,20
-unstructured_ingest-0.0.2.dist-info/RECORD,,
+unstructured_ingest-0.0.2.dev0.dist-info/METADATA,sha256=dnWewzRLiYQlU6Fglws3oQnUkgzAVCnKM7BPMtls9YU,21573
+unstructured_ingest-0.0.2.dev0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+unstructured_ingest-0.0.2.dev0.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-0.0.2.dev0.dist-info/top_level.txt,sha256=QaTxTcjfM5Hr9sZJ6weOJvSe5ESQc0F8AWkhHInTCf8,20
+unstructured_ingest-0.0.2.dev0.dist-info/RECORD,,

unstructured_ingest/v2/cli/configs/filter.py

@@ -1,28 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.cli.utils import DelimitedString
-
-
-@dataclass
-class FilterCliConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--file-glob"],
-                default=None,
-                type=DelimitedString(),
-                help="A comma-separated list of file globs to limit which types of "
-                "local files are accepted, e.g. '*.html,*.txt'",
-            ),
-            click.Option(
-                ["--max-file-size"],
-                default=None,
-                type=click.IntRange(min=1),
-                help="Max file size to process in bytes",
-            ),
-        ]
-        return options

unstructured_ingest/v2/pipeline/steps/filter.py

@@ -1,40 +0,0 @@
-import asyncio
-from dataclasses import dataclass
-from typing import Callable, Optional
-
-from unstructured_ingest.v2.interfaces.file_data import FileData
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
-from unstructured_ingest.v2.pipeline.utils import sterilize_dict
-from unstructured_ingest.v2.processes.filter import Filterer
-
-STEP_ID = "filter"
-
-
-@dataclass
-class FilterStep(PipelineStep):
-    process: Filterer
-    identifier: str = STEP_ID
-
-    def __post_init__(self):
-        config = (
-            sterilize_dict(self.process.config.to_dict(redact_sensitive=True))
-            if self.process.config
-            else None
-        )
-        logger.info(f"Created {self.identifier} with configs: {config}")
-
-    async def _run_async(self, fn: Callable, file_data_path: str, **kwargs) -> Optional[dict]:
-        file_data = FileData.from_file(path=file_data_path)
-        fn_kwargs = {"file_data": file_data}
-        if not asyncio.iscoroutinefunction(fn):
-            resp = fn(**fn_kwargs)
-        elif semaphore := self.context.semaphore:
-            async with semaphore:
-                resp = await fn(**fn_kwargs)
-        else:
-            resp = await fn(**fn_kwargs)
-
-        if resp:
-            return {"file_data_path": file_data_path}
-        return None

unstructured_ingest/v2/processes/filter.py

@@ -1,54 +0,0 @@
-import fnmatch
-from abc import ABC
-from dataclasses import dataclass, field
-from typing import Any, Callable, Optional
-
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
-from unstructured_ingest.v2.interfaces import FileData
-from unstructured_ingest.v2.interfaces.process import BaseProcess
-from unstructured_ingest.v2.logger import logger
-
-
-@dataclass
-class FiltererConfig(EnhancedDataClassJsonMixin):
-    file_glob: Optional[list[str]] = None
-    max_file_size: Optional[int] = None
-
-
-@dataclass
-class Filterer(BaseProcess, ABC):
-    config: FiltererConfig = field(default_factory=lambda: FiltererConfig())
-    filters: list[Callable[[FileData], bool]] = field(init=False, default_factory=list)
-
-    def __post_init__(self):
-        # Populate the filters based on values in config
-        if self.config.file_glob is not None:
-            self.filters.append(self.glob_filter)
-        if self.config.max_file_size:
-            self.filters.append(self.file_size_filter)
-
-    def is_async(self) -> bool:
-        return False
-
-    def file_size_filter(self, file_data: FileData) -> bool:
-        if filesize_bytes := file_data.metadata.filesize_bytes:
-            return filesize_bytes <= self.config.max_file_size
-        return True
-
-    def glob_filter(self, file_data: FileData) -> bool:
-        patterns = self.config.file_glob
-        path = file_data.source_identifiers.fullpath
-        for pattern in patterns:
-            if fnmatch.filter([path], pattern):
-                return True
-        logger.debug(f"The file {path!r} is discarded as it does not match any given glob.")
-        return False
-
-    def run(self, file_data: FileData, **kwargs: Any) -> Optional[FileData]:
-        for filter in self.filters:
-            if not filter(file_data):
-                logger.debug(
-                    f"filtered out file data due to {filter.__name__}: {file_data.identifier}"
-                )
-                return None
-        return file_data