unstructured-ingest 0.3.13__py3-none-any.whl → 0.3.15__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/sql/databricks_delta_tables.py

@@ -0,0 +1,213 @@
+import json
+from contextlib import contextmanager
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Generator, Optional
+
+import numpy as np
+import pandas as pd
+from pydantic import Field, Secret
+
+from unstructured_ingest.utils.data_prep import split_dataframe
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+)
+from unstructured_ingest.v2.processes.connectors.sql.sql import (
+    SQLAccessConfig,
+    SQLConnectionConfig,
+    SQLUploader,
+    SQLUploaderConfig,
+    SQLUploadStager,
+    SQLUploadStagerConfig,
+)
+
+if TYPE_CHECKING:
+    from databricks.sdk.core import oauth_service_principal
+    from databricks.sql.client import Connection as DeltaTableConnection
+    from databricks.sql.client import Cursor as DeltaTableCursor
+
+CONNECTOR_TYPE = "databricks_delta_tables"
+
+
+class DatabrickDeltaTablesAccessConfig(SQLAccessConfig):
+    token: Optional[str] = Field(default=None, description="Databricks Personal Access Token")
+    client_id: Optional[str] = Field(default=None, description="Client ID of the OAuth app.")
+    client_secret: Optional[str] = Field(
+        default=None, description="Client Secret of the OAuth app."
+    )
+
+
+class DatabrickDeltaTablesConnectionConfig(SQLConnectionConfig):
+    access_config: Secret[DatabrickDeltaTablesAccessConfig]
+    server_hostname: str = Field(description="server hostname connection config value")
+    http_path: str = Field(description="http path connection config value")
+    user_agent: str = "unstructuredio_oss"
+
+    @requires_dependencies(["databricks"], extras="databricks-delta-tables")
+    def get_credentials_provider(self) -> "oauth_service_principal":
+        from databricks.sdk.core import Config, oauth_service_principal
+
+        host = f"https://{self.server_hostname}"
+        access_configs = self.access_config.get_secret_value()
+        if (client_id := access_configs.client_id) and (
+            client_secret := access_configs.client_secret
+        ):
+            return oauth_service_principal(
+                Config(
+                    host=host,
+                    client_id=client_id,
+                    client_secret=client_secret,
+                )
+            )
+        return False
+
+    def model_post_init(self, __context: Any) -> None:
+        access_config = self.access_config.get_secret_value()
+        if access_config.token and access_config.client_secret and access_config.client_id:
+            raise ValueError(
+                "One one for of auth can be provided, either token or client id and secret"
+            )
+        if not access_config.token and not (
+            access_config.client_secret and access_config.client_id
+        ):
+            raise ValueError(
+                "One form of auth must be provided, either token or client id and secret"
+            )
+
+    @contextmanager
+    @requires_dependencies(["databricks"], extras="databricks-delta-tables")
+    def get_connection(self, **connect_kwargs) -> Generator["DeltaTableConnection", None, None]:
+        from databricks.sql import connect
+
+        connect_kwargs = connect_kwargs or {}
+        connect_kwargs["_user_agent_entry"] = self.user_agent
+        connect_kwargs["server_hostname"] = connect_kwargs.get(
+            "server_hostname", self.server_hostname
+        )
+        connect_kwargs["http_path"] = connect_kwargs.get("http_path", self.http_path)
+
+        if credential_provider := self.get_credentials_provider():
+            connect_kwargs["credentials_provider"] = credential_provider
+        else:
+            connect_kwargs["access_token"] = self.access_config.get_secret_value().token
+        with connect(**connect_kwargs) as connection:
+            yield connection
+
+    @contextmanager
+    def get_cursor(self, **connect_kwargs) -> Generator["DeltaTableCursor", None, None]:
+        with self.get_connection(**connect_kwargs) as connection:
+            cursor = connection.cursor()
+            yield cursor
+
+
+class DatabrickDeltaTablesUploadStagerConfig(SQLUploadStagerConfig):
+    pass
+
+
+class DatabrickDeltaTablesUploadStager(SQLUploadStager):
+    upload_stager_config: DatabrickDeltaTablesUploadStagerConfig
+
+
+class DatabrickDeltaTablesUploaderConfig(SQLUploaderConfig):
+    catalog: str = Field(description="Name of the catalog in the Databricks Unity Catalog service")
+    database: str = Field(description="Database name", default="default")
+    table_name: str = Field(description="Table name")
+
+
+@dataclass
+class DatabrickDeltaTablesUploader(SQLUploader):
+    upload_config: DatabrickDeltaTablesUploaderConfig
+    connection_config: DatabrickDeltaTablesConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    @contextmanager
+    def get_cursor(self) -> Generator[Any, None, None]:
+        with self.connection_config.get_cursor() as cursor:
+            cursor.execute(f"USE CATALOG '{self.upload_config.catalog}'")
+            yield cursor
+
+    def precheck(self) -> None:
+        with self.connection_config.get_cursor() as cursor:
+            cursor.execute("SHOW CATALOGS")
+            catalogs = [r[0] for r in cursor.fetchall()]
+            if self.upload_config.catalog not in catalogs:
+                raise ValueError(
+                    "Catalog {} not found in {}".format(
+                        self.upload_config.catalog, ", ".join(catalogs)
+                    )
+                )
+            cursor.execute(f"USE CATALOG '{self.upload_config.catalog}'")
+            cursor.execute("SHOW DATABASES")
+            databases = [r[0] for r in cursor.fetchall()]
+            if self.upload_config.database not in databases:
+                raise ValueError(
+                    "Database {} not found in {}".format(
+                        self.upload_config.database, ", ".join(databases)
+                    )
+                )
+            cursor.execute("SHOW TABLES")
+            table_names = [r[1] for r in cursor.fetchall()]
+            if self.upload_config.table_name not in table_names:
+                raise ValueError(
+                    "Table {} not found in {}".format(
+                        self.upload_config.table_name, ", ".join(table_names)
+                    )
+                )
+
+    def create_statement(self, columns: list[str], values: tuple[Any, ...]) -> str:
+        values_list = []
+        for v in values:
+            if isinstance(v, dict):
+                values_list.append(json.dumps(v))
+            elif isinstance(v, list):
+                if v and isinstance(v[0], (int, float)):
+                    values_list.append("ARRAY({})".format(", ".join([str(val) for val in v])))
+                else:
+                    values_list.append("ARRAY({})".format(", ".join([f"'{val}'" for val in v])))
+            else:
+                values_list.append(f"'{v}'")
+        statement = "INSERT INTO {table_name} ({columns}) VALUES({values})".format(
+            table_name=self.upload_config.table_name,
+            columns=", ".join(columns),
+            values=", ".join(values_list),
+        )
+        return statement
+
+    def upload_dataframe(self, df: pd.DataFrame, file_data: FileData) -> None:
+        if self.can_delete():
+            self.delete_by_record_id(file_data=file_data)
+        else:
+            logger.warning(
+                f"table doesn't contain expected "
+                f"record id column "
+                f"{self.upload_config.record_id_key}, skipping delete"
+            )
+        df.replace({np.nan: None}, inplace=True)
+        self._fit_to_schema(df=df)
+
+        columns = list(df.columns)
+        logger.info(
+            f"writing a total of {len(df)} elements via"
+            f" document batches to destination"
+            f" table named {self.upload_config.table_name}"
+            # f" with batch size {self.upload_config.batch_size}"
+        )
+        # TODO: currently variable binding not supporting for list types,
+        #  update once that gets resolved in SDK
+        for rows in split_dataframe(df=df, chunk_size=self.upload_config.batch_size):
+            with self.get_cursor() as cursor:
+                values = self.prepare_data(columns, tuple(rows.itertuples(index=False, name=None)))
+                for v in values:
+                    stmt = self.create_statement(columns=columns, values=v)
+                    cursor.execute(stmt)
+
+
+databricks_delta_tables_destination_entry = DestinationRegistryEntry(
+    connection_config=DatabrickDeltaTablesConnectionConfig,
+    uploader=DatabrickDeltaTablesUploader,
+    uploader_config=DatabrickDeltaTablesUploaderConfig,
+    upload_stager=DatabrickDeltaTablesUploadStager,
+    upload_stager_config=DatabrickDeltaTablesUploadStagerConfig,
+)

unstructured_ingest/v2/processes/connectors/sql/sql.py

@@ -129,8 +129,13 @@ class SQLIndexer(Indexer, ABC):
     connection_config: SQLConnectionConfig
     index_config: SQLIndexerConfig
 
-    def _get_doc_ids(self) -> list[str]:
+    @contextmanager
+    def get_cursor(self) -> Generator[Any, None, None]:
         with self.connection_config.get_cursor() as cursor:
+            yield cursor
+
+    def _get_doc_ids(self) -> list[str]:
+        with self.get_cursor() as cursor:
             cursor.execute(
                 f"SELECT {self.index_config.id_column} FROM {self.index_config.table_name}"
             )
@@ -140,7 +145,7 @@ class SQLIndexer(Indexer, ABC):
 
     def precheck(self) -> None:
         try:
-            with self.connection_config.get_cursor() as cursor:
+            with self.get_cursor() as cursor:
                 cursor.execute("SELECT 1;")
         except Exception as e:
             logger.error(f"failed to validate connection: {e}", exc_info=True)
@@ -182,6 +187,11 @@ class SQLDownloader(Downloader, ABC):
     connection_config: SQLConnectionConfig
     download_config: SQLDownloaderConfig
 
+    @contextmanager
+    def get_cursor(self) -> Generator[Any, None, None]:
+        with self.connection_config.get_cursor() as cursor:
+            yield cursor
+
     @abstractmethod
     def query_db(self, file_data: SqlBatchFileData) -> tuple[list[tuple], list[str]]:
         pass
@@ -323,12 +333,17 @@ class SQLUploader(Uploader):
 
     def precheck(self) -> None:
         try:
-            with self.connection_config.get_cursor() as cursor:
+            with self.get_cursor() as cursor:
                 cursor.execute("SELECT 1;")
         except Exception as e:
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")
 
+    @contextmanager
+    def get_cursor(self) -> Generator[Any, None, None]:
+        with self.connection_config.get_cursor() as cursor:
+            yield cursor
+
     def prepare_data(
         self, columns: list[str], data: tuple[tuple[Any, ...], ...]
     ) -> list[tuple[Any, ...]]:
@@ -346,7 +361,7 @@ class SQLUploader(Uploader):
             output.append(tuple(parsed))
         return output
 
-    def _fit_to_schema(self, df: pd.DataFrame, columns: list[str]) -> pd.DataFrame:
+    def _fit_to_schema(self, df: pd.DataFrame) -> pd.DataFrame:
         columns = set(df.columns)
         schema_fields = set(columns)
         columns_to_drop = columns - schema_fields
@@ -367,6 +382,7 @@ class SQLUploader(Uploader):
 
         for column in missing_columns:
             df[column] = pd.Series()
+        return df
 
     def upload_dataframe(self, df: pd.DataFrame, file_data: FileData) -> None:
         if self.can_delete():
@@ -378,7 +394,7 @@ class SQLUploader(Uploader):
                 f"{self.upload_config.record_id_key}, skipping delete"
             )
         df.replace({np.nan: None}, inplace=True)
-        self._fit_to_schema(df=df, columns=self.get_table_columns())
+        self._fit_to_schema(df=df)
 
         columns = list(df.columns)
         stmt = "INSERT INTO {table_name} ({columns}) VALUES({values})".format(
@@ -393,7 +409,7 @@ class SQLUploader(Uploader):
             f" with batch size {self.upload_config.batch_size}"
         )
         for rows in split_dataframe(df=df, chunk_size=self.upload_config.batch_size):
-            with self.connection_config.get_cursor() as cursor:
+            with self.get_cursor() as cursor:
                 values = self.prepare_data(columns, tuple(rows.itertuples(index=False, name=None)))
                 # For debugging purposes:
                 # for val in values:
@@ -406,7 +422,7 @@ class SQLUploader(Uploader):
                 cursor.executemany(stmt, values)
 
     def get_table_columns(self) -> list[str]:
-        with self.connection_config.get_cursor() as cursor:
+        with self.get_cursor() as cursor:
             cursor.execute(f"SELECT * from {self.upload_config.table_name}")
             return [desc[0] for desc in cursor.description]
 
@@ -420,10 +436,11 @@ class SQLUploader(Uploader):
             f"from table {self.upload_config.table_name}"
         )
         stmt = f"DELETE FROM {self.upload_config.table_name} WHERE {self.upload_config.record_id_key} = {self.values_delimiter}"  # noqa: E501
-        with self.connection_config.get_cursor() as cursor:
+        with self.get_cursor() as cursor:
             cursor.execute(stmt, [file_data.identifier])
             rowcount = cursor.rowcount
-            logger.info(f"deleted {rowcount} rows from table {self.upload_config.table_name}")
+            if rowcount > 0:
+                logger.info(f"deleted {rowcount} rows from table {self.upload_config.table_name}")
 
     def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:
         df = pd.DataFrame(data)

{unstructured_ingest-0.3.13.dist-info → unstructured_ingest-0.3.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.3.13
+Version: 0.3.15
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -23,12 +23,12 @@ Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
 Requires-Dist: ndjson
-Requires-Dist: tqdm
 Requires-Dist: pydantic>=2.7
-Requires-Dist: click
-Requires-Dist: python-dateutil
 Requires-Dist: pandas
 Requires-Dist: dataclasses-json
+Requires-Dist: tqdm
+Requires-Dist: click
+Requires-Dist: python-dateutil
 Requires-Dist: opentelemetry-sdk
 Provides-Extra: airtable
 Requires-Dist: pyairtable; extra == "airtable"
@@ -40,8 +40,8 @@ Requires-Dist: fsspec; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: bedrock
-Requires-Dist: boto3; extra == "bedrock"
 Requires-Dist: aioboto3; extra == "bedrock"
+Requires-Dist: boto3; extra == "bedrock"
 Provides-Extra: biomed
 Requires-Dist: bs4; extra == "biomed"
 Requires-Dist: requests; extra == "biomed"
@@ -59,11 +59,13 @@ Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
 Provides-Extra: csv
 Requires-Dist: unstructured[tsv]; extra == "csv"
+Provides-Extra: databricks-delta-tables
+Requires-Dist: databricks-sql-connector; extra == "databricks-delta-tables"
 Provides-Extra: databricks-volumes
 Requires-Dist: databricks-sdk; extra == "databricks-volumes"
 Provides-Extra: delta-table
-Requires-Dist: deltalake; extra == "delta-table"
 Requires-Dist: boto3; extra == "delta-table"
+Requires-Dist: deltalake; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord.py; extra == "discord"
 Provides-Extra: doc
@@ -82,8 +84,8 @@ Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Provides-Extra: embed-octoai
-Requires-Dist: openai; extra == "embed-octoai"
 Requires-Dist: tiktoken; extra == "embed-octoai"
+Requires-Dist: openai; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
@@ -91,12 +93,12 @@ Requires-Dist: voyageai; extra == "embed-voyageai"
 Provides-Extra: epub
 Requires-Dist: unstructured[epub]; extra == "epub"
 Provides-Extra: gcs
-Requires-Dist: gcsfs; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
+Requires-Dist: gcsfs; extra == "gcs"
 Requires-Dist: fsspec; extra == "gcs"
 Provides-Extra: github
-Requires-Dist: requests; extra == "github"
 Requires-Dist: pygithub>1.58.0; extra == "github"
+Requires-Dist: requests; extra == "github"
 Provides-Extra: gitlab
 Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
@@ -121,30 +123,30 @@ Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: msg
 Requires-Dist: unstructured[msg]; extra == "msg"
 Provides-Extra: neo4j
-Requires-Dist: neo4j; extra == "neo4j"
 Requires-Dist: cymple; extra == "neo4j"
+Requires-Dist: neo4j; extra == "neo4j"
 Requires-Dist: networkx; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: htmlBuilder; extra == "notion"
-Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
+Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
+Requires-Dist: notion-client; extra == "notion"
 Provides-Extra: odt
 Requires-Dist: unstructured[odt]; extra == "odt"
 Provides-Extra: onedrive
+Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
 Requires-Dist: msal; extra == "onedrive"
-Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Provides-Extra: openai
-Requires-Dist: openai; extra == "openai"
 Requires-Dist: tiktoken; extra == "openai"
+Requires-Dist: openai; extra == "openai"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: org
 Requires-Dist: unstructured[org]; extra == "org"
 Provides-Extra: outlook
-Requires-Dist: msal; extra == "outlook"
 Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
+Requires-Dist: msal; extra == "outlook"
 Provides-Extra: pdf
 Requires-Dist: unstructured[pdf]; extra == "pdf"
 Provides-Extra: pinecone
@@ -168,16 +170,16 @@ Requires-Dist: unstructured[rst]; extra == "rst"
 Provides-Extra: rtf
 Requires-Dist: unstructured[rtf]; extra == "rtf"
 Provides-Extra: s3
-Requires-Dist: s3fs; extra == "s3"
 Requires-Dist: fsspec; extra == "s3"
+Requires-Dist: s3fs; extra == "s3"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
 Requires-Dist: paramiko; extra == "sftp"
 Requires-Dist: fsspec; extra == "sftp"
 Provides-Extra: sharepoint
-Requires-Dist: msal; extra == "sharepoint"
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
+Requires-Dist: msal; extra == "sharepoint"
 Provides-Extra: singlestore
 Requires-Dist: singlestoredb; extra == "singlestore"
 Provides-Extra: slack
@@ -190,9 +192,9 @@ Requires-Dist: together; extra == "togetherai"
 Provides-Extra: tsv
 Requires-Dist: unstructured[tsv]; extra == "tsv"
 Provides-Extra: vectara
-Requires-Dist: requests; extra == "vectara"
 Requires-Dist: httpx; extra == "vectara"
 Requires-Dist: aiofiles; extra == "vectara"
+Requires-Dist: requests; extra == "vectara"
 Provides-Extra: weaviate
 Requires-Dist: weaviate-client; extra == "weaviate"
 Provides-Extra: wikipedia

{unstructured_ingest-0.3.13.dist-info → unstructured_ingest-0.3.15.dist-info}/RECORD

@@ -5,42 +5,43 @@ test/integration/chunkers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJW
 test/integration/chunkers/test_chunkers.py,sha256=USkltQN_mVVCxI0FkJsrS1gnLXlVr-fvsc0tPaK2sWI,1062
 test/integration/connectors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/connectors/conftest.py,sha256=vYs4WDlCuieAwwErkJxCk4a1lGvr3qpeiAm-YaDznSo,1018
-test/integration/connectors/test_astradb.py,sha256=D_M8DvfHi36sp4irPQ5tTRxf0ujYenn_q-h0DU_lkzU,7201
-test/integration/connectors/test_azure_ai_search.py,sha256=EGV-G_Lq3h6pHhhmmQGWjIU1Mgsqg3ICZSgF0z3lZZs,9608
-test/integration/connectors/test_chroma.py,sha256=KQCzBJsOHAOtg0Ehp0tNtuYchFtiSmhHDKyOju33kJg,3686
-test/integration/connectors/test_confluence.py,sha256=adJxIggjuO-jgMimBZdv_AqWeBFlQoodELucIYwWC98,3546
-test/integration/connectors/test_delta_table.py,sha256=xsnJmwlWVQrccYeAtpt2lm0DYm2jGxiKXeERQXqCDCM,6884
-test/integration/connectors/test_lancedb.py,sha256=U2HfIrf6iJ7lYMn-vz0j-LesVyDY-jc9QrQhlJVhG9Q,9183
-test/integration/connectors/test_milvus.py,sha256=aRT5SpJHY4NA8pG_LcVTJwYwvLw2W_OOE-NIfDq03SE,7015
-test/integration/connectors/test_mongodb.py,sha256=UZ4eo61MisCw4s0p7HWaediN7M-lSddMDs71RFgdmJs,12347
-test/integration/connectors/test_neo4j.py,sha256=Esiq_Z9k1JLrWNXPmLBsX3LLwyEozwKoxX7iwMEJjRM,8252
-test/integration/connectors/test_notion.py,sha256=eNYJOMxSWJENsh6e79sFtuC1cVJRDWevO4ctshHN-sU,5087
-test/integration/connectors/test_onedrive.py,sha256=Bp9Ayv59JnfsjSwqbQ-zYvg-XAPGgZfKJ45Asc0y1bM,3808
-test/integration/connectors/test_pinecone.py,sha256=suPFi40d6rHXurQQLIpCzW5XRTdgzlP-f-KLPhGCUHo,10208
-test/integration/connectors/test_qdrant.py,sha256=hyuqSJDaylkQVxWh7byD8jo8bwPuBxSa8MWRD3sBu-Y,7906
-test/integration/connectors/test_redis.py,sha256=Q_KAZPNE9NIoRN2UsbXtc1fe_aJg66RbSQtS3OKNpc0,4327
-test/integration/connectors/test_s3.py,sha256=eA77LubtIhu6jkTrZ-rHIaJD2v9DTNhrkRcR7nmwi5g,7343
-test/integration/connectors/test_vectara.py,sha256=_FQHbhxL3f1rLV9MrHOvcljm_4qTVf5xl-Q7MplE_xs,8688
+test/integration/connectors/test_astradb.py,sha256=2DNNNum7cTKjsRvYaCu4doAGjhSN8vl-iHprFMDfQgk,7951
+test/integration/connectors/test_azure_ai_search.py,sha256=MxFwk84vI_HT4taQTGrNpJ8ewGPqHSGrx626j8hC_Pw,9695
+test/integration/connectors/test_chroma.py,sha256=NuQv0PWPM0_LQfdPeUd6IYKqaKKXWmVaHGWjq5aBfOY,3721
+test/integration/connectors/test_confluence.py,sha256=nVNeJ3ucLsQsj4Yo8QEyARVOnYy7HVxa_m7IHaIFtnM,3594
+test/integration/connectors/test_delta_table.py,sha256=4qm2Arfc9Eb7SOZOnOlLF-vNpHy6Eqvr5Q45svfX1PY,6911
+test/integration/connectors/test_lancedb.py,sha256=8MBxK_CUtOt87-4B7svDDK82NFII5psceo5cNN8HJMs,9228
+test/integration/connectors/test_milvus.py,sha256=7mI6zznN0PTxDL9DLogH1k3dxx6R8DgGzlpyevsFu2w,7173
+test/integration/connectors/test_mongodb.py,sha256=0A6DvF-iTCSZzOefisd_i20j9li8uNWTF2wyLGwlhco,12446
+test/integration/connectors/test_neo4j.py,sha256=r4TRYtTXeeOdcRcfa_gvslhSKvoIWrwN1FRJ5XRoH4k,8456
+test/integration/connectors/test_notion.py,sha256=ueXyVqYWzP4LuZYe6PauptkXNG6qkoV3srltFOSSKTA,5403
+test/integration/connectors/test_onedrive.py,sha256=TcMaa5BIp8J6engS4UZ2t19WQP0NNz2rkpBB47m7A3Y,3835
+test/integration/connectors/test_pinecone.py,sha256=acKEu1vnAk0Ht3FhCnGtOEKaj_YlgCzZB7wRU17ehQ0,12407
+test/integration/connectors/test_qdrant.py,sha256=Yme3ZZ5zIbaZ-yYLUqN2oy0hsrcAfvlleRLYWMSYeSE,8049
+test/integration/connectors/test_redis.py,sha256=1aKwOb-K4zCxZwHmgW_WzGJwqLntbWTbpGQ-rtUwN9o,4360
+test/integration/connectors/test_s3.py,sha256=E1dypeag_E3OIfpQWIz3jb7ctRHRD63UtyTrzyvJzpc,7473
+test/integration/connectors/test_vectara.py,sha256=4kKOOTGUjeZw2jKRcgVDI7ifbRPRZfjjVO4d_7H5C6I,8710
 test/integration/connectors/databricks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-test/integration/connectors/databricks/test_volumes_native.py,sha256=5gE6v5I3gRK_Xc9OORm4BL0JDJAQpegzvjdqPlLwDuE,9434
+test/integration/connectors/databricks/test_volumes_native.py,sha256=KqiapQAV0s_Zv0CO8BwYoiCk30dwrSZzuigUWNRIem0,9559
 test/integration/connectors/discord/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-test/integration/connectors/discord/test_discord.py,sha256=u1VxwlkHC1QTlfGjNvseQ1j94lUe_SAz-I1cAYl_YwY,3100
+test/integration/connectors/discord/test_discord.py,sha256=DDvlOA0lnojMigoCNnKUBUusV_Y0nI4q9lGnlb1gAQA,3176
 test/integration/connectors/duckdb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/connectors/duckdb/conftest.py,sha256=rlBHMJTiJ2a5xbvIxTOyhhcuTBc9DO-yTzD6Kf8X3hY,301
-test/integration/connectors/duckdb/test_duckdb.py,sha256=tZfHJYNILVqwT20XD-aJUFZ67TnJvHLpfAxNvNiE51o,2891
-test/integration/connectors/duckdb/test_motherduck.py,sha256=45anpo0bhbHJh84jOIoKULKrneCMSoq7GuFaBy-HS7g,3206
+test/integration/connectors/duckdb/test_duckdb.py,sha256=jGxmZhnWwoSf63EHftAejXqNY8zhe4S-yGzZzB1ZAWY,2979
+test/integration/connectors/duckdb/test_motherduck.py,sha256=vqR3uL3Cs6Cfl7ItCoMljvHmTUbci36YDslu6rswVjA,3224
 test/integration/connectors/elasticsearch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/connectors/elasticsearch/conftest.py,sha256=-i4_7MkIxSQENz7nuD2uHuhGU9mZ33vpeTPhHtRpQfs,989
-test/integration/connectors/elasticsearch/test_elasticsearch.py,sha256=Lm8topVssTsqcI8H2Tzohuxb9j-CFHv9orM6WfAqCZw,11933
-test/integration/connectors/elasticsearch/test_opensearch.py,sha256=fWpZrhzRiVpm9AOlZvgZRCjyXSYvWG7-8j06x-HR3PY,11311
+test/integration/connectors/elasticsearch/test_elasticsearch.py,sha256=TsSEPsyaTUoEvFBadinrdM0b5C4FoUtEwCv24OUbpO8,12072
+test/integration/connectors/elasticsearch/test_opensearch.py,sha256=7b7z0GqoBsBqA3IK35N6axmwEMjzJ1l3Fg2WT2c7uqs,11450
 test/integration/connectors/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-test/integration/connectors/sql/test_postgres.py,sha256=DXyHMZBQgrV2HyVflkoBpT1mewSnvw3ugoHtGR5o8OM,6876
-test/integration/connectors/sql/test_singlestore.py,sha256=pzCPo8IW3c9VH-f3UdJS5MjPjkHarJPSepAxV0ZVajo,6059
-test/integration/connectors/sql/test_snowflake.py,sha256=MiTzepeeJlv147CyzCGyd16MRk5QeUw4g4L3TTi5gVY,7400
-test/integration/connectors/sql/test_sqlite.py,sha256=rSkjv3KpslAvt_8LQecJUT0lOLtuZSvhtlW2deJovLI,5862
+test/integration/connectors/sql/test_databricks_delta_tables.py,sha256=UjVjw5hVoMSNJoYdoYympYow25gvcDAEHLmUmOJKz7I,5036
+test/integration/connectors/sql/test_postgres.py,sha256=bGDyzLRpgrXO7nl0U8nF2zSNr6ykUG-w8T4daIqUCG4,6970
+test/integration/connectors/sql/test_singlestore.py,sha256=XeU2s4Kt_3tGyaDYYKTgYjdOyb8j2dnz4TgSMwFUjWs,6153
+test/integration/connectors/sql/test_snowflake.py,sha256=LEwsRDoC6-rRiwYsqeo5B9Eo6RYygLLGAUsrtrgI9pM,7494
+test/integration/connectors/sql/test_sqlite.py,sha256=MHvhFRx1y_LTgfS-aPz-Zn9yOGsm-TF_s0t1seBzV1k,5956
 test/integration/connectors/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-test/integration/connectors/utils/constants.py,sha256=0zSPnsZVqJuNhXduXvdXFQLZTRIQa5Fo_1qjBYVCfb8,209
-test/integration/connectors/utils/docker.py,sha256=8uOTJ3AVG1dxK4OiLvOLfRxL_TsYQX2KKCID9TZ7-Ac,4995
+test/integration/connectors/utils/constants.py,sha256=JhTk6YNw7JVpkk-Pl8zn2YYkExeL1oE9VBWm_kMYGfo,369
+test/integration/connectors/utils/docker.py,sha256=4g1STiSbYN5qcmDTXyPxVJgwx97O6wk7n-DJ-zgzgag,4971
 test/integration/connectors/utils/docker_compose.py,sha256=GVTB6Cel05c0VQ2n4AwkQQx_cBfz13ZTs1HpbaYipNU,2223
 test/integration/connectors/utils/validation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/connectors/utils/validation/destination.py,sha256=ZvMSvqz9in35xaoUJGx9rG8oWCU3FYlfLLQ6sfdI0pw,2649
@@ -49,8 +50,8 @@ test/integration/connectors/utils/validation/source.py,sha256=VALU5ms_JBu_eFkp2W
 test/integration/connectors/utils/validation/utils.py,sha256=xYYvAbqP6_lZyH09_JjB4w2Sf8aQPvDVT5vZTs05ILs,1428
 test/integration/connectors/weaviate/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/connectors/weaviate/conftest.py,sha256=6Q6QdrLJmGHowRFSmoVSzup2EX6qASfS2Z5tqlpTm9M,387
-test/integration/connectors/weaviate/test_cloud.py,sha256=07VxNRxWWcgTstFfpoZ1FlVnEhcBnQlo5nosWKjKz_4,979
-test/integration/connectors/weaviate/test_local.py,sha256=SK6iEwQUKiCd0X99BEk8GlQoLaCcJcFPt09NN526Ct0,4508
+test/integration/connectors/weaviate/test_cloud.py,sha256=U1ZS6a7wTPX7h3XGvaJHaT-Uwg4IeGgzxx1YBywgVhM,1284
+test/integration/connectors/weaviate/test_local.py,sha256=bSJwS6rWxPf3BoOXKzZi2AOuT51py9V3tao6IBy1Rgk,4538
 test/integration/embedders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/embedders/conftest.py,sha256=B2W771RbijR7G_GybsCzRyIvOzXqzbKZdRIlNDd5AGY,334
 test/integration/embedders/test_azure_openai.py,sha256=6tFpKFBFRXD49imhhRzsvy3MPtuZ4L1PtnKyMVBRAqc,1808
@@ -96,7 +97,7 @@ test/unit/v2/partitioners/test_partitioner.py,sha256=iIYg7IpftV3LusoO4H8tr1IHY1U
 test/unit/v2/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/unit/v2/utils/data_generator.py,sha256=UoYVNjG4S4wlaA9gceQ82HIpF9_6I1UTHD1_GrQBHp0,973
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=-DFDA_b8O_Ur3J_Y_DIl2WMm2x49cF3ddoY_tPbNlUM,43
+unstructured_ingest/__version__.py,sha256=31lJzr6gfqqAcVEa6C2kjStzBSJPXWUyP7eRpa8Y7gI,43
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/interfaces.py,sha256=OYVUP0bzBJpT-Lz92BDyz_hLBvyfxkuSwWHhUdnUayA,31493
 unstructured_ingest/logger.py,sha256=S5nSqGcABoQyeicgRnBQFjDScCaTvFVivOCvbo-laL0,4479
@@ -386,7 +387,7 @@ unstructured_ingest/v2/interfaces/file_data.py,sha256=7MyRlj5dijQsCR6W18wQ8fEgJi
 unstructured_ingest/v2/interfaces/indexer.py,sha256=gsa1MLhFa82BzD2h4Yb7ons0VxRwKINZOrzvHAahwVU,846
 unstructured_ingest/v2/interfaces/process.py,sha256=BgglTu5K93FnDDopZKKr_rkK2LTZOguR6kcQjKHjF40,392
 unstructured_ingest/v2/interfaces/processor.py,sha256=VX7JqXlbG1plxMK8THWhWINPbTICaaUEk4XUXhnOixY,3303
-unstructured_ingest/v2/interfaces/upload_stager.py,sha256=HSSq_htv009-5yA8QqIi6rRnkfI1fnDkX5JRom8rNDY,3566
+unstructured_ingest/v2/interfaces/upload_stager.py,sha256=nbMuo_U6Gqn9bDJrAJTCjrZXKMw_G28OZOuNsT23i0k,3608
 unstructured_ingest/v2/interfaces/uploader.py,sha256=T2oHbN-d4Px1w1oATKKYZA10aUssqytEpiaqBM92r0Q,1600
 unstructured_ingest/v2/pipeline/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/v2/pipeline/interfaces.py,sha256=-Y6gPnl-SbNxIx5-dQCmiYSPKUMjivrRlBLIKIUWVeM,8658
@@ -424,22 +425,23 @@ unstructured_ingest/v2/processes/connectors/kdbai.py,sha256=VRDAiou_7oWOIAgQTdOG
 unstructured_ingest/v2/processes/connectors/local.py,sha256=ZvWTj6ZYkwnvQMNFsZWoaQyp9zp0WVqAywMaHJ2kcAc,7153
 unstructured_ingest/v2/processes/connectors/milvus.py,sha256=wmcu9NVy3gYlQGT25inN5w_QrhFoL8-hRq0pJFSNw8g,8866
 unstructured_ingest/v2/processes/connectors/mongodb.py,sha256=cL0QUQZF_s2brh3nNNeAywXVpaIiND4b5JTAFlYjLjw,14273
-unstructured_ingest/v2/processes/connectors/neo4j.py,sha256=QTw_Kq1_kHMdqsaBST6yW8vl-SYXVQFlIofDP1W_IuI,14250
+unstructured_ingest/v2/processes/connectors/neo4j.py,sha256=HU1IwchTM7Q1kkeIFVe-Lg6gInMItBpgkDkVwuTvkGY,14259
 unstructured_ingest/v2/processes/connectors/onedrive.py,sha256=d6gC40YmfqBNXxizAt4MO4OOu5BoCZ7SAe1AbNwTP0E,18322
 unstructured_ingest/v2/processes/connectors/outlook.py,sha256=KgNGM8hImRhy6_SpswRP2VwRD4VOrqqJoySgxf2oduI,9290
-unstructured_ingest/v2/processes/connectors/pinecone.py,sha256=cohF7gBj0opSGKXlENSdGfTtyIKMHd1pwu4ydeb7JAY,10605
+unstructured_ingest/v2/processes/connectors/pinecone.py,sha256=bQDCch7OGiQgpWO3n3ncLuQ4XCWqDc7ZWEB-Qrqkss8,10730
 unstructured_ingest/v2/processes/connectors/redisdb.py,sha256=p0AY4ukBNpwAemV4bWzpScvVbLTVlI3DzsCNUKiBI5M,6757
 unstructured_ingest/v2/processes/connectors/salesforce.py,sha256=2CiO2ZZiZ1Y1-nB7wcDlDVcpW2B7ut9wCj66rkkqho0,11616
 unstructured_ingest/v2/processes/connectors/sharepoint.py,sha256=Ndn2Wm7RupfjAtlLxxQwJueeE0V8aGMbNVPuFq9nqdQ,19730
 unstructured_ingest/v2/processes/connectors/slack.py,sha256=Z73VmQ3oUY09KoLEi5OBdQeDt4ONEY_02SglWQc6HXE,9252
 unstructured_ingest/v2/processes/connectors/utils.py,sha256=8kd0g7lo9NqnpaIkjeO-Ut6erhwUNH_gS9koevpe3WE,878
 unstructured_ingest/v2/processes/connectors/vectara.py,sha256=BlI_4nkpNR99aYxDd9eusm5LQsVB9EI0r-5Kc1D7pgQ,12255
-unstructured_ingest/v2/processes/connectors/databricks/__init__.py,sha256=jO71UTC7bLA_N12CrLWJzh_yZML5gfT7VohxzCpUGWg,1848
-unstructured_ingest/v2/processes/connectors/databricks/volumes.py,sha256=MDnTUjFlqOP4rmQA5wkgT2DhwjhFhUwPpUPGSzqCOOE,7577
+unstructured_ingest/v2/processes/connectors/databricks/__init__.py,sha256=Oh8SwTWi66gO8BsNF6vRMoQVuegyBPPCpVozkOHEf3A,2136
+unstructured_ingest/v2/processes/connectors/databricks/volumes.py,sha256=Yj4fIxgGo9Qh1x_6-INdbrPGHCuZElu-QKNfSVtW7F4,7694
 unstructured_ingest/v2/processes/connectors/databricks/volumes_aws.py,sha256=TA2e_1SIr4VaEI62873eyReCNfgmQ51_2Pko2I04pPM,2747
 unstructured_ingest/v2/processes/connectors/databricks/volumes_azure.py,sha256=cb-EUW0T-linZMkbU6AcKEGWnFHQvhpO5Abtps4P2X0,3532
 unstructured_ingest/v2/processes/connectors/databricks/volumes_gcp.py,sha256=tR8NubkyHw49IpW_42g6w1Koxlm56EPiPf1lB-eoRSI,2783
 unstructured_ingest/v2/processes/connectors/databricks/volumes_native.py,sha256=dJLD1fueXf8_0AfC4cg0G7siJZVefz68iuEx2Kq7rMs,2890
+unstructured_ingest/v2/processes/connectors/databricks/volumes_table.py,sha256=muj7G2JFO_WwAPub14k0VqDmN3c56t9MA60rM48wal8,4750
 unstructured_ingest/v2/processes/connectors/duckdb/__init__.py,sha256=5sVvJCWhU-YkjHIwk4W6BZCanFYK5W4xTpWtQ8xzeB4,561
 unstructured_ingest/v2/processes/connectors/duckdb/base.py,sha256=XTV9Pox3_xVmI8YVQWC9Bn6PugbPM49kp4Scv1OXFys,2649
 unstructured_ingest/v2/processes/connectors/duckdb/duckdb.py,sha256=oUHHaLpO2pWW2Lu4Mc-XFjrA0ze97205WQ_xP95ua4M,4296
@@ -538,20 +540,21 @@ unstructured_ingest/v2/processes/connectors/qdrant/cloud.py,sha256=accJ4sNWBVWV-
 unstructured_ingest/v2/processes/connectors/qdrant/local.py,sha256=cGEyv3Oy6y4BQ4DU8yhJWMpL82QYwBVdPTxxNuV127U,1588
 unstructured_ingest/v2/processes/connectors/qdrant/qdrant.py,sha256=BHI7HYSdbS05j2vrjyDvLzVG1WfsM8osKeq-lttlybQ,5437
 unstructured_ingest/v2/processes/connectors/qdrant/server.py,sha256=odvCZWZp8DmRxLXMR7tHhW-c7UQbix1_zpFdfXfCvKI,1613
-unstructured_ingest/v2/processes/connectors/sql/__init__.py,sha256=E16CXRBw8fZKTuXIECns5wif_I07oncBHskVxHC4p7w,1448
+unstructured_ingest/v2/processes/connectors/sql/__init__.py,sha256=mxcrncrjeP-C2jqQoTOOpGjV3Bmyfg4efT5lq_c-V1E,1760
+unstructured_ingest/v2/processes/connectors/sql/databricks_delta_tables.py,sha256=s_W6wSvyIXZ9mdAxvgSXFeFSze9E7pwIvc38p1hVDLM,8839
 unstructured_ingest/v2/processes/connectors/sql/postgres.py,sha256=BATfX1PQGT2kl8jAbdNKXTojYKJxh3pJV9-h3OBnHGo,5124
 unstructured_ingest/v2/processes/connectors/sql/singlestore.py,sha256=-2E9dsdNhjAiuzeSBytBbAhljOhvQ8kN8wvlUESvLo8,5465
 unstructured_ingest/v2/processes/connectors/sql/snowflake.py,sha256=8qCm1XiJmVxy8TSeoxwmQrE2W1x8S8At2ctrS_lJ8-I,7780
-unstructured_ingest/v2/processes/connectors/sql/sql.py,sha256=FtI5DSMd1QUgoYLn8NAVoETc4qwCbFEwSulqziXyODY,15292
+unstructured_ingest/v2/processes/connectors/sql/sql.py,sha256=ZGpeBfiOEzVaSiQxwqJkMC00Eu6TQhsrZKHnOHM0Xug,15667
 unstructured_ingest/v2/processes/connectors/sql/sqlite.py,sha256=Q5RAqn5Ccw-pbeKZLkiMn5IVw6EemCMukXzLlS7pDhc,5162
 unstructured_ingest/v2/processes/connectors/weaviate/__init__.py,sha256=NMiwnVWan69KnzVELvaqX34tMhCytIa-C8EDsXVKsEo,856
 unstructured_ingest/v2/processes/connectors/weaviate/cloud.py,sha256=bXtfEYLquR-BszZ5S_lQ4JbETNs9Vozgpfm8x9egAmE,6251
 unstructured_ingest/v2/processes/connectors/weaviate/embedded.py,sha256=S8Zg8StuZT-k7tCg1D5YShO1-vJYYk9-M1bE1fIqx64,3014
 unstructured_ingest/v2/processes/connectors/weaviate/local.py,sha256=LuTBKPseVewsz8VqxRPRLfGEm3BeI9nBZxpy7ZU5tOA,2201
 unstructured_ingest/v2/processes/connectors/weaviate/weaviate.py,sha256=X1yv1H_orDQ-J965EMXhR2XaURqe8vovSi9n1fk85B4,10499
-unstructured_ingest-0.3.13.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-0.3.13.dist-info/METADATA,sha256=RixQYNz-MYaC8p8BWxHv12aeOosZeetLkJ3mkHjzhL4,7813
-unstructured_ingest-0.3.13.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-unstructured_ingest-0.3.13.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-0.3.13.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
-unstructured_ingest-0.3.13.dist-info/RECORD,,
+unstructured_ingest-0.3.15.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-0.3.15.dist-info/METADATA,sha256=rZFAbiv0HZ-VUWVk4MP2vANZuzsxJLhK2_QWZ5zTjRA,7929
+unstructured_ingest-0.3.15.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+unstructured_ingest-0.3.15.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-0.3.15.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
+unstructured_ingest-0.3.15.dist-info/RECORD,,