unstructured-ingest 0.4.0__py3-none-any.whl → 0.4.2__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/sql/vastdb.py

@@ -0,0 +1,270 @@
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Optional
+
+import numpy as np
+import pandas as pd
+from pydantic import Field, Secret
+
+from unstructured_ingest.error import DestinationConnectionError
+from unstructured_ingest.utils.data_prep import split_dataframe
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.constants import RECORD_ID_LABEL
+from unstructured_ingest.v2.interfaces import (
+    FileData,
+)
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
+from unstructured_ingest.v2.processes.connectors.sql.sql import (
+    _COLUMNS,
+    SQLAccessConfig,
+    SqlBatchFileData,
+    SQLConnectionConfig,
+    SQLDownloader,
+    SQLDownloaderConfig,
+    SQLIndexer,
+    SQLIndexerConfig,
+    SQLUploader,
+    SQLUploaderConfig,
+    SQLUploadStager,
+    SQLUploadStagerConfig,
+)
+from unstructured_ingest.v2.utils import get_enhanced_element_id
+
+if TYPE_CHECKING:
+    from vastdb import connect as VastdbConnect
+    from vastdb import transaction as VastdbTransaction
+    from vastdb.table import Table as VastdbTable
+
+CONNECTOR_TYPE = "vastdb"
+
+
+class VastdbAccessConfig(SQLAccessConfig):
+    endpoint: Optional[str] = Field(default=None, description="DB endpoint")
+    access_key_id: Optional[str] = Field(default=None, description="access key id")
+    access_key_secret: Optional[str] = Field(default=None, description="access key secret")
+
+
+class VastdbConnectionConfig(SQLConnectionConfig):
+    access_config: Secret[VastdbAccessConfig] = Field(
+        default=VastdbAccessConfig(), validate_default=True
+    )
+    vastdb_bucket: str
+    vastdb_schema: str
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+
+    @requires_dependencies(["vastdb"], extras="vastdb")
+    @contextmanager
+    def get_connection(self) -> "VastdbConnect":
+        from vastdb import connect
+
+        access_config = self.access_config.get_secret_value()
+        connection = connect(
+            endpoint=access_config.endpoint,
+            access=access_config.access_key_id,
+            secret=access_config.access_key_secret,
+        )
+        yield connection
+
+    @contextmanager
+    def get_cursor(self) -> "VastdbTransaction":
+        with self.get_connection() as connection:
+            with connection.transaction() as transaction:
+                yield transaction
+
+    @contextmanager
+    def get_table(self, table_name: str) -> "VastdbTable":
+        with self.get_cursor() as cursor:
+            bucket = cursor.bucket(self.vastdb_bucket)
+            schema = bucket.schema(self.vastdb_schema)
+            table = schema.table(table_name)
+            yield table
+
+
+class VastdbIndexerConfig(SQLIndexerConfig):
+    pass
+
+
+@dataclass
+class VastdbIndexer(SQLIndexer):
+    connection_config: VastdbConnectionConfig
+    index_config: VastdbIndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def _get_doc_ids(self) -> list[str]:
+        with self.connection_config.get_table(self.index_config.table_name) as table:
+            reader = table.select(columns=[self.index_config.id_column])
+            results = reader.read_all()  # Build a PyArrow Table from the RecordBatchReader
+            ids = sorted([result[self.index_config.id_column] for result in results.to_pylist()])
+            return ids
+
+    def precheck(self) -> None:
+        try:
+            with self.connection_config.get_table(self.index_config.table_name) as table:
+                table.select()
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise DestinationConnectionError(f"failed to validate connection: {e}")
+
+
+class VastdbDownloaderConfig(SQLDownloaderConfig):
+    pass
+
+
+@dataclass
+class VastdbDownloader(SQLDownloader):
+    connection_config: VastdbConnectionConfig
+    download_config: VastdbDownloaderConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    @requires_dependencies(["ibis"], extras="vastdb")
+    def query_db(self, file_data: SqlBatchFileData) -> tuple[list[tuple], list[str]]:
+        from ibis import _  # imports the Ibis deferred expression
+
+        table_name = file_data.additional_metadata.table_name
+        id_column = file_data.additional_metadata.id_column
+        ids = tuple([item.identifier for item in file_data.batch_items])
+
+        with self.connection_config.get_table(table_name) as table:
+
+            predicate = _[id_column].isin(ids)
+
+            if self.download_config.fields:
+                # Vastdb requires the id column to be included in the fields
+                fields = self.download_config.fields + [id_column]
+                # dict.fromkeys to remove duplicates and keep order
+                reader = table.select(columns=list(dict.fromkeys(fields)), predicate=predicate)
+            else:
+                reader = table.select(predicate=predicate)
+            results = reader.read_all()
+            df = results.to_pandas()
+            return [tuple(r) for r in df.to_numpy()], results.column_names
+
+
+class VastdbUploadStagerConfig(SQLUploadStagerConfig):
+    rename_columns_map: Optional[dict] = Field(
+        default=None,
+        description="Map of column names to rename, ex: {'old_name': 'new_name'}",
+    )
+    additional_columns: Optional[list[str]] = Field(
+        default_factory=list, description="Additional columns to include in the upload"
+    )
+
+
+class VastdbUploadStager(SQLUploadStager):
+    upload_stager_config: VastdbUploadStagerConfig
+
+    def conform_dict(self, element_dict: dict, file_data: FileData) -> dict:
+        data = element_dict.copy()
+        metadata: dict[str, Any] = data.pop("metadata", {})
+        data_source = metadata.pop("data_source", {})
+        coordinates = metadata.pop("coordinates", {})
+
+        data.update(metadata)
+        data.update(data_source)
+        data.update(coordinates)
+
+        data["id"] = get_enhanced_element_id(element_dict=data, file_data=file_data)
+
+        # remove extraneous, not supported columns
+        # but also allow for additional columns
+        approved_columns = set(_COLUMNS).union(self.upload_stager_config.additional_columns)
+        element = {k: v for k, v in data.items() if k in approved_columns}
+        element[RECORD_ID_LABEL] = file_data.identifier
+        return element
+
+    def conform_dataframe(self, df: pd.DataFrame) -> pd.DataFrame:
+        df = super().conform_dataframe(df=df)
+        if self.upload_stager_config.rename_columns_map:
+            df.rename(columns=self.upload_stager_config.rename_columns_map, inplace=True)
+        return df
+
+
+class VastdbUploaderConfig(SQLUploaderConfig):
+    pass
+
+
+@dataclass
+class VastdbUploader(SQLUploader):
+    upload_config: VastdbUploaderConfig = field(default_factory=VastdbUploaderConfig)
+    connection_config: VastdbConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def precheck(self) -> None:
+        try:
+            with self.connection_config.get_table(self.upload_config.table_name) as table:
+                table.select()
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise DestinationConnectionError(f"failed to validate connection: {e}")
+
+    @requires_dependencies(["pyarrow"], extras="vastdb")
+    def upload_dataframe(self, df: pd.DataFrame, file_data: FileData) -> None:
+        import pyarrow as pa
+
+        if self.can_delete():
+            self.delete_by_record_id(file_data=file_data)
+        else:
+            logger.warning(
+                f"table doesn't contain expected "
+                f"record id column "
+                f"{self.upload_config.record_id_key}, skipping delete"
+            )
+        df.replace({np.nan: None}, inplace=True)
+        df = self._fit_to_schema(df=df)
+
+        logger.info(
+            f"writing a total of {len(df)} elements via"
+            f" document batches to destination"
+            f" table named {self.upload_config.table_name}"
+            f" with batch size {self.upload_config.batch_size}"
+        )
+
+        for rows in split_dataframe(df=df, chunk_size=self.upload_config.batch_size):
+
+            with self.connection_config.get_table(self.upload_config.table_name) as table:
+                pa_table = pa.Table.from_pandas(rows)
+                table.insert(pa_table)
+
+    def get_table_columns(self) -> list[str]:
+        if self._columns is None:
+            with self.connection_config.get_table(self.upload_config.table_name) as table:
+                self._columns = table.columns().names
+        return self._columns
+
+    @requires_dependencies(["ibis"], extras="vastdb")
+    def delete_by_record_id(self, file_data: FileData) -> None:
+        from ibis import _  # imports the Ibis deferred expression
+
+        logger.debug(
+            f"deleting any content with data "
+            f"{self.upload_config.record_id_key}={file_data.identifier} "
+            f"from table {self.upload_config.table_name}"
+        )
+        predicate = _[self.upload_config.record_id_key].isin([file_data.identifier])
+        with self.connection_config.get_table(self.upload_config.table_name) as table:
+            # Get the internal row id
+            rows_to_delete = table.select(
+                columns=[], predicate=predicate, internal_row_id=True
+            ).read_all()
+            table.delete(rows_to_delete)
+
+
+vastdb_source_entry = SourceRegistryEntry(
+    connection_config=VastdbConnectionConfig,
+    indexer_config=VastdbIndexerConfig,
+    indexer=VastdbIndexer,
+    downloader_config=VastdbDownloaderConfig,
+    downloader=VastdbDownloader,
+)
+
+vastdb_destination_entry = DestinationRegistryEntry(
+    connection_config=VastdbConnectionConfig,
+    uploader=VastdbUploader,
+    uploader_config=VastdbUploaderConfig,
+    upload_stager=VastdbUploadStager,
+    upload_stager_config=VastdbUploadStagerConfig,
+)

{unstructured_ingest-0.4.0.dist-info → unstructured_ingest-0.4.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.4.0
+Version: 0.4.2
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,32 +22,31 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: click
 Requires-Dist: pydantic>=2.7
+Requires-Dist: click
+Requires-Dist: tqdm
+Requires-Dist: dataclasses-json
 Requires-Dist: pandas
-Requires-Dist: ndjson
 Requires-Dist: opentelemetry-sdk
 Requires-Dist: python-dateutil
-Requires-Dist: tqdm
-Requires-Dist: dataclasses-json
 Provides-Extra: airtable
 Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
 Requires-Dist: astrapy; extra == "astradb"
 Provides-Extra: azure
-Requires-Dist: fsspec; extra == "azure"
 Requires-Dist: adlfs; extra == "azure"
+Requires-Dist: fsspec; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: bedrock
-Requires-Dist: boto3; extra == "bedrock"
 Requires-Dist: aioboto3; extra == "bedrock"
+Requires-Dist: boto3; extra == "bedrock"
 Provides-Extra: biomed
 Requires-Dist: bs4; extra == "biomed"
 Requires-Dist: requests; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: fsspec; extra == "box"
 Requires-Dist: boxfs; extra == "box"
+Requires-Dist: fsspec; extra == "box"
 Provides-Extra: chroma
 Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
@@ -64,8 +63,8 @@ Requires-Dist: databricks-sql-connector; extra == "databricks-delta-tables"
 Provides-Extra: databricks-volumes
 Requires-Dist: databricks-sdk; extra == "databricks-volumes"
 Provides-Extra: delta-table
-Requires-Dist: boto3; extra == "delta-table"
 Requires-Dist: deltalake; extra == "delta-table"
+Requires-Dist: boto3; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord.py; extra == "discord"
 Provides-Extra: doc
@@ -73,8 +72,8 @@ Requires-Dist: unstructured[docx]; extra == "doc"
 Provides-Extra: docx
 Requires-Dist: unstructured[docx]; extra == "docx"
 Provides-Extra: dropbox
-Requires-Dist: fsspec; extra == "dropbox"
 Requires-Dist: dropboxdrivefs; extra == "dropbox"
+Requires-Dist: fsspec; extra == "dropbox"
 Provides-Extra: duckdb
 Requires-Dist: duckdb; extra == "duckdb"
 Provides-Extra: elasticsearch
@@ -84,8 +83,8 @@ Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Provides-Extra: embed-octoai
-Requires-Dist: tiktoken; extra == "embed-octoai"
 Requires-Dist: openai; extra == "embed-octoai"
+Requires-Dist: tiktoken; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
@@ -93,8 +92,8 @@ Requires-Dist: voyageai; extra == "embed-voyageai"
 Provides-Extra: epub
 Requires-Dist: unstructured[epub]; extra == "epub"
 Provides-Extra: gcs
-Requires-Dist: fsspec; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
+Requires-Dist: fsspec; extra == "gcs"
 Requires-Dist: gcsfs; extra == "gcs"
 Provides-Extra: github
 Requires-Dist: pygithub>1.58.0; extra == "github"
@@ -104,8 +103,8 @@ Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: hubspot-api-client; extra == "hubspot"
 Requires-Dist: urllib3; extra == "hubspot"
+Requires-Dist: hubspot-api-client; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -123,23 +122,23 @@ Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: msg
 Requires-Dist: unstructured[msg]; extra == "msg"
 Provides-Extra: neo4j
+Requires-Dist: networkx; extra == "neo4j"
 Requires-Dist: cymple; extra == "neo4j"
 Requires-Dist: neo4j; extra == "neo4j"
-Requires-Dist: networkx; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
+Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
 Provides-Extra: odt
 Requires-Dist: unstructured[odt]; extra == "odt"
 Provides-Extra: onedrive
-Requires-Dist: bs4; extra == "onedrive"
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
+Requires-Dist: bs4; extra == "onedrive"
 Requires-Dist: msal; extra == "onedrive"
 Provides-Extra: openai
-Requires-Dist: tiktoken; extra == "openai"
 Requires-Dist: openai; extra == "openai"
+Requires-Dist: tiktoken; extra == "openai"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: org
@@ -175,8 +174,8 @@ Requires-Dist: s3fs; extra == "s3"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
-Requires-Dist: fsspec; extra == "sftp"
 Requires-Dist: paramiko; extra == "sftp"
+Requires-Dist: fsspec; extra == "sftp"
 Provides-Extra: sharepoint
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Requires-Dist: msal; extra == "sharepoint"
@@ -191,10 +190,14 @@ Provides-Extra: togetherai
 Requires-Dist: together; extra == "togetherai"
 Provides-Extra: tsv
 Requires-Dist: unstructured[tsv]; extra == "tsv"
+Provides-Extra: vastdb
+Requires-Dist: ibis; extra == "vastdb"
+Requires-Dist: pyarrow; extra == "vastdb"
+Requires-Dist: vastdb; extra == "vastdb"
 Provides-Extra: vectara
-Requires-Dist: httpx; extra == "vectara"
-Requires-Dist: requests; extra == "vectara"
 Requires-Dist: aiofiles; extra == "vectara"
+Requires-Dist: requests; extra == "vectara"
+Requires-Dist: httpx; extra == "vectara"
 Provides-Extra: weaviate
 Requires-Dist: weaviate-client; extra == "weaviate"
 Provides-Extra: wikipedia

{unstructured_ingest-0.4.0.dist-info → unstructured_ingest-0.4.2.dist-info}/RECORD

@@ -34,7 +34,7 @@ test/integration/connectors/elasticsearch/conftest.py,sha256=-i4_7MkIxSQENz7nuD2
 test/integration/connectors/elasticsearch/test_elasticsearch.py,sha256=TsSEPsyaTUoEvFBadinrdM0b5C4FoUtEwCv24OUbpO8,12072
 test/integration/connectors/elasticsearch/test_opensearch.py,sha256=7b7z0GqoBsBqA3IK35N6axmwEMjzJ1l3Fg2WT2c7uqs,11450
 test/integration/connectors/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-test/integration/connectors/sql/test_databricks_delta_tables.py,sha256=UjVjw5hVoMSNJoYdoYympYow25gvcDAEHLmUmOJKz7I,5036
+test/integration/connectors/sql/test_databricks_delta_tables.py,sha256=aC8B7peVYR8L2QaR18arqR6ffA197IsVkM2quCOVNSo,5046
 test/integration/connectors/sql/test_postgres.py,sha256=bGDyzLRpgrXO7nl0U8nF2zSNr6ykUG-w8T4daIqUCG4,6970
 test/integration/connectors/sql/test_singlestore.py,sha256=XeU2s4Kt_3tGyaDYYKTgYjdOyb8j2dnz4TgSMwFUjWs,6153
 test/integration/connectors/sql/test_snowflake.py,sha256=LEwsRDoC6-rRiwYsqeo5B9Eo6RYygLLGAUsrtrgI9pM,7494
@@ -45,7 +45,7 @@ test/integration/connectors/utils/docker.py,sha256=4g1STiSbYN5qcmDTXyPxVJgwx97O6
 test/integration/connectors/utils/docker_compose.py,sha256=GVTB6Cel05c0VQ2n4AwkQQx_cBfz13ZTs1HpbaYipNU,2223
 test/integration/connectors/utils/validation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/connectors/utils/validation/destination.py,sha256=ZvMSvqz9in35xaoUJGx9rG8oWCU3FYlfLLQ6sfdI0pw,2649
-test/integration/connectors/utils/validation/equality.py,sha256=8riUASLHccqnRZLoEv4rz5_ba54PFEHcxfxuJaJy6X0,2570
+test/integration/connectors/utils/validation/equality.py,sha256=R6d_1c-Si5518WJcBcshF_wBRnywnZ0ORQ-NL0xNmGo,2602
 test/integration/connectors/utils/validation/source.py,sha256=VALU5ms_JBu_eFkp2WQ7oZtJKozJ8MZSJ7h7ZA3Fz_Q,12296
 test/integration/connectors/utils/validation/utils.py,sha256=xYYvAbqP6_lZyH09_JjB4w2Sf8aQPvDVT5vZTs05ILs,1428
 test/integration/connectors/weaviate/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -83,8 +83,10 @@ test/unit/v2/chunkers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
 test/unit/v2/chunkers/test_chunkers.py,sha256=HSr3_lsoMw1nkDhkjO0-NOTEomRdR9oxCrSXvcMFecE,1772
 test/unit/v2/connectors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/unit/v2/connectors/test_confluence.py,sha256=bXrn_kRb4IQdqkk4rc-P2gJAtPba7n7pNplQgfbqZDY,1047
+test/unit/v2/connectors/databricks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+test/unit/v2/connectors/databricks/test_volumes_table.py,sha256=-R_EJHqv1BseGRK9VRAZhF-2EXA64LAlhycoyIu556U,1078
 test/unit/v2/connectors/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-test/unit/v2/connectors/sql/test_sql.py,sha256=6BfOxNqIvf6OoxGmCSucEqDkUOmaKgA6E1FvwOyB9pc,2412
+test/unit/v2/connectors/sql/test_sql.py,sha256=51-AKUBxw6ThO68bjenLopUUuxM88YZb2rMUV8L6YwY,2464
 test/unit/v2/embedders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/unit/v2/embedders/test_bedrock.py,sha256=sW-Vv-u3Yiw8rHPOfE5x_reywXlnozxO49rIMx6_xjo,1071
 test/unit/v2/embedders/test_huggingface.py,sha256=mkVPym7TZkRJchwHedujgFXWdL9sVMi1W90jpmZ_vxg,1543
@@ -99,7 +101,7 @@ test/unit/v2/partitioners/test_partitioner.py,sha256=iIYg7IpftV3LusoO4H8tr1IHY1U
 test/unit/v2/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/unit/v2/utils/data_generator.py,sha256=UoYVNjG4S4wlaA9gceQ82HIpF9_6I1UTHD1_GrQBHp0,973
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=uG7rkjX1YTBF6-J_mvj3gBQn-ZgW3qMoKBfjt0oZDS8,42
+unstructured_ingest/__version__.py,sha256=Y85nIpRVpjjjl2MW3ZwhLs55JjhABkZJeXfKDAbsRxM,42
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/interfaces.py,sha256=OYVUP0bzBJpT-Lz92BDyz_hLBvyfxkuSwWHhUdnUayA,31493
 unstructured_ingest/logger.py,sha256=S5nSqGcABoQyeicgRnBQFjDScCaTvFVivOCvbo-laL0,4479
@@ -358,9 +360,11 @@ unstructured_ingest/runner/writers/fsspec/s3.py,sha256=kHJq2O3864QBd_tL2SKb0mdyw
 unstructured_ingest/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/utils/chunking.py,sha256=9b3sXMA6L8RW5xAkKQbwdtVudGLAcj_sgT6Grh5tyYM,1870
 unstructured_ingest/utils/compression.py,sha256=NNiY-2S2Gf3at7zC1PYxMijaEza9vVSzRn5mdFf6mHo,4434
-unstructured_ingest/utils/data_prep.py,sha256=FB0LXNvxXQWQSXyn4VecRzcFavIG6kh49yQ-PGsc76k,6837
+unstructured_ingest/utils/data_prep.py,sha256=X3d8Kos1zqX-HQAicF_8TB0BrstRtHrbMzu_1s7mj7M,7191
 unstructured_ingest/utils/dep_check.py,sha256=SXXcUna2H0RtxA6j1S2NGkvQa9JP2DujWhmyBa7776Y,2400
 unstructured_ingest/utils/google_filetype.py,sha256=YVspEkiiBrRUSGVeVbsavvLvTmizdy2e6TsjigXTSRU,468
+unstructured_ingest/utils/html.py,sha256=gORKKCkva71JBbOilYtAn_MLLCqV8VKmSjSbpwEOlno,4257
+unstructured_ingest/utils/ndjson.py,sha256=nz8VUOPEgAFdhaDOpuveknvCU4x82fVwqE01qAbElH0,1201
 unstructured_ingest/utils/string_and_date_utils.py,sha256=kijtPlGAbH376vVjFSo5H_ZhW-FEcMC2sCNsSNwDOjo,1729
 unstructured_ingest/utils/table.py,sha256=aWjcowDVSClNpEAdR6PY3H7khKu4T6T3QqQE6GjmQ_M,3469
 unstructured_ingest/v2/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
@@ -389,19 +393,19 @@ unstructured_ingest/v2/interfaces/file_data.py,sha256=7MyRlj5dijQsCR6W18wQ8fEgJi
 unstructured_ingest/v2/interfaces/indexer.py,sha256=gsa1MLhFa82BzD2h4Yb7ons0VxRwKINZOrzvHAahwVU,846
 unstructured_ingest/v2/interfaces/process.py,sha256=BgglTu5K93FnDDopZKKr_rkK2LTZOguR6kcQjKHjF40,392
 unstructured_ingest/v2/interfaces/processor.py,sha256=VX7JqXlbG1plxMK8THWhWINPbTICaaUEk4XUXhnOixY,3303
-unstructured_ingest/v2/interfaces/upload_stager.py,sha256=nbMuo_U6Gqn9bDJrAJTCjrZXKMw_G28OZOuNsT23i0k,3608
+unstructured_ingest/v2/interfaces/upload_stager.py,sha256=9EV9863ODDv0Y5liDT3xh2yiVuFiaVVyCcnwCy6nfkM,3172
 unstructured_ingest/v2/interfaces/uploader.py,sha256=T2oHbN-d4Px1w1oATKKYZA10aUssqytEpiaqBM92r0Q,1600
 unstructured_ingest/v2/pipeline/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/v2/pipeline/interfaces.py,sha256=-Y6gPnl-SbNxIx5-dQCmiYSPKUMjivrRlBLIKIUWVeM,8658
 unstructured_ingest/v2/pipeline/otel.py,sha256=K3pQvWVgWzyOWMKCBUofsH7wTZPJ0Ysw5sLjMBLW41I,1088
 unstructured_ingest/v2/pipeline/pipeline.py,sha256=7Yg8_xwlSX6lA-oPGlTcn6KXZ9kc51zsoJxME5TiUlw,15956
 unstructured_ingest/v2/pipeline/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-unstructured_ingest/v2/pipeline/steps/chunk.py,sha256=wmGYmydm3CfoaRd0vXzou-o9h7-3pc2OOeVbzH6QZAI,3212
+unstructured_ingest/v2/pipeline/steps/chunk.py,sha256=LK2ldM24TE4ukX_Z6Z81LpF53orMaRkddM3uhLtT5EQ,3221
 unstructured_ingest/v2/pipeline/steps/download.py,sha256=nZ4B0d9p-6TgWqrBoKUQPlr8m6dz1RGNr_3OjUhRpWg,8259
-unstructured_ingest/v2/pipeline/steps/embed.py,sha256=gafBvg-9CECHxxR0ptm00IwtJyLw1bTXL0yMDz13j0I,3190
+unstructured_ingest/v2/pipeline/steps/embed.py,sha256=iL6X0G5AvKnlfI-3XRWudlb0-6rD_PqyzA3MFmmcn6M,3199
 unstructured_ingest/v2/pipeline/steps/filter.py,sha256=pju7knTSbB2ll1jC9DPePRDnHlOlvEcU1-sjk6xYGGc,1211
 unstructured_ingest/v2/pipeline/steps/index.py,sha256=uIiGZeI9pFxkwS91IldXE37UUwAopsinfUgGNL7WJaw,3555
-unstructured_ingest/v2/pipeline/steps/partition.py,sha256=5Rr1-J6iGnmFgd02P9v3ExYCqq7QlT0mWZaheuDG6Zk,3273
+unstructured_ingest/v2/pipeline/steps/partition.py,sha256=IJQWaOTcyFlH2bz8WbmynE5Zkd5D8ELOKTnSCnt9Wcc,3282
 unstructured_ingest/v2/pipeline/steps/stage.py,sha256=VR8SLUJdVva61aieVKyxUHzupTCQbQeaMA0CKu4Fx7o,2347
 unstructured_ingest/v2/pipeline/steps/uncompress.py,sha256=p2nPFGbcpivPAZO5jDogTfn0iaL5bCFsgBNMejxVbzE,1768
 unstructured_ingest/v2/pipeline/steps/upload.py,sha256=We4OAtStuZwWKKBCOPhfeAz_vjQt5hgsy_jRCxPzxo4,2010
@@ -417,7 +421,7 @@ unstructured_ingest/v2/processes/connectors/airtable.py,sha256=eeZJe-bBNxt5Sa-XE
 unstructured_ingest/v2/processes/connectors/astradb.py,sha256=xhUMoUdnrfAY1isZGqsV4lZUsnZNpbvgLyQWQbR4hVo,14814
 unstructured_ingest/v2/processes/connectors/azure_ai_search.py,sha256=ngPDpU0oZ6m5sxIlB6u5ebQpqCS_SJ-_amCC1KQ03EQ,11529
 unstructured_ingest/v2/processes/connectors/chroma.py,sha256=VHCnM56qNXuHzovJihrNfJnZbWLJShOe8j12PJFrbL0,7219
-unstructured_ingest/v2/processes/connectors/confluence.py,sha256=dE2kEQJT2QamFLsdCHpac_D5G4zTN7RIFRIEvt6XBPc,8238
+unstructured_ingest/v2/processes/connectors/confluence.py,sha256=OdoMK5ZD2HOncquj9c_Xct7bFa6kSGW3qZwfiN1LqtQ,11399
 unstructured_ingest/v2/processes/connectors/couchbase.py,sha256=i7vuNKsUkN93JRVmg4--MO0ZgbjvhIqt46oYqk9zFSQ,12250
 unstructured_ingest/v2/processes/connectors/delta_table.py,sha256=SotSXZQ85_6TO906YvFi3yTml8jE9A_zV6nBJ4oTx8A,7075
 unstructured_ingest/v2/processes/connectors/discord.py,sha256=-e4-cBK4TnHkknK1qIb86AIVMy81lBgC288_iLpTzM8,5246
@@ -443,22 +447,22 @@ unstructured_ingest/v2/processes/connectors/databricks/volumes_aws.py,sha256=TA2
 unstructured_ingest/v2/processes/connectors/databricks/volumes_azure.py,sha256=cb-EUW0T-linZMkbU6AcKEGWnFHQvhpO5Abtps4P2X0,3532
 unstructured_ingest/v2/processes/connectors/databricks/volumes_gcp.py,sha256=tR8NubkyHw49IpW_42g6w1Koxlm56EPiPf1lB-eoRSI,2783
 unstructured_ingest/v2/processes/connectors/databricks/volumes_native.py,sha256=dJLD1fueXf8_0AfC4cg0G7siJZVefz68iuEx2Kq7rMs,2890
-unstructured_ingest/v2/processes/connectors/databricks/volumes_table.py,sha256=muj7G2JFO_WwAPub14k0VqDmN3c56t9MA60rM48wal8,4750
+unstructured_ingest/v2/processes/connectors/databricks/volumes_table.py,sha256=2KNLwDZJDhsMAUGCzktEIn4Lvb0nxLWabBOPJbgyoEE,5010
 unstructured_ingest/v2/processes/connectors/duckdb/__init__.py,sha256=5sVvJCWhU-YkjHIwk4W6BZCanFYK5W4xTpWtQ8xzeB4,561
-unstructured_ingest/v2/processes/connectors/duckdb/base.py,sha256=XTV9Pox3_xVmI8YVQWC9Bn6PugbPM49kp4Scv1OXFys,2649
+unstructured_ingest/v2/processes/connectors/duckdb/base.py,sha256=0YBdOpTX5mbRLhP00lRHSMpl2-LfuRpqB1XPMJMxn04,2647
 unstructured_ingest/v2/processes/connectors/duckdb/duckdb.py,sha256=oUHHaLpO2pWW2Lu4Mc-XFjrA0ze97205WQ_xP95ua4M,4296
 unstructured_ingest/v2/processes/connectors/duckdb/motherduck.py,sha256=mU5x6SnbFgRsVicNGh4y4gtR6ek7eQFinI0dQQmzMds,4481
 unstructured_ingest/v2/processes/connectors/elasticsearch/__init__.py,sha256=Zzc0JNPP-eFqpwWw1Gp-XC8H-s__IgkYKzoagECycZY,829
 unstructured_ingest/v2/processes/connectors/elasticsearch/elasticsearch.py,sha256=MEKU64OsiQmbLPb3ken-WWCIV6-pnFbs_6kjJweG-SY,18813
 unstructured_ingest/v2/processes/connectors/elasticsearch/opensearch.py,sha256=qRz8Fyr2RSZIPZGkhPeme6AZxM0aX-c_xOa1ZtSr2Kg,6781
 unstructured_ingest/v2/processes/connectors/fsspec/__init__.py,sha256=TtdeImM7Ypl_n6sl7I1JqX6bGSG0t_FqvCqE3Cy24og,1846
-unstructured_ingest/v2/processes/connectors/fsspec/azure.py,sha256=c_ivsh8zWcz2YzKCPyLkyflg7a7nCtMRZejJF8F3g_o,6933
-unstructured_ingest/v2/processes/connectors/fsspec/box.py,sha256=hJ8Xli89DAgvIHI2CG8uqYvgFlmQn-AoFDRipOFprDc,5658
-unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py,sha256=IoSm1DfWPqJveXUBPR_pZCILQ404KKksR9Fv8kuxfZA,5707
-unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py,sha256=rvI0_a5vFB7jU3aThnHxG-AnCxQkV6Jz-TiSpxEjb-s,12764
-unstructured_ingest/v2/processes/connectors/fsspec/gcs.py,sha256=xiAewsD5wbkdXDZhwUNTFTORhJV2eLlO4mqqLSf7v0I,6954
-unstructured_ingest/v2/processes/connectors/fsspec/s3.py,sha256=8mg25go9bZypbBDb-VSkMOJvh6bEKcRHv_qKrXkqDZg,6400
-unstructured_ingest/v2/processes/connectors/fsspec/sftp.py,sha256=JsxXKXNI20mdgwR_A6Rnf4u8fsFwLe3AkJmIe_3NEKY,6150
+unstructured_ingest/v2/processes/connectors/fsspec/azure.py,sha256=fwbHYoRrN0ZRuLdLb5X-Z7nr11rMSY8VhWMhfR3ljQo,6933
+unstructured_ingest/v2/processes/connectors/fsspec/box.py,sha256=VXxEfgJbW8DCOrqLW7mQkSeWqH-HczidTNIE28SgERY,5658
+unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py,sha256=GflyMNCKxYRj6hgO1btyrZ4hx3lXOwbWjHViRw1LIWw,5707
+unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py,sha256=0Z--cPh17W_j4jQkSe2BeeD_j0Tt147Z01gqqF58Z9A,14421
+unstructured_ingest/v2/processes/connectors/fsspec/gcs.py,sha256=uOfm2tLc0r5U3CNkfauuwhGOhP7RJpjyBpHWMDXCk7c,6954
+unstructured_ingest/v2/processes/connectors/fsspec/s3.py,sha256=LcfIU-QgW5oVMF4jMUVm7HSgVcSrQamY6mgXdQuiSjc,6400
+unstructured_ingest/v2/processes/connectors/fsspec/sftp.py,sha256=3cVwVH3fT_JEYzIbl48-NDXdbo7XWX4C4_eqTvgWIro,6150
 unstructured_ingest/v2/processes/connectors/fsspec/utils.py,sha256=jec_Qfe2hbfahBuY-u8FnvHuv933AI5HwPFjOL3kEEY,456
 unstructured_ingest/v2/processes/connectors/kafka/__init__.py,sha256=mQJ9Ex-QCfhz-BB5YWTfbPf7xGLd1i7FpjRr0ukbhNw,754
 unstructured_ingest/v2/processes/connectors/kafka/cloud.py,sha256=GdAeQ8Uz-6v1C5byBHtjfevVfbzW3obScBFFLRTb0ps,3441
@@ -542,21 +546,22 @@ unstructured_ingest/v2/processes/connectors/qdrant/cloud.py,sha256=accJ4sNWBVWV-
 unstructured_ingest/v2/processes/connectors/qdrant/local.py,sha256=cGEyv3Oy6y4BQ4DU8yhJWMpL82QYwBVdPTxxNuV127U,1588
 unstructured_ingest/v2/processes/connectors/qdrant/qdrant.py,sha256=BHI7HYSdbS05j2vrjyDvLzVG1WfsM8osKeq-lttlybQ,5437
 unstructured_ingest/v2/processes/connectors/qdrant/server.py,sha256=odvCZWZp8DmRxLXMR7tHhW-c7UQbix1_zpFdfXfCvKI,1613
-unstructured_ingest/v2/processes/connectors/sql/__init__.py,sha256=mxcrncrjeP-C2jqQoTOOpGjV3Bmyfg4efT5lq_c-V1E,1760
-unstructured_ingest/v2/processes/connectors/sql/databricks_delta_tables.py,sha256=s_W6wSvyIXZ9mdAxvgSXFeFSze9E7pwIvc38p1hVDLM,8839
+unstructured_ingest/v2/processes/connectors/sql/__init__.py,sha256=NSEZwJDHh_9kFc31LnG14iRtYF3meK2UfUlQfYnwYEQ,2059
+unstructured_ingest/v2/processes/connectors/sql/databricks_delta_tables.py,sha256=SRNplobVKd8fSeauYbLzBNlMb3HRHhinFS281B8aYtY,8854
 unstructured_ingest/v2/processes/connectors/sql/postgres.py,sha256=BATfX1PQGT2kl8jAbdNKXTojYKJxh3pJV9-h3OBnHGo,5124
-unstructured_ingest/v2/processes/connectors/sql/singlestore.py,sha256=-2E9dsdNhjAiuzeSBytBbAhljOhvQ8kN8wvlUESvLo8,5465
+unstructured_ingest/v2/processes/connectors/sql/singlestore.py,sha256=OPBDQ2c_5KjWHEFfqXxf3pQ2tWC-N4MtslMulMgP1Wc,5503
 unstructured_ingest/v2/processes/connectors/sql/snowflake.py,sha256=QE-WBqrPVjCgcxR5EdVD9iTHBjgDSSSQgWYvq5N61qU,7746
-unstructured_ingest/v2/processes/connectors/sql/sql.py,sha256=MZPj-o4OdIrcMPmzMk851w21i9Ms0fm7dTiqegp773k,15813
-unstructured_ingest/v2/processes/connectors/sql/sqlite.py,sha256=Q5RAqn5Ccw-pbeKZLkiMn5IVw6EemCMukXzLlS7pDhc,5162
+unstructured_ingest/v2/processes/connectors/sql/sql.py,sha256=O2XBu_E2WqNia9OUTdhTWkYo0xhoMMm6ZuanTz-0V9s,16192
+unstructured_ingest/v2/processes/connectors/sql/sqlite.py,sha256=PRjN_S7UQv0k4ZpSyclW1AJrsrugyxbR-GoOrHvBpks,5200
+unstructured_ingest/v2/processes/connectors/sql/vastdb.py,sha256=4DckpVAXpmMTcoKrWiJbnFQQlcrwMA-GMaDsAYchTUs,9992
 unstructured_ingest/v2/processes/connectors/weaviate/__init__.py,sha256=NMiwnVWan69KnzVELvaqX34tMhCytIa-C8EDsXVKsEo,856
 unstructured_ingest/v2/processes/connectors/weaviate/cloud.py,sha256=bXtfEYLquR-BszZ5S_lQ4JbETNs9Vozgpfm8x9egAmE,6251
 unstructured_ingest/v2/processes/connectors/weaviate/embedded.py,sha256=S8Zg8StuZT-k7tCg1D5YShO1-vJYYk9-M1bE1fIqx64,3014
 unstructured_ingest/v2/processes/connectors/weaviate/local.py,sha256=LuTBKPseVewsz8VqxRPRLfGEm3BeI9nBZxpy7ZU5tOA,2201
 unstructured_ingest/v2/processes/connectors/weaviate/weaviate.py,sha256=X1yv1H_orDQ-J965EMXhR2XaURqe8vovSi9n1fk85B4,10499
-unstructured_ingest-0.4.0.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-0.4.0.dist-info/METADATA,sha256=YCJmfdIJGlE9lfszi5ybee8wf0rTrB58YmBakeuZ8ac,7928
-unstructured_ingest-0.4.0.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-unstructured_ingest-0.4.0.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-0.4.0.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
-unstructured_ingest-0.4.0.dist-info/RECORD,,
+unstructured_ingest-0.4.2.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-0.4.2.dist-info/METADATA,sha256=-3ILUK1wZ1fDgJcT22FO9ZhM_NKKHNBCLvgWBgzvVOY,8051
+unstructured_ingest-0.4.2.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+unstructured_ingest-0.4.2.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-0.4.2.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
+unstructured_ingest-0.4.2.dist-info/RECORD,,