unstructured-ingest 0.2.1__py3-none-any.whl → 0.2.2__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/qdrant/qdrant.py

@@ -0,0 +1,168 @@
+import asyncio
+import json
+import uuid
+from abc import ABC, abstractmethod
+from contextlib import asynccontextmanager
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.error import DestinationConnectionError, WriteError
+from unstructured_ingest.utils.data_prep import batch_generator, flatten_dict
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    FileData,
+    Uploader,
+    UploaderConfig,
+    UploadStager,
+    UploadStagerConfig,
+)
+from unstructured_ingest.v2.logger import logger
+
+if TYPE_CHECKING:
+    from qdrant_client import AsyncQdrantClient
+
+
+class QdrantAccessConfig(AccessConfig, ABC):
+    pass
+
+
+class QdrantConnectionConfig(ConnectionConfig, ABC):
+    access_config: Secret[QdrantAccessConfig] = Field(
+        default_factory=QdrantAccessConfig, validate_default=True, description="Access Config"
+    )
+
+    @abstractmethod
+    def get_client_kwargs(self) -> dict:
+        pass
+
+    @requires_dependencies(["qdrant_client"], extras="qdrant")
+    @asynccontextmanager
+    async def get_client(self) -> AsyncGenerator["AsyncQdrantClient", None]:
+        from qdrant_client.async_qdrant_client import AsyncQdrantClient
+
+        client_kwargs = self.get_client_kwargs()
+        client = AsyncQdrantClient(**client_kwargs)
+        try:
+            yield client
+        finally:
+            await client.close()
+
+
+class QdrantUploadStagerConfig(UploadStagerConfig):
+    pass
+
+
+@dataclass
+class QdrantUploadStager(UploadStager, ABC):
+    upload_stager_config: QdrantUploadStagerConfig = field(
+        default_factory=lambda: QdrantUploadStagerConfig()
+    )
+
+    @staticmethod
+    def conform_dict(data: dict) -> dict:
+        """Prepares dictionary in the format that Chroma requires"""
+        return {
+            "id": str(uuid.uuid4()),
+            "vector": data.pop("embeddings", {}),
+            "payload": {
+                "text": data.pop("text", None),
+                "element_serialized": json.dumps(data),
+                **flatten_dict(
+                    data,
+                    separator="-",
+                    flatten_lists=True,
+                ),
+            },
+        }
+
+    def run(
+        self,
+        elements_filepath: Path,
+        file_data: FileData,
+        output_dir: Path,
+        output_filename: str,
+        **kwargs: Any,
+    ) -> Path:
+        with open(elements_filepath) as elements_file:
+            elements_contents = json.load(elements_file)
+
+        conformed_elements = [self.conform_dict(data=element) for element in elements_contents]
+        output_path = Path(output_dir) / Path(f"{output_filename}.json")
+
+        with open(output_path, "w") as output_file:
+            json.dump(conformed_elements, output_file)
+        return output_path
+
+
+class QdrantUploaderConfig(UploaderConfig):
+    collection_name: str = Field(description="Name of the collection.")
+    batch_size: int = Field(default=50, description="Number of records per batch.")
+    num_processes: Optional[int] = Field(
+        default=1,
+        description="Optional limit on number of threads to use for upload.",
+        deprecated=True,
+    )
+
+
+@dataclass
+class QdrantUploader(Uploader, ABC):
+    upload_config: QdrantUploaderConfig
+    connection_config: QdrantConnectionConfig
+
+    @DestinationConnectionError.wrap
+    def precheck(self) -> None:
+        async def check_connection():
+            async with self.connection_config.get_client() as async_client:
+                await async_client.get_collections()
+
+        asyncio.run(check_connection())
+
+    def is_async(self):
+        return True
+
+    async def run_async(
+        self,
+        path: Path,
+        file_data: FileData,
+        **kwargs: Any,
+    ) -> None:
+        with path.open("r") as file:
+            elements: list[dict] = json.load(file)
+
+        logger.debug("Loaded %i elements from %s", len(elements), path)
+
+        batches = list(batch_generator(elements, batch_size=self.upload_config.batch_size))
+        logger.debug(
+            "Elements split into %i batches of size %i.",
+            len(batches),
+            self.upload_config.batch_size,
+        )
+        await asyncio.gather(*[self._upsert_batch(batch) for batch in batches])
+
+    async def _upsert_batch(self, batch: list[dict]) -> None:
+        from qdrant_client import models
+
+        points: list[models.PointStruct] = [models.PointStruct(**item) for item in batch]
+        try:
+            logger.debug(
+                "Upserting %i points to the '%s' collection.",
+                len(points),
+                self.upload_config.collection_name,
+            )
+            async with self.connection_config.get_client() as async_client:
+                await async_client.upsert(
+                    self.upload_config.collection_name, points=points, wait=True
+                )
+        except Exception as api_error:
+            logger.error(
+                "Failed to upsert points to the collection due to the following error %s", api_error
+            )
+
+            raise WriteError(f"Qdrant error: {api_error}") from api_error
+
+        logger.debug("Successfully upsert points to the collection.")

unstructured_ingest/v2/processes/connectors/qdrant/server.py

@@ -0,0 +1,60 @@
+from dataclasses import dataclass
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.processes.connectors.qdrant.qdrant import (
+    QdrantAccessConfig,
+    QdrantConnectionConfig,
+    QdrantUploader,
+    QdrantUploaderConfig,
+    QdrantUploadStager,
+    QdrantUploadStagerConfig,
+)
+
+CONNECTOR_TYPE = "qdrant-server"
+
+
+class ServerQdrantAccessConfig(QdrantAccessConfig):
+    pass
+
+
+class ServerQdrantConnectionConfig(QdrantConnectionConfig):
+    url: str = Field(default=None, description="url of Qdrant server")
+    access_config: Secret[ServerQdrantAccessConfig] = Field(
+        default_factory=ServerQdrantAccessConfig, validate_default=True
+    )
+
+    def get_client_kwargs(self) -> dict:
+        return {
+            "url": self.url,
+        }
+
+
+class ServerQdrantUploadStagerConfig(QdrantUploadStagerConfig):
+    pass
+
+
+@dataclass
+class ServerQdrantUploadStager(QdrantUploadStager):
+    upload_stager_config: ServerQdrantUploadStagerConfig
+
+
+class ServerQdrantUploaderConfig(QdrantUploaderConfig):
+    pass
+
+
+@dataclass
+class ServerQdrantUploader(QdrantUploader):
+    connection_config: ServerQdrantConnectionConfig
+    upload_config: ServerQdrantUploaderConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+qdrant_server_destination_entry = DestinationRegistryEntry(
+    connection_config=ServerQdrantConnectionConfig,
+    uploader=ServerQdrantUploader,
+    uploader_config=ServerQdrantUploaderConfig,
+    upload_stager=ServerQdrantUploadStager,
+    upload_stager_config=ServerQdrantUploadStagerConfig,
+)

unstructured_ingest/v2/processes/connectors/sql/snowflake.py

@@ -70,7 +70,9 @@ class SnowflakeConnectionConfig(SQLConnectionConfig):
         connect_kwargs["password"] = self.access_config.get_secret_value().password
         # https://peps.python.org/pep-0249/#paramstyle
         connect_kwargs["paramstyle"] = "qmark"
-        connection = connect(**connect_kwargs)
+        # remove anything that is none
+        active_kwargs = {k: v for k, v in connect_kwargs.items() if v is not None}
+        connection = connect(**active_kwargs)
         try:
             yield connection
         finally:

unstructured_ingest/v2/processes/connectors/sql/sql.py

@@ -300,10 +300,8 @@ class SQLUploader(Uploader):
 
     def precheck(self) -> None:
         try:
-            connection = self.connection_config.get_connection()
-            cursor = connection.cursor()
-            cursor.execute("SELECT 1;")
-            cursor.close()
+            with self.connection_config.get_cursor() as cursor:
+                cursor.execute("SELECT 1;")
         except Exception as e:
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")

unstructured_ingest/v2/processes/partitioner.py

@@ -9,7 +9,7 @@ from unstructured_ingest.utils.data_prep import flatten_dict
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.unstructured_api import call_api
+from unstructured_ingest.v2.unstructured_api import call_api_async
 
 
 class PartitionerConfig(BaseModel):
@@ -47,7 +47,11 @@ class PartitionerConfig(BaseModel):
     )
     metadata_exclude: list[str] = Field(
         default_factory=list,
-        description="If set, drop the specified metadata " "fields if they exist.",
+        description="If set, drop the specified metadata fields if they exist.",
+    )
+    element_exclude: list[str] = Field(
+        default_factory=list,
+        description="If set, drop the specified element types, if they exist.",
     )
     metadata_include: list[str] = Field(
         default_factory=list,
@@ -100,6 +104,13 @@ class Partitioner(BaseProcess, ABC):
 
     def postprocess(self, elements: list[dict]) -> list[dict]:
         element_dicts = [e.copy() for e in elements]
+        if self.config.element_exclude:
+            element_dicts = list(
+                filter(
+                    lambda element: element["type"] not in self.config.element_exclude,
+                    element_dicts,
+                )
+            )
         for elem in element_dicts:
             if self.config.metadata_exclude:
                 ex_list = self.config.metadata_exclude
@@ -156,7 +167,7 @@ class Partitioner(BaseProcess, ABC):
         metadata = metadata or {}
         logger.debug(f"partitioning file {filename} with metadata: {metadata}")
 
-        elements = await call_api(
+        elements = await call_api_async(
             server_url=self.config.partition_endpoint,
             api_key=self.config.api_key.get_secret_value(),
             filename=filename,

unstructured_ingest/v2/unstructured_api.py

@@ -1,6 +1,4 @@
-import asyncio
 from dataclasses import fields
-from functools import partial
 from pathlib import Path
 from typing import TYPE_CHECKING, Optional
 
@@ -53,7 +51,7 @@ def create_partition_request(filename: Path, parameters_dict: dict) -> "Partitio
     return PartitionRequest(partition_parameters=partition_params)
 
 
-async def call_api(
+async def call_api_async(
     server_url: Optional[str], api_key: Optional[str], filename: Path, api_parameters: dict
 ) -> list[dict]:
     """Call the Unstructured API using unstructured-client.
@@ -73,15 +71,31 @@ async def call_api(
         api_key_auth=api_key,
     )
     partition_request = create_partition_request(filename=filename, parameters_dict=api_parameters)
+    res = await client.general.partition_async(request=partition_request)
 
-    # TODO when client supports async, run without using run_in_executor
-    # isolate the IO heavy call
-    loop = asyncio.get_event_loop()
+    return res.elements or []
+
+
+def call_api(
+    server_url: Optional[str], api_key: Optional[str], filename: Path, api_parameters: dict
+) -> list[dict]:
+    """Call the Unstructured API using unstructured-client.
+
+    Args:
+        server_url: The base URL where the API is hosted
+        api_key: The user's API key (can be empty if this is a self hosted API)
+        filename: Path to the file being partitioned
+        api_parameters: A dict containing the requested API parameters
 
-    # Note(austin) - The partition calls needs request to be a keyword arg
-    # We have to use partial to do this, we can't pass request=request into run_in_executor
-    partition_call = partial(client.general.partition, request=partition_request)
+    Returns: A list of the file's elements, or an empty list if there was an error
+    """
+    from unstructured_client import UnstructuredClient
 
-    res = await loop.run_in_executor(None, partition_call)
+    client = UnstructuredClient(
+        server_url=server_url,
+        api_key_auth=api_key,
+    )
+    partition_request = create_partition_request(filename=filename, parameters_dict=api_parameters)
+    res = client.general.partition(request=partition_request)
 
     return res.elements or []

{unstructured_ingest-0.2.1.dist-info → unstructured_ingest-0.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.2.1
+Version: 0.2.2
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,20 +22,20 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
+Requires-Dist: opentelemetry-sdk
+Requires-Dist: dataclasses-json
+Requires-Dist: click
 Requires-Dist: tqdm
 Requires-Dist: python-dateutil
 Requires-Dist: pandas
-Requires-Dist: click
 Requires-Dist: pydantic>=2.7
-Requires-Dist: dataclasses-json
-Requires-Dist: opentelemetry-sdk
 Provides-Extra: airtable
 Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
 Requires-Dist: astrapy; extra == "astradb"
 Provides-Extra: azure
-Requires-Dist: fsspec; extra == "azure"
 Requires-Dist: adlfs; extra == "azure"
+Requires-Dist: fsspec; extra == "azure"
 Provides-Extra: azure-cognitive-search
 Requires-Dist: azure-search-documents; extra == "azure-cognitive-search"
 Provides-Extra: bedrock
@@ -44,15 +44,15 @@ Provides-Extra: biomed
 Requires-Dist: requests; extra == "biomed"
 Requires-Dist: bs4; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: fsspec; extra == "box"
 Requires-Dist: boxfs; extra == "box"
+Requires-Dist: fsspec; extra == "box"
 Provides-Extra: chroma
 Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
 Requires-Dist: clarifai; extra == "clarifai"
 Provides-Extra: confluence
-Requires-Dist: atlassian-python-api; extra == "confluence"
 Requires-Dist: requests; extra == "confluence"
+Requires-Dist: atlassian-python-api; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
 Provides-Extra: csv
@@ -60,8 +60,8 @@ Requires-Dist: unstructured[tsv]; extra == "csv"
 Provides-Extra: databricks-volumes
 Requires-Dist: databricks-sdk; extra == "databricks-volumes"
 Provides-Extra: delta-table
-Requires-Dist: fsspec; extra == "delta-table"
 Requires-Dist: deltalake; extra == "delta-table"
+Requires-Dist: fsspec; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord-py; extra == "discord"
 Provides-Extra: doc
@@ -69,8 +69,8 @@ Requires-Dist: unstructured[docx]; extra == "doc"
 Provides-Extra: docx
 Requires-Dist: unstructured[docx]; extra == "docx"
 Provides-Extra: dropbox
-Requires-Dist: fsspec; extra == "dropbox"
 Requires-Dist: dropboxdrivefs; extra == "dropbox"
+Requires-Dist: fsspec; extra == "dropbox"
 Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
 Provides-Extra: embed-huggingface
@@ -78,8 +78,8 @@ Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Provides-Extra: embed-octoai
-Requires-Dist: openai; extra == "embed-octoai"
 Requires-Dist: tiktoken; extra == "embed-octoai"
+Requires-Dist: openai; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
@@ -88,18 +88,18 @@ Provides-Extra: epub
 Requires-Dist: unstructured[epub]; extra == "epub"
 Provides-Extra: gcs
 Requires-Dist: gcsfs; extra == "gcs"
-Requires-Dist: fsspec; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
+Requires-Dist: fsspec; extra == "gcs"
 Provides-Extra: github
-Requires-Dist: pygithub>1.58.0; extra == "github"
 Requires-Dist: requests; extra == "github"
+Requires-Dist: pygithub>1.58.0; extra == "github"
 Provides-Extra: gitlab
 Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: urllib3; extra == "hubspot"
 Requires-Dist: hubspot-api-client; extra == "hubspot"
+Requires-Dist: urllib3; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -115,26 +115,26 @@ Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: msg
 Requires-Dist: unstructured[msg]; extra == "msg"
 Provides-Extra: notion
+Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
-Requires-Dist: notion-client; extra == "notion"
 Provides-Extra: odt
 Requires-Dist: unstructured[odt]; extra == "odt"
 Provides-Extra: onedrive
+Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
-Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Provides-Extra: openai
-Requires-Dist: openai; extra == "openai"
 Requires-Dist: tiktoken; extra == "openai"
+Requires-Dist: openai; extra == "openai"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: org
 Requires-Dist: unstructured[org]; extra == "org"
 Provides-Extra: outlook
-Requires-Dist: msal; extra == "outlook"
 Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
+Requires-Dist: msal; extra == "outlook"
 Provides-Extra: pdf
 Requires-Dist: unstructured[pdf]; extra == "pdf"
 Provides-Extra: pinecone
@@ -150,28 +150,28 @@ Requires-Dist: qdrant-client; extra == "qdrant"
 Provides-Extra: reddit
 Requires-Dist: praw; extra == "reddit"
 Provides-Extra: remote
-Requires-Dist: unstructured-client>=0.25.8; extra == "remote"
+Requires-Dist: unstructured-client>=0.26.1; extra == "remote"
 Provides-Extra: rst
 Requires-Dist: unstructured[rst]; extra == "rst"
 Provides-Extra: rtf
 Requires-Dist: unstructured[rtf]; extra == "rtf"
 Provides-Extra: s3
-Requires-Dist: fsspec; extra == "s3"
 Requires-Dist: s3fs; extra == "s3"
+Requires-Dist: fsspec; extra == "s3"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
-Requires-Dist: fsspec; extra == "sftp"
 Requires-Dist: paramiko; extra == "sftp"
+Requires-Dist: fsspec; extra == "sftp"
 Provides-Extra: sharepoint
-Requires-Dist: msal; extra == "sharepoint"
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
+Requires-Dist: msal; extra == "sharepoint"
 Provides-Extra: singlestore
 Requires-Dist: singlestoredb; extra == "singlestore"
 Provides-Extra: slack
 Requires-Dist: slack-sdk[optional]; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: snowflake; extra == "snowflake"
+Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Provides-Extra: togetherai
 Requires-Dist: together; extra == "togetherai"
 Provides-Extra: tsv

{unstructured_ingest-0.2.1.dist-info → unstructured_ingest-0.2.2.dist-info}/RECORD

@@ -5,7 +5,11 @@ test/integration/chunkers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJW
 test/integration/chunkers/test_chunkers.py,sha256=pqn1Rqh36jZTJL4qpU0iuOMFAEQ-LrKAPOgWtQMAt_I,1482
 test/integration/connectors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/connectors/conftest.py,sha256=6dVNMBrL6WIO4KXA-0nf2tNrPYk_tsor8uomi6fbi3Q,727
+test/integration/connectors/test_confluence.py,sha256=xcPmZ_vi_pkCt-tUPn10P49FH9i_9YUbrAPO6fYk5rU,3521
 test/integration/connectors/test_delta_table.py,sha256=4_KPyQJpd6DmyIjjtXWPMw6NNf7xULRkxmqfbvmZ80g,5018
+test/integration/connectors/test_kafka.py,sha256=tWSs7Lkdam6sYJDvOinQUKvxtsknFI0SKuGou8e1H-0,2272
+test/integration/connectors/test_onedrive.py,sha256=KIkBwKh1hnv203VCL2UABnDkS_bP4NxOFm1AL8EPGLA,3554
+test/integration/connectors/test_qdrant.py,sha256=ASvO-BNyhv8m8or28KljrJy27Da0uaTNeoR5w_QsvFg,5121
 test/integration/connectors/test_s3.py,sha256=1ErPRpNmbg-88ig80SfIyxujF7xnAWtI42WSue4sgKU,5850
 test/integration/connectors/databricks_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/connectors/databricks_tests/test_volumes_native.py,sha256=k4lALbwNtlyuI3wd3OHoBULI21E3Ck2Fo8EJXaVfwgw,5812
@@ -16,9 +20,9 @@ test/integration/connectors/sql/test_snowflake.py,sha256=XXU2-2z_k8jHWP684v2IuaG
 test/integration/connectors/sql/test_sqlite.py,sha256=51QrFufAq-XxNjHAkmPWxdJUkGdIRRIGKeRT09A5pkA,5704
 test/integration/connectors/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/connectors/utils/constants.py,sha256=0zSPnsZVqJuNhXduXvdXFQLZTRIQa5Fo_1qjBYVCfb8,209
-test/integration/connectors/utils/docker.py,sha256=-wknXRVlzr3BVPdEhCyJgsdNjO9aSb2xjb-mQ306j7Q,2256
+test/integration/connectors/utils/docker.py,sha256=JxfX8u46YwpqUnVGd4syI0SrqGqvGQx9yBN0Xq-bIKE,2328
 test/integration/connectors/utils/docker_compose.py,sha256=GVTB6Cel05c0VQ2n4AwkQQx_cBfz13ZTs1HpbaYipNU,2223
-test/integration/connectors/utils/validation.py,sha256=gnflehoYbFkSBJdXQV-7HwcrlL_Cuqni2ri1YmArjT0,12019
+test/integration/connectors/utils/validation.py,sha256=5rQOBJyu1etvuwJmkH6xvKUPF08AKwJRxlN4L7-nw9w,13894
 test/integration/embedders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/embedders/conftest.py,sha256=B2W771RbijR7G_GybsCzRyIvOzXqzbKZdRIlNDd5AGY,334
 test/integration/embedders/test_bedrock.py,sha256=0oBRNS_DtFDGQ22Z1T3t6VOJ31PrItgvnJpqcLe9Fg4,1903
@@ -47,9 +51,9 @@ test/unit/embed/test_openai.py,sha256=0O1yshDcE0BMKv1yJqrNuiNLSdPhLpKqJ-D_wmnids
 test/unit/embed/test_vertexai.py,sha256=Pl7COc9E3tf_yGidkTEmTizNGyZF1F5zuL2TgPTMnfI,1048
 test/unit/embed/test_voyageai.py,sha256=DviCOJFhe5H4e26-kNyX3JNe8h3qB5Yl0KOe8rQEMrc,981
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=Hmm5OuicK0ynl_R5DSnpRYWJpEXwe7guJdsAMHH7K60,42
+unstructured_ingest/__version__.py,sha256=h73ZYWteeEB3jWqVrz7Z3m26fzblpJHqT2R5UK5aQ5w,42
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
-unstructured_ingest/interfaces.py,sha256=m03BgenxSA34HbW157L7V9TGxK_dTG7N2AnAhF31W-U,31364
+unstructured_ingest/interfaces.py,sha256=OYVUP0bzBJpT-Lz92BDyz_hLBvyfxkuSwWHhUdnUayA,31493
 unstructured_ingest/logger.py,sha256=S5nSqGcABoQyeicgRnBQFjDScCaTvFVivOCvbo-laL0,4479
 unstructured_ingest/main.py,sha256=82G_7eG4PNhc_xIqj4Y_sFbDV9VI-nwSfsfJQMzovMk,169
 unstructured_ingest/processor.py,sha256=XKKrvbxsb--5cDzz4hB3-GfWZYyIjJ2ah8FpzQKF_DM,2760
@@ -123,7 +127,7 @@ unstructured_ingest/connector/gitlab.py,sha256=OEilnSFabWT3XY0riNxVTXc9tS3f1lMyH
 unstructured_ingest/connector/google_drive.py,sha256=Sl6r-IcbV_7s8LeMg2z8qiko2r5RAyRnDzBxMwvY6ng,13053
 unstructured_ingest/connector/hubspot.py,sha256=jL-bqU4EJIqbG0YRk9IR3MKsHi_WHf86Fy6r1fVeCz4,9271
 unstructured_ingest/connector/jira.py,sha256=kxjGhbVSH8FJNPMGJbnpZEV5zZRfGFckVJFiOzExphQ,15690
-unstructured_ingest/connector/kafka.py,sha256=D0XPf0IZ4XkLjnYy7I1FDc5XucNehJLlj-yCKjBMr0w,10053
+unstructured_ingest/connector/kafka.py,sha256=VSlZWqYODbITvFKnR87MyLYO33vLRdFTpclwsQfdfBM,10052
 unstructured_ingest/connector/local.py,sha256=ayEz7gtnr1ioiYWmJ5ElSlSC8_ZFk1rk-9sX2htRq6c,4479
 unstructured_ingest/connector/mongodb.py,sha256=UD8T1V435YvGY68dpL-fyFesD7bcLckptgXzzfgnILI,9771
 unstructured_ingest/connector/onedrive.py,sha256=-yy3scFHVIUiPAAQdmJXel3_BMZnZc9qUI8HwecuoJ4,8911
@@ -314,7 +318,7 @@ unstructured_ingest/v2/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LG
 unstructured_ingest/v2/logger.py,sha256=wcln4s5Nyp2fjjJux9iM3d6t9aQFNJ2H1IAZXmIknjI,4323
 unstructured_ingest/v2/main.py,sha256=WFdLEqEXRy6E9_G-dF20MK2AtgX51Aan1sp_N67U2B8,172
 unstructured_ingest/v2/otel.py,sha256=2fGj1c7cVcC3J8NwL6MNYhyPEAXiB33DsilvRDkrdLo,4130
-unstructured_ingest/v2/unstructured_api.py,sha256=HqOaQ80YTdAnFj_2Ce108g7Pp3-F9Qg329Uw2OXtRmA,3375
+unstructured_ingest/v2/unstructured_api.py,sha256=f_6NK0QOVwjAFJvlyvzu0IaXb6QQgRNJleYxB1KvzKE,3856
 unstructured_ingest/v2/utils.py,sha256=BT_j15e4rX40wQbt8LUXbqfPhA3rJn1PHTI_G_A_EHY,1720
 unstructured_ingest/v2/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/v2/cli/cli.py,sha256=qHXIs-PcvMgDZhP1AR9iDMxh8FXBMJCEDksPBfiMULE,648
@@ -351,26 +355,28 @@ unstructured_ingest/v2/pipeline/steps/stage.py,sha256=cphKgHScLz2rNLZRI5Olsb6dAH
 unstructured_ingest/v2/pipeline/steps/uncompress.py,sha256=CFSy4tGp6BAvF0oIwWFN8v4zFzh5pRDeESjEn5iP9hE,1756
 unstructured_ingest/v2/pipeline/steps/upload.py,sha256=zlgXgwReX9TBOdfTpS9hETah4SeSmzPB2g8dAGfLIvM,1987
 unstructured_ingest/v2/processes/__init__.py,sha256=FaHWSCGyc7GWVnAsNEUUj7L8hT8gCVY3_hUE2VzWtUg,462
-unstructured_ingest/v2/processes/chunker.py,sha256=1bfJ2qgl6qu2HvClzHbC7-q5QtUp7mrlNxZxnPGYTm0,5479
+unstructured_ingest/v2/processes/chunker.py,sha256=31-7ojsM2coIt2rMR0KOb82IxLVJfNHbqYUOsDkhxN8,5491
 unstructured_ingest/v2/processes/connector_registry.py,sha256=vkEe6jpgdYtZCxMj59s5atWGgmPuxAEXRUoTt-MJ7wc,2198
 unstructured_ingest/v2/processes/embedder.py,sha256=PQn0IO8xbGRQHpcT2VVl-J8gTJ5HGGEP9gdEAwMVK3U,6498
 unstructured_ingest/v2/processes/filter.py,sha256=kjUmMw2SDq2bme0JCAOxs6cJriIG6Ty09KOznS-xz08,2145
-unstructured_ingest/v2/processes/partitioner.py,sha256=2Lhztd730soVC2TOqrn_ba7CGZna8AHHpqJY2ZUYVxE,7776
+unstructured_ingest/v2/processes/partitioner.py,sha256=agpHwB9FR8OZVQqE7zFEb0IcDPCOPA_BZjLzLF71nOY,8194
 unstructured_ingest/v2/processes/uncompress.py,sha256=Z_XfsITGdyaRwhtNUc7bMj5Y2jLuBge8KoK4nxhqKag,2425
-unstructured_ingest/v2/processes/connectors/__init__.py,sha256=zMO50wOGWOJrCTdh19Najj-i5tfMUyf977TKz4yN04A,5249
+unstructured_ingest/v2/processes/connectors/__init__.py,sha256=t3mBUb7F4LgNY8dyoDnzQVaRqGajOaXUqAEBUO5hr4A,5902
 unstructured_ingest/v2/processes/connectors/airtable.py,sha256=Yi7PEv_FejZ9_y3BPY3gu5YGVfeLh-9YX-qLyQHjJsY,8921
 unstructured_ingest/v2/processes/connectors/astradb.py,sha256=k6zaxm05-ESpRV6w1jgrtfE10-I2Z50kafURxxJVzdk,14043
 unstructured_ingest/v2/processes/connectors/azure_cognitive_search.py,sha256=S55v7TXu30rEdgythMBB_2VcuomyMPmcPtLYykbhw_E,8466
 unstructured_ingest/v2/processes/connectors/chroma.py,sha256=skrxRPHZ8y3JxNa0dt5SVitHiDQ5WVxLvY_kh2-QUrQ,8029
+unstructured_ingest/v2/processes/connectors/confluence.py,sha256=bcqE-5ODFpHy43g72dOHXVNUxK1Z5N0_dy2UrCESr7Y,7007
 unstructured_ingest/v2/processes/connectors/couchbase.py,sha256=SONLywyEfoAlLc-HPabXeGzoiwKnekMHIbRMXd4CGXs,12146
 unstructured_ingest/v2/processes/connectors/delta_table.py,sha256=ZZfdNTw1W0ISQGWCtM1JuIME26FYzuPBOqRKql0wlLg,7013
 unstructured_ingest/v2/processes/connectors/elasticsearch.py,sha256=ojxMUHkLa6ZG50aTGn2YWhDHZ1n38uFRn5p8_ghAIvM,16762
+unstructured_ingest/v2/processes/connectors/gitlab.py,sha256=yBgCeLy9iCVI8bBDcHHuHB0H3BO05e9E1OccbHwvKAo,9724
 unstructured_ingest/v2/processes/connectors/google_drive.py,sha256=7xOQthcqBd9auJxB0nxZlhh1vdjXpMX_CtQZa6YfZz0,13088
 unstructured_ingest/v2/processes/connectors/kdbai.py,sha256=8bGHbZctJ_Tl1AUSMnI7CCZ7CgEtTRVcRuvlB1HPlqQ,5907
 unstructured_ingest/v2/processes/connectors/local.py,sha256=a3stgnIkhBbXPIQD0O-RaRM-Eb-szHj9Yy4Fz881-9c,6723
 unstructured_ingest/v2/processes/connectors/milvus.py,sha256=ZUlyAQyTt0U1JoapFYHQW3IIaGYY50b3URDSLEAFjtk,7687
 unstructured_ingest/v2/processes/connectors/mongodb.py,sha256=A0pt6JcNTD5bEu79jZ8KhnHcBQ2VUJ2AjtQAtdFr_Lo,13175
-unstructured_ingest/v2/processes/connectors/onedrive.py,sha256=ZiUo-dFo1LMOvFwphSLRZiR1PcrN8GWLTHhsh4TU6n0,9207
+unstructured_ingest/v2/processes/connectors/onedrive.py,sha256=4Xs3akCaBC2JW3MyVnKctrOgB7xOjiEMudFL1Fg7lsc,15981
 unstructured_ingest/v2/processes/connectors/opensearch.py,sha256=dfDSNrWIEk19wuHdlMJpp_SLMOteNPlkDBPlAwu1LVY,6767
 unstructured_ingest/v2/processes/connectors/outlook.py,sha256=NK67Pd8Nk5oUIXTK-sK18K7rZ_Cl0UuCbeF2ExBEZho,9294
 unstructured_ingest/v2/processes/connectors/pinecone.py,sha256=k_GH55S_OQ6-wCLC6gkhRrNpXIFECYZ_2Gjz_XRtY6Y,7561
@@ -380,7 +386,7 @@ unstructured_ingest/v2/processes/connectors/slack.py,sha256=b9IanzUApUexiJzuNg7P
 unstructured_ingest/v2/processes/connectors/utils.py,sha256=8kd0g7lo9NqnpaIkjeO-Ut6erhwUNH_gS9koevpe3WE,878
 unstructured_ingest/v2/processes/connectors/weaviate.py,sha256=Ss0YyD5T6k-00eJ6dr5lSo2H0LcOjVTMmozehyTvnAo,8866
 unstructured_ingest/v2/processes/connectors/databricks/__init__.py,sha256=jO71UTC7bLA_N12CrLWJzh_yZML5gfT7VohxzCpUGWg,1848
-unstructured_ingest/v2/processes/connectors/databricks/volumes.py,sha256=8FasrRcoqa9zrhmnbfYN_rBBTH6xBXM50TzGsUMEm98,6581
+unstructured_ingest/v2/processes/connectors/databricks/volumes.py,sha256=p7sjCYZb7JmY3v3Xy1gm-q0O7oamLTsSFf2EWXYfXYQ,6447
 unstructured_ingest/v2/processes/connectors/databricks/volumes_aws.py,sha256=I1MJwe5LOxoPLjwo00H0XbXO6u_SJHWYgsj4s6ePoyI,2754
 unstructured_ingest/v2/processes/connectors/databricks/volumes_azure.py,sha256=P4rfcE3td7WyuuguRgUnGQytCMDpfeYrrpshBZuVynY,3539
 unstructured_ingest/v2/processes/connectors/databricks/volumes_gcp.py,sha256=UUotY_-HpgSEJkvdQfZTlbxY7CRLZ4ctL8TlryeFvxk,2790
@@ -389,20 +395,29 @@ unstructured_ingest/v2/processes/connectors/fsspec/__init__.py,sha256=TtdeImM7Yp
 unstructured_ingest/v2/processes/connectors/fsspec/azure.py,sha256=Y01BuVRql0Kvzc_cdaZE9dDGYjJzrwJu-etfUrEGcUU,7061
 unstructured_ingest/v2/processes/connectors/fsspec/box.py,sha256=Cjk0LUxqOCDbme0GmnD_5_b1hfStjI23cKw6BquKNrg,5488
 unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py,sha256=NNAxIRdOQxUncfwhu7J7SnQRM6BSStNOyQZi-4E51iY,5816
-unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py,sha256=5uZ_nGBXNQgwvfjNcor6mwzbYOHeja4-EV3nNCXvxaQ,11512
+unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py,sha256=yuWZL9x0C2m7IXRhjaHomu8OpyWl0G2e-FMRKdnsM8s,10964
 unstructured_ingest/v2/processes/connectors/fsspec/gcs.py,sha256=-_pYHbsBG9FyRyNIaf_xyFbPiiR7pnWEEg_8mp0rIZ8,7053
 unstructured_ingest/v2/processes/connectors/fsspec/s3.py,sha256=je1BDqFWlyMfPa4oAMMNFQLLQtCY9quuqx3xjTwF8OQ,6251
 unstructured_ingest/v2/processes/connectors/fsspec/sftp.py,sha256=dwpyqDq0qceCBWX3zM1hiUlgXB4hzX6ObOr-sh-5CJs,6926
 unstructured_ingest/v2/processes/connectors/fsspec/utils.py,sha256=jec_Qfe2hbfahBuY-u8FnvHuv933AI5HwPFjOL3kEEY,456
+unstructured_ingest/v2/processes/connectors/kafka/__init__.py,sha256=yWYxcmNypbbmTteXHGFg8ivQb-njuDX-alfaGWLJ0OQ,478
+unstructured_ingest/v2/processes/connectors/kafka/cloud.py,sha256=TC31QQWpmJ6-ciNo3GtgDLgaImouEPUYs9256QMW_tU,2341
+unstructured_ingest/v2/processes/connectors/kafka/kafka.py,sha256=FmnZjXCvTkFH3J9t5rPSIBCXulB2T0i0sHh6NLDmY9Y,6843
+unstructured_ingest/v2/processes/connectors/kafka/local.py,sha256=vGUR6Y-g0poU_MVM2e2fmMCO5mUuZcf0n0mMW70qGbE,1942
+unstructured_ingest/v2/processes/connectors/qdrant/__init__.py,sha256=xM19uYzAuGizVoZIM_hnVZ5AcBN69aOBGpqZcpWPtuE,760
+unstructured_ingest/v2/processes/connectors/qdrant/cloud.py,sha256=accJ4sNWBVWV-KiVBDBDBYYx5A9CUoikP5NCErRmfik,1624
+unstructured_ingest/v2/processes/connectors/qdrant/local.py,sha256=cGEyv3Oy6y4BQ4DU8yhJWMpL82QYwBVdPTxxNuV127U,1588
+unstructured_ingest/v2/processes/connectors/qdrant/qdrant.py,sha256=14qDTLrLBugsfvetFPx4ueS8zrk53wBLISuNRD4P-B8,5350
+unstructured_ingest/v2/processes/connectors/qdrant/server.py,sha256=odvCZWZp8DmRxLXMR7tHhW-c7UQbix1_zpFdfXfCvKI,1613
 unstructured_ingest/v2/processes/connectors/sql/__init__.py,sha256=D43wrV2ADvQsToIYwbEWnZ7mhzlsYcZMFCqf6jIC7dQ,1333
 unstructured_ingest/v2/processes/connectors/sql/postgres.py,sha256=__Wf5lkCQGhbtEH_2DxfNmQyWP-UKC9o_KEawG81jY0,4905
 unstructured_ingest/v2/processes/connectors/sql/singlestore.py,sha256=YrmhAL1RQ1c5-2fnR3UAyj_4KfvjYTQ2cWzpvsdJOnU,5535
-unstructured_ingest/v2/processes/connectors/sql/snowflake.py,sha256=0s0oBfMttPg5JL6jn8SsoCeTSRoXXdVy2bJAZv_hiSk,5576
-unstructured_ingest/v2/processes/connectors/sql/sql.py,sha256=rWDkefUnYkzJT0mhIcHxieECdaIWLTvbDcOcZgLA4FQ,11636
+unstructured_ingest/v2/processes/connectors/sql/snowflake.py,sha256=jl524VudwmFK63emCT7DmZan_EWJAMiGir5_zoO9FuY,5697
+unstructured_ingest/v2/processes/connectors/sql/sql.py,sha256=Jwu3ZC4PGEw9la72cOwC3tclYAoBXFQTII9Mhh8ziP4,11571
 unstructured_ingest/v2/processes/connectors/sql/sqlite.py,sha256=9605K36nQ5-gBxzt1daYKYotON1SE85RETusqCJrbdk,5230
-unstructured_ingest-0.2.1.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-0.2.1.dist-info/METADATA,sha256=NBV3OAonxt8Y0Tra7LWqQBoLSROwA106sf8vDCsXu2k,7271
-unstructured_ingest-0.2.1.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-unstructured_ingest-0.2.1.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-0.2.1.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
-unstructured_ingest-0.2.1.dist-info/RECORD,,
+unstructured_ingest-0.2.2.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-0.2.2.dist-info/METADATA,sha256=0sygNKxP0BY-jzQIlvm0h3xyJWShW9nVlNRAUXb8XC8,7288
+unstructured_ingest-0.2.2.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+unstructured_ingest-0.2.2.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-0.2.2.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
+unstructured_ingest-0.2.2.dist-info/RECORD,,