unstructured-ingest 0.5.13__py3-none-any.whl → 0.5.15__py3-none-any.whl

test/integration/connectors/test_confluence.py

@@ -30,7 +30,7 @@ async def test_confluence_source(temp_dir):
     spaces = ["testteamsp", "MFS"]
 
     # Create connection and indexer configurations
-    access_config = ConfluenceAccessConfig(password=api_token)
+    access_config = ConfluenceAccessConfig(api_token=api_token)
     connection_config = ConfluenceConnectionConfig(
         url=confluence_url,
         username=user_email,
@@ -77,7 +77,7 @@ async def test_confluence_source_large(temp_dir):
     spaces = ["testteamsp1"]
 
     # Create connection and indexer configurations
-    access_config = ConfluenceAccessConfig(password=api_token)
+    access_config = ConfluenceAccessConfig(api_token=api_token)
     connection_config = ConfluenceConnectionConfig(
         url=confluence_url,
         username=user_email,

test/integration/connectors/test_zendesk.py

@@ -11,7 +11,7 @@ from test.integration.connectors.utils.validation.source import (
 )
 from test.integration.utils import requires_env
 from unstructured_ingest.v2.errors import UserAuthError
-from unstructured_ingest.v2.processes.connectors.zendesk import (
+from unstructured_ingest.v2.processes.connectors.zendesk.zendesk import (
     CONNECTOR_TYPE,
     ZendeskAccessConfig,
     ZendeskConnectionConfig,

test/unit/v2/connectors/test_confluence.py

@@ -11,7 +11,7 @@ def test_connection_config_multiple_auth():
     with pytest.raises(ValidationError):
         ConfluenceConnectionConfig(
             access_config=ConfluenceAccessConfig(
-                password="api_token",
+                password="password",
                 token="access_token",
             ),
             username="user_email",
@@ -19,14 +19,46 @@ def test_connection_config_multiple_auth():
         )
 
 
+def test_connection_config_multiple_auth2():
+    with pytest.raises(ValidationError):
+        ConfluenceConnectionConfig(
+            access_config=ConfluenceAccessConfig(
+                api_token="api_token",
+                token="access_token",
+            ),
+            username="user_email",
+            url="url",
+        )
+
+
+def test_connection_config_multiple_auth3():
+    with pytest.raises(ValidationError):
+        ConfluenceConnectionConfig(
+            access_config=ConfluenceAccessConfig(
+                api_token="api_token",
+                password="password",
+            ),
+            username="user_email",
+            url="url",
+        )
+
+
 def test_connection_config_no_auth():
     with pytest.raises(ValidationError):
         ConfluenceConnectionConfig(access_config=ConfluenceAccessConfig(), url="url")
 
 
-def test_connection_config_basic_auth():
+def test_connection_config_password_auth():
+    ConfluenceConnectionConfig(
+        access_config=ConfluenceAccessConfig(password="password"),
+        url="url",
+        username="user_email",
+    )
+
+
+def test_connection_config_api_token_auth():
     ConfluenceConnectionConfig(
-        access_config=ConfluenceAccessConfig(password="api_token"),
+        access_config=ConfluenceAccessConfig(api_token="api_token"),
         url="url",
         username="user_email",
     )

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.5.13"  # pragma: no cover
+__version__ = "0.5.15"  # pragma: no cover

unstructured_ingest/embed/huggingface.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Optional
 
 from pydantic import Field
@@ -15,14 +15,11 @@ if TYPE_CHECKING:
 
 
 class HuggingFaceEmbeddingConfig(EmbeddingConfig):
-    embedder_model_name: Optional[str] = Field(
-        default="sentence-transformers/all-MiniLM-L6-v2", alias="model_name"
-    )
+    embedder_model_name: Optional[str] = Field(default="all-MiniLM-L6-v2", alias="model_name")
     embedder_model_kwargs: Optional[dict] = Field(
         default_factory=lambda: {"device": "cpu"}, alias="model_kwargs"
     )
     encode_kwargs: Optional[dict] = Field(default_factory=lambda: {"normalize_embeddings": False})
-    cache_folder: Optional[str] = Field(default=None)
 
     @requires_dependencies(
         ["sentence_transformers"],
@@ -33,7 +30,6 @@ class HuggingFaceEmbeddingConfig(EmbeddingConfig):
 
         return SentenceTransformer(
             model_name_or_path=self.embedder_model_name,
-            cache_folder=self.cache_folder,
             **self.embedder_model_kwargs,
         )
 
@@ -45,7 +41,7 @@ class HuggingFaceEmbeddingConfig(EmbeddingConfig):
 
 @dataclass
 class HuggingFaceEmbeddingEncoder(BaseEmbeddingEncoder):
-    config: HuggingFaceEmbeddingConfig
+    config: HuggingFaceEmbeddingConfig = field(default_factory=HuggingFaceEmbeddingConfig)
 
     def _embed_query(self, query: str) -> list[float]:
         return self._embed_documents(texts=[query])[0]

unstructured_ingest/utils/data_prep.py

@@ -2,7 +2,7 @@ import itertools
 import json
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Generator, Iterable, Optional, Sequence, TypeVar, cast
+from typing import Any, Generator, Iterable, Optional, Sequence, TypeVar, Union, cast
 
 import pandas as pd
 
@@ -163,7 +163,9 @@ def write_data(path: Path, data: list[dict], indent: Optional[int] = 2) -> None:
             raise IOError("Unsupported file type: {path}")
 
 
-def get_data(path: Path) -> list[dict]:
+def get_data(path: Union[Path, str]) -> list[dict]:
+    if isinstance(path, str):
+        path = Path(path)
     try:
         return get_data_by_suffix(path=path)
     except Exception as e:

unstructured_ingest/v2/interfaces/file_data.py

@@ -102,7 +102,7 @@ def file_data_from_file(path: str) -> FileData:
     try:
         return BatchFileData.from_file(path=path)
     except ValidationError:
-        logger.debug(f"{path} not valid for batch file data")
+        logger.debug(f"{path} not detected as batch file data")
 
     return FileData.from_file(path=path)
 

unstructured_ingest/v2/interfaces/upload_stager.py

@@ -1,4 +1,3 @@
-import json
 from abc import ABC
 from dataclasses import dataclass
 from pathlib import Path
@@ -7,6 +6,7 @@ from typing import Any, TypeVar
 from pydantic import BaseModel
 
 from unstructured_ingest.utils import ndjson
+from unstructured_ingest.utils.data_prep import get_data, write_data
 from unstructured_ingest.v2.interfaces.file_data import FileData
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 
@@ -43,16 +43,13 @@ class UploadStager(BaseProcess, ABC):
                     writer.f.flush()
 
     def process_whole(self, input_file: Path, output_file: Path, file_data: FileData) -> None:
-        with input_file.open() as in_f:
-            elements_contents = json.load(in_f)
+        elements_contents = get_data(path=input_file)
 
         conformed_elements = [
             self.conform_dict(element_dict=element, file_data=file_data)
             for element in elements_contents
         ]
-
-        with open(output_file, "w") as out_f:
-            json.dump(conformed_elements, out_f, indent=2)
+        write_data(path=output_file, data=conformed_elements)
 
     def run(
         self,

unstructured_ingest/v2/pipeline/pipeline.py

@@ -108,6 +108,13 @@ class Pipeline:
         uploader_connector_type = self.uploader_step.process.connector_type
         registry_entry = destination_registry[uploader_connector_type]
         if registry_entry.upload_stager and self.stager_step is None:
+            try:
+                self.stager_step = UploadStageStep(
+                    process=registry_entry.upload_stager(), context=self.context
+                )
+                return
+            except Exception as e:
+                logger.debug(f"failed to instantiate required stager on user's behalf: {e}")
             raise ValueError(
                 f"pipeline with uploader type {self.uploader_step.process.__class__.__name__} "
                 f"expects a stager of type {registry_entry.upload_stager.__name__} "

unstructured_ingest/v2/pipeline/steps/chunk.py

@@ -38,7 +38,7 @@ class ChunkStep(PipelineStep):
         return not filepath.exists()
 
     def get_output_filepath(self, filename: Path) -> Path:
-        hashed_output_file = f"{self.get_hash(extras=[filename.name])}.json"
+        hashed_output_file = f"{self.get_hash(extras=[filename.name])}.ndjson"
         filepath = (self.cache_dir / hashed_output_file).resolve()
         filepath.parent.mkdir(parents=True, exist_ok=True)
         return filepath

unstructured_ingest/v2/pipeline/steps/embed.py

@@ -38,7 +38,7 @@ class EmbedStep(PipelineStep):
         return not filepath.exists()
 
     def get_output_filepath(self, filename: Path) -> Path:
-        hashed_output_file = f"{self.get_hash(extras=[filename.name])}.json"
+        hashed_output_file = f"{self.get_hash(extras=[filename.name])}.ndjson"
         filepath = (self.cache_dir / hashed_output_file).resolve()
         filepath.parent.mkdir(parents=True, exist_ok=True)
         return filepath

unstructured_ingest/v2/pipeline/steps/partition.py

@@ -38,7 +38,7 @@ class PartitionStep(PipelineStep):
         return not filepath.exists()
 
     def get_output_filepath(self, filename: Path) -> Path:
-        hashed_output_file = f"{self.get_hash(extras=[filename.name])}.json"
+        hashed_output_file = f"{self.get_hash(extras=[filename.name])}.ndjson"
         filepath = (self.cache_dir / hashed_output_file).resolve()
         filepath.parent.mkdir(parents=True, exist_ok=True)
         return filepath

unstructured_ingest/v2/processes/connectors/__init__.py

@@ -64,6 +64,8 @@ from .slack import CONNECTOR_TYPE as SLACK_CONNECTOR_TYPE
 from .slack import slack_source_entry
 from .vectara import CONNECTOR_TYPE as VECTARA_CONNECTOR_TYPE
 from .vectara import vectara_destination_entry
+from .zendesk.zendesk import CONNECTOR_TYPE as ZENDESK_CONNECTOR_TYPE
+from .zendesk.zendesk import zendesk_source_entry
 
 add_source_entry(source_type=ASTRA_DB_CONNECTOR_TYPE, entry=astra_db_source_entry)
 add_destination_entry(destination_type=ASTRA_DB_CONNECTOR_TYPE, entry=astra_db_destination_entry)
@@ -119,3 +121,4 @@ add_source_entry(source_type=DISCORD_CONNECTOR_TYPE, entry=discord_source_entry)
 add_destination_entry(destination_type=REDIS_CONNECTOR_TYPE, entry=redis_destination_entry)
 
 add_source_entry(source_type=JIRA_CONNECTOR_TYPE, entry=jira_source_entry)
+add_source_entry(source_type=ZENDESK_CONNECTOR_TYPE, entry=zendesk_source_entry)

unstructured_ingest/v2/processes/connectors/confluence.py

@@ -35,7 +35,11 @@ CONNECTOR_TYPE = "confluence"
 
 class ConfluenceAccessConfig(AccessConfig):
     password: Optional[str] = Field(
-        description="Confluence password or Cloud API token",
+        description="Confluence password",
+        default=None,
+    )
+    api_token: Optional[str] = Field(
+        description="Confluence Cloud API token",
         default=None,
     )
     token: Optional[str] = Field(
@@ -57,7 +61,12 @@ class ConfluenceConnectionConfig(ConnectionConfig):
 
     def model_post_init(self, __context):
         access_configs = self.access_config.get_secret_value()
-        basic_auth = self.username and access_configs.password
+        if access_configs.password and access_configs.api_token:
+            raise ValueError(
+                "both password and api_token provided, only one allowed, "
+                "see: https://atlassian-python-api.readthedocs.io/"
+            )
+        basic_auth = bool(self.username and (access_configs.password or access_configs.api_token))
         pat_auth = access_configs.token
         if self.cloud and not basic_auth:
             raise ValueError(
@@ -74,6 +83,14 @@ class ConfluenceConnectionConfig(ConnectionConfig):
                 "no form of auth provided, see: https://atlassian-python-api.readthedocs.io/"
             )
 
+    def password_or_api_token(self) -> str:
+        # Confluence takes either password or API token under the same field: password
+        # This ambiguity led to confusion, so we are making it specific what you are passing in
+        access_configs = self.access_config.get_secret_value()
+        if access_configs.password:
+            return access_configs.password
+        return access_configs.api_token
+
     @requires_dependencies(["atlassian"], extras="confluence")
     @contextmanager
     def get_client(self) -> "Confluence":
@@ -83,7 +100,7 @@ class ConfluenceConnectionConfig(ConnectionConfig):
         with Confluence(
             url=self.url,
             username=self.username,
-            password=access_configs.password,
+            password=self.password_or_api_token(),
             token=access_configs.token,
             cloud=self.cloud,
         ) as client:

unstructured_ingest/v2/processes/connectors/databricks/volumes_aws.py

@@ -17,6 +17,10 @@ from unstructured_ingest.v2.processes.connectors.databricks.volumes import (
     DatabricksVolumesUploader,
     DatabricksVolumesUploaderConfig,
 )
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 CONNECTOR_TYPE = "databricks_volumes_aws"
 
@@ -76,6 +80,8 @@ databricks_aws_volumes_destination_entry = DestinationRegistryEntry(
     connection_config=DatabricksAWSVolumesConnectionConfig,
     uploader=DatabricksAWSVolumesUploader,
     uploader_config=DatabricksAWSVolumesUploaderConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )
 
 databricks_aws_volumes_source_entry = SourceRegistryEntry(

unstructured_ingest/v2/processes/connectors/databricks/volumes_azure.py

@@ -17,6 +17,10 @@ from unstructured_ingest.v2.processes.connectors.databricks.volumes import (
     DatabricksVolumesUploader,
     DatabricksVolumesUploaderConfig,
 )
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 CONNECTOR_TYPE = "databricks_volumes_azure"
 
@@ -91,6 +95,8 @@ databricks_azure_volumes_destination_entry = DestinationRegistryEntry(
     connection_config=DatabricksAzureVolumesConnectionConfig,
     uploader=DatabricksAzureVolumesUploader,
     uploader_config=DatabricksAzureVolumesUploaderConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )
 
 databricks_azure_volumes_source_entry = SourceRegistryEntry(

unstructured_ingest/v2/processes/connectors/databricks/volumes_gcp.py

@@ -17,6 +17,10 @@ from unstructured_ingest.v2.processes.connectors.databricks.volumes import (
     DatabricksVolumesUploader,
     DatabricksVolumesUploaderConfig,
 )
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 CONNECTOR_TYPE = "databricks_volumes_gcp"
 
@@ -74,6 +78,8 @@ databricks_gcp_volumes_destination_entry = DestinationRegistryEntry(
     connection_config=DatabricksGoogleVolumesConnectionConfig,
     uploader=DatabricksGoogleVolumesUploader,
     uploader_config=DatabricksGoogleVolumesUploaderConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )
 
 databricks_gcp_volumes_source_entry = SourceRegistryEntry(

unstructured_ingest/v2/processes/connectors/databricks/volumes_native.py

@@ -17,6 +17,10 @@ from unstructured_ingest.v2.processes.connectors.databricks.volumes import (
     DatabricksVolumesUploader,
     DatabricksVolumesUploaderConfig,
 )
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 CONNECTOR_TYPE = "databricks_volumes"
 
@@ -75,6 +79,8 @@ databricks_native_volumes_destination_entry = DestinationRegistryEntry(
     connection_config=DatabricksNativeVolumesConnectionConfig,
     uploader=DatabricksNativeVolumesUploader,
     uploader_config=DatabricksNativeVolumesUploaderConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )
 
 databricks_native_volumes_source_entry = SourceRegistryEntry(

unstructured_ingest/v2/processes/connectors/fsspec/azure.py

@@ -26,6 +26,10 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecUploaderConfig,
 )
 from unstructured_ingest.v2.processes.connectors.fsspec.utils import json_serial, sterilize_dict
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 if TYPE_CHECKING:
     from adlfs import AzureBlobFileSystem
@@ -194,4 +198,6 @@ azure_destination_entry = DestinationRegistryEntry(
     uploader=AzureUploader,
     uploader_config=AzureUploaderConfig,
     connection_config=AzureConnectionConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )

unstructured_ingest/v2/processes/connectors/fsspec/box.py

@@ -28,6 +28,10 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecUploaderConfig,
 )
 from unstructured_ingest.v2.processes.connectors.utils import conform_string_to_dict
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 if TYPE_CHECKING:
     from boxfs import BoxFileSystem
@@ -167,4 +171,6 @@ box_destination_entry = DestinationRegistryEntry(
     uploader=BoxUploader,
     uploader_config=BoxUploaderConfig,
     connection_config=BoxConnectionConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )

unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py

@@ -31,6 +31,10 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecUploader,
     FsspecUploaderConfig,
 )
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 if TYPE_CHECKING:
     pass
@@ -228,4 +232,6 @@ dropbox_destination_entry = DestinationRegistryEntry(
     uploader=DropboxUploader,
     uploader_config=DropboxUploaderConfig,
     connection_config=DropboxConnectionConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )

unstructured_ingest/v2/processes/connectors/fsspec/gcs.py

@@ -28,6 +28,10 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecUploader,
     FsspecUploaderConfig,
 )
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 if TYPE_CHECKING:
     from gcsfs import GCSFileSystem
@@ -194,4 +198,6 @@ gcs_destination_entry = DestinationRegistryEntry(
     uploader=GcsUploader,
     uploader_config=GcsUploaderConfig,
     connection_config=GcsConnectionConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )

unstructured_ingest/v2/processes/connectors/fsspec/s3.py

@@ -26,6 +26,10 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecUploader,
     FsspecUploaderConfig,
 )
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 CONNECTOR_TYPE = "s3"
 
@@ -182,4 +186,6 @@ s3_destination_entry = DestinationRegistryEntry(
     uploader=S3Uploader,
     uploader_config=S3UploaderConfig,
     connection_config=S3ConnectionConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )

unstructured_ingest/v2/processes/connectors/fsspec/sftp.py

@@ -26,6 +26,10 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecUploader,
     FsspecUploaderConfig,
 )
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 if TYPE_CHECKING:
     from fsspec.implementations.sftp import SFTPFileSystem
@@ -168,4 +172,6 @@ sftp_destination_entry = DestinationRegistryEntry(
     uploader=SftpUploader,
     uploader_config=SftpUploaderConfig,
     connection_config=SftpConnectionConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )

unstructured_ingest/v2/processes/connectors/local.py

@@ -27,6 +27,10 @@ from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 CONNECTOR_TYPE = "local"
 
@@ -213,5 +217,8 @@ local_source_entry = SourceRegistryEntry(
 )
 
 local_destination_entry = DestinationRegistryEntry(
-    uploader=LocalUploader, uploader_config=LocalUploaderConfig
+    uploader=LocalUploader,
+    uploader_config=LocalUploaderConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )

unstructured_ingest/v2/processes/connectors/zendesk/__init__.py

@@ -1,31 +0,0 @@
-from unstructured_ingest.v2.processes.connector_registry import (
-    add_source_entry,
-)
-
-from .zendesk import (
-    CONNECTOR_TYPE,
-    ZendeskAccessConfig,
-    ZendeskClient,
-    ZendeskConnectionConfig,
-    ZendeskDownloader,
-    ZendeskDownloaderConfig,
-    ZendeskIndexer,
-    ZendeskIndexerConfig,
-    ZendeskTicket,
-    zendesk_source_entry,
-)
-
-__all__ = [
-    "add_source_entry",
-    "zendesk_source_entry",
-    "ZendeskAccessConfig",
-    "ZendeskClient",
-    "ZendeskConnectionConfig",
-    "ZendeskDownloader",
-    "ZendeskDownloaderConfig",
-    "ZendeskIndexer",
-    "ZendeskIndexerConfig",
-    "ZendeskTicket",
-]
-
-add_source_entry(source_type=CONNECTOR_TYPE, entry=zendesk_source_entry)

unstructured_ingest/v2/processes/embedder.py

@@ -1,4 +1,3 @@
-import json
 from abc import ABC
 from dataclasses import dataclass
 from pathlib import Path
@@ -6,6 +5,7 @@ from typing import TYPE_CHECKING, Any, Literal, Optional
 
 from pydantic import BaseModel, Field, SecretStr
 
+from unstructured_ingest.utils.data_prep import get_data
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 
 if TYPE_CHECKING:
@@ -192,9 +192,8 @@ class Embedder(BaseProcess, ABC):
     def run(self, elements_filepath: Path, **kwargs: Any) -> list[dict]:
         # TODO update base embedder classes to support async
         embedder = self.config.get_embedder()
-        with elements_filepath.open("r") as elements_file:
-            elements = json.load(elements_file)
+        elements = get_data(path=elements_filepath)
         if not elements:
-            return [e.to_dict() for e in elements]
+            return []
         embedded_elements = embedder.embed_documents(elements=elements)
         return embedded_elements

unstructured_ingest/v2/processes/utils/blob_storage.py

@@ -0,0 +1,31 @@
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from unstructured_ingest.utils.data_prep import get_data, write_data
+from unstructured_ingest.v2.interfaces import FileData, UploadStager, UploadStagerConfig
+
+
+class BlobStoreUploadStagerConfig(UploadStagerConfig):
+    pass
+
+
+@dataclass
+class BlobStoreUploadStager(UploadStager):
+    upload_stager_config: BlobStoreUploadStagerConfig = field(
+        default_factory=BlobStoreUploadStagerConfig
+    )
+
+    def run(
+        self,
+        elements_filepath: Path,
+        file_data: FileData,
+        output_dir: Path,
+        output_filename: str,
+        **kwargs: Any,
+    ) -> Path:
+        output_file = self.get_output_path(output_filename=output_filename, output_dir=output_dir)
+        # Always save as json
+        data = get_data(elements_filepath)
+        write_data(path=output_file.with_suffix(".json"), data=data)
+        return output_file.with_suffix(".json")

{unstructured_ingest-0.5.13.dist-info → unstructured_ingest-0.5.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: unstructured-ingest
-Version: 0.5.13
+Version: 0.5.15
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,13 +22,13 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: python-dateutil
-Requires-Dist: dataclasses_json
 Requires-Dist: pandas
+Requires-Dist: tqdm
 Requires-Dist: pydantic>=2.7
+Requires-Dist: python-dateutil
 Requires-Dist: click
 Requires-Dist: opentelemetry-sdk
-Requires-Dist: tqdm
+Requires-Dist: dataclasses_json
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.30.0; extra == "remote"
 Provides-Extra: csv
@@ -71,11 +71,11 @@ Requires-Dist: fsspec; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: biomed
-Requires-Dist: requests; extra == "biomed"
 Requires-Dist: bs4; extra == "biomed"
+Requires-Dist: requests; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: boxfs; extra == "box"
 Requires-Dist: fsspec; extra == "box"
+Requires-Dist: boxfs; extra == "box"
 Provides-Extra: chroma
 Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
@@ -102,15 +102,15 @@ Requires-Dist: gcsfs; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
 Requires-Dist: fsspec; extra == "gcs"
 Provides-Extra: github
-Requires-Dist: requests; extra == "github"
 Requires-Dist: pygithub>1.58.0; extra == "github"
+Requires-Dist: requests; extra == "github"
 Provides-Extra: gitlab
 Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: hubspot-api-client; extra == "hubspot"
 Requires-Dist: urllib3; extra == "hubspot"
+Requires-Dist: hubspot-api-client; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -128,14 +128,14 @@ Requires-Dist: cymple; extra == "neo4j"
 Requires-Dist: neo4j-rust-ext; extra == "neo4j"
 Requires-Dist: networkx; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: notion-client; extra == "notion"
+Requires-Dist: httpx; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
+Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
-Requires-Dist: httpx; extra == "notion"
 Provides-Extra: onedrive
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
-Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
+Requires-Dist: msal; extra == "onedrive"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: outlook
@@ -152,8 +152,8 @@ Requires-Dist: praw; extra == "reddit"
 Provides-Extra: redis
 Requires-Dist: redis; extra == "redis"
 Provides-Extra: s3
-Requires-Dist: fsspec; extra == "s3"
 Requires-Dist: s3fs; extra == "s3"
+Requires-Dist: fsspec; extra == "s3"
 Provides-Extra: sharepoint
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Requires-Dist: msal; extra == "sharepoint"
@@ -165,8 +165,8 @@ Requires-Dist: fsspec; extra == "sftp"
 Provides-Extra: slack
 Requires-Dist: slack_sdk[optional]; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Requires-Dist: psycopg2-binary; extra == "snowflake"
+Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Provides-Extra: wikipedia
 Requires-Dist: wikipedia; extra == "wikipedia"
 Provides-Extra: weaviate
@@ -178,22 +178,22 @@ Requires-Dist: databricks-sql-connector; extra == "databricks-delta-tables"
 Provides-Extra: singlestore
 Requires-Dist: singlestoredb; extra == "singlestore"
 Provides-Extra: vectara
+Requires-Dist: httpx; extra == "vectara"
 Requires-Dist: requests; extra == "vectara"
 Requires-Dist: aiofiles; extra == "vectara"
-Requires-Dist: httpx; extra == "vectara"
 Provides-Extra: vastdb
-Requires-Dist: ibis; extra == "vastdb"
 Requires-Dist: vastdb; extra == "vastdb"
+Requires-Dist: ibis; extra == "vastdb"
 Requires-Dist: pyarrow; extra == "vastdb"
 Provides-Extra: zendesk
+Requires-Dist: httpx; extra == "zendesk"
 Requires-Dist: bs4; extra == "zendesk"
 Requires-Dist: aiofiles; extra == "zendesk"
-Requires-Dist: httpx; extra == "zendesk"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
-Requires-Dist: tiktoken; extra == "embed-octoai"
 Requires-Dist: openai; extra == "embed-octoai"
+Requires-Dist: tiktoken; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
@@ -201,11 +201,11 @@ Requires-Dist: voyageai; extra == "embed-voyageai"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Provides-Extra: openai
-Requires-Dist: tiktoken; extra == "openai"
 Requires-Dist: openai; extra == "openai"
+Requires-Dist: tiktoken; extra == "openai"
 Provides-Extra: bedrock
-Requires-Dist: boto3; extra == "bedrock"
 Requires-Dist: aioboto3; extra == "bedrock"
+Requires-Dist: boto3; extra == "bedrock"
 Provides-Extra: togetherai
 Requires-Dist: together; extra == "togetherai"
 Dynamic: author

{unstructured_ingest-0.5.13.dist-info → unstructured_ingest-0.5.15.dist-info}/RECORD

@@ -8,7 +8,7 @@ test/integration/connectors/conftest.py,sha256=vYs4WDlCuieAwwErkJxCk4a1lGvr3qpei
 test/integration/connectors/test_astradb.py,sha256=c9Lk0dvJVVdzHcokvsc4XMNJ4SIO1k2vGtT5py0cFVM,9753
 test/integration/connectors/test_azure_ai_search.py,sha256=MxFwk84vI_HT4taQTGrNpJ8ewGPqHSGrx626j8hC_Pw,9695
 test/integration/connectors/test_chroma.py,sha256=1uGHbZXkXKGb8wl3p7c9G-L1MViUe283Hw5u3dg8OgI,4532
-test/integration/connectors/test_confluence.py,sha256=Ju0gRQbD2g9l9iRf2HDZKi7RyPnBGtFRWcGpsqhO3F8,3588
+test/integration/connectors/test_confluence.py,sha256=W93znOusdvFXta8q0dqQ1rKhLafRVIqrfaFqk2FY-fo,3590
 test/integration/connectors/test_delta_table.py,sha256=4qm2Arfc9Eb7SOZOnOlLF-vNpHy6Eqvr5Q45svfX1PY,6911
 test/integration/connectors/test_dropbox.py,sha256=jzpZ6wawLa4sC1BVoHWZJ3cHjL4DWWUEX5ee7bXUOOM,4945
 test/integration/connectors/test_google_drive.py,sha256=ubjn3wvMhgpGHQs-wT_5icGgTIx2coS6hwNkAHOCEI8,10306
@@ -25,7 +25,7 @@ test/integration/connectors/test_redis.py,sha256=1aKwOb-K4zCxZwHmgW_WzGJwqLntbWT
 test/integration/connectors/test_s3.py,sha256=E1dypeag_E3OIfpQWIz3jb7ctRHRD63UtyTrzyvJzpc,7473
 test/integration/connectors/test_sharepoint.py,sha256=weGby5YD6se7R7KLEq96hxUZYPzwoqZqXXTPhtQWZsQ,7646
 test/integration/connectors/test_vectara.py,sha256=4kKOOTGUjeZw2jKRcgVDI7ifbRPRZfjjVO4d_7H5C6I,8710
-test/integration/connectors/test_zendesk.py,sha256=6Xsxxav9b1NBp_zd66S_sE4Nn5iO6Et4a5zgGR2-Y04,4159
+test/integration/connectors/test_zendesk.py,sha256=Jc1GcMBnCrpzm6_6tJi-FdYxSs15xnp94a7kVwrObMc,4167
 test/integration/connectors/databricks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/connectors/databricks/test_volumes_native.py,sha256=KqiapQAV0s_Zv0CO8BwYoiCk30dwrSZzuigUWNRIem0,9559
 test/integration/connectors/discord/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -89,7 +89,7 @@ test/unit/v2/test_utils.py,sha256=TWVAeE0OrcHgPyzGPtEnQakICsVrDeVhIKPMRQPX554,26
 test/unit/v2/chunkers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/unit/v2/chunkers/test_chunkers.py,sha256=HSr3_lsoMw1nkDhkjO0-NOTEomRdR9oxCrSXvcMFecE,1772
 test/unit/v2/connectors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-test/unit/v2/connectors/test_confluence.py,sha256=bXrn_kRb4IQdqkk4rc-P2gJAtPba7n7pNplQgfbqZDY,1047
+test/unit/v2/connectors/test_confluence.py,sha256=lN6nnU5qOtmsjIGcz65roepm76w4vPF7AmSzi9vqV78,1919
 test/unit/v2/connectors/test_jira.py,sha256=XEBBDSdNZWUVO5JbpiSsjazJYmbLsgXUOW-APqPRKLg,12113
 test/unit/v2/connectors/databricks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/unit/v2/connectors/databricks/test_volumes_table.py,sha256=-R_EJHqv1BseGRK9VRAZhF-2EXA64LAlhycoyIu556U,1078
@@ -111,7 +111,7 @@ test/unit/v2/partitioners/test_partitioner.py,sha256=iIYg7IpftV3LusoO4H8tr1IHY1U
 test/unit/v2/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/unit/v2/utils/data_generator.py,sha256=UoYVNjG4S4wlaA9gceQ82HIpF9_6I1UTHD1_GrQBHp0,973
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=SoXJvWNbP_dJpjjbYHn-IaGnsULa9X8yicnEFO_W3yI,43
+unstructured_ingest/__version__.py,sha256=noAC1JV7rAfkk9NQctRgYOifiiASnPhPSbtOr9y3Hkk,43
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/interfaces.py,sha256=7DOnDpGvUNlCoFR7UPRGmOarqH5sFtuUOO5vf8X3oTM,31489
 unstructured_ingest/logger.py,sha256=S5nSqGcABoQyeicgRnBQFjDScCaTvFVivOCvbo-laL0,4479
@@ -281,7 +281,7 @@ unstructured_ingest/connector/notion/types/database_properties/verification.py,s
 unstructured_ingest/embed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/embed/azure_openai.py,sha256=_-I-nwd-wdCiKkSdYBL4UKrTZ2UPWsM_0T69fcObs_I,1707
 unstructured_ingest/embed/bedrock.py,sha256=tZumLLXafSr1zIFVjckapRoiiY-7u65GPuWmwsdhY0I,7726
-unstructured_ingest/embed/huggingface.py,sha256=EWU1kd5Cm6ajgCw6hP5w_4pniGSgxnR0wM9vjuPQ6Yk,2334
+unstructured_ingest/embed/huggingface.py,sha256=-ZD17O_H_UnK80fqig6y6wNKJckjx0HuAkY5vgPvk8M,2259
 unstructured_ingest/embed/interfaces.py,sha256=_-CqasY6R5nnNUY-X6PS5lz8dsmGaUw5zIGRdPfx16o,4918
 unstructured_ingest/embed/mixedbreadai.py,sha256=-Y0J27G9CL1t3ZTIeNjTjRviErSMAzJRf2zgDgMHUmg,4499
 unstructured_ingest/embed/octoai.py,sha256=hNLEskDEP-2qWExUgVz2Eyw3KTIFwdUE9elbJ5qp4Ao,3855
@@ -370,7 +370,7 @@ unstructured_ingest/runner/writers/fsspec/s3.py,sha256=kHJq2O3864QBd_tL2SKb0mdyw
 unstructured_ingest/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/utils/chunking.py,sha256=9b3sXMA6L8RW5xAkKQbwdtVudGLAcj_sgT6Grh5tyYM,1870
 unstructured_ingest/utils/compression.py,sha256=NNiY-2S2Gf3at7zC1PYxMijaEza9vVSzRn5mdFf6mHo,4434
-unstructured_ingest/utils/data_prep.py,sha256=X3d8Kos1zqX-HQAicF_8TB0BrstRtHrbMzu_1s7mj7M,7191
+unstructured_ingest/utils/data_prep.py,sha256=AKtsdu9stYA63CV1C5B_fFWigqy-giVv-euumitos-A,7266
 unstructured_ingest/utils/dep_check.py,sha256=SXXcUna2H0RtxA6j1S2NGkvQa9JP2DujWhmyBa7776Y,2400
 unstructured_ingest/utils/google_filetype.py,sha256=YVspEkiiBrRUSGVeVbsavvLvTmizdy2e6TsjigXTSRU,468
 unstructured_ingest/utils/html.py,sha256=DGRDMqGbwH8RiF94Qh6NiqVkbbjZfe1h26dIehC-X7M,6340
@@ -399,39 +399,39 @@ unstructured_ingest/v2/cli/utils/model_conversion.py,sha256=7eEIkk1KU51-ZNiIfI1K
 unstructured_ingest/v2/interfaces/__init__.py,sha256=Xp7-345QpM6MG7V7G4ZrVERjADAUBiPAY88PKaMRyqY,1005
 unstructured_ingest/v2/interfaces/connector.py,sha256=qUFFJ3qgDMenTCZMtVRjq1DIwsVak6pxNjQOH2eVkMw,1623
 unstructured_ingest/v2/interfaces/downloader.py,sha256=Qi_wISgUACZKEPu5p1kUaG3uiCXcr3zWg9z9uRDwoOk,2927
-unstructured_ingest/v2/interfaces/file_data.py,sha256=7MyRlj5dijQsCR6W18wQ8fEgJigGKwoOYc10g9A6PSo,3834
+unstructured_ingest/v2/interfaces/file_data.py,sha256=kowOhvYy0q_-khX3IuR111AfjkdQezEfxjzK6QDH7oA,3836
 unstructured_ingest/v2/interfaces/indexer.py,sha256=i0oftyifXefxfKa4a3sCfSwkzWGSPE6EvC9sg6fwZgk,833
 unstructured_ingest/v2/interfaces/process.py,sha256=S3A_9gkwwGC-iQxvnpj3Er6IJAjAT5npzpSgxuFAzUM,449
 unstructured_ingest/v2/interfaces/processor.py,sha256=VX7JqXlbG1plxMK8THWhWINPbTICaaUEk4XUXhnOixY,3303
-unstructured_ingest/v2/interfaces/upload_stager.py,sha256=9EV9863ODDv0Y5liDT3xh2yiVuFiaVVyCcnwCy6nfkM,3172
+unstructured_ingest/v2/interfaces/upload_stager.py,sha256=Bzhb994gVqFI8KBd6lx3Rcr5UH0ZhU66jOD3WAUr20Q,3151
 unstructured_ingest/v2/interfaces/uploader.py,sha256=AMgp0uaJ5XeqiyURLIUnWyoIqhUT9Ak5P_LT9-qasYk,2107
 unstructured_ingest/v2/pipeline/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/v2/pipeline/interfaces.py,sha256=-Y6gPnl-SbNxIx5-dQCmiYSPKUMjivrRlBLIKIUWVeM,8658
 unstructured_ingest/v2/pipeline/otel.py,sha256=K3pQvWVgWzyOWMKCBUofsH7wTZPJ0Ysw5sLjMBLW41I,1088
-unstructured_ingest/v2/pipeline/pipeline.py,sha256=UeOk5SywJZIn3kCnHclQ2cP7JJIXb4NDjpwzsCP_cF0,16523
+unstructured_ingest/v2/pipeline/pipeline.py,sha256=m3m9F9wZsCEhsFK_0WZv5_ENl2M42VHBV6Vc39t90v8,16842
 unstructured_ingest/v2/pipeline/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-unstructured_ingest/v2/pipeline/steps/chunk.py,sha256=LK2ldM24TE4ukX_Z6Z81LpF53orMaRkddM3uhLtT5EQ,3221
+unstructured_ingest/v2/pipeline/steps/chunk.py,sha256=s2BY2v1cs_ImTsPrg8J-92k-fV73b61nDiSy4p9736k,3223
 unstructured_ingest/v2/pipeline/steps/download.py,sha256=nZ4B0d9p-6TgWqrBoKUQPlr8m6dz1RGNr_3OjUhRpWg,8259
-unstructured_ingest/v2/pipeline/steps/embed.py,sha256=iL6X0G5AvKnlfI-3XRWudlb0-6rD_PqyzA3MFmmcn6M,3199
+unstructured_ingest/v2/pipeline/steps/embed.py,sha256=HPQgEWvVrpThUD1FB9k7XNiARXkd6rb4lnpxTGmEQxI,3201
 unstructured_ingest/v2/pipeline/steps/filter.py,sha256=pju7knTSbB2ll1jC9DPePRDnHlOlvEcU1-sjk6xYGGc,1211
 unstructured_ingest/v2/pipeline/steps/index.py,sha256=m0BbUwe_7s_gFxR9K31IJdAf3_GgKXXajGJec5jcSXA,3557
-unstructured_ingest/v2/pipeline/steps/partition.py,sha256=IJQWaOTcyFlH2bz8WbmynE5Zkd5D8ELOKTnSCnt9Wcc,3282
+unstructured_ingest/v2/pipeline/steps/partition.py,sha256=yE4HFFyORhnzH25PoJG6MNquMXqpzAznyf9NoZYBV5E,3284
 unstructured_ingest/v2/pipeline/steps/stage.py,sha256=VR8SLUJdVva61aieVKyxUHzupTCQbQeaMA0CKu4Fx7o,2347
 unstructured_ingest/v2/pipeline/steps/uncompress.py,sha256=p2nPFGbcpivPAZO5jDogTfn0iaL5bCFsgBNMejxVbzE,1768
 unstructured_ingest/v2/pipeline/steps/upload.py,sha256=We4OAtStuZwWKKBCOPhfeAz_vjQt5hgsy_jRCxPzxo4,2010
 unstructured_ingest/v2/processes/__init__.py,sha256=FaHWSCGyc7GWVnAsNEUUj7L8hT8gCVY3_hUE2VzWtUg,462
 unstructured_ingest/v2/processes/chunker.py,sha256=31-7ojsM2coIt2rMR0KOb82IxLVJfNHbqYUOsDkhxN8,5491
 unstructured_ingest/v2/processes/connector_registry.py,sha256=vkEe6jpgdYtZCxMj59s5atWGgmPuxAEXRUoTt-MJ7wc,2198
-unstructured_ingest/v2/processes/embedder.py,sha256=4x-Rt5UCvwdgihDAr24hvTGDEd1CdKF9xJrf3aMU-ck,7926
+unstructured_ingest/v2/processes/embedder.py,sha256=gvlCQDsbQVgcp-2f0Qq4RiFbcr8gJwIS-imgloE-UOc,7887
 unstructured_ingest/v2/processes/filter.py,sha256=kjUmMw2SDq2bme0JCAOxs6cJriIG6Ty09KOznS-xz08,2145
 unstructured_ingest/v2/processes/partitioner.py,sha256=HxopDSbovLh_1epeGeVtuWEX7v5KG35BowwKIJ_y4e8,9910
 unstructured_ingest/v2/processes/uncompress.py,sha256=Z_XfsITGdyaRwhtNUc7bMj5Y2jLuBge8KoK4nxhqKag,2425
-unstructured_ingest/v2/processes/connectors/__init__.py,sha256=rkEQVVgcHoY3jwgW_5PH_NzdXIEwtBLs9Dk4VzmTZMA,6387
+unstructured_ingest/v2/processes/connectors/__init__.py,sha256=ebLvZes84qRx4eS20SkvlVH6WIIM76hifyUgkUJ-dfg,6588
 unstructured_ingest/v2/processes/connectors/airtable.py,sha256=eeZJe-bBNxt5Sa-XEFCdcGeJCguJU5WN2Mv9kLp5dVQ,8917
 unstructured_ingest/v2/processes/connectors/astradb.py,sha256=E6fB4anCd_gtSzVUsZ5pDrfdxs5AWERQM_NEfeenfEs,18202
 unstructured_ingest/v2/processes/connectors/azure_ai_search.py,sha256=ngPDpU0oZ6m5sxIlB6u5ebQpqCS_SJ-_amCC1KQ03EQ,11529
 unstructured_ingest/v2/processes/connectors/chroma.py,sha256=VHCnM56qNXuHzovJihrNfJnZbWLJShOe8j12PJFrbL0,7219
-unstructured_ingest/v2/processes/connectors/confluence.py,sha256=wTZewdbmCHaQuEJZ7Wf0NBOo8fS_n1I0DDwlhN96woE,11243
+unstructured_ingest/v2/processes/connectors/confluence.py,sha256=gSs4-AxL0gfeWdJfP7JfCrQSQNLoJRkvHquKK9RJvpQ,12043
 unstructured_ingest/v2/processes/connectors/couchbase.py,sha256=i7vuNKsUkN93JRVmg4--MO0ZgbjvhIqt46oYqk9zFSQ,12250
 unstructured_ingest/v2/processes/connectors/delta_table.py,sha256=xvLWTSFEC3gyGTwEISXxWmUoAfCgzdgZkETMMBOPHuI,7153
 unstructured_ingest/v2/processes/connectors/discord.py,sha256=-e4-cBK4TnHkknK1qIb86AIVMy81lBgC288_iLpTzM8,5246
@@ -439,7 +439,7 @@ unstructured_ingest/v2/processes/connectors/gitlab.py,sha256=ufE65Z8q_tC4oppGg5B
 unstructured_ingest/v2/processes/connectors/google_drive.py,sha256=QzcHNelUbnubsDtanFIgDCRzmYTuP-GjJ_g9y8fButE,19623
 unstructured_ingest/v2/processes/connectors/jira.py,sha256=-f_vIWNw6Xr8rMNdAcfCC2cmhB-QndnZk5XymHo60FU,17094
 unstructured_ingest/v2/processes/connectors/kdbai.py,sha256=VRDAiou_7oWOIAgQTdOGQWxudzQEDopXM8XkfkQ2j6g,5004
-unstructured_ingest/v2/processes/connectors/local.py,sha256=ZvWTj6ZYkwnvQMNFsZWoaQyp9zp0WVqAywMaHJ2kcAc,7153
+unstructured_ingest/v2/processes/connectors/local.py,sha256=FWPRjjUsnQjyZMChuZGuMU04AB5X0sFEOcAXhx1r9sk,7381
 unstructured_ingest/v2/processes/connectors/milvus.py,sha256=wmcu9NVy3gYlQGT25inN5w_QrhFoL8-hRq0pJFSNw8g,8866
 unstructured_ingest/v2/processes/connectors/mongodb.py,sha256=cL0QUQZF_s2brh3nNNeAywXVpaIiND4b5JTAFlYjLjw,14273
 unstructured_ingest/v2/processes/connectors/neo4j.py,sha256=vxf6Xuh-OMS09Y-mIF0PIwrFauqRtoI7vjeLBXsFwTk,18744
@@ -456,10 +456,10 @@ unstructured_ingest/v2/processes/connectors/assets/__init__.py,sha256=47DEQpj8HB
 unstructured_ingest/v2/processes/connectors/assets/weaviate_collection_config.json,sha256=SJlIO0kXxy866tWQ8bEzvwLwflsoUMIS-OKlxMvHIuE,504
 unstructured_ingest/v2/processes/connectors/databricks/__init__.py,sha256=Oh8SwTWi66gO8BsNF6vRMoQVuegyBPPCpVozkOHEf3A,2136
 unstructured_ingest/v2/processes/connectors/databricks/volumes.py,sha256=Yj4fIxgGo9Qh1x_6-INdbrPGHCuZElu-QKNfSVtW7F4,7694
-unstructured_ingest/v2/processes/connectors/databricks/volumes_aws.py,sha256=TA2e_1SIr4VaEI62873eyReCNfgmQ51_2Pko2I04pPM,2747
-unstructured_ingest/v2/processes/connectors/databricks/volumes_azure.py,sha256=cb-EUW0T-linZMkbU6AcKEGWnFHQvhpO5Abtps4P2X0,3532
-unstructured_ingest/v2/processes/connectors/databricks/volumes_gcp.py,sha256=tR8NubkyHw49IpW_42g6w1Koxlm56EPiPf1lB-eoRSI,2783
-unstructured_ingest/v2/processes/connectors/databricks/volumes_native.py,sha256=dJLD1fueXf8_0AfC4cg0G7siJZVefz68iuEx2Kq7rMs,2890
+unstructured_ingest/v2/processes/connectors/databricks/volumes_aws.py,sha256=h6qDxQhWlT7H4K1CEfKag1stTiD1o97VckJZERsofqU,2970
+unstructured_ingest/v2/processes/connectors/databricks/volumes_azure.py,sha256=gjICJJwhDHBLt_L-LrMlvJ3DL1DYtwFpyMLb_zYvOIg,3755
+unstructured_ingest/v2/processes/connectors/databricks/volumes_gcp.py,sha256=Uss3XPPaq1AsqJOEy4RJgBJw2-bTjrXH2PgtVNYd2w0,3006
+unstructured_ingest/v2/processes/connectors/databricks/volumes_native.py,sha256=g1qYnIrML4TjN7rmC0MGrD5JzAprb6SymBHlEdOumz0,3113
 unstructured_ingest/v2/processes/connectors/databricks/volumes_table.py,sha256=2KNLwDZJDhsMAUGCzktEIn4Lvb0nxLWabBOPJbgyoEE,5010
 unstructured_ingest/v2/processes/connectors/duckdb/__init__.py,sha256=5sVvJCWhU-YkjHIwk4W6BZCanFYK5W4xTpWtQ8xzeB4,561
 unstructured_ingest/v2/processes/connectors/duckdb/base.py,sha256=IHaY1mWuidt6GDEJhB1c_orwmjeyXuRCVJ88djYDciM,2793
@@ -469,13 +469,13 @@ unstructured_ingest/v2/processes/connectors/elasticsearch/__init__.py,sha256=Zzc
 unstructured_ingest/v2/processes/connectors/elasticsearch/elasticsearch.py,sha256=MEKU64OsiQmbLPb3ken-WWCIV6-pnFbs_6kjJweG-SY,18813
 unstructured_ingest/v2/processes/connectors/elasticsearch/opensearch.py,sha256=qRz8Fyr2RSZIPZGkhPeme6AZxM0aX-c_xOa1ZtSr2Kg,6781
 unstructured_ingest/v2/processes/connectors/fsspec/__init__.py,sha256=TtdeImM7Ypl_n6sl7I1JqX6bGSG0t_FqvCqE3Cy24og,1846
-unstructured_ingest/v2/processes/connectors/fsspec/azure.py,sha256=fwbHYoRrN0ZRuLdLb5X-Z7nr11rMSY8VhWMhfR3ljQo,6933
-unstructured_ingest/v2/processes/connectors/fsspec/box.py,sha256=VXxEfgJbW8DCOrqLW7mQkSeWqH-HczidTNIE28SgERY,5658
-unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py,sha256=DUAxkMDosLhRYfITuXh-Jpupwd20_3VK4P-FK-wgg7k,8137
+unstructured_ingest/v2/processes/connectors/fsspec/azure.py,sha256=kw0UfGI2fx3oQ8jVpzF45pH8Qg_QP_que5C_VXgnktc,7156
+unstructured_ingest/v2/processes/connectors/fsspec/box.py,sha256=aJCtCHRBAauLwdWEQe704Cm4UHv-ukTXV2bT3SBENVk,5881
+unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py,sha256=epf2okPKqF4R-u_zxEYDJK4g0qhFqf1ejuz8JSJaNyU,8360
 unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py,sha256=0Z--cPh17W_j4jQkSe2BeeD_j0Tt147Z01gqqF58Z9A,14421
-unstructured_ingest/v2/processes/connectors/fsspec/gcs.py,sha256=uOfm2tLc0r5U3CNkfauuwhGOhP7RJpjyBpHWMDXCk7c,6954
-unstructured_ingest/v2/processes/connectors/fsspec/s3.py,sha256=LcfIU-QgW5oVMF4jMUVm7HSgVcSrQamY6mgXdQuiSjc,6400
-unstructured_ingest/v2/processes/connectors/fsspec/sftp.py,sha256=3cVwVH3fT_JEYzIbl48-NDXdbo7XWX4C4_eqTvgWIro,6150
+unstructured_ingest/v2/processes/connectors/fsspec/gcs.py,sha256=nlDSKHs8mbXCY5Bok1hGH8UZJCdtnyhZWiRwn180ohk,7177
+unstructured_ingest/v2/processes/connectors/fsspec/s3.py,sha256=qO4WDZPoxmYMbUkaSvrxXaLn3UxzyMVhpj5wVyXqmi4,6623
+unstructured_ingest/v2/processes/connectors/fsspec/sftp.py,sha256=ZimcBJL-Har7GOESb9blzDb8pzPZcmh16YvvHYxYkJM,6373
 unstructured_ingest/v2/processes/connectors/fsspec/utils.py,sha256=jec_Qfe2hbfahBuY-u8FnvHuv933AI5HwPFjOL3kEEY,456
 unstructured_ingest/v2/processes/connectors/kafka/__init__.py,sha256=mQJ9Ex-QCfhz-BB5YWTfbPf7xGLd1i7FpjRr0ukbhNw,754
 unstructured_ingest/v2/processes/connectors/kafka/cloud.py,sha256=GdAeQ8Uz-6v1C5byBHtjfevVfbzW3obScBFFLRTb0ps,3441
@@ -572,12 +572,14 @@ unstructured_ingest/v2/processes/connectors/weaviate/cloud.py,sha256=bXtfEYLquR-
 unstructured_ingest/v2/processes/connectors/weaviate/embedded.py,sha256=S8Zg8StuZT-k7tCg1D5YShO1-vJYYk9-M1bE1fIqx64,3014
 unstructured_ingest/v2/processes/connectors/weaviate/local.py,sha256=LuTBKPseVewsz8VqxRPRLfGEm3BeI9nBZxpy7ZU5tOA,2201
 unstructured_ingest/v2/processes/connectors/weaviate/weaviate.py,sha256=UZ_s8dnVNx9BWFG2fPah4VbQbgEDF4nP78bQeU3jg08,12821
-unstructured_ingest/v2/processes/connectors/zendesk/__init__.py,sha256=XMNocKJ3FHDfy36p_KHhH7ALi0-ji6NhGuQNCV2E4vY,699
+unstructured_ingest/v2/processes/connectors/zendesk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/v2/processes/connectors/zendesk/client.py,sha256=MNyI6SUuxZHf_6zONoC6jR2f9BvyTYoMyGKDOhl4kgs,7897
 unstructured_ingest/v2/processes/connectors/zendesk/zendesk.py,sha256=vQHZa5YYiDPXXPRAPMnPXhh0QzXeiBVx_YIWskZBQIc,15465
-unstructured_ingest-0.5.13.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-0.5.13.dist-info/METADATA,sha256=K95xEzr8Tq75w90-ivlwvfFhRkNRTPnNmtIiRXDXhjs,8465
-unstructured_ingest-0.5.13.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-unstructured_ingest-0.5.13.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-0.5.13.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
-unstructured_ingest-0.5.13.dist-info/RECORD,,
+unstructured_ingest/v2/processes/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+unstructured_ingest/v2/processes/utils/blob_storage.py,sha256=EWvK4HRYubr9i1UyMhv5cU9u0UzVkCDC_BIm4Uxab7Y,964
+unstructured_ingest-0.5.15.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-0.5.15.dist-info/METADATA,sha256=TimVS8ZngyfFUMhuD317dXl6nlI9acBRC_LxZWZczuU,8465
+unstructured_ingest-0.5.15.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+unstructured_ingest-0.5.15.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-0.5.15.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
+unstructured_ingest-0.5.15.dist-info/RECORD,,