unstructured-ingest 0.5.14__tar.gz → 0.5.15__tar.gz

{unstructured_ingest-0.5.14/unstructured_ingest.egg-info → unstructured_ingest-0.5.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: unstructured-ingest
-Version: 0.5.14
+Version: 0.5.15
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,13 +22,13 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: pydantic>=2.7
-Requires-Dist: opentelemetry-sdk
-Requires-Dist: pandas
+Requires-Dist: python-dateutil
 Requires-Dist: tqdm
 Requires-Dist: dataclasses_json
+Requires-Dist: pydantic>=2.7
 Requires-Dist: click
-Requires-Dist: python-dateutil
+Requires-Dist: opentelemetry-sdk
+Requires-Dist: pandas
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.30.0; extra == "remote"
 Provides-Extra: csv
@@ -66,8 +66,8 @@ Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
 Requires-Dist: astrapy; extra == "astradb"
 Provides-Extra: azure
-Requires-Dist: fsspec; extra == "azure"
 Requires-Dist: adlfs; extra == "azure"
+Requires-Dist: fsspec; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: biomed
@@ -81,13 +81,13 @@ Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
 Requires-Dist: clarifai; extra == "clarifai"
 Provides-Extra: confluence
-Requires-Dist: atlassian-python-api; extra == "confluence"
 Requires-Dist: requests; extra == "confluence"
+Requires-Dist: atlassian-python-api; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
 Provides-Extra: delta-table
-Requires-Dist: deltalake; extra == "delta-table"
 Requires-Dist: boto3; extra == "delta-table"
+Requires-Dist: deltalake; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord.py; extra == "discord"
 Provides-Extra: dropbox
@@ -98,8 +98,8 @@ Requires-Dist: duckdb; extra == "duckdb"
 Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
 Provides-Extra: gcs
-Requires-Dist: bs4; extra == "gcs"
 Requires-Dist: gcsfs; extra == "gcs"
+Requires-Dist: bs4; extra == "gcs"
 Requires-Dist: fsspec; extra == "gcs"
 Provides-Extra: github
 Requires-Dist: requests; extra == "github"
@@ -109,8 +109,8 @@ Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: urllib3; extra == "hubspot"
 Requires-Dist: hubspot-api-client; extra == "hubspot"
+Requires-Dist: urllib3; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -124,14 +124,14 @@ Requires-Dist: pymilvus; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: neo4j
-Requires-Dist: neo4j-rust-ext; extra == "neo4j"
 Requires-Dist: cymple; extra == "neo4j"
 Requires-Dist: networkx; extra == "neo4j"
+Requires-Dist: neo4j-rust-ext; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: backoff; extra == "notion"
+Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
+Requires-Dist: backoff; extra == "notion"
 Requires-Dist: notion-client; extra == "notion"
-Requires-Dist: htmlBuilder; extra == "notion"
 Provides-Extra: onedrive
 Requires-Dist: bs4; extra == "onedrive"
 Requires-Dist: msal; extra == "onedrive"
@@ -165,8 +165,8 @@ Requires-Dist: fsspec; extra == "sftp"
 Provides-Extra: slack
 Requires-Dist: slack_sdk[optional]; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: psycopg2-binary; extra == "snowflake"
 Requires-Dist: snowflake-connector-python; extra == "snowflake"
+Requires-Dist: psycopg2-binary; extra == "snowflake"
 Provides-Extra: wikipedia
 Requires-Dist: wikipedia; extra == "wikipedia"
 Provides-Extra: weaviate
@@ -178,16 +178,16 @@ Requires-Dist: databricks-sql-connector; extra == "databricks-delta-tables"
 Provides-Extra: singlestore
 Requires-Dist: singlestoredb; extra == "singlestore"
 Provides-Extra: vectara
-Requires-Dist: requests; extra == "vectara"
 Requires-Dist: httpx; extra == "vectara"
+Requires-Dist: requests; extra == "vectara"
 Requires-Dist: aiofiles; extra == "vectara"
 Provides-Extra: vastdb
 Requires-Dist: ibis; extra == "vastdb"
 Requires-Dist: vastdb; extra == "vastdb"
 Requires-Dist: pyarrow; extra == "vastdb"
 Provides-Extra: zendesk
-Requires-Dist: bs4; extra == "zendesk"
 Requires-Dist: httpx; extra == "zendesk"
+Requires-Dist: bs4; extra == "zendesk"
 Requires-Dist: aiofiles; extra == "zendesk"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
@@ -204,8 +204,8 @@ Provides-Extra: openai
 Requires-Dist: tiktoken; extra == "openai"
 Requires-Dist: openai; extra == "openai"
 Provides-Extra: bedrock
-Requires-Dist: boto3; extra == "bedrock"
 Requires-Dist: aioboto3; extra == "bedrock"
+Requires-Dist: boto3; extra == "bedrock"
 Provides-Extra: togetherai
 Requires-Dist: together; extra == "togetherai"
 Dynamic: author

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/test/integration/connectors/test_confluence.py

@@ -30,7 +30,7 @@ async def test_confluence_source(temp_dir):
     spaces = ["testteamsp", "MFS"]
 
     # Create connection and indexer configurations
-    access_config = ConfluenceAccessConfig(password=api_token)
+    access_config = ConfluenceAccessConfig(api_token=api_token)
     connection_config = ConfluenceConnectionConfig(
         url=confluence_url,
         username=user_email,
@@ -77,7 +77,7 @@ async def test_confluence_source_large(temp_dir):
     spaces = ["testteamsp1"]
 
     # Create connection and indexer configurations
-    access_config = ConfluenceAccessConfig(password=api_token)
+    access_config = ConfluenceAccessConfig(api_token=api_token)
     connection_config = ConfluenceConnectionConfig(
         url=confluence_url,
         username=user_email,

unstructured_ingest-0.5.15/test/unit/v2/connectors/test_confluence.py

@@ -0,0 +1,71 @@
+import pytest
+from pydantic import ValidationError
+
+from unstructured_ingest.v2.processes.connectors.confluence import (
+    ConfluenceAccessConfig,
+    ConfluenceConnectionConfig,
+)
+
+
+def test_connection_config_multiple_auth():
+    with pytest.raises(ValidationError):
+        ConfluenceConnectionConfig(
+            access_config=ConfluenceAccessConfig(
+                password="password",
+                token="access_token",
+            ),
+            username="user_email",
+            url="url",
+        )
+
+
+def test_connection_config_multiple_auth2():
+    with pytest.raises(ValidationError):
+        ConfluenceConnectionConfig(
+            access_config=ConfluenceAccessConfig(
+                api_token="api_token",
+                token="access_token",
+            ),
+            username="user_email",
+            url="url",
+        )
+
+
+def test_connection_config_multiple_auth3():
+    with pytest.raises(ValidationError):
+        ConfluenceConnectionConfig(
+            access_config=ConfluenceAccessConfig(
+                api_token="api_token",
+                password="password",
+            ),
+            username="user_email",
+            url="url",
+        )
+
+
+def test_connection_config_no_auth():
+    with pytest.raises(ValidationError):
+        ConfluenceConnectionConfig(access_config=ConfluenceAccessConfig(), url="url")
+
+
+def test_connection_config_password_auth():
+    ConfluenceConnectionConfig(
+        access_config=ConfluenceAccessConfig(password="password"),
+        url="url",
+        username="user_email",
+    )
+
+
+def test_connection_config_api_token_auth():
+    ConfluenceConnectionConfig(
+        access_config=ConfluenceAccessConfig(api_token="api_token"),
+        url="url",
+        username="user_email",
+    )
+
+
+def test_connection_config_pat_auth():
+    ConfluenceConnectionConfig(
+        access_config=ConfluenceAccessConfig(token="access_token"),
+        url="url",
+    )

unstructured_ingest-0.5.15/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.5.15"  # pragma: no cover

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/unstructured_ingest/embed/huggingface.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Optional
 
 from pydantic import Field
@@ -15,14 +15,11 @@ if TYPE_CHECKING:
 
 
 class HuggingFaceEmbeddingConfig(EmbeddingConfig):
-    embedder_model_name: Optional[str] = Field(
-        default="sentence-transformers/all-MiniLM-L6-v2", alias="model_name"
-    )
+    embedder_model_name: Optional[str] = Field(default="all-MiniLM-L6-v2", alias="model_name")
     embedder_model_kwargs: Optional[dict] = Field(
         default_factory=lambda: {"device": "cpu"}, alias="model_kwargs"
     )
     encode_kwargs: Optional[dict] = Field(default_factory=lambda: {"normalize_embeddings": False})
-    cache_folder: Optional[str] = Field(default=None)
 
     @requires_dependencies(
         ["sentence_transformers"],
@@ -33,7 +30,6 @@ class HuggingFaceEmbeddingConfig(EmbeddingConfig):
 
         return SentenceTransformer(
             model_name_or_path=self.embedder_model_name,
-            cache_folder=self.cache_folder,
             **self.embedder_model_kwargs,
         )
 
@@ -45,7 +41,7 @@ class HuggingFaceEmbeddingConfig(EmbeddingConfig):
 
 @dataclass
 class HuggingFaceEmbeddingEncoder(BaseEmbeddingEncoder):
-    config: HuggingFaceEmbeddingConfig
+    config: HuggingFaceEmbeddingConfig = field(default_factory=HuggingFaceEmbeddingConfig)
 
     def _embed_query(self, query: str) -> list[float]:
         return self._embed_documents(texts=[query])[0]

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/unstructured_ingest/utils/data_prep.py

@@ -2,7 +2,7 @@ import itertools
 import json
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Generator, Iterable, Optional, Sequence, TypeVar, cast
+from typing import Any, Generator, Iterable, Optional, Sequence, TypeVar, Union, cast
 
 import pandas as pd
 
@@ -163,7 +163,9 @@ def write_data(path: Path, data: list[dict], indent: Optional[int] = 2) -> None:
             raise IOError("Unsupported file type: {path}")
 
 
-def get_data(path: Path) -> list[dict]:
+def get_data(path: Union[Path, str]) -> list[dict]:
+    if isinstance(path, str):
+        path = Path(path)
     try:
         return get_data_by_suffix(path=path)
     except Exception as e:

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/unstructured_ingest/v2/interfaces/file_data.py

@@ -102,7 +102,7 @@ def file_data_from_file(path: str) -> FileData:
     try:
         return BatchFileData.from_file(path=path)
     except ValidationError:
-        logger.debug(f"{path} not valid for batch file data")
+        logger.debug(f"{path} not detected as batch file data")
 
     return FileData.from_file(path=path)
 

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/unstructured_ingest/v2/interfaces/upload_stager.py

@@ -1,4 +1,3 @@
-import json
 from abc import ABC
 from dataclasses import dataclass
 from pathlib import Path
@@ -7,6 +6,7 @@ from typing import Any, TypeVar
 from pydantic import BaseModel
 
 from unstructured_ingest.utils import ndjson
+from unstructured_ingest.utils.data_prep import get_data, write_data
 from unstructured_ingest.v2.interfaces.file_data import FileData
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 
@@ -43,16 +43,13 @@ class UploadStager(BaseProcess, ABC):
                     writer.f.flush()
 
     def process_whole(self, input_file: Path, output_file: Path, file_data: FileData) -> None:
-        with input_file.open() as in_f:
-            elements_contents = json.load(in_f)
+        elements_contents = get_data(path=input_file)
 
         conformed_elements = [
             self.conform_dict(element_dict=element, file_data=file_data)
             for element in elements_contents
         ]
-
-        with open(output_file, "w") as out_f:
-            json.dump(conformed_elements, out_f, indent=2)
+        write_data(path=output_file, data=conformed_elements)
 
     def run(
         self,

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/unstructured_ingest/v2/pipeline/pipeline.py

@@ -108,6 +108,13 @@ class Pipeline:
         uploader_connector_type = self.uploader_step.process.connector_type
         registry_entry = destination_registry[uploader_connector_type]
         if registry_entry.upload_stager and self.stager_step is None:
+            try:
+                self.stager_step = UploadStageStep(
+                    process=registry_entry.upload_stager(), context=self.context
+                )
+                return
+            except Exception as e:
+                logger.debug(f"failed to instantiate required stager on user's behalf: {e}")
             raise ValueError(
                 f"pipeline with uploader type {self.uploader_step.process.__class__.__name__} "
                 f"expects a stager of type {registry_entry.upload_stager.__name__} "

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/unstructured_ingest/v2/pipeline/steps/chunk.py

@@ -38,7 +38,7 @@ class ChunkStep(PipelineStep):
         return not filepath.exists()
 
     def get_output_filepath(self, filename: Path) -> Path:
-        hashed_output_file = f"{self.get_hash(extras=[filename.name])}.json"
+        hashed_output_file = f"{self.get_hash(extras=[filename.name])}.ndjson"
         filepath = (self.cache_dir / hashed_output_file).resolve()
         filepath.parent.mkdir(parents=True, exist_ok=True)
         return filepath

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/unstructured_ingest/v2/pipeline/steps/embed.py

@@ -38,7 +38,7 @@ class EmbedStep(PipelineStep):
         return not filepath.exists()
 
     def get_output_filepath(self, filename: Path) -> Path:
-        hashed_output_file = f"{self.get_hash(extras=[filename.name])}.json"
+        hashed_output_file = f"{self.get_hash(extras=[filename.name])}.ndjson"
         filepath = (self.cache_dir / hashed_output_file).resolve()
         filepath.parent.mkdir(parents=True, exist_ok=True)
         return filepath

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/unstructured_ingest/v2/pipeline/steps/partition.py

@@ -38,7 +38,7 @@ class PartitionStep(PipelineStep):
         return not filepath.exists()
 
     def get_output_filepath(self, filename: Path) -> Path:
-        hashed_output_file = f"{self.get_hash(extras=[filename.name])}.json"
+        hashed_output_file = f"{self.get_hash(extras=[filename.name])}.ndjson"
         filepath = (self.cache_dir / hashed_output_file).resolve()
         filepath.parent.mkdir(parents=True, exist_ok=True)
         return filepath

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/unstructured_ingest/v2/processes/connectors/confluence.py

@@ -35,7 +35,11 @@ CONNECTOR_TYPE = "confluence"
 
 class ConfluenceAccessConfig(AccessConfig):
     password: Optional[str] = Field(
-        description="Confluence password or Cloud API token",
+        description="Confluence password",
+        default=None,
+    )
+    api_token: Optional[str] = Field(
+        description="Confluence Cloud API token",
         default=None,
     )
     token: Optional[str] = Field(
@@ -57,7 +61,12 @@ class ConfluenceConnectionConfig(ConnectionConfig):
 
     def model_post_init(self, __context):
         access_configs = self.access_config.get_secret_value()
-        basic_auth = self.username and access_configs.password
+        if access_configs.password and access_configs.api_token:
+            raise ValueError(
+                "both password and api_token provided, only one allowed, "
+                "see: https://atlassian-python-api.readthedocs.io/"
+            )
+        basic_auth = bool(self.username and (access_configs.password or access_configs.api_token))
         pat_auth = access_configs.token
         if self.cloud and not basic_auth:
             raise ValueError(
@@ -74,6 +83,14 @@ class ConfluenceConnectionConfig(ConnectionConfig):
                 "no form of auth provided, see: https://atlassian-python-api.readthedocs.io/"
             )
 
+    def password_or_api_token(self) -> str:
+        # Confluence takes either password or API token under the same field: password
+        # This ambiguity led to confusion, so we are making it specific what you are passing in
+        access_configs = self.access_config.get_secret_value()
+        if access_configs.password:
+            return access_configs.password
+        return access_configs.api_token
+
     @requires_dependencies(["atlassian"], extras="confluence")
     @contextmanager
     def get_client(self) -> "Confluence":
@@ -83,7 +100,7 @@ class ConfluenceConnectionConfig(ConnectionConfig):
         with Confluence(
             url=self.url,
             username=self.username,
-            password=access_configs.password,
+            password=self.password_or_api_token(),
             token=access_configs.token,
             cloud=self.cloud,
         ) as client:

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/unstructured_ingest/v2/processes/connectors/databricks/volumes_aws.py

@@ -17,6 +17,10 @@ from unstructured_ingest.v2.processes.connectors.databricks.volumes import (
     DatabricksVolumesUploader,
     DatabricksVolumesUploaderConfig,
 )
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 CONNECTOR_TYPE = "databricks_volumes_aws"
 
@@ -76,6 +80,8 @@ databricks_aws_volumes_destination_entry = DestinationRegistryEntry(
     connection_config=DatabricksAWSVolumesConnectionConfig,
     uploader=DatabricksAWSVolumesUploader,
     uploader_config=DatabricksAWSVolumesUploaderConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )
 
 databricks_aws_volumes_source_entry = SourceRegistryEntry(

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/unstructured_ingest/v2/processes/connectors/databricks/volumes_azure.py

@@ -17,6 +17,10 @@ from unstructured_ingest.v2.processes.connectors.databricks.volumes import (
     DatabricksVolumesUploader,
     DatabricksVolumesUploaderConfig,
 )
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 CONNECTOR_TYPE = "databricks_volumes_azure"
 
@@ -91,6 +95,8 @@ databricks_azure_volumes_destination_entry = DestinationRegistryEntry(
     connection_config=DatabricksAzureVolumesConnectionConfig,
     uploader=DatabricksAzureVolumesUploader,
     uploader_config=DatabricksAzureVolumesUploaderConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )
 
 databricks_azure_volumes_source_entry = SourceRegistryEntry(

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/unstructured_ingest/v2/processes/connectors/databricks/volumes_gcp.py

@@ -17,6 +17,10 @@ from unstructured_ingest.v2.processes.connectors.databricks.volumes import (
     DatabricksVolumesUploader,
     DatabricksVolumesUploaderConfig,
 )
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 CONNECTOR_TYPE = "databricks_volumes_gcp"
 
@@ -74,6 +78,8 @@ databricks_gcp_volumes_destination_entry = DestinationRegistryEntry(
     connection_config=DatabricksGoogleVolumesConnectionConfig,
     uploader=DatabricksGoogleVolumesUploader,
     uploader_config=DatabricksGoogleVolumesUploaderConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )
 
 databricks_gcp_volumes_source_entry = SourceRegistryEntry(

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/unstructured_ingest/v2/processes/connectors/databricks/volumes_native.py

@@ -17,6 +17,10 @@ from unstructured_ingest.v2.processes.connectors.databricks.volumes import (
     DatabricksVolumesUploader,
     DatabricksVolumesUploaderConfig,
 )
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 CONNECTOR_TYPE = "databricks_volumes"
 
@@ -75,6 +79,8 @@ databricks_native_volumes_destination_entry = DestinationRegistryEntry(
     connection_config=DatabricksNativeVolumesConnectionConfig,
     uploader=DatabricksNativeVolumesUploader,
     uploader_config=DatabricksNativeVolumesUploaderConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )
 
 databricks_native_volumes_source_entry = SourceRegistryEntry(

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/unstructured_ingest/v2/processes/connectors/fsspec/azure.py

@@ -26,6 +26,10 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecUploaderConfig,
 )
 from unstructured_ingest.v2.processes.connectors.fsspec.utils import json_serial, sterilize_dict
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 if TYPE_CHECKING:
     from adlfs import AzureBlobFileSystem
@@ -194,4 +198,6 @@ azure_destination_entry = DestinationRegistryEntry(
     uploader=AzureUploader,
     uploader_config=AzureUploaderConfig,
     connection_config=AzureConnectionConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/unstructured_ingest/v2/processes/connectors/fsspec/box.py

@@ -28,6 +28,10 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecUploaderConfig,
 )
 from unstructured_ingest.v2.processes.connectors.utils import conform_string_to_dict
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 if TYPE_CHECKING:
     from boxfs import BoxFileSystem
@@ -167,4 +171,6 @@ box_destination_entry = DestinationRegistryEntry(
     uploader=BoxUploader,
     uploader_config=BoxUploaderConfig,
     connection_config=BoxConnectionConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py

@@ -31,6 +31,10 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecUploader,
     FsspecUploaderConfig,
 )
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 if TYPE_CHECKING:
     pass
@@ -228,4 +232,6 @@ dropbox_destination_entry = DestinationRegistryEntry(
     uploader=DropboxUploader,
     uploader_config=DropboxUploaderConfig,
     connection_config=DropboxConnectionConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/unstructured_ingest/v2/processes/connectors/fsspec/gcs.py

@@ -28,6 +28,10 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecUploader,
     FsspecUploaderConfig,
 )
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 if TYPE_CHECKING:
     from gcsfs import GCSFileSystem
@@ -194,4 +198,6 @@ gcs_destination_entry = DestinationRegistryEntry(
     uploader=GcsUploader,
     uploader_config=GcsUploaderConfig,
     connection_config=GcsConnectionConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/unstructured_ingest/v2/processes/connectors/fsspec/s3.py

@@ -26,6 +26,10 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecUploader,
     FsspecUploaderConfig,
 )
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 CONNECTOR_TYPE = "s3"
 
@@ -182,4 +186,6 @@ s3_destination_entry = DestinationRegistryEntry(
     uploader=S3Uploader,
     uploader_config=S3UploaderConfig,
     connection_config=S3ConnectionConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/unstructured_ingest/v2/processes/connectors/fsspec/sftp.py

@@ -26,6 +26,10 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecUploader,
     FsspecUploaderConfig,
 )
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 if TYPE_CHECKING:
     from fsspec.implementations.sftp import SFTPFileSystem
@@ -168,4 +172,6 @@ sftp_destination_entry = DestinationRegistryEntry(
     uploader=SftpUploader,
     uploader_config=SftpUploaderConfig,
     connection_config=SftpConnectionConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/unstructured_ingest/v2/processes/connectors/local.py

@@ -27,6 +27,10 @@ from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
+from unstructured_ingest.v2.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
 
 CONNECTOR_TYPE = "local"
 
@@ -213,5 +217,8 @@ local_source_entry = SourceRegistryEntry(
 )
 
 local_destination_entry = DestinationRegistryEntry(
-    uploader=LocalUploader, uploader_config=LocalUploaderConfig
+    uploader=LocalUploader,
+    uploader_config=LocalUploaderConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
 )

{unstructured_ingest-0.5.14 → unstructured_ingest-0.5.15}/unstructured_ingest/v2/processes/embedder.py

@@ -1,4 +1,3 @@
-import json
 from abc import ABC
 from dataclasses import dataclass
 from pathlib import Path
@@ -6,6 +5,7 @@ from typing import TYPE_CHECKING, Any, Literal, Optional
 
 from pydantic import BaseModel, Field, SecretStr
 
+from unstructured_ingest.utils.data_prep import get_data
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 
 if TYPE_CHECKING:
@@ -192,9 +192,8 @@ class Embedder(BaseProcess, ABC):
     def run(self, elements_filepath: Path, **kwargs: Any) -> list[dict]:
         # TODO update base embedder classes to support async
         embedder = self.config.get_embedder()
-        with elements_filepath.open("r") as elements_file:
-            elements = json.load(elements_file)
+        elements = get_data(path=elements_filepath)
         if not elements:
-            return [e.to_dict() for e in elements]
+            return []
         embedded_elements = embedder.embed_documents(elements=elements)
         return embedded_elements