unstructured-ingest 0.5.5__py3-none-any.whl → 0.5.7__py3-none-any.whl

test/integration/connectors/test_pinecone.py

@@ -351,3 +351,37 @@ def test_pinecone_stager(
         stager=stager,
         tmp_dir=tmp_path,
     )
+
+
+@requires_env(API_KEY)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
+def test_pinecone_create_destination(pinecone_index):
+    uploader = PineconeUploader(
+        connection_config=PineconeConnectionConfig(
+            access_config=PineconeAccessConfig(api_key=get_api_key())
+        ),
+        upload_config=PineconeUploaderConfig(),
+    )
+
+    random_id = str(uuid4()).split("-")[0]
+
+    index_name = f"test-create-destination-{random_id}"
+
+    assert not uploader.index_exists(index_name=index_name)
+
+    try:
+        uploader.create_destination(destination_name=index_name, vector_length=1536)
+    except Exception as e:
+        error_body = getattr(e, "body", None)
+        raise pytest.fail(f"failed to create destination: {e} {error_body}")
+
+    assert uploader.index_exists(index_name=index_name), "destination was not created successfully"
+
+    try:
+        pc = uploader.connection_config.get_client()
+        logger.info(f"deleting index for test create destination: {index_name}")
+        pc.delete_index(name=index_name)
+    except Exception as e:
+        raise pytest.fail(f"failed to cleanup / delete the destination: {e}")
+
+    assert not uploader.index_exists(index_name=index_name), "cleanup failed"

test/integration/connectors/weaviate/test_local.py

@@ -78,7 +78,6 @@ def run_uploader_and_validate(
     validate_count(expected_count=expected_count)
 
 
-@pytest.mark.asyncio
 @pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 def test_weaviate_local_destination(upload_file: Path, collection: str, tmp_path: Path):
     file_data = FileData(
@@ -142,11 +141,12 @@ def test_weaviate_local_create_destination(weaviate_instance):
         upload_config=LocalWeaviateUploaderConfig(),
         connection_config=LocalWeaviateConnectionConfig(),
     )
-    collection_name = "system_created"
+    collection_name = "system_created-123"
+    formatted_collection_name = "System_created_123"
     created = uploader.create_destination(destination_name=collection_name)
     assert created
     with uploader.connection_config.get_client() as weaviate_client:
-        assert weaviate_client.collections.exists(name=collection_name)
+        assert weaviate_client.collections.exists(name=formatted_collection_name)
 
     created = uploader.create_destination(destination_name=collection_name)
     assert not created

test/integration/embedders/test_mixedbread.py

@@ -50,7 +50,7 @@ def test_raw_mixedbread_embedder(embedder_file: Path):
         embedder=embedder,
         embedder_file=embedder_file,
         expected_dimension=1024,
-        expected_is_unit_vector=False,
+        expected_is_unit_vector=True,
     )
 
 
@@ -67,5 +67,5 @@ async def test_raw_async_mixedbread_embedder(embedder_file: Path):
         embedder=embedder,
         embedder_file=embedder_file,
         expected_dimension=1024,
-        expected_is_unit_vector=False,
+        expected_is_unit_vector=True,
     )

test/integration/embedders/test_voyageai.py

@@ -61,3 +61,19 @@ async def test_raw_async_voyageai_embedder(embedder_file: Path):
     await validate_raw_embedder_async(
         embedder=embedder, embedder_file=embedder_file, expected_dimension=1024
     )
+
+
+@requires_env(API_KEY)
+def test_voyageai_multimodal_embedder(embedder_file: Path):
+    api_key = get_api_key()
+    embedder_config = EmbedderConfig(
+        embedding_provider="voyageai",
+        embedding_api_key=api_key,
+        embedding_model_name="voyage-multimodal-3",
+    )
+    embedder = Embedder(config=embedder_config)
+    results = embedder.run(elements_filepath=embedder_file)
+    assert results
+    with embedder_file.open("r") as f:
+        original_elements = json.load(f)
+    validate_embedding_output(original_elements=original_elements, output_elements=results)

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.5.5"  # pragma: no cover
+__version__ = "0.5.7"  # pragma: no cover

unstructured_ingest/embed/interfaces.py

@@ -49,7 +49,7 @@ class BaseEmbeddingEncoder(BaseEncoder, ABC):
     def is_unit_vector(self) -> bool:
         """Denotes if the embedding vector is a unit vector."""
         exemplary_embedding = self.get_exemplary_embedding()
-        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
+        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0, rtol=1e-03)
 
     def get_client(self):
         raise NotImplementedError
@@ -103,7 +103,7 @@ class AsyncBaseEmbeddingEncoder(BaseEncoder, ABC):
     async def is_unit_vector(self) -> bool:
         """Denotes if the embedding vector is a unit vector."""
         exemplary_embedding = await self.get_exemplary_embedding()
-        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
+        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0, rtol=1e-03)
 
     def get_client(self):
         raise NotImplementedError

unstructured_ingest/embed/voyageai.py

@@ -96,7 +96,11 @@ class VoyageAIEmbeddingEncoder(BaseEmbeddingEncoder):
         return self.config.get_client()
 
     def embed_batch(self, client: "VoyageAIClient", batch: list[str]) -> list[list[float]]:
-        response = client.embed(texts=batch, model=self.config.embedder_model_name)
+        if self.config.embedder_model_name == "voyage-multimodal-3":
+            batch = [[text] for text in batch]
+            response = client.multimodal_embed(inputs=batch, model=self.config.embedder_model_name)
+        else:
+            response = client.embed(texts=batch, model=self.config.embedder_model_name)
         return response.embeddings
 
 
@@ -113,5 +117,11 @@ class AsyncVoyageAIEmbeddingEncoder(AsyncBaseEmbeddingEncoder):
     async def embed_batch(
         self, client: "AsyncVoyageAIClient", batch: list[str]
     ) -> list[list[float]]:
-        response = await client.embed(texts=batch, model=self.config.embedder_model_name)
+        if self.config.embedder_model_name == "voyage-multimodal-3":
+            batch = [[text] for text in batch]
+            response = await client.multimodal_embed(
+                inputs=batch, model=self.config.embedder_model_name
+            )
+        else:
+            response = await client.embed(texts=batch, model=self.config.embedder_model_name)
         return response.embeddings

unstructured_ingest/v2/interfaces/process.py

@@ -8,7 +8,7 @@ class BaseProcess(ABC):
     def is_async(self) -> bool:
         return False
 
-    def init(self, *kwargs: Any) -> None:
+    def init(self, **kwargs: Any) -> None:
         pass
 
     def precheck(self) -> None:

unstructured_ingest/v2/interfaces/uploader.py

@@ -1,7 +1,7 @@
 from abc import ABC
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Optional, TypeVar
+from typing import Any, TypeVar
 
 from pydantic import BaseModel
 
@@ -61,6 +61,6 @@ class Uploader(BaseProcess, BaseConnector, ABC):
 @dataclass
 class VectorDBUploader(Uploader, ABC):
     def create_destination(
-        self, destination_name: str = "elements", vector_length: Optional[int] = None, **kwargs: Any
+        self, vector_length: int, destination_name: str = "elements", **kwargs: Any
     ) -> bool:
         return False

unstructured_ingest/v2/pipeline/pipeline.py

@@ -126,14 +126,32 @@ class Pipeline:
                 for kk, vv in v.items():
                     logger.error(f"{k}: [{kk}] {vv}")
 
+    def _run_initialization(self):
+        failures = {}
+        init_kwargs = {}
+        for step in self._get_ordered_steps():
+            try:
+                step.process.init(**init_kwargs)
+                step.process.precheck()
+                # Make sure embedder dimensions available for downstream steps
+                if isinstance(step.process, Embedder):
+                    embed_dimensions = step.process.config.get_embedder().dimension
+                    init_kwargs["vector_length"] = embed_dimensions
+
+            except Exception as e:
+                failures[step.process.__class__.__name__] = f"[{type(e).__name__}] {e}"
+        if failures:
+            for k, v in failures.items():
+                logger.error(f"Step initialization failure: {k}: {v}")
+            raise PipelineError("Initialization failed")
+
     def run(self):
         otel_handler = OtelHandler(otel_endpoint=self.context.otel_endpoint, log_out=logger.info)
         try:
             with otel_handler.get_tracer().start_as_current_span(
                 "ingest process", record_exception=True
             ):
-                self._run_inits()
-                self._run_prechecks()
+                self._run_initialization()
                 self._run()
         finally:
             self.log_statuses()
@@ -154,43 +172,20 @@ class Pipeline:
         final = [f for f in flat if f]
         return final or None
 
-    def _get_all_steps(self) -> list[PipelineStep]:
-        steps = [self.indexer_step, self.downloader_step, self.partitioner_step, self.uploader_step]
+    def _get_ordered_steps(self) -> list[PipelineStep]:
+        steps = [self.indexer_step, self.downloader_step]
+        if self.uncompress_step:
+            steps.append(self.uncompress_step)
+        steps.append(self.partitioner_step)
         if self.chunker_step:
             steps.append(self.chunker_step)
         if self.embedder_step:
             steps.append(self.embedder_step)
-        if self.uncompress_step:
-            steps.append(self.uncompress_step)
         if self.stager_step:
             steps.append(self.stager_step)
+        steps.append(self.uploader_step)
         return steps
 
-    def _run_inits(self):
-        failures = {}
-
-        for step in self._get_all_steps():
-            try:
-                step.process.init()
-            except Exception as e:
-                failures[step.process.__class__.__name__] = f"[{type(e).__name__}] {e}"
-        if failures:
-            for k, v in failures.items():
-                logger.error(f"Step init failure: {k}: {v}")
-            raise PipelineError("Init failed")
-
-    def _run_prechecks(self):
-        failures = {}
-        for step in self._get_all_steps():
-            try:
-                step.process.precheck()
-            except Exception as e:
-                failures[step.process.__class__.__name__] = f"[{type(e).__name__}] {e}"
-        if failures:
-            for k, v in failures.items():
-                logger.error(f"Step precheck failure: {k}: {v}")
-            raise PipelineError("Precheck failed")
-
     def apply_filter(self, records: list[dict]) -> list[dict]:
         if not self.filter_step:
             return records

unstructured_ingest/v2/processes/connectors/pinecone.py

@@ -1,6 +1,7 @@
 import json
+import re
 from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Any, Optional
+from typing import TYPE_CHECKING, Any, Literal, Optional
 
 from pydantic import Field, Secret
 
@@ -13,10 +14,10 @@ from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
     FileData,
-    Uploader,
     UploaderConfig,
     UploadStager,
     UploadStagerConfig,
+    VectorDBUploader,
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
@@ -41,7 +42,7 @@ class PineconeAccessConfig(AccessConfig):
 
 
 class PineconeConnectionConfig(ConnectionConfig):
-    index_name: str = Field(description="Name of the index to connect to.")
+    index_name: Optional[str] = Field(description="Name of the index to connect to.", default=None)
     access_config: Secret[PineconeAccessConfig] = Field(
         default=PineconeAccessConfig(), validate_default=True
     )
@@ -160,18 +161,101 @@ class PineconeUploadStager(UploadStager):
 
 
 @dataclass
-class PineconeUploader(Uploader):
+class PineconeUploader(VectorDBUploader):
     upload_config: PineconeUploaderConfig
     connection_config: PineconeConnectionConfig
     connector_type: str = CONNECTOR_TYPE
 
+    def init(self, **kwargs: Any) -> None:
+        self.create_destination(**kwargs)
+
+    def index_exists(self, index_name: Optional[str]) -> bool:
+        from pinecone.exceptions import NotFoundException
+
+        index_name = index_name or self.connection_config.index_name
+        pc = self.connection_config.get_client()
+        try:
+            pc.describe_index(index_name)
+            return True
+        except NotFoundException:
+            return False
+        except Exception as e:
+            logger.error(f"failed to check if pinecone index exists : {e}")
+            raise DestinationConnectionError(f"failed to check if pinecone index exists : {e}")
+
     def precheck(self):
         try:
-            self.connection_config.get_index()
+            # just a connection check here. not an actual index_exists check
+            self.index_exists("just-checking-our-connection")
+
+            if self.connection_config.index_name and not self.index_exists(
+                self.connection_config.index_name
+            ):
+                raise DestinationConnectionError(
+                    f"index {self.connection_config.index_name} does not exist"
+                )
         except Exception as e:
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")
 
+    def format_destination_name(self, destination_name: str) -> str:
+        # Pinecone naming requirements:
+        # can only contain lowercase letters, numbers, and hyphens
+        # must be 45 characters or less
+        formatted = re.sub(r"[^a-z0-9]", "-", destination_name.lower())
+        return formatted
+
+    def create_destination(
+        self,
+        vector_length: int,
+        destination_name: str = "elements",
+        destination_type: Literal["pod", "serverless"] = "serverless",
+        serverless_cloud: str = "aws",
+        serverless_region: str = "us-west-2",
+        pod_environment: str = "us-east1-gcp",
+        pod_type: str = "p1.x1",
+        pod_count: int = 1,
+        **kwargs: Any,
+    ) -> bool:
+        from pinecone import PodSpec, ServerlessSpec
+
+        index_name = destination_name or self.connection_config.index_name
+        index_name = self.format_destination_name(index_name)
+        self.connection_config.index_name = index_name
+
+        if not self.index_exists(index_name):
+
+            logger.info(f"creating pinecone index {index_name}")
+
+            pc = self.connection_config.get_client()
+
+            if destination_type == "serverless":
+                pc.create_index(
+                    name=destination_name,
+                    dimension=vector_length,
+                    spec=ServerlessSpec(cloud=serverless_cloud, region=serverless_region),
+                    **kwargs,
+                )
+
+                return True
+
+            elif destination_type == "pod":
+                pc.create_index(
+                    name=destination_name,
+                    dimension=vector_length,
+                    spec=PodSpec(environment=pod_environment, pod_type=pod_type, pods=pod_count),
+                    **kwargs,
+                )
+
+                return True
+
+            else:
+                raise ValueError(f"unexpected destination type: {destination_type}")
+
+        else:
+            logger.debug(f"index {index_name} already exists, skipping creation")
+            return False
+
     def pod_delete_by_record_id(self, file_data: FileData) -> None:
         logger.debug(
             f"deleting any content with metadata "
@@ -266,6 +350,10 @@ class PineconeUploader(Uploader):
         )
         # Determine if serverless or pod based index
         pinecone_client = self.connection_config.get_client()
+
+        if not self.connection_config.index_name:
+            raise ValueError("No index name specified")
+
         index_description = pinecone_client.describe_index(name=self.connection_config.index_name)
         if "serverless" in index_description.get("spec"):
             self.serverless_delete_by_record_id(file_data=file_data)

unstructured_ingest/v2/processes/connectors/weaviate/weaviate.py

@@ -1,4 +1,5 @@
 import json
+import re
 from abc import ABC, abstractmethod
 from contextlib import contextmanager
 from dataclasses import dataclass, field
@@ -229,19 +230,29 @@ class WeaviateUploader(VectorDBUploader, ABC):
             logger.error(f"Failed to validate connection {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")
 
-    def init(self, *kwargs: Any) -> None:
-        self.create_destination()
+    def init(self, **kwargs: Any) -> None:
+        self.create_destination(**kwargs)
+
+    def format_destination_name(self, destination_name: str) -> str:
+        # Weaviate naming requirements:
+        # must be alphanumeric and underscores only
+        formatted = re.sub(r"[^a-zA-Z0-9]", "_", destination_name)
+        # must begin with capital letter
+        return formatted.capitalize()
 
     def create_destination(
         self, destination_name: str = "elements", vector_length: Optional[int] = None, **kwargs: Any
     ) -> bool:
+        destination_name = self.format_destination_name(destination_name)
         collection_name = self.upload_config.collection or destination_name
         self.upload_config.collection = collection_name
+
         connectors_dir = Path(__file__).parents[1]
         collection_config_file = connectors_dir / "assets" / "weaviate_collection_config.json"
         with collection_config_file.open() as f:
             collection_config = json.load(f)
         collection_config["class"] = collection_name
+
         if not self._collection_exists():
             logger.info(
                 f"creating default weaviate collection '{collection_name}' with default configs"

unstructured_ingest/v2/processes/embedder.py

@@ -186,7 +186,7 @@ class EmbedderConfig(BaseModel):
 class Embedder(BaseProcess, ABC):
     config: EmbedderConfig
 
-    def init(self, *kwargs: Any) -> None:
+    def init(self, **kwargs: Any) -> None:
         self.config.get_embedder().initialize()
 
     def run(self, elements_filepath: Path, **kwargs: Any) -> list[dict]:

{unstructured_ingest-0.5.5.dist-info → unstructured_ingest-0.5.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: unstructured-ingest
-Version: 0.5.5
+Version: 0.5.7
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,13 +22,13 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: tqdm
-Requires-Dist: pydantic>=2.7
-Requires-Dist: click
 Requires-Dist: opentelemetry-sdk
+Requires-Dist: pydantic>=2.7
+Requires-Dist: dataclasses_json
 Requires-Dist: python-dateutil
+Requires-Dist: tqdm
+Requires-Dist: click
 Requires-Dist: pandas
-Requires-Dist: dataclasses_json
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.26.1; extra == "remote"
 Provides-Extra: csv
@@ -66,16 +66,16 @@ Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
 Requires-Dist: astrapy; extra == "astradb"
 Provides-Extra: azure
-Requires-Dist: fsspec; extra == "azure"
 Requires-Dist: adlfs; extra == "azure"
+Requires-Dist: fsspec; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: biomed
-Requires-Dist: requests; extra == "biomed"
 Requires-Dist: bs4; extra == "biomed"
+Requires-Dist: requests; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: fsspec; extra == "box"
 Requires-Dist: boxfs; extra == "box"
+Requires-Dist: fsspec; extra == "box"
 Provides-Extra: chroma
 Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
@@ -98,19 +98,19 @@ Requires-Dist: duckdb; extra == "duckdb"
 Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
 Provides-Extra: gcs
-Requires-Dist: gcsfs; extra == "gcs"
-Requires-Dist: fsspec; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
+Requires-Dist: fsspec; extra == "gcs"
+Requires-Dist: gcsfs; extra == "gcs"
 Provides-Extra: github
-Requires-Dist: pygithub>1.58.0; extra == "github"
 Requires-Dist: requests; extra == "github"
+Requires-Dist: pygithub>1.58.0; extra == "github"
 Provides-Extra: gitlab
 Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: urllib3; extra == "hubspot"
 Requires-Dist: hubspot-api-client; extra == "hubspot"
+Requires-Dist: urllib3; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -124,23 +124,23 @@ Requires-Dist: pymilvus; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: neo4j
-Requires-Dist: cymple; extra == "neo4j"
 Requires-Dist: neo4j-rust-ext; extra == "neo4j"
+Requires-Dist: cymple; extra == "neo4j"
 Requires-Dist: networkx; extra == "neo4j"
 Provides-Extra: notion
 Requires-Dist: notion-client; extra == "notion"
-Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
+Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
 Provides-Extra: onedrive
-Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
+Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: outlook
-Requires-Dist: msal; extra == "outlook"
 Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
+Requires-Dist: msal; extra == "outlook"
 Provides-Extra: pinecone
 Requires-Dist: pinecone-client>=3.7.1; extra == "pinecone"
 Provides-Extra: postgres
@@ -152,11 +152,11 @@ Requires-Dist: praw; extra == "reddit"
 Provides-Extra: redis
 Requires-Dist: redis; extra == "redis"
 Provides-Extra: s3
-Requires-Dist: fsspec; extra == "s3"
 Requires-Dist: s3fs; extra == "s3"
+Requires-Dist: fsspec; extra == "s3"
 Provides-Extra: sharepoint
-Requires-Dist: msal; extra == "sharepoint"
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
+Requires-Dist: msal; extra == "sharepoint"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
@@ -178,9 +178,9 @@ Requires-Dist: databricks-sql-connector; extra == "databricks-delta-tables"
 Provides-Extra: singlestore
 Requires-Dist: singlestoredb; extra == "singlestore"
 Provides-Extra: vectara
-Requires-Dist: aiofiles; extra == "vectara"
-Requires-Dist: requests; extra == "vectara"
 Requires-Dist: httpx; extra == "vectara"
+Requires-Dist: requests; extra == "vectara"
+Requires-Dist: aiofiles; extra == "vectara"
 Provides-Extra: vastdb
 Requires-Dist: pyarrow; extra == "vastdb"
 Requires-Dist: ibis; extra == "vastdb"
@@ -188,8 +188,8 @@ Requires-Dist: vastdb; extra == "vastdb"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
-Requires-Dist: openai; extra == "embed-octoai"
 Requires-Dist: tiktoken; extra == "embed-octoai"
+Requires-Dist: openai; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
@@ -197,11 +197,11 @@ Requires-Dist: voyageai; extra == "embed-voyageai"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Provides-Extra: openai
-Requires-Dist: openai; extra == "openai"
 Requires-Dist: tiktoken; extra == "openai"
+Requires-Dist: openai; extra == "openai"
 Provides-Extra: bedrock
-Requires-Dist: aioboto3; extra == "bedrock"
 Requires-Dist: boto3; extra == "bedrock"
+Requires-Dist: aioboto3; extra == "bedrock"
 Provides-Extra: togetherai
 Requires-Dist: together; extra == "togetherai"
 Dynamic: author

{unstructured_ingest-0.5.5.dist-info → unstructured_ingest-0.5.7.dist-info}/RECORD

@@ -17,7 +17,7 @@ test/integration/connectors/test_mongodb.py,sha256=0A6DvF-iTCSZzOefisd_i20j9li8u
 test/integration/connectors/test_neo4j.py,sha256=r4TRYtTXeeOdcRcfa_gvslhSKvoIWrwN1FRJ5XRoH4k,8456
 test/integration/connectors/test_notion.py,sha256=ueXyVqYWzP4LuZYe6PauptkXNG6qkoV3srltFOSSKTA,5403
 test/integration/connectors/test_onedrive.py,sha256=iwiDK0kWCfQbIEPnWUzzAA5PiCsHcmFZSxEcIZy_6cc,5229
-test/integration/connectors/test_pinecone.py,sha256=acKEu1vnAk0Ht3FhCnGtOEKaj_YlgCzZB7wRU17ehQ0,12407
+test/integration/connectors/test_pinecone.py,sha256=9FC0frer7gtDzk5A6OhGsV8S4ggYfa5ReEO9t7L3Am0,13649
 test/integration/connectors/test_qdrant.py,sha256=Yme3ZZ5zIbaZ-yYLUqN2oy0hsrcAfvlleRLYWMSYeSE,8049
 test/integration/connectors/test_redis.py,sha256=1aKwOb-K4zCxZwHmgW_WzGJwqLntbWTbpGQ-rtUwN9o,4360
 test/integration/connectors/test_s3.py,sha256=E1dypeag_E3OIfpQWIz3jb7ctRHRD63UtyTrzyvJzpc,7473
@@ -54,18 +54,18 @@ test/integration/connectors/utils/validation/utils.py,sha256=xYYvAbqP6_lZyH09_Jj
 test/integration/connectors/weaviate/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/connectors/weaviate/conftest.py,sha256=6Q6QdrLJmGHowRFSmoVSzup2EX6qASfS2Z5tqlpTm9M,387
 test/integration/connectors/weaviate/test_cloud.py,sha256=U1ZS6a7wTPX7h3XGvaJHaT-Uwg4IeGgzxx1YBywgVhM,1284
-test/integration/connectors/weaviate/test_local.py,sha256=gXMpnzVcrNQdptDjx0haPWBU-dm1MQTkalgxocI3-L8,5287
+test/integration/connectors/weaviate/test_local.py,sha256=NMQh9kV_BoIrpXe5abGkUSJYsY2ipRSqyFS4EzH1o7s,5333
 test/integration/embedders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/embedders/conftest.py,sha256=B2W771RbijR7G_GybsCzRyIvOzXqzbKZdRIlNDd5AGY,334
 test/integration/embedders/test_azure_openai.py,sha256=YQ3uq2-NuxtTyGsSgMNa10pcITLKMJ4E1scTGFgwujw,1790
 test/integration/embedders/test_bedrock.py,sha256=vmjoi1uUk-LX4Yz0ZPn6Ry1JdVEsyIhLhPbSPmkeT9o,3553
 test/integration/embedders/test_huggingface.py,sha256=qFblyXounVNRaNkk3gbKoBqU5E2dNecgKU2Bz2LyOa8,989
-test/integration/embedders/test_mixedbread.py,sha256=lLz_cooyC38VSo-FMHbhKpHvYs3QzA20NOIvM5oooaw,1998
+test/integration/embedders/test_mixedbread.py,sha256=oesaTY8H7es72vhctmNVU0oWkHNJQckHd_KD-K6kWxI,1996
 test/integration/embedders/test_octoai.py,sha256=qs-bqZ7iGWO_BzUZvKJmOHBT3cmFSkEYbleWhj3snJc,2197
 test/integration/embedders/test_openai.py,sha256=9XioXuvdnbh_3vRmRwpMsi1D5heCcY7KA4nHb5vOU_M,2127
 test/integration/embedders/test_togetherai.py,sha256=hsg3c3SGJGd93unz4-VLYmFXxLA1vmrD5xK5Gj-g0R4,2205
 test/integration/embedders/test_vertexai.py,sha256=4-E4plJXFf1b02RhOqOCBHR2GA4gTnc8K4AnHm6EgPU,1830
-test/integration/embedders/test_voyageai.py,sha256=Gm3sVjhsym1ASIDfr-sZoCbpsNMaAk_l4E3-dtjRCQ4,1832
+test/integration/embedders/test_voyageai.py,sha256=hf8JP8eSL1MMFsmQ9rErM8oxCcwO6kC1WfzzBn7bnME,2414
 test/integration/embedders/utils.py,sha256=Sqqg-X31ZV1hojqPQBaZgM2lb2u8cG6s6OnH9JRsFjs,2717
 test/integration/partitioners/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/partitioners/test_partitioner.py,sha256=6sdZhhtqEICBPqEgpKrCQIfJ-7hKcwuTFqjWs1mbQf8,2787
@@ -107,7 +107,7 @@ test/unit/v2/partitioners/test_partitioner.py,sha256=iIYg7IpftV3LusoO4H8tr1IHY1U
 test/unit/v2/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/unit/v2/utils/data_generator.py,sha256=UoYVNjG4S4wlaA9gceQ82HIpF9_6I1UTHD1_GrQBHp0,973
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=hq-SHV5wD53er4rg53emQu0ZaUQDdyW7b_j9iCQj9es,42
+unstructured_ingest/__version__.py,sha256=SJI27PQ23gz4_g984Mn5VF7Lgitn3vm0GQDyvqnYbdc,42
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/interfaces.py,sha256=7DOnDpGvUNlCoFR7UPRGmOarqH5sFtuUOO5vf8X3oTM,31489
 unstructured_ingest/logger.py,sha256=S5nSqGcABoQyeicgRnBQFjDScCaTvFVivOCvbo-laL0,4479
@@ -278,13 +278,13 @@ unstructured_ingest/embed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJW
 unstructured_ingest/embed/azure_openai.py,sha256=_-I-nwd-wdCiKkSdYBL4UKrTZ2UPWsM_0T69fcObs_I,1707
 unstructured_ingest/embed/bedrock.py,sha256=tZumLLXafSr1zIFVjckapRoiiY-7u65GPuWmwsdhY0I,7726
 unstructured_ingest/embed/huggingface.py,sha256=EWU1kd5Cm6ajgCw6hP5w_4pniGSgxnR0wM9vjuPQ6Yk,2334
-unstructured_ingest/embed/interfaces.py,sha256=C_R_RZJafm3gq8Q0T-HJs-OlbnXpUrCKl4D7Ryv9-Ss,4894
+unstructured_ingest/embed/interfaces.py,sha256=_-CqasY6R5nnNUY-X6PS5lz8dsmGaUw5zIGRdPfx16o,4918
 unstructured_ingest/embed/mixedbreadai.py,sha256=-Y0J27G9CL1t3ZTIeNjTjRviErSMAzJRf2zgDgMHUmg,4499
 unstructured_ingest/embed/octoai.py,sha256=hNLEskDEP-2qWExUgVz2Eyw3KTIFwdUE9elbJ5qp4Ao,3855
 unstructured_ingest/embed/openai.py,sha256=Fe_17y-YpkiGcfrOxZFmgjV-Y-u8svhDVYyAjV-GeBM,3279
 unstructured_ingest/embed/togetherai.py,sha256=i1qeX2fwWtUf1vdGOGnpA_bJB__VzU1NQsR8k-KhxIw,2983
 unstructured_ingest/embed/vertexai.py,sha256=EcXhhm1IbCZVq4KA0sbJjyABu8jpF2ZL3JCqmuxPsjo,3688
-unstructured_ingest/embed/voyageai.py,sha256=tuUZ28B_t2sVDVC6EiHdQEQycMEY2nybE3DFcpxSqh0,3992
+unstructured_ingest/embed/voyageai.py,sha256=lsdiTHVE3CMUX4gXdn2AaRJcKPcKptzgYdF2McvQcvA,4496
 unstructured_ingest/enhanced_dataclass/__init__.py,sha256=gDZOUsv5eo-8jm4Yu7DdDwi101aGbfG7JctTdOYnTOM,151
 unstructured_ingest/enhanced_dataclass/core.py,sha256=d6aUkDynuKX87cHx9_N5UDUWrvISR4jYRFRTvd_avlI,3038
 unstructured_ingest/enhanced_dataclass/dataclasses.py,sha256=aZMsoCzAGRb8Rmh3BTSBFtNr6FmFTY93KYGLk3gYJKQ,1949
@@ -397,14 +397,14 @@ unstructured_ingest/v2/interfaces/connector.py,sha256=qUFFJ3qgDMenTCZMtVRjq1DIws
 unstructured_ingest/v2/interfaces/downloader.py,sha256=Lj3nTY1hPA71GfNeedFVCdHdZsHLle8qrx5RtXAy9GY,2940
 unstructured_ingest/v2/interfaces/file_data.py,sha256=7MyRlj5dijQsCR6W18wQ8fEgJigGKwoOYc10g9A6PSo,3834
 unstructured_ingest/v2/interfaces/indexer.py,sha256=i0oftyifXefxfKa4a3sCfSwkzWGSPE6EvC9sg6fwZgk,833
-unstructured_ingest/v2/interfaces/process.py,sha256=6Ll0O9ATcdm36dx2_TOg9PfCEJrADgyd8OQK3TTNzZM,448
+unstructured_ingest/v2/interfaces/process.py,sha256=S3A_9gkwwGC-iQxvnpj3Er6IJAjAT5npzpSgxuFAzUM,449
 unstructured_ingest/v2/interfaces/processor.py,sha256=VX7JqXlbG1plxMK8THWhWINPbTICaaUEk4XUXhnOixY,3303
 unstructured_ingest/v2/interfaces/upload_stager.py,sha256=9EV9863ODDv0Y5liDT3xh2yiVuFiaVVyCcnwCy6nfkM,3172
-unstructured_ingest/v2/interfaces/uploader.py,sha256=rrZLTjmTcrDL-amQIKzIP6j2fW-iZPCVsUaL0rljcME,2090
+unstructured_ingest/v2/interfaces/uploader.py,sha256=diMkAD5HY8IYpeP1DoFeRD_SexAgOEl1nUcimNnyATc,2063
 unstructured_ingest/v2/pipeline/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/v2/pipeline/interfaces.py,sha256=-Y6gPnl-SbNxIx5-dQCmiYSPKUMjivrRlBLIKIUWVeM,8658
 unstructured_ingest/v2/pipeline/otel.py,sha256=K3pQvWVgWzyOWMKCBUofsH7wTZPJ0Ysw5sLjMBLW41I,1088
-unstructured_ingest/v2/pipeline/pipeline.py,sha256=b37fQGm_lGutQ3Jc0qePB15lkBiFavH9tCso3inm-3I,16564
+unstructured_ingest/v2/pipeline/pipeline.py,sha256=UeOk5SywJZIn3kCnHclQ2cP7JJIXb4NDjpwzsCP_cF0,16523
 unstructured_ingest/v2/pipeline/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/v2/pipeline/steps/chunk.py,sha256=LK2ldM24TE4ukX_Z6Z81LpF53orMaRkddM3uhLtT5EQ,3221
 unstructured_ingest/v2/pipeline/steps/download.py,sha256=nZ4B0d9p-6TgWqrBoKUQPlr8m6dz1RGNr_3OjUhRpWg,8259
@@ -418,7 +418,7 @@ unstructured_ingest/v2/pipeline/steps/upload.py,sha256=We4OAtStuZwWKKBCOPhfeAz_v
 unstructured_ingest/v2/processes/__init__.py,sha256=FaHWSCGyc7GWVnAsNEUUj7L8hT8gCVY3_hUE2VzWtUg,462
 unstructured_ingest/v2/processes/chunker.py,sha256=31-7ojsM2coIt2rMR0KOb82IxLVJfNHbqYUOsDkhxN8,5491
 unstructured_ingest/v2/processes/connector_registry.py,sha256=vkEe6jpgdYtZCxMj59s5atWGgmPuxAEXRUoTt-MJ7wc,2198
-unstructured_ingest/v2/processes/embedder.py,sha256=64mWxKMzDtrm0_QWDUA3J8gChPDEVLP6bFnac_JPBRY,7925
+unstructured_ingest/v2/processes/embedder.py,sha256=4x-Rt5UCvwdgihDAr24hvTGDEd1CdKF9xJrf3aMU-ck,7926
 unstructured_ingest/v2/processes/filter.py,sha256=kjUmMw2SDq2bme0JCAOxs6cJriIG6Ty09KOznS-xz08,2145
 unstructured_ingest/v2/processes/partitioner.py,sha256=ZC9mt85I3o_SLR4DvE7vPBGphMET994phFkTuT-L9B8,9998
 unstructured_ingest/v2/processes/uncompress.py,sha256=Z_XfsITGdyaRwhtNUc7bMj5Y2jLuBge8KoK4nxhqKag,2425
@@ -440,7 +440,7 @@ unstructured_ingest/v2/processes/connectors/mongodb.py,sha256=cL0QUQZF_s2brh3nNN
 unstructured_ingest/v2/processes/connectors/neo4j.py,sha256=ijp5hjmDpLoIHL9UJzV4_4vVtQBlQ2R_vLatlUYivX4,17464
 unstructured_ingest/v2/processes/connectors/onedrive.py,sha256=EM9fq67RsiudZvZbi6nDXkS-i6W0xLvbkNvD0G-Ni5E,17779
 unstructured_ingest/v2/processes/connectors/outlook.py,sha256=KgNGM8hImRhy6_SpswRP2VwRD4VOrqqJoySgxf2oduI,9290
-unstructured_ingest/v2/processes/connectors/pinecone.py,sha256=U5gSa8S08JvCwmAhE8aV0yxGTIFnUlKVsQDybE8Fqb8,10746
+unstructured_ingest/v2/processes/connectors/pinecone.py,sha256=93WmYO9OT8er9DSlh8odbJCtjcLsVMlqyXlYADgDEjc,14013
 unstructured_ingest/v2/processes/connectors/redisdb.py,sha256=p0AY4ukBNpwAemV4bWzpScvVbLTVlI3DzsCNUKiBI5M,6757
 unstructured_ingest/v2/processes/connectors/salesforce.py,sha256=2CiO2ZZiZ1Y1-nB7wcDlDVcpW2B7ut9wCj66rkkqho0,11616
 unstructured_ingest/v2/processes/connectors/sharepoint.py,sha256=2T9Bm1H_ALwHhG_YP7vsuUUW-mUg61zcaae3aa9BnN4,4827
@@ -566,10 +566,10 @@ unstructured_ingest/v2/processes/connectors/weaviate/__init__.py,sha256=NMiwnVWa
 unstructured_ingest/v2/processes/connectors/weaviate/cloud.py,sha256=bXtfEYLquR-BszZ5S_lQ4JbETNs9Vozgpfm8x9egAmE,6251
 unstructured_ingest/v2/processes/connectors/weaviate/embedded.py,sha256=S8Zg8StuZT-k7tCg1D5YShO1-vJYYk9-M1bE1fIqx64,3014
 unstructured_ingest/v2/processes/connectors/weaviate/local.py,sha256=LuTBKPseVewsz8VqxRPRLfGEm3BeI9nBZxpy7ZU5tOA,2201
-unstructured_ingest/v2/processes/connectors/weaviate/weaviate.py,sha256=yJza_jBSEFnzZRq5L6vJ0Mm3uS1uxkOiKIimPpUyQds,12418
-unstructured_ingest-0.5.5.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-0.5.5.dist-info/METADATA,sha256=6HKF0p1ZqHp-ZBAbflvLIHGWPk6i8QvmgA8ltgOPytI,8316
-unstructured_ingest-0.5.5.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-unstructured_ingest-0.5.5.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-0.5.5.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
-unstructured_ingest-0.5.5.dist-info/RECORD,,
+unstructured_ingest/v2/processes/connectors/weaviate/weaviate.py,sha256=yfABJKJGCvPuZ2XCNtDOuCtiscdEAmBCSPPNZnbTKDk,12821
+unstructured_ingest-0.5.7.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-0.5.7.dist-info/METADATA,sha256=Zr_UTJd0V_0vUjwukPd2BgrEh47hqfLSiwivBPAxJos,8316
+unstructured_ingest-0.5.7.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+unstructured_ingest-0.5.7.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-0.5.7.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
+unstructured_ingest-0.5.7.dist-info/RECORD,,