unstructured-ingest 0.5.6__tar.gz → 0.5.8__tar.gz

{unstructured_ingest-0.5.6/unstructured_ingest.egg-info → unstructured_ingest-0.5.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: unstructured-ingest
-Version: 0.5.6
+Version: 0.5.8
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -23,12 +23,12 @@ Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
 Requires-Dist: python-dateutil
+Requires-Dist: pydantic>=2.7
+Requires-Dist: dataclasses_json
 Requires-Dist: pandas
-Requires-Dist: click
 Requires-Dist: tqdm
-Requires-Dist: pydantic>=2.7
+Requires-Dist: click
 Requires-Dist: opentelemetry-sdk
-Requires-Dist: dataclasses_json
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.26.1; extra == "remote"
 Provides-Extra: csv
@@ -71,8 +71,8 @@ Requires-Dist: fsspec; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: biomed
-Requires-Dist: requests; extra == "biomed"
 Requires-Dist: bs4; extra == "biomed"
+Requires-Dist: requests; extra == "biomed"
 Provides-Extra: box
 Requires-Dist: boxfs; extra == "box"
 Requires-Dist: fsspec; extra == "box"
@@ -86,8 +86,8 @@ Requires-Dist: requests; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
 Provides-Extra: delta-table
-Requires-Dist: deltalake; extra == "delta-table"
 Requires-Dist: boto3; extra == "delta-table"
+Requires-Dist: deltalake; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord.py; extra == "discord"
 Provides-Extra: dropbox
@@ -98,8 +98,8 @@ Requires-Dist: duckdb; extra == "duckdb"
 Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
 Provides-Extra: gcs
-Requires-Dist: gcsfs; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
+Requires-Dist: gcsfs; extra == "gcs"
 Requires-Dist: fsspec; extra == "gcs"
 Provides-Extra: github
 Requires-Dist: pygithub>1.58.0; extra == "github"
@@ -109,8 +109,8 @@ Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: urllib3; extra == "hubspot"
 Requires-Dist: hubspot-api-client; extra == "hubspot"
+Requires-Dist: urllib3; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -124,18 +124,18 @@ Requires-Dist: pymilvus; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: neo4j
-Requires-Dist: neo4j-rust-ext; extra == "neo4j"
 Requires-Dist: networkx; extra == "neo4j"
 Requires-Dist: cymple; extra == "neo4j"
+Requires-Dist: neo4j-rust-ext; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
-Requires-Dist: htmlBuilder; extra == "notion"
+Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
+Requires-Dist: htmlBuilder; extra == "notion"
 Provides-Extra: onedrive
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
-Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
+Requires-Dist: msal; extra == "onedrive"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: outlook
@@ -183,13 +183,13 @@ Requires-Dist: httpx; extra == "vectara"
 Requires-Dist: requests; extra == "vectara"
 Provides-Extra: vastdb
 Requires-Dist: ibis; extra == "vastdb"
-Requires-Dist: pyarrow; extra == "vastdb"
 Requires-Dist: vastdb; extra == "vastdb"
+Requires-Dist: pyarrow; extra == "vastdb"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
-Requires-Dist: tiktoken; extra == "embed-octoai"
 Requires-Dist: openai; extra == "embed-octoai"
+Requires-Dist: tiktoken; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
@@ -197,11 +197,11 @@ Requires-Dist: voyageai; extra == "embed-voyageai"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Provides-Extra: openai
-Requires-Dist: tiktoken; extra == "openai"
 Requires-Dist: openai; extra == "openai"
+Requires-Dist: tiktoken; extra == "openai"
 Provides-Extra: bedrock
-Requires-Dist: aioboto3; extra == "bedrock"
 Requires-Dist: boto3; extra == "bedrock"
+Requires-Dist: aioboto3; extra == "bedrock"
 Provides-Extra: togetherai
 Requires-Dist: together; extra == "togetherai"
 Dynamic: author

{unstructured_ingest-0.5.6 → unstructured_ingest-0.5.8}/test/integration/embedders/test_mixedbread.py

@@ -50,7 +50,7 @@ def test_raw_mixedbread_embedder(embedder_file: Path):
         embedder=embedder,
         embedder_file=embedder_file,
         expected_dimension=1024,
-        expected_is_unit_vector=False,
+        expected_is_unit_vector=True,
     )
 
 
@@ -67,5 +67,5 @@ async def test_raw_async_mixedbread_embedder(embedder_file: Path):
         embedder=embedder,
         embedder_file=embedder_file,
         expected_dimension=1024,
-        expected_is_unit_vector=False,
+        expected_is_unit_vector=True,
     )

{unstructured_ingest-0.5.6 → unstructured_ingest-0.5.8}/test/integration/embedders/test_voyageai.py

@@ -61,3 +61,19 @@ async def test_raw_async_voyageai_embedder(embedder_file: Path):
     await validate_raw_embedder_async(
         embedder=embedder, embedder_file=embedder_file, expected_dimension=1024
     )
+
+
+@requires_env(API_KEY)
+def test_voyageai_multimodal_embedder(embedder_file: Path):
+    api_key = get_api_key()
+    embedder_config = EmbedderConfig(
+        embedding_provider="voyageai",
+        embedding_api_key=api_key,
+        embedding_model_name="voyage-multimodal-3",
+    )
+    embedder = Embedder(config=embedder_config)
+    results = embedder.run(elements_filepath=embedder_file)
+    assert results
+    with embedder_file.open("r") as f:
+        original_elements = json.load(f)
+    validate_embedding_output(original_elements=original_elements, output_elements=results)

unstructured_ingest-0.5.8/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.5.8"  # pragma: no cover

{unstructured_ingest-0.5.6 → unstructured_ingest-0.5.8}/unstructured_ingest/connector/pinecone.py

@@ -63,14 +63,14 @@ class PineconeDestinationConnector(IngestDocSessionHandleMixin, BaseDestinationC
     @property
     def pinecone_index(self):
         if self._index is None:
-            self._index = self.create_index()
+            self._index = self.get_index()
         return self._index
 
     def initialize(self):
         pass
 
     @requires_dependencies(["pinecone"], extras="pinecone")
-    def create_index(self) -> "PineconeIndex":
+    def get_index(self) -> "PineconeIndex":
         from pinecone import Pinecone
         from unstructured import __version__ as unstructured_version
 
@@ -83,6 +83,16 @@ class PineconeDestinationConnector(IngestDocSessionHandleMixin, BaseDestinationC
         logger.debug(f"connected to index: {pc.describe_index(self.connector_config.index_name)}")
         return index
 
+    @requires_dependencies(["pinecone"], extras="pinecone")
+    def create_index(self) -> "PineconeIndex":
+        logger.warning(
+            "create_index (a misleading name as of now) will be deprecated soon. "
+            + "Use get_index instead. This is due to unstructured supporting actual "
+            + "index creation/provisioning now. "
+            + "(Support for v2 connectors only. you are currently using a v1 connector.)"
+        )
+        return self.get_index()
+
     @DestinationConnectionError.wrap
     def check_connection(self):
         _ = self.pinecone_index

{unstructured_ingest-0.5.6 → unstructured_ingest-0.5.8}/unstructured_ingest/embed/interfaces.py

@@ -49,7 +49,7 @@ class BaseEmbeddingEncoder(BaseEncoder, ABC):
     def is_unit_vector(self) -> bool:
         """Denotes if the embedding vector is a unit vector."""
         exemplary_embedding = self.get_exemplary_embedding()
-        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
+        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0, rtol=1e-03)
 
     def get_client(self):
         raise NotImplementedError
@@ -103,7 +103,7 @@ class AsyncBaseEmbeddingEncoder(BaseEncoder, ABC):
     async def is_unit_vector(self) -> bool:
         """Denotes if the embedding vector is a unit vector."""
         exemplary_embedding = await self.get_exemplary_embedding()
-        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
+        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0, rtol=1e-03)
 
     def get_client(self):
         raise NotImplementedError

{unstructured_ingest-0.5.6 → unstructured_ingest-0.5.8}/unstructured_ingest/embed/voyageai.py

@@ -96,7 +96,11 @@ class VoyageAIEmbeddingEncoder(BaseEmbeddingEncoder):
         return self.config.get_client()
 
     def embed_batch(self, client: "VoyageAIClient", batch: list[str]) -> list[list[float]]:
-        response = client.embed(texts=batch, model=self.config.embedder_model_name)
+        if self.config.embedder_model_name == "voyage-multimodal-3":
+            batch = [[text] for text in batch]
+            response = client.multimodal_embed(inputs=batch, model=self.config.embedder_model_name)
+        else:
+            response = client.embed(texts=batch, model=self.config.embedder_model_name)
         return response.embeddings
 
 
@@ -113,5 +117,11 @@ class AsyncVoyageAIEmbeddingEncoder(AsyncBaseEmbeddingEncoder):
     async def embed_batch(
         self, client: "AsyncVoyageAIClient", batch: list[str]
     ) -> list[list[float]]:
-        response = await client.embed(texts=batch, model=self.config.embedder_model_name)
+        if self.config.embedder_model_name == "voyage-multimodal-3":
+            batch = [[text] for text in batch]
+            response = await client.multimodal_embed(
+                inputs=batch, model=self.config.embedder_model_name
+            )
+        else:
+            response = await client.embed(texts=batch, model=self.config.embedder_model_name)
         return response.embeddings

{unstructured_ingest-0.5.6 → unstructured_ingest-0.5.8}/unstructured_ingest/v2/interfaces/uploader.py

@@ -38,7 +38,9 @@ class Uploader(BaseProcess, BaseConnector, ABC):
     def run_batch(self, contents: list[UploadContent], **kwargs: Any) -> None:
         raise NotImplementedError()
 
-    def create_destination(self, destination_name: str = "elements", **kwargs: Any) -> bool:
+    def create_destination(
+        self, destination_name: str = "unstructuredautocreated", **kwargs: Any
+    ) -> bool:
         # Update the uploader config if needed with a new destination that gets created.
         # Return a flag on if anything was created or not.
         return False
@@ -61,6 +63,6 @@ class Uploader(BaseProcess, BaseConnector, ABC):
 @dataclass
 class VectorDBUploader(Uploader, ABC):
     def create_destination(
-        self, vector_length: int, destination_name: str = "elements", **kwargs: Any
+        self, vector_length: int, destination_name: str = "unstructuredautocreated", **kwargs: Any
     ) -> bool:
         return False

{unstructured_ingest-0.5.6 → unstructured_ingest-0.5.8}/unstructured_ingest/v2/pipeline/steps/index.py

@@ -11,7 +11,7 @@ from unstructured_ingest.v2.utils import serialize_base_model_json
 
 IndexerT = TypeVar("IndexerT", bound=Indexer)
 
-STEP_ID = "index"
+STEP_ID = "indexer"
 
 
 @dataclass

{unstructured_ingest-0.5.6 → unstructured_ingest-0.5.8}/unstructured_ingest/v2/processes/connectors/pinecone.py

@@ -208,7 +208,7 @@ class PineconeUploader(VectorDBUploader):
     def create_destination(
         self,
         vector_length: int,
-        destination_name: str = "elements",
+        destination_name: str = "unstructuredautocreated",
         destination_type: Literal["pod", "serverless"] = "serverless",
         serverless_cloud: str = "aws",
         serverless_region: str = "us-west-2",
@@ -219,7 +219,7 @@ class PineconeUploader(VectorDBUploader):
     ) -> bool:
         from pinecone import PodSpec, ServerlessSpec
 
-        index_name = destination_name or self.connection_config.index_name
+        index_name = self.connection_config.index_name or destination_name
         index_name = self.format_destination_name(index_name)
         self.connection_config.index_name = index_name
 
@@ -228,13 +228,11 @@ class PineconeUploader(VectorDBUploader):
             logger.info(f"creating pinecone index {index_name}")
 
             pc = self.connection_config.get_client()
-
             if destination_type == "serverless":
                 pc.create_index(
-                    name=destination_name,
+                    name=index_name,
                     dimension=vector_length,
                     spec=ServerlessSpec(cloud=serverless_cloud, region=serverless_region),
-                    **kwargs,
                 )
 
                 return True
@@ -244,7 +242,6 @@ class PineconeUploader(VectorDBUploader):
                     name=destination_name,
                     dimension=vector_length,
                     spec=PodSpec(environment=pod_environment, pod_type=pod_type, pods=pod_count),
-                    **kwargs,
                 )
 
                 return True

{unstructured_ingest-0.5.6 → unstructured_ingest-0.5.8}/unstructured_ingest/v2/processes/connectors/weaviate/weaviate.py

@@ -241,10 +241,13 @@ class WeaviateUploader(VectorDBUploader, ABC):
         return formatted.capitalize()
 
     def create_destination(
-        self, destination_name: str = "elements", vector_length: Optional[int] = None, **kwargs: Any
+        self,
+        destination_name: str = "unstructuredautocreated",
+        vector_length: Optional[int] = None,
+        **kwargs: Any,
     ) -> bool:
-        destination_name = self.format_destination_name(destination_name)
         collection_name = self.upload_config.collection or destination_name
+        collection_name = self.format_destination_name(collection_name)
         self.upload_config.collection = collection_name
 
         connectors_dir = Path(__file__).parents[1]
@@ -254,9 +257,7 @@ class WeaviateUploader(VectorDBUploader, ABC):
         collection_config["class"] = collection_name
 
         if not self._collection_exists():
-            logger.info(
-                f"creating default weaviate collection '{collection_name}' with default configs"
-            )
+            logger.info(f"creating weaviate collection '{collection_name}' with default configs")
             with self.connection_config.get_client() as weaviate_client:
                 weaviate_client.collections.create_from_dict(config=collection_config)
                 return True

{unstructured_ingest-0.5.6 → unstructured_ingest-0.5.8/unstructured_ingest.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: unstructured-ingest
-Version: 0.5.6
+Version: 0.5.8
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -23,12 +23,12 @@ Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
 Requires-Dist: python-dateutil
+Requires-Dist: pydantic>=2.7
+Requires-Dist: dataclasses_json
 Requires-Dist: pandas
-Requires-Dist: click
 Requires-Dist: tqdm
-Requires-Dist: pydantic>=2.7
+Requires-Dist: click
 Requires-Dist: opentelemetry-sdk
-Requires-Dist: dataclasses_json
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.26.1; extra == "remote"
 Provides-Extra: csv
@@ -71,8 +71,8 @@ Requires-Dist: fsspec; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: biomed
-Requires-Dist: requests; extra == "biomed"
 Requires-Dist: bs4; extra == "biomed"
+Requires-Dist: requests; extra == "biomed"
 Provides-Extra: box
 Requires-Dist: boxfs; extra == "box"
 Requires-Dist: fsspec; extra == "box"
@@ -86,8 +86,8 @@ Requires-Dist: requests; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
 Provides-Extra: delta-table
-Requires-Dist: deltalake; extra == "delta-table"
 Requires-Dist: boto3; extra == "delta-table"
+Requires-Dist: deltalake; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord.py; extra == "discord"
 Provides-Extra: dropbox
@@ -98,8 +98,8 @@ Requires-Dist: duckdb; extra == "duckdb"
 Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
 Provides-Extra: gcs
-Requires-Dist: gcsfs; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
+Requires-Dist: gcsfs; extra == "gcs"
 Requires-Dist: fsspec; extra == "gcs"
 Provides-Extra: github
 Requires-Dist: pygithub>1.58.0; extra == "github"
@@ -109,8 +109,8 @@ Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: urllib3; extra == "hubspot"
 Requires-Dist: hubspot-api-client; extra == "hubspot"
+Requires-Dist: urllib3; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -124,18 +124,18 @@ Requires-Dist: pymilvus; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: neo4j
-Requires-Dist: neo4j-rust-ext; extra == "neo4j"
 Requires-Dist: networkx; extra == "neo4j"
 Requires-Dist: cymple; extra == "neo4j"
+Requires-Dist: neo4j-rust-ext; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
-Requires-Dist: htmlBuilder; extra == "notion"
+Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
+Requires-Dist: htmlBuilder; extra == "notion"
 Provides-Extra: onedrive
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
-Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
+Requires-Dist: msal; extra == "onedrive"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: outlook
@@ -183,13 +183,13 @@ Requires-Dist: httpx; extra == "vectara"
 Requires-Dist: requests; extra == "vectara"
 Provides-Extra: vastdb
 Requires-Dist: ibis; extra == "vastdb"
-Requires-Dist: pyarrow; extra == "vastdb"
 Requires-Dist: vastdb; extra == "vastdb"
+Requires-Dist: pyarrow; extra == "vastdb"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
-Requires-Dist: tiktoken; extra == "embed-octoai"
 Requires-Dist: openai; extra == "embed-octoai"
+Requires-Dist: tiktoken; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
@@ -197,11 +197,11 @@ Requires-Dist: voyageai; extra == "embed-voyageai"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Provides-Extra: openai
-Requires-Dist: tiktoken; extra == "openai"
 Requires-Dist: openai; extra == "openai"
+Requires-Dist: tiktoken; extra == "openai"
 Provides-Extra: bedrock
-Requires-Dist: aioboto3; extra == "bedrock"
 Requires-Dist: boto3; extra == "bedrock"
+Requires-Dist: aioboto3; extra == "bedrock"
 Provides-Extra: togetherai
 Requires-Dist: together; extra == "togetherai"
 Dynamic: author

unstructured_ingest-0.5.6/unstructured_ingest/__version__.py

@@ -1 +0,0 @@
-__version__ = "0.5.6"  # pragma: no cover