unstructured-ingest 0.5.1__tar.gz → 0.5.2__tar.gz

{unstructured_ingest-0.5.1/unstructured_ingest.egg-info → unstructured_ingest-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.5.1
+Version: 0.5.2
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,13 +22,13 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: opentelemetry-sdk
-Requires-Dist: python-dateutil
-Requires-Dist: tqdm
 Requires-Dist: pydantic>=2.7
+Requires-Dist: click
 Requires-Dist: pandas
+Requires-Dist: opentelemetry-sdk
 Requires-Dist: dataclasses_json
-Requires-Dist: click
+Requires-Dist: tqdm
+Requires-Dist: python-dateutil
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.26.1; extra == "remote"
 Provides-Extra: csv
@@ -66,41 +66,41 @@ Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
 Requires-Dist: astrapy; extra == "astradb"
 Provides-Extra: azure
-Requires-Dist: fsspec; extra == "azure"
 Requires-Dist: adlfs; extra == "azure"
+Requires-Dist: fsspec; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: biomed
 Requires-Dist: bs4; extra == "biomed"
 Requires-Dist: requests; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: fsspec; extra == "box"
 Requires-Dist: boxfs; extra == "box"
+Requires-Dist: fsspec; extra == "box"
 Provides-Extra: chroma
 Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
 Requires-Dist: clarifai; extra == "clarifai"
 Provides-Extra: confluence
-Requires-Dist: atlassian-python-api; extra == "confluence"
 Requires-Dist: requests; extra == "confluence"
+Requires-Dist: atlassian-python-api; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
 Provides-Extra: delta-table
-Requires-Dist: deltalake; extra == "delta-table"
 Requires-Dist: boto3; extra == "delta-table"
+Requires-Dist: deltalake; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord.py; extra == "discord"
 Provides-Extra: dropbox
-Requires-Dist: fsspec; extra == "dropbox"
 Requires-Dist: dropboxdrivefs; extra == "dropbox"
+Requires-Dist: fsspec; extra == "dropbox"
 Provides-Extra: duckdb
 Requires-Dist: duckdb; extra == "duckdb"
 Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
 Provides-Extra: gcs
-Requires-Dist: fsspec; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
 Requires-Dist: gcsfs; extra == "gcs"
+Requires-Dist: fsspec; extra == "gcs"
 Provides-Extra: github
 Requires-Dist: requests; extra == "github"
 Requires-Dist: pygithub>1.58.0; extra == "github"
@@ -109,8 +109,8 @@ Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: hubspot-api-client; extra == "hubspot"
 Requires-Dist: urllib3; extra == "hubspot"
+Requires-Dist: hubspot-api-client; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -124,18 +124,18 @@ Requires-Dist: pymilvus; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: neo4j
-Requires-Dist: networkx; extra == "neo4j"
-Requires-Dist: cymple; extra == "neo4j"
 Requires-Dist: neo4j; extra == "neo4j"
+Requires-Dist: cymple; extra == "neo4j"
+Requires-Dist: networkx; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: httpx; extra == "notion"
 Requires-Dist: notion-client; extra == "notion"
-Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
+Requires-Dist: httpx; extra == "notion"
+Requires-Dist: htmlBuilder; extra == "notion"
 Provides-Extra: onedrive
-Requires-Dist: bs4; extra == "onedrive"
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Requires-Dist: msal; extra == "onedrive"
+Requires-Dist: bs4; extra == "onedrive"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: outlook
@@ -152,16 +152,16 @@ Requires-Dist: praw; extra == "reddit"
 Provides-Extra: redis
 Requires-Dist: redis; extra == "redis"
 Provides-Extra: s3
-Requires-Dist: fsspec; extra == "s3"
 Requires-Dist: s3fs; extra == "s3"
+Requires-Dist: fsspec; extra == "s3"
 Provides-Extra: sharepoint
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Requires-Dist: msal; extra == "sharepoint"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
-Requires-Dist: fsspec; extra == "sftp"
 Requires-Dist: paramiko; extra == "sftp"
+Requires-Dist: fsspec; extra == "sftp"
 Provides-Extra: slack
 Requires-Dist: slack_sdk[optional]; extra == "slack"
 Provides-Extra: snowflake
@@ -178,9 +178,9 @@ Requires-Dist: databricks-sql-connector; extra == "databricks-delta-tables"
 Provides-Extra: singlestore
 Requires-Dist: singlestoredb; extra == "singlestore"
 Provides-Extra: vectara
+Requires-Dist: requests; extra == "vectara"
 Requires-Dist: aiofiles; extra == "vectara"
 Requires-Dist: httpx; extra == "vectara"
-Requires-Dist: requests; extra == "vectara"
 Provides-Extra: vastdb
 Requires-Dist: pyarrow; extra == "vastdb"
 Requires-Dist: ibis; extra == "vastdb"

unstructured_ingest-0.5.2/test/integration/connectors/test_google_drive.py

@@ -0,0 +1,257 @@
+import os
+import uuid
+
+import pytest
+from googleapiclient.errors import HttpError
+
+from test.integration.connectors.utils.constants import (
+    SOURCE_TAG,
+    UNCATEGORIZED_TAG,
+)
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
+    get_all_file_data,
+    run_all_validations,
+    update_fixtures,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.error import (
+    SourceConnectionError,
+)
+from unstructured_ingest.v2.interfaces import Downloader, Indexer
+from unstructured_ingest.v2.processes.connectors.google_drive import (
+    CONNECTOR_TYPE,
+    GoogleDriveAccessConfig,
+    GoogleDriveConnectionConfig,
+    GoogleDriveDownloader,
+    GoogleDriveDownloaderConfig,
+    GoogleDriveIndexer,
+    GoogleDriveIndexerConfig,
+)
+
+
+@pytest.fixture
+def google_drive_connection_config():
+    """
+    Build a valid GoogleDriveConnectionConfig using the environment variables.
+    Expects:
+      - GOOGLE_DRIVE_ID
+      - GOOGLE_DRIVE_SERVICE_KEY
+    """
+    drive_id = os.getenv("GOOGLE_DRIVE_ID")
+    service_key = os.getenv("GOOGLE_DRIVE_SERVICE_KEY")
+    if not drive_id or not service_key:
+        pytest.skip("Google Drive credentials not provided in environment variables.")
+
+    access_config = GoogleDriveAccessConfig(service_account_key=service_key)
+    return GoogleDriveConnectionConfig(drive_id=drive_id, access_config=access_config)
+
+
+@pytest.fixture
+def google_drive_empty_folder(google_drive_connection_config):
+    """
+    Creates an empty folder on Google Drive for testing the "empty folder" case.
+    The folder is deleted after the test.
+    """
+    from google.oauth2 import service_account
+    from googleapiclient.discovery import build
+
+    access_config = google_drive_connection_config.access_config.get_secret_value()
+    creds = service_account.Credentials.from_service_account_info(access_config.service_account_key)
+    service = build("drive", "v3", credentials=creds)
+
+    # Create an empty folder.
+    file_metadata = {
+        "name": f"utic-empty-folder-{uuid.uuid4()}",
+        "mimeType": "application/vnd.google-apps.folder",
+    }
+    folder = service.files().create(body=file_metadata, fields="id, name").execute()
+    folder_id = folder.get("id")
+    try:
+        yield folder_id
+    finally:
+        service.files().delete(fileId=folder_id).execute()
+
+
+@requires_env("GOOGLE_DRIVE_SERVICE_KEY")
+@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE)
+def test_google_drive_source(temp_dir):
+    # Retrieve environment variables
+    service_account_key = os.environ["GOOGLE_DRIVE_SERVICE_KEY"]
+
+    # Create connection and indexer configurations
+    access_config = GoogleDriveAccessConfig(service_account_key=service_account_key)
+    connection_config = GoogleDriveConnectionConfig(
+        drive_id="1XidSOO76VpZ4m0i3gJN2m1X0Obol3UAi",
+        access_config=access_config,
+    )
+    index_config = GoogleDriveIndexerConfig(recursive=True)
+
+    download_config = GoogleDriveDownloaderConfig(download_dir=temp_dir)
+
+    # Instantiate indexer and downloader
+    indexer = GoogleDriveIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+    )
+    downloader = GoogleDriveDownloader(
+        connection_config=connection_config,
+        download_config=download_config,
+    )
+
+    # Run the source connector validation
+    source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="google_drive_source",
+            expected_num_files=1,
+            validate_downloaded_files=True,
+        ),
+    )
+
+
+@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
+def source_connector_validation(
+    indexer: Indexer,
+    downloader: Downloader,
+    configs: SourceValidationConfigs,
+    overwrite_fixtures: bool = os.getenv("OVERWRITE_FIXTURES", "False").lower() == "true",
+) -> None:
+    # Run common validations on the process of running a source connector, supporting dynamic
+    # validators that get passed in along with comparisons on the saved expected values.
+    # If overwrite_fixtures is st to True, will ignore all validators but instead overwrite the
+    # expected values with what gets generated by this test.
+    all_predownload_file_data = []
+    all_postdownload_file_data = []
+    indexer.precheck()
+    download_dir = downloader.download_config.download_dir
+    test_output_dir = configs.test_output_dir()
+
+    for file_data in indexer.run():
+        assert file_data
+        predownload_file_data = file_data.model_copy(deep=True)
+        all_predownload_file_data.append(predownload_file_data)
+        resp = downloader.run(file_data=file_data)
+        if isinstance(resp, list):
+            for r in resp:
+                postdownload_file_data = r["file_data"].model_copy(deep=True)
+                all_postdownload_file_data.append(postdownload_file_data)
+        else:
+            postdownload_file_data = resp["file_data"].model_copy(deep=True)
+            all_postdownload_file_data.append(postdownload_file_data)
+
+    if not overwrite_fixtures:
+        print("Running validation")
+        run_all_validations(
+            configs=configs,
+            predownload_file_data=all_predownload_file_data,
+            postdownload_file_data=all_postdownload_file_data,
+            download_dir=download_dir,
+            test_output_dir=test_output_dir,
+        )
+    else:
+        print("Running fixtures update")
+        update_fixtures(
+            output_dir=test_output_dir,
+            download_dir=download_dir,
+            all_file_data=get_all_file_data(
+                all_predownload_file_data=all_predownload_file_data,
+                all_postdownload_file_data=all_postdownload_file_data,
+            ),
+            save_downloads=configs.validate_downloaded_files,
+            save_filedata=configs.validate_file_data,
+        )
+
+
+# Precheck fails when the drive ID has an appended parameter (simulate copy-paste error)
+@pytest.mark.tags("google-drive", "precheck")
+@requires_env("GOOGLE_DRIVE_ID", "GOOGLE_DRIVE_SERVICE_KEY")
+def test_google_drive_precheck_invalid_parameter(google_drive_connection_config):
+    # Append a query parameter as often happens when copying from a URL.
+    invalid_drive_id = google_drive_connection_config.drive_id + "?usp=sharing"
+    connection_config = GoogleDriveConnectionConfig(
+        drive_id=invalid_drive_id,
+        access_config=google_drive_connection_config.access_config,
+    )
+    index_config = GoogleDriveIndexerConfig(recursive=True)
+    indexer = GoogleDriveIndexer(connection_config=connection_config, index_config=index_config)
+    with pytest.raises(SourceConnectionError) as excinfo:
+        indexer.precheck()
+    assert "invalid" in str(excinfo.value).lower() or "not found" in str(excinfo.value).lower()
+
+
+# Precheck fails due to lack of permission (simulate via monkeypatching).
+@pytest.mark.tags("google-drive", "precheck")
+@requires_env("GOOGLE_DRIVE_ID", "GOOGLE_DRIVE_SERVICE_KEY")
+def test_google_drive_precheck_no_permission(google_drive_connection_config, monkeypatch):
+    index_config = GoogleDriveIndexerConfig(recursive=True)
+    indexer = GoogleDriveIndexer(
+        connection_config=google_drive_connection_config,
+        index_config=index_config,
+    )
+
+    # Monkeypatch get_root_info to always raise an HTTP 403 error.
+    def fake_get_root_info(files_client, object_id):
+        raise HttpError(
+            resp=type("Response", (), {"status": 403, "reason": "Forbidden"})(),
+            content=b"Forbidden",
+        )
+
+    monkeypatch.setattr(indexer, "get_root_info", fake_get_root_info)
+    with pytest.raises(SourceConnectionError) as excinfo:
+        indexer.precheck()
+    assert "forbidden" in str(excinfo.value).lower() or "permission" in str(excinfo.value).lower()
+
+
+# Precheck fails when the folder is empty.
+# @pytest.mark.tags("google-drive", "precheck")
+# @requires_env("GOOGLE_DRIVE_ID", "GOOGLE_DRIVE_SERVICE_KEY")
+# def test_google_drive_precheck_empty_folder(
+#     google_drive_connection_config, google_drive_empty_folder
+# ):
+#     # Use the empty folder's ID as the target.
+#     connection_config = GoogleDriveConnectionConfig(
+#         drive_id=google_drive_empty_folder,
+#         access_config=google_drive_connection_config.access_config,
+#     )
+
+#     index_config = GoogleDriveIndexerConfig(recursive=True)
+#     indexer = GoogleDriveIndexer(connection_config=connection_config, index_config=index_config)
+#     with pytest.raises(SourceConnectionError) as excinfo:
+#         indexer.precheck()
+#     assert "empty folder" in str(excinfo.value).lower()
+
+
+@pytest.mark.tags("google-drive", "count", "integration")
+@requires_env("GOOGLE_DRIVE_ID", "GOOGLE_DRIVE_SERVICE_KEY")
+def test_google_drive_count_files(google_drive_connection_config):
+    """
+    This test verifies that the count_files_recursively method returns the expected count of files.
+    According to the test credentials, there are 3 files in the root directory and 1 nested file,
+    so the total count should be 4.
+    """
+    # I assumed that we're applying the same extension filter as with other tests
+    # However there's 6 files in total in the test dir
+    extensions_filter = ["pdf", "docx"]
+    with google_drive_connection_config.get_client() as client:
+        count = GoogleDriveIndexer.count_files_recursively(
+            client, google_drive_connection_config.drive_id, extensions_filter
+        )
+    assert count == 4, f"Expected file count of 4, but got {count}"
+
+
+# Precheck fails with a completely invalid drive ID.
+@pytest.mark.tags("google-drive", "precheck")
+@requires_env("GOOGLE_DRIVE_ID", "GOOGLE_DRIVE_SERVICE_KEY")
+def test_google_drive_precheck_invalid_drive_id(google_drive_connection_config):
+    invalid_drive_id = "invalid_drive_id"
+    connection_config = GoogleDriveConnectionConfig(
+        drive_id=invalid_drive_id,
+        access_config=google_drive_connection_config.access_config,
+    )
+    index_config = GoogleDriveIndexerConfig(recursive=True)
+    indexer = GoogleDriveIndexer(connection_config=connection_config, index_config=index_config)
+    with pytest.raises(SourceConnectionError) as excinfo:
+        indexer.precheck()
+    assert "invalid" in str(excinfo.value).lower() or "not found" in str(excinfo.value).lower()

unstructured_ingest-0.5.2/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.5.2"  # pragma: no cover

{unstructured_ingest-0.5.1 → unstructured_ingest-0.5.2}/unstructured_ingest/embed/bedrock.py

@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, AsyncIterable
 from pydantic import Field, SecretStr
 
 from unstructured_ingest.embed.interfaces import (
+    EMBEDDINGS_KEY,
     AsyncBaseEmbeddingEncoder,
     BaseEmbeddingEncoder,
     EmbeddingConfig,
@@ -145,9 +146,12 @@ class BedrockEmbeddingEncoder(BaseEmbeddingEncoder):
             return response_body.get("embedding")
 
     def embed_documents(self, elements: list[dict]) -> list[dict]:
-        embeddings = [self.embed_query(query=e.get("text", "")) for e in elements]
-        elements_with_embeddings = self._add_embeddings_to_elements(elements, embeddings)
-        return elements_with_embeddings
+        elements = elements.copy()
+        elements_with_text = [e for e in elements if e.get("text")]
+        embeddings = [self.embed_query(query=e["text"]) for e in elements_with_text]
+        for element, embedding in zip(elements_with_text, embeddings):
+            element[EMBEDDINGS_KEY] = embedding
+        return elements
 
 
 @dataclass
@@ -186,8 +190,11 @@ class AsyncBedrockEmbeddingEncoder(AsyncBaseEmbeddingEncoder):
             raise ValueError(f"Error raised by inference endpoint: {e}")
 
     async def embed_documents(self, elements: list[dict]) -> list[dict]:
+        elements = elements.copy()
+        elements_with_text = [e for e in elements if e.get("text")]
         embeddings = await asyncio.gather(
-            *[self.embed_query(query=e.get("text", "")) for e in elements]
+            *[self.embed_query(query=e.get("text", "")) for e in elements_with_text]
         )
-        elements_with_embeddings = self._add_embeddings_to_elements(elements, embeddings)
-        return elements_with_embeddings
+        for element, embedding in zip(elements_with_text, embeddings):
+            element[EMBEDDINGS_KEY] = embedding
+        return elements

{unstructured_ingest-0.5.1 → unstructured_ingest-0.5.2}/unstructured_ingest/embed/huggingface.py

@@ -3,7 +3,11 @@ from typing import TYPE_CHECKING, Optional
 
 from pydantic import Field
 
-from unstructured_ingest.embed.interfaces import BaseEmbeddingEncoder, EmbeddingConfig
+from unstructured_ingest.embed.interfaces import (
+    EMBEDDINGS_KEY,
+    BaseEmbeddingEncoder,
+    EmbeddingConfig,
+)
 from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
@@ -52,6 +56,9 @@ class HuggingFaceEmbeddingEncoder(BaseEmbeddingEncoder):
         return embeddings.tolist()
 
     def embed_documents(self, elements: list[dict]) -> list[dict]:
-        embeddings = self._embed_documents([e.get("text", "") for e in elements])
-        elements_with_embeddings = self._add_embeddings_to_elements(elements, embeddings)
-        return elements_with_embeddings
+        elements = elements.copy()
+        elements_with_text = [e for e in elements if e.get("text")]
+        embeddings = self._embed_documents([e["text"] for e in elements_with_text])
+        for element, embedding in zip(elements_with_text, embeddings):
+            element[EMBEDDINGS_KEY] = embedding
+        return elements

{unstructured_ingest-0.5.1 → unstructured_ingest-0.5.2}/unstructured_ingest/embed/interfaces.py

@@ -6,6 +6,8 @@ from typing import Optional
 import numpy as np
 from pydantic import BaseModel, Field
 
+EMBEDDINGS_KEY = "embeddings"
+
 
 class EmbeddingConfig(BaseModel):
     batch_size: Optional[int] = Field(
@@ -26,27 +28,6 @@ class BaseEncoder(ABC):
         if possible"""
         return e
 
-    @staticmethod
-    def _add_embeddings_to_elements(
-        elements: list[dict], embeddings: list[list[float]]
-    ) -> list[dict]:
-        """
-        Add embeddings to elements.
-
-        Args:
-            elements (list[Element]): List of elements.
-            embeddings (list[list[float]]): List of embeddings.
-
-        Returns:
-            list[Element]: Elements with embeddings added.
-        """
-        assert len(elements) == len(embeddings)
-        elements_w_embedding = []
-        for i, element in enumerate(elements):
-            element["embeddings"] = embeddings[i]
-            elements_w_embedding.append(element)
-        return elements
-
 
 @dataclass
 class BaseEmbeddingEncoder(BaseEncoder, ABC):

{unstructured_ingest-0.5.1 → unstructured_ingest-0.5.2}/unstructured_ingest/embed/mixedbreadai.py

@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING
 from pydantic import Field, SecretStr
 
 from unstructured_ingest.embed.interfaces import (
+    EMBEDDINGS_KEY,
     AsyncBaseEmbeddingEncoder,
     BaseEmbeddingEncoder,
     EmbeddingConfig,
@@ -134,8 +135,12 @@ class MixedbreadAIEmbeddingEncoder(BaseEmbeddingEncoder):
         Returns:
             list[Element]: Elements with embeddings.
         """
-        embeddings = self._embed([e.get("text", "") for e in elements])
-        return self._add_embeddings_to_elements(elements, embeddings)
+        elements = elements.copy()
+        elements_with_text = [e for e in elements if e.get("text")]
+        embeddings = self._embed([e["text"] for e in elements_with_text])
+        for element, embedding in zip(elements_with_text, embeddings):
+            element[EMBEDDINGS_KEY] = embedding
+        return elements
 
     def embed_query(self, query: str) -> list[float]:
         """
@@ -209,8 +214,12 @@ class AsyncMixedbreadAIEmbeddingEncoder(AsyncBaseEmbeddingEncoder):
         Returns:
             list[Element]: Elements with embeddings.
         """
-        embeddings = await self._embed([e.get("text", "") for e in elements])
-        return self._add_embeddings_to_elements(elements, embeddings)
+        elements = elements.copy()
+        elements_with_text = [e for e in elements if e.get("text")]
+        embeddings = await self._embed([e["text"] for e in elements_with_text])
+        for element, embedding in zip(elements_with_text, embeddings):
+            element[EMBEDDINGS_KEY] = embedding
+        return elements
 
     async def embed_query(self, query: str) -> list[float]:
         """

{unstructured_ingest-0.5.1 → unstructured_ingest-0.5.2}/unstructured_ingest/embed/octoai.py

@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING
 from pydantic import Field, SecretStr
 
 from unstructured_ingest.embed.interfaces import (
+    EMBEDDINGS_KEY,
     AsyncBaseEmbeddingEncoder,
     BaseEmbeddingEncoder,
     EmbeddingConfig,
@@ -89,7 +90,9 @@ class OctoAIEmbeddingEncoder(BaseEmbeddingEncoder):
         return response.data[0].embedding
 
     def embed_documents(self, elements: list[dict]) -> list[dict]:
-        texts = [e.get("text", "") for e in elements]
+        elements = elements.copy()
+        elements_with_text = [e for e in elements if e.get("text")]
+        texts = [e["text"] for e in elements_with_text]
         embeddings = []
         client = self.config.get_client()
         try:
@@ -100,8 +103,9 @@ class OctoAIEmbeddingEncoder(BaseEmbeddingEncoder):
                 embeddings.extend([data.embedding for data in response.data])
         except Exception as e:
             raise self.wrap_error(e=e)
-        elements_with_embeddings = self._add_embeddings_to_elements(elements, embeddings)
-        return elements_with_embeddings
+        for element, embedding in zip(elements_with_text, embeddings):
+            element[EMBEDDINGS_KEY] = embedding
+        return elements
 
 
 @dataclass
@@ -122,7 +126,9 @@ class AsyncOctoAIEmbeddingEncoder(AsyncBaseEmbeddingEncoder):
         return response.data[0].embedding
 
     async def embed_documents(self, elements: list[dict]) -> list[dict]:
-        texts = [e.get("text", "") for e in elements]
+        elements = elements.copy()
+        elements_with_text = [e for e in elements if e.get("text")]
+        texts = [e["text"] for e in elements_with_text]
         client = self.config.get_async_client()
         embeddings = []
         try:
@@ -133,5 +139,6 @@ class AsyncOctoAIEmbeddingEncoder(AsyncBaseEmbeddingEncoder):
                 embeddings.extend([data.embedding for data in response.data])
         except Exception as e:
             raise self.wrap_error(e=e)
-        elements_with_embeddings = self._add_embeddings_to_elements(elements, embeddings)
-        return elements_with_embeddings
+        for element, embedding in zip(elements_with_text, embeddings):
+            element[EMBEDDINGS_KEY] = embedding
+        return elements

{unstructured_ingest-0.5.1 → unstructured_ingest-0.5.2}/unstructured_ingest/embed/openai.py

@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING
 from pydantic import Field, SecretStr
 
 from unstructured_ingest.embed.interfaces import (
+    EMBEDDINGS_KEY,
     AsyncBaseEmbeddingEncoder,
     BaseEmbeddingEncoder,
     EmbeddingConfig,
@@ -82,7 +83,9 @@ class OpenAIEmbeddingEncoder(BaseEmbeddingEncoder):
 
     def embed_documents(self, elements: list[dict]) -> list[dict]:
         client = self.config.get_client()
-        texts = [e.get("text", "") for e in elements]
+        elements = elements.copy()
+        elements_with_text = [e for e in elements if e.get("text")]
+        texts = [e["text"] for e in elements_with_text]
         embeddings = []
         try:
             for batch in batch_generator(texts, batch_size=self.config.batch_size or len(texts)):
@@ -92,8 +95,9 @@ class OpenAIEmbeddingEncoder(BaseEmbeddingEncoder):
                 embeddings.extend([data.embedding for data in response.data])
         except Exception as e:
             raise self.wrap_error(e=e)
-        elements_with_embeddings = self._add_embeddings_to_elements(elements, embeddings)
-        return elements_with_embeddings
+        for element, embedding in zip(elements_with_text, embeddings):
+            element[EMBEDDINGS_KEY] = embedding
+        return elements
 
 
 @dataclass
@@ -115,7 +119,9 @@ class AsyncOpenAIEmbeddingEncoder(AsyncBaseEmbeddingEncoder):
 
     async def embed_documents(self, elements: list[dict]) -> list[dict]:
         client = self.config.get_async_client()
-        texts = [e.get("text", "") for e in elements]
+        elements = elements.copy()
+        elements_with_text = [e for e in elements if e.get("text")]
+        texts = [e["text"] for e in elements_with_text]
         embeddings = []
         try:
             for batch in batch_generator(texts, batch_size=self.config.batch_size or len(texts)):
@@ -125,5 +131,6 @@ class AsyncOpenAIEmbeddingEncoder(AsyncBaseEmbeddingEncoder):
                 embeddings.extend([data.embedding for data in response.data])
         except Exception as e:
             raise self.wrap_error(e=e)
-        elements_with_embeddings = self._add_embeddings_to_elements(elements, embeddings)
-        return elements_with_embeddings
+        for element, embedding in zip(elements_with_text, embeddings):
+            element[EMBEDDINGS_KEY] = embedding
+        return elements

{unstructured_ingest-0.5.1 → unstructured_ingest-0.5.2}/unstructured_ingest/embed/togetherai.py

@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING
 from pydantic import Field, SecretStr
 
 from unstructured_ingest.embed.interfaces import (
+    EMBEDDINGS_KEY,
     AsyncBaseEmbeddingEncoder,
     BaseEmbeddingEncoder,
     EmbeddingConfig,
@@ -67,8 +68,12 @@ class TogetherAIEmbeddingEncoder(BaseEmbeddingEncoder):
         return self._embed_documents(elements=[query])[0]
 
     def embed_documents(self, elements: list[dict]) -> list[dict]:
-        embeddings = self._embed_documents([e.get("text", "") for e in elements])
-        return self._add_embeddings_to_elements(elements, embeddings)
+        elements = elements.copy()
+        elements_with_text = [e for e in elements if e.get("text")]
+        embeddings = self._embed_documents([e["text"] for e in elements_with_text])
+        for element, embedding in zip(elements_with_text, embeddings):
+            element[EMBEDDINGS_KEY] = embedding
+        return elements
 
     def _embed_documents(self, elements: list[str]) -> list[list[float]]:
         client = self.config.get_client()
@@ -98,8 +103,12 @@ class AsyncTogetherAIEmbeddingEncoder(AsyncBaseEmbeddingEncoder):
         return embedding[0]
 
     async def embed_documents(self, elements: list[dict]) -> list[dict]:
-        embeddings = await self._embed_documents([e.get("text", "") for e in elements])
-        return self._add_embeddings_to_elements(elements, embeddings)
+        elements = elements.copy()
+        elements_with_text = [e for e in elements if e.get("text")]
+        embeddings = await self._embed_documents([e["text"] for e in elements_with_text])
+        for element, embedding in zip(elements_with_text, embeddings):
+            element[EMBEDDINGS_KEY] = embedding
+        return elements
 
     async def _embed_documents(self, elements: list[str]) -> list[list[float]]:
         client = self.config.get_async_client()