unstructured-ingest 0.0.21__tar.gz → 0.0.22__tar.gz

unstructured_ingest-0.0.22/PKG-INFO

@@ -0,0 +1,186 @@
+Metadata-Version: 2.1
+Name: unstructured-ingest
+Version: 0.0.22
+Summary: A library that prepares raw documents for downstream ML tasks.
+Home-page: https://github.com/Unstructured-IO/unstructured-ingest
+Author: Unstructured Technologies
+Author-email: devops@unstructuredai.io
+License: Apache-2.0
+Keywords: NLP PDF HTML CV XML parsing preprocessing
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9.0,<3.13
+Description-Content-Type: text/markdown
+License-File: LICENSE.md
+Requires-Dist: pandas
+Requires-Dist: pydantic>=2.7
+Requires-Dist: click
+Requires-Dist: opentelemetry-sdk
+Requires-Dist: dataclasses_json
+Requires-Dist: tqdm
+Requires-Dist: python-dateutil
+Provides-Extra: remote
+Requires-Dist: unstructured-client>=0.25.8; extra == "remote"
+Provides-Extra: csv
+Requires-Dist: unstructured[tsv]; extra == "csv"
+Provides-Extra: doc
+Requires-Dist: unstructured[docx]; extra == "doc"
+Provides-Extra: docx
+Requires-Dist: unstructured[docx]; extra == "docx"
+Provides-Extra: epub
+Requires-Dist: unstructured[epub]; extra == "epub"
+Provides-Extra: md
+Requires-Dist: unstructured[md]; extra == "md"
+Provides-Extra: msg
+Requires-Dist: unstructured[msg]; extra == "msg"
+Provides-Extra: odt
+Requires-Dist: unstructured[odt]; extra == "odt"
+Provides-Extra: org
+Requires-Dist: unstructured[org]; extra == "org"
+Provides-Extra: pdf
+Requires-Dist: unstructured[pdf]; extra == "pdf"
+Provides-Extra: ppt
+Requires-Dist: unstructured[pptx]; extra == "ppt"
+Provides-Extra: pptx
+Requires-Dist: unstructured[pptx]; extra == "pptx"
+Provides-Extra: rtf
+Requires-Dist: unstructured[rtf]; extra == "rtf"
+Provides-Extra: rst
+Requires-Dist: unstructured[rst]; extra == "rst"
+Provides-Extra: tsv
+Requires-Dist: unstructured[tsv]; extra == "tsv"
+Provides-Extra: xlsx
+Requires-Dist: unstructured[xlsx]; extra == "xlsx"
+Provides-Extra: airtable
+Requires-Dist: pyairtable; extra == "airtable"
+Provides-Extra: astradb
+Requires-Dist: astrapy; extra == "astradb"
+Provides-Extra: azure
+Requires-Dist: adlfs; extra == "azure"
+Requires-Dist: fsspec; extra == "azure"
+Provides-Extra: azure-cognitive-search
+Requires-Dist: azure-search-documents; extra == "azure-cognitive-search"
+Provides-Extra: biomed
+Requires-Dist: requests; extra == "biomed"
+Requires-Dist: bs4; extra == "biomed"
+Provides-Extra: box
+Requires-Dist: fsspec; extra == "box"
+Requires-Dist: boxfs; extra == "box"
+Provides-Extra: chroma
+Requires-Dist: chromadb; extra == "chroma"
+Provides-Extra: clarifai
+Requires-Dist: clarifai; extra == "clarifai"
+Provides-Extra: confluence
+Requires-Dist: requests; extra == "confluence"
+Requires-Dist: atlassian-python-api; extra == "confluence"
+Provides-Extra: couchbase
+Requires-Dist: couchbase; extra == "couchbase"
+Provides-Extra: delta-table
+Requires-Dist: deltalake; extra == "delta-table"
+Requires-Dist: fsspec; extra == "delta-table"
+Provides-Extra: discord
+Requires-Dist: discord-py; extra == "discord"
+Provides-Extra: dropbox
+Requires-Dist: fsspec; extra == "dropbox"
+Requires-Dist: dropboxdrivefs; extra == "dropbox"
+Provides-Extra: elasticsearch
+Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
+Provides-Extra: gcs
+Requires-Dist: gcsfs; extra == "gcs"
+Requires-Dist: fsspec; extra == "gcs"
+Requires-Dist: bs4; extra == "gcs"
+Provides-Extra: github
+Requires-Dist: requests; extra == "github"
+Requires-Dist: pygithub>1.58.0; extra == "github"
+Provides-Extra: gitlab
+Requires-Dist: python-gitlab; extra == "gitlab"
+Provides-Extra: google-drive
+Requires-Dist: google-api-python-client; extra == "google-drive"
+Provides-Extra: hubspot
+Requires-Dist: urllib3; extra == "hubspot"
+Requires-Dist: hubspot-api-client; extra == "hubspot"
+Provides-Extra: jira
+Requires-Dist: atlassian-python-api; extra == "jira"
+Provides-Extra: kafka
+Requires-Dist: confluent-kafka; extra == "kafka"
+Provides-Extra: kdbai
+Requires-Dist: kdbai-client; extra == "kdbai"
+Provides-Extra: milvus
+Requires-Dist: pymilvus; extra == "milvus"
+Provides-Extra: mongodb
+Requires-Dist: pymongo; extra == "mongodb"
+Provides-Extra: notion
+Requires-Dist: httpx; extra == "notion"
+Requires-Dist: htmlBuilder; extra == "notion"
+Requires-Dist: notion-client; extra == "notion"
+Requires-Dist: backoff; extra == "notion"
+Provides-Extra: onedrive
+Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
+Requires-Dist: msal; extra == "onedrive"
+Requires-Dist: bs4; extra == "onedrive"
+Provides-Extra: opensearch
+Requires-Dist: opensearch-py; extra == "opensearch"
+Provides-Extra: outlook
+Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
+Requires-Dist: msal; extra == "outlook"
+Provides-Extra: pinecone
+Requires-Dist: pinecone-client>=3.7.1; extra == "pinecone"
+Provides-Extra: postgres
+Requires-Dist: psycopg2-binary; extra == "postgres"
+Provides-Extra: qdrant
+Requires-Dist: qdrant-client; extra == "qdrant"
+Provides-Extra: reddit
+Requires-Dist: praw; extra == "reddit"
+Provides-Extra: s3
+Requires-Dist: s3fs; extra == "s3"
+Requires-Dist: fsspec; extra == "s3"
+Provides-Extra: sharepoint
+Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
+Requires-Dist: msal; extra == "sharepoint"
+Provides-Extra: salesforce
+Requires-Dist: simple-salesforce; extra == "salesforce"
+Provides-Extra: sftp
+Requires-Dist: paramiko; extra == "sftp"
+Requires-Dist: fsspec; extra == "sftp"
+Provides-Extra: slack
+Requires-Dist: slack_sdk; extra == "slack"
+Provides-Extra: wikipedia
+Requires-Dist: wikipedia; extra == "wikipedia"
+Provides-Extra: weaviate
+Requires-Dist: weaviate-client; extra == "weaviate"
+Provides-Extra: databricks-volumes
+Requires-Dist: databricks-sdk; extra == "databricks-volumes"
+Provides-Extra: singlestore
+Requires-Dist: singlestoredb; extra == "singlestore"
+Provides-Extra: vectara
+Requires-Dist: requests; extra == "vectara"
+Provides-Extra: embed-huggingface
+Requires-Dist: sentence-transformers; extra == "embed-huggingface"
+Provides-Extra: embed-octoai
+Requires-Dist: tiktoken; extra == "embed-octoai"
+Requires-Dist: openai; extra == "embed-octoai"
+Provides-Extra: embed-vertexai
+Requires-Dist: vertexai; extra == "embed-vertexai"
+Provides-Extra: embed-voyageai
+Requires-Dist: voyageai; extra == "embed-voyageai"
+Provides-Extra: embed-mixedbreadai
+Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
+Provides-Extra: openai
+Requires-Dist: tiktoken; extra == "openai"
+Requires-Dist: openai; extra == "openai"
+Provides-Extra: bedrock
+Requires-Dist: boto3; extra == "bedrock"
+
+# Unstructured Ingest  
+
+For details, see the [Unstructured Ingest overview](https://docs.unstructured.io/ingestion/overview) in the Unstructured documentation.

{unstructured-ingest-0.0.21 → unstructured_ingest-0.0.22}/test/test_utils_v2.py

@@ -34,7 +34,7 @@ model = MockBaseModel(
 
 def test_serialize_base_model():
 
-    serialized_dict = model.dict()
+    serialized_dict = model.model_dump()
     assert isinstance(serialized_dict["secret_str"], _SecretBase)
     assert isinstance(serialized_dict["secret_child_base"], _SecretBase)
 
@@ -57,7 +57,7 @@ def test_serialize_base_model():
 
 
 def test_serialize_base_model_json():
-    serialized_json = model.json()
+    serialized_json = model.model_dump_json()
     serialized_dict = json.loads(serialized_json)
     expected_dict = {
         "secret_str": "**********",

unstructured_ingest-0.0.22/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.0.22"  # pragma: no cover

unstructured_ingest-0.0.22/unstructured_ingest/embed/bedrock.py

@@ -0,0 +1,107 @@
+import json
+import os
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+import numpy as np
+from pydantic import Field, SecretStr
+
+from unstructured_ingest.embed.interfaces import BaseEmbeddingEncoder, EmbeddingConfig
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+if TYPE_CHECKING:
+    from botocore.client import BaseClient
+
+    class BedrockClient(BaseClient):
+        def invoke_model(self, body: str, modelId: str, trace: str) -> dict:
+            pass
+
+
+class BedrockEmbeddingConfig(EmbeddingConfig):
+    aws_access_key_id: SecretStr
+    aws_secret_access_key: SecretStr
+    region_name: str = "us-west-2"
+    embed_model_name: str = Field(default="amazon.titan-embed-text-v1", alias="model_name")
+
+    @requires_dependencies(
+        ["boto3", "numpy", "botocore"],
+        extras="bedrock",
+    )
+    def get_client(self) -> "BedrockClient":
+        # delay import only when needed
+        import boto3
+
+        bedrock_client = boto3.client(
+            service_name="bedrock-runtime",
+            aws_access_key_id=self.aws_access_key_id.get_secret_value(),
+            aws_secret_access_key=self.aws_secret_access_key.get_secret_value(),
+            region_name=self.region_name,
+        )
+
+        return bedrock_client
+
+
+@dataclass
+class BedrockEmbeddingEncoder(BaseEmbeddingEncoder):
+    config: BedrockEmbeddingConfig
+
+    def get_exemplary_embedding(self) -> list[float]:
+        return self.embed_query(query="Q")
+
+    def num_of_dimensions(self) -> tuple[int, ...]:
+        exemplary_embedding = self.get_exemplary_embedding()
+        return np.shape(exemplary_embedding)
+
+    def is_unit_vector(self) -> bool:
+        exemplary_embedding = self.get_exemplary_embedding()
+        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
+
+    def embed_query(self, query: str) -> list[float]:
+        """Call out to Bedrock embedding endpoint."""
+        # replace newlines, which can negatively affect performance.
+        text = query.replace(os.linesep, " ")
+
+        # format input body for provider
+        provider = self.config.embed_model_name.split(".")[0]
+        input_body = {}
+        if provider == "cohere":
+            if "input_type" not in input_body:
+                input_body["input_type"] = "search_document"
+            input_body["texts"] = [text]
+        else:
+            # includes common provider == "amazon"
+            input_body["inputText"] = text
+        body = json.dumps(input_body)
+
+        try:
+            bedrock_client = self.config.get_client()
+            # invoke bedrock API
+            response = bedrock_client.invoke_model(
+                body=body,
+                modelId=self.config.embed_model_name,
+                accept="application/json",
+                contentType="application/json",
+            )
+
+            # format output based on provider
+            response_body = json.loads(response.get("body").read())
+            if provider == "cohere":
+                return response_body.get("embeddings")[0]
+            else:
+                # includes common provider == "amazon"
+                return response_body.get("embedding")
+        except Exception as e:
+            raise ValueError(f"Error raised by inference endpoint: {e}")
+
+    def embed_documents(self, elements: list[dict]) -> list[dict]:
+        embeddings = [self.embed_query(query=e.get("text", "")) for e in elements]
+        elements_with_embeddings = self._add_embeddings_to_elements(elements, embeddings)
+        return elements_with_embeddings
+
+    def _add_embeddings_to_elements(self, elements, embeddings) -> list[dict]:
+        assert len(elements) == len(embeddings)
+        elements_w_embedding = []
+        for i, element in enumerate(elements):
+            element["embeddings"] = embeddings[i]
+            elements_w_embedding.append(element)
+        return elements

{unstructured-ingest-0.0.21 → unstructured_ingest-0.0.22}/unstructured_ingest/embed/huggingface.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, List, Optional
+from typing import TYPE_CHECKING, Optional
 
 import numpy as np
 from pydantic import Field
@@ -8,7 +8,7 @@ from unstructured_ingest.embed.interfaces import BaseEmbeddingEncoder, Embedding
 from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
-    from langchain_huggingface.embeddings import HuggingFaceEmbeddings
+    from sentence_transformers import SentenceTransformer
 
 
 class HuggingFaceEmbeddingConfig(EmbeddingConfig):
@@ -19,51 +19,51 @@ class HuggingFaceEmbeddingConfig(EmbeddingConfig):
         default_factory=lambda: {"device": "cpu"}, alias="model_kwargs"
     )
     encode_kwargs: Optional[dict] = Field(default_factory=lambda: {"normalize_embeddings": False})
-    cache_folder: Optional[dict] = Field(default=None)
+    cache_folder: Optional[str] = Field(default=None)
 
     @requires_dependencies(
-        ["langchain_huggingface"],
+        ["sentence_transformers"],
         extras="embed-huggingface",
     )
-    def get_client(self) -> "HuggingFaceEmbeddings":
-        """Creates a langchain Huggingface python client to embed elements."""
-        from langchain_huggingface.embeddings import HuggingFaceEmbeddings
-
-        client = HuggingFaceEmbeddings(
-            model_name=self.embedder_model_name,
-            model_kwargs=self.embedder_model_kwargs,
-            encode_kwargs=self.encode_kwargs,
+    def get_client(self) -> "SentenceTransformer":
+        from sentence_transformers import SentenceTransformer
+
+        return SentenceTransformer(
+            model_name_or_path=self.embedder_model_name,
             cache_folder=self.cache_folder,
+            **self.embedder_model_kwargs,
         )
-        return client
 
 
 @dataclass
 class HuggingFaceEmbeddingEncoder(BaseEmbeddingEncoder):
     config: HuggingFaceEmbeddingConfig
 
-    def get_exemplary_embedding(self) -> List[float]:
+    def get_exemplary_embedding(self) -> list[float]:
         return self.embed_query(query="Q")
 
-    def num_of_dimensions(self):
+    def num_of_dimensions(self) -> tuple[int, ...]:
         exemplary_embedding = self.get_exemplary_embedding()
         return np.shape(exemplary_embedding)
 
-    def is_unit_vector(self):
+    def is_unit_vector(self) -> bool:
         exemplary_embedding = self.get_exemplary_embedding()
         return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
 
-    def embed_query(self, query):
-        client = self.config.get_client()
-        return client.embed_query(str(query))
+    def embed_query(self, query: str) -> list[float]:
+        return self._embed_documents(texts=[query])[0]
 
-    def embed_documents(self, elements: List[dict]) -> List[dict]:
+    def _embed_documents(self, texts: list[str]) -> list[list[float]]:
         client = self.config.get_client()
-        embeddings = client.embed_documents([e.get("text", "") for e in elements])
+        embeddings = client.encode(texts, **self.config.encode_kwargs)
+        return embeddings.tolist()
+
+    def embed_documents(self, elements: list[dict]) -> list[dict]:
+        embeddings = self._embed_documents([e.get("text", "") for e in elements])
         elements_with_embeddings = self._add_embeddings_to_elements(elements, embeddings)
         return elements_with_embeddings
 
-    def _add_embeddings_to_elements(self, elements: list[dict], embeddings: list) -> List[dict]:
+    def _add_embeddings_to_elements(self, elements: list[dict], embeddings: list) -> list[dict]:
         assert len(elements) == len(embeddings)
         elements_w_embedding = []
 

{unstructured-ingest-0.0.21 → unstructured_ingest-0.0.22}/unstructured_ingest/embed/interfaces.py

@@ -1,6 +1,5 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from typing import List, Tuple
 
 from pydantic import BaseModel
 
@@ -19,7 +18,7 @@ class BaseEmbeddingEncoder(ABC):
 
     @property
     @abstractmethod
-    def num_of_dimensions(self) -> Tuple[int]:
+    def num_of_dimensions(self) -> tuple[int, ...]:
         """Number of dimensions for the embedding vector."""
 
     @property
@@ -28,9 +27,17 @@ class BaseEmbeddingEncoder(ABC):
         """Denotes if the embedding vector is a unit vector."""
 
     @abstractmethod
-    def embed_documents(self, elements: List[dict]) -> List[dict]:
+    def embed_documents(self, elements: list[dict]) -> list[dict]:
         pass
 
     @abstractmethod
-    def embed_query(self, query: str) -> List[float]:
+    def embed_query(self, query: str) -> list[float]:
         pass
+
+    def _embed_documents(self, elements: list[str]) -> list[list[float]]:
+        results = []
+        for text in elements:
+            response = self.embed_query(query=text)
+            results.append(response)
+
+        return results

{unstructured-ingest-0.0.21 → unstructured_ingest-0.0.22}/unstructured_ingest/embed/mixedbreadai.py

@@ -1,6 +1,6 @@
 import os
 from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, List, Optional
+from typing import TYPE_CHECKING, Optional
 
 import numpy as np
 from pydantic import Field, SecretStr
@@ -67,10 +67,10 @@ class MixedbreadAIEmbeddingEncoder(BaseEmbeddingEncoder):
 
     config: MixedbreadAIEmbeddingConfig
 
-    _exemplary_embedding: Optional[List[float]] = field(init=False, default=None)
+    _exemplary_embedding: Optional[list[float]] = field(init=False, default=None)
     _request_options: Optional["RequestOptions"] = field(init=False, default=None)
 
-    def get_exemplary_embedding(self) -> List[float]:
+    def get_exemplary_embedding(self) -> list[float]:
         """Get an exemplary embedding to determine dimensions and unit vector status."""
         return self._embed(["Q"])[0]
 
@@ -91,7 +91,7 @@ class MixedbreadAIEmbeddingEncoder(BaseEmbeddingEncoder):
         )
 
     @property
-    def num_of_dimensions(self):
+    def num_of_dimensions(self) -> tuple[int, ...]:
         """Get the number of dimensions for the embeddings."""
         exemplary_embedding = self.get_exemplary_embedding()
         return np.shape(exemplary_embedding)
@@ -102,15 +102,15 @@ class MixedbreadAIEmbeddingEncoder(BaseEmbeddingEncoder):
         exemplary_embedding = self.get_exemplary_embedding()
         return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
 
-    def _embed(self, texts: List[str]) -> List[List[float]]:
+    def _embed(self, texts: list[str]) -> list[list[float]]:
         """
         Embed a list of texts using the Mixedbread AI API.
 
         Args:
-            texts (List[str]): List of texts to embed.
+            texts (list[str]): List of texts to embed.
 
         Returns:
-            List[List[float]]: List of embeddings.
+            list[list[float]]: List of embeddings.
         """
         batch_size = BATCH_SIZE
         batch_itr = range(0, len(texts), batch_size)
@@ -132,17 +132,17 @@ class MixedbreadAIEmbeddingEncoder(BaseEmbeddingEncoder):
 
     @staticmethod
     def _add_embeddings_to_elements(
-        elements: List[dict], embeddings: List[List[float]]
-    ) -> List[dict]:
+        elements: list[dict], embeddings: list[list[float]]
+    ) -> list[dict]:
         """
         Add embeddings to elements.
 
         Args:
-            elements (List[Element]): List of elements.
-            embeddings (List[List[float]]): List of embeddings.
+            elements (list[Element]): List of elements.
+            embeddings (list[list[float]]): List of embeddings.
 
         Returns:
-            List[Element]: Elements with embeddings added.
+            list[Element]: Elements with embeddings added.
         """
         assert len(elements) == len(embeddings)
         elements_w_embedding = []
@@ -151,20 +151,20 @@ class MixedbreadAIEmbeddingEncoder(BaseEmbeddingEncoder):
             elements_w_embedding.append(element)
         return elements
 
-    def embed_documents(self, elements: List[dict]) -> List[dict]:
+    def embed_documents(self, elements: list[dict]) -> list[dict]:
         """
         Embed a list of document elements.
 
         Args:
-            elements (List[Element]): List of document elements.
+            elements (list[Element]): List of document elements.
 
         Returns:
-            List[Element]: Elements with embeddings.
+            list[Element]: Elements with embeddings.
         """
         embeddings = self._embed([e.get("text", "") for e in elements])
         return self._add_embeddings_to_elements(elements, embeddings)
 
-    def embed_query(self, query: str) -> List[float]:
+    def embed_query(self, query: str) -> list[float]:
         """
         Embed a query string.
 
@@ -172,6 +172,6 @@ class MixedbreadAIEmbeddingEncoder(BaseEmbeddingEncoder):
             query (str): Query string to embed.
 
         Returns:
-            List[float]: Embedding of the query.
+            list[float]: Embedding of the query.
         """
         return self._embed([query])[0]

{unstructured-ingest-0.0.21 → unstructured_ingest-0.0.22}/unstructured_ingest/embed/octoai.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, List, Optional
+from typing import TYPE_CHECKING, Optional
 
 import numpy as np
 from pydantic import Field, SecretStr
@@ -31,16 +31,16 @@ class OctoAiEmbeddingConfig(EmbeddingConfig):
 class OctoAIEmbeddingEncoder(BaseEmbeddingEncoder):
     config: OctoAiEmbeddingConfig
     # Uses the OpenAI SDK
-    _exemplary_embedding: Optional[List[float]] = field(init=False, default=None)
+    _exemplary_embedding: Optional[list[float]] = field(init=False, default=None)
 
-    def get_exemplary_embedding(self) -> List[float]:
+    def get_exemplary_embedding(self) -> list[float]:
         return self.embed_query("Q")
 
-    def num_of_dimensions(self):
+    def num_of_dimensions(self) -> tuple[int, ...]:
         exemplary_embedding = self.get_exemplary_embedding()
         return np.shape(exemplary_embedding)
 
-    def is_unit_vector(self):
+    def is_unit_vector(self) -> bool:
         exemplary_embedding = self.get_exemplary_embedding()
         return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
 
@@ -49,12 +49,12 @@ class OctoAIEmbeddingEncoder(BaseEmbeddingEncoder):
         response = client.embeddings.create(input=query, model=self.config.embedder_model_name)
         return response.data[0].embedding
 
-    def embed_documents(self, elements: List[dict]) -> List[dict]:
+    def embed_documents(self, elements: list[dict]) -> list[dict]:
         embeddings = [self.embed_query(e.get("text", "")) for e in elements]
         elements_with_embeddings = self._add_embeddings_to_elements(elements, embeddings)
         return elements_with_embeddings
 
-    def _add_embeddings_to_elements(self, elements, embeddings) -> List[dict]:
+    def _add_embeddings_to_elements(self, elements, embeddings) -> list[dict]:
         assert len(elements) == len(embeddings)
         elements_w_embedding = []
         for i, element in enumerate(elements):

{unstructured-ingest-0.0.21 → unstructured_ingest-0.0.22}/unstructured_ingest/embed/openai.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, List
+from typing import TYPE_CHECKING
 
 import numpy as np
 from pydantic import Field, SecretStr
@@ -8,51 +8,46 @@ from unstructured_ingest.embed.interfaces import BaseEmbeddingEncoder, Embedding
 from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
-    from langchain_openai.embeddings import OpenAIEmbeddings
+    from openai import OpenAI
 
 
 class OpenAIEmbeddingConfig(EmbeddingConfig):
     api_key: SecretStr
     embedder_model_name: str = Field(default="text-embedding-ada-002", alias="model_name")
 
-    @requires_dependencies(["langchain_openai"], extras="openai")
-    def get_client(self) -> "OpenAIEmbeddings":
-        """Creates a langchain OpenAI python client to embed elements."""
-        from langchain_openai import OpenAIEmbeddings
+    @requires_dependencies(["openai"], extras="openai")
+    def get_client(self) -> "OpenAI":
+        from openai import OpenAI
 
-        openai_client = OpenAIEmbeddings(
-            openai_api_key=self.api_key.get_secret_value(),
-            model=self.embedder_model_name,  # type:ignore
-        )
-        return openai_client
+        return OpenAI(api_key=self.api_key.get_secret_value())
 
 
 @dataclass
 class OpenAIEmbeddingEncoder(BaseEmbeddingEncoder):
     config: OpenAIEmbeddingConfig
 
-    def get_exemplary_embedding(self) -> List[float]:
+    def get_exemplary_embedding(self) -> list[float]:
         return self.embed_query(query="Q")
 
-    def num_of_dimensions(self):
+    def num_of_dimensions(self) -> tuple[int, ...]:
         exemplary_embedding = self.get_exemplary_embedding()
         return np.shape(exemplary_embedding)
 
-    def is_unit_vector(self):
+    def is_unit_vector(self) -> bool:
         exemplary_embedding = self.get_exemplary_embedding()
         return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
 
-    def embed_query(self, query):
+    def embed_query(self, query: str) -> list[float]:
         client = self.config.get_client()
-        return client.embed_query(str(query))
+        response = client.embeddings.create(input=query, model=self.config.embedder_model_name)
+        return response.data[0].embedding
 
-    def embed_documents(self, elements: List[dict]) -> List[dict]:
-        client = self.config.get_client()
-        embeddings = client.embed_documents([e.get("text", "") for e in elements])
+    def embed_documents(self, elements: list[dict]) -> list[dict]:
+        embeddings = self._embed_documents([e.get("text", "") for e in elements])
         elements_with_embeddings = self._add_embeddings_to_elements(elements, embeddings)
         return elements_with_embeddings
 
-    def _add_embeddings_to_elements(self, elements, embeddings) -> List[dict]:
+    def _add_embeddings_to_elements(self, elements, embeddings) -> list[dict]:
         assert len(elements) == len(embeddings)
         elements_w_embedding = []
         for i, element in enumerate(elements):