unstructured-ingest 1.0.15__py3-none-any.whl → 1.0.17__py3-none-any.whl

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.15"  # pragma: no cover
+__version__ = "1.0.17"  # pragma: no cover

unstructured_ingest/embed/azure_openai.py

@@ -16,15 +16,18 @@ if TYPE_CHECKING:
 
 class AzureOpenAIEmbeddingConfig(OpenAIEmbeddingConfig):
     api_version: str = Field(description="Azure API version", default="2024-06-01")
-    azure_endpoint: str
-    embedder_model_name: str = Field(default="text-embedding-ada-002", alias="model_name")
+    azure_endpoint: str = Field(description="Azure endpoint")
+    embedder_model_name: str = Field(
+        default="text-embedding-ada-002", alias="model_name", description="Azure OpenAI model name"
+    )
 
     @requires_dependencies(["openai"], extras="openai")
     def get_client(self) -> "AzureOpenAI":
         from openai import AzureOpenAI
 
+        api_key = self.api_key.get_secret_value() if self.api_key else None
         return AzureOpenAI(
-            api_key=self.api_key.get_secret_value(),
+            api_key=api_key,
             api_version=self.api_version,
             azure_endpoint=self.azure_endpoint,
         )
@@ -33,8 +36,9 @@ class AzureOpenAIEmbeddingConfig(OpenAIEmbeddingConfig):
     def get_async_client(self) -> "AsyncAzureOpenAI":
         from openai import AsyncAzureOpenAI
 
+        api_key = self.api_key.get_secret_value() if self.api_key else None
         return AsyncAzureOpenAI(
-            api_key=self.api_key.get_secret_value(),
+            api_key=api_key,
             api_version=self.api_version,
             azure_endpoint=self.azure_endpoint,
         )

unstructured_ingest/embed/bedrock.py

@@ -58,10 +58,14 @@ def conform_query(query: str, provider: str) -> dict:
 
 
 class BedrockEmbeddingConfig(EmbeddingConfig):
-    aws_access_key_id: SecretStr
-    aws_secret_access_key: SecretStr
-    region_name: str = "us-west-2"
-    embedder_model_name: str = Field(default="amazon.titan-embed-text-v1", alias="model_name")
+    aws_access_key_id: SecretStr = Field(description="aws access key id")
+    aws_secret_access_key: SecretStr = Field(description="aws secret access key")
+    region_name: str = Field(description="aws region name", default="us-west-2")
+    embedder_model_name: str = Field(
+        default="amazon.titan-embed-text-v1",
+        alias="model_name",
+        description="AWS Bedrock model name",
+    )
 
     def wrap_error(self, e: Exception) -> Exception:
         if is_internal_error(e=e):

unstructured_ingest/embed/huggingface.py

@@ -15,11 +15,18 @@ if TYPE_CHECKING:
 
 
 class HuggingFaceEmbeddingConfig(EmbeddingConfig):
-    embedder_model_name: Optional[str] = Field(default="all-MiniLM-L6-v2", alias="model_name")
+    embedder_model_name: Optional[str] = Field(
+        default="all-MiniLM-L6-v2", alias="model_name", description="HuggingFace model name"
+    )
     embedder_model_kwargs: Optional[dict] = Field(
-        default_factory=lambda: {"device": "cpu"}, alias="model_kwargs"
+        default_factory=lambda: {"device": "cpu"},
+        alias="model_kwargs",
+        description="additional model parameters",
+    )
+    encode_kwargs: Optional[dict] = Field(
+        default_factory=lambda: {"normalize_embeddings": False},
+        description="additional embedding parameters",
     )
-    encode_kwargs: Optional[dict] = Field(default_factory=lambda: {"normalize_embeddings": False})
 
     @requires_dependencies(
         ["sentence_transformers"],

unstructured_ingest/embed/mixedbreadai.py

@@ -34,10 +34,13 @@ class MixedbreadAIEmbeddingConfig(EmbeddingConfig):
 
     api_key: SecretStr = Field(
         default_factory=lambda: SecretStr(os.environ.get("MXBAI_API_KEY")),
+        description="API key for Mixedbread AI",
     )
 
     embedder_model_name: str = Field(
-        default="mixedbread-ai/mxbai-embed-large-v1", alias="model_name"
+        default="mixedbread-ai/mxbai-embed-large-v1",
+        alias="model_name",
+        description="Mixedbread AI model name",
     )
 
     @requires_dependencies(

unstructured_ingest/embed/octoai.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 from pydantic import Field, SecretStr
 
@@ -24,9 +24,13 @@ if TYPE_CHECKING:
 
 
 class OctoAiEmbeddingConfig(EmbeddingConfig):
-    api_key: SecretStr
-    embedder_model_name: str = Field(default="thenlper/gte-large", alias="model_name")
-    base_url: str = Field(default="https://text.octoai.run/v1")
+    api_key: Optional[SecretStr] = Field(description="API key for OctoAI", default=None)
+    embedder_model_name: str = Field(
+        default="thenlper/gte-large", alias="model_name", description="octoai model name"
+    )
+    base_url: str = Field(
+        default="https://text.octoai.run/v1", description="optional override for the base url"
+    )
 
     def wrap_error(self, e: Exception) -> Exception:
         if is_internal_error(e=e):
@@ -73,7 +77,8 @@ class OctoAiEmbeddingConfig(EmbeddingConfig):
         """Creates an OpenAI python client to embed elements. Uses the OpenAI SDK."""
         from openai import OpenAI
 
-        return OpenAI(api_key=self.api_key.get_secret_value(), base_url=self.base_url)
+        api_key = self.api_key.get_secret_value() if self.api_key else None
+        return OpenAI(api_key=api_key, base_url=self.base_url)
 
     @requires_dependencies(
         ["openai", "tiktoken"],
@@ -83,7 +88,8 @@ class OctoAiEmbeddingConfig(EmbeddingConfig):
         """Creates an OpenAI python client to embed elements. Uses the OpenAI SDK."""
         from openai import AsyncOpenAI
 
-        return AsyncOpenAI(api_key=self.api_key.get_secret_value(), base_url=self.base_url)
+        api_key = self.api_key.get_secret_value() if self.api_key else None
+        return AsyncOpenAI(api_key=api_key, base_url=self.base_url)
 
 
 @dataclass

unstructured_ingest/embed/openai.py

@@ -24,9 +24,11 @@ if TYPE_CHECKING:
 
 
 class OpenAIEmbeddingConfig(EmbeddingConfig):
-    api_key: SecretStr
-    embedder_model_name: str = Field(default="text-embedding-ada-002", alias="model_name")
-    base_url: Optional[str] = None
+    api_key: Optional[SecretStr] = Field(description="API key for OpenAI", default=None)
+    embedder_model_name: str = Field(
+        default="text-embedding-ada-002", alias="model_name", description="OpenAI model name"
+    )
+    base_url: Optional[str] = Field(default=None, description="optional override for the base url")
 
     @requires_dependencies(["openai"], extras="openai")
     def wrap_error(self, e: Exception) -> Exception:
@@ -86,13 +88,15 @@ class OpenAIEmbeddingConfig(EmbeddingConfig):
     def get_client(self) -> "OpenAI":
         from openai import OpenAI
 
-        return OpenAI(api_key=self.api_key.get_secret_value(), base_url=self.base_url)
+        api_key = self.api_key.get_secret_value() if self.api_key else None
+        return OpenAI(api_key=api_key, base_url=self.base_url)
 
     @requires_dependencies(["openai"], extras="openai")
     def get_async_client(self) -> "AsyncOpenAI":
         from openai import AsyncOpenAI
 
-        return AsyncOpenAI(api_key=self.api_key.get_secret_value(), base_url=self.base_url)
+        api_key = self.api_key.get_secret_value() if self.api_key else None
+        return AsyncOpenAI(api_key=api_key, base_url=self.base_url)
 
 
 @dataclass

unstructured_ingest/embed/togetherai.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Optional
 
 from pydantic import Field, SecretStr
 
@@ -20,9 +20,11 @@ if TYPE_CHECKING:
 
 
 class TogetherAIEmbeddingConfig(EmbeddingConfig):
-    api_key: SecretStr
+    api_key: Optional[SecretStr] = Field(description="API key for Together AI", default=None)
     embedder_model_name: str = Field(
-        default="togethercomputer/m2-bert-80M-8k-retrieval", alias="model_name"
+        default="togethercomputer/m2-bert-80M-8k-retrieval",
+        alias="model_name",
+        description="Together AI model name",
     )
 
     def wrap_error(self, e: Exception) -> Exception:
@@ -56,13 +58,15 @@ class TogetherAIEmbeddingConfig(EmbeddingConfig):
     def get_client(self) -> "Together":
         from together import Together
 
-        return Together(api_key=self.api_key.get_secret_value())
+        api_key = self.api_key.get_secret_value() if self.api_key else None
+        return Together(api_key=api_key)
 
     @requires_dependencies(["together"], extras="togetherai")
     def get_async_client(self) -> "AsyncTogether":
         from together import AsyncTogether
 
-        return AsyncTogether(api_key=self.api_key.get_secret_value())
+        api_key = self.api_key.get_secret_value() if self.api_key else None
+        return AsyncTogether(api_key=api_key)
 
 
 @dataclass

unstructured_ingest/embed/vertexai.py

@@ -32,9 +32,9 @@ ApiKeyType = Secret[Annotated[dict, BeforeValidator(conform_string_to_dict)]]
 
 
 class VertexAIEmbeddingConfig(EmbeddingConfig):
-    api_key: ApiKeyType
+    api_key: ApiKeyType = Field(description="API key for Vertex AI")
     embedder_model_name: Optional[str] = Field(
-        default="textembedding-gecko@001", alias="model_name"
+        default="textembedding-gecko@001", alias="model_name", description="Vertex AI model name"
     )
 
     def wrap_error(self, e: Exception) -> Exception:

unstructured_ingest/embed/voyageai.py

@@ -26,11 +26,14 @@ class VoyageAIEmbeddingConfig(EmbeddingConfig):
         le=128,
         description="Batch size for embedding requests. VoyageAI has a limit of 128.",
     )
-    api_key: SecretStr
-    embedder_model_name: str = Field(default="voyage-3", alias="model_name")
-    truncation: Optional[bool] = Field(default=None)
-    max_retries: int = 0
-    timeout_in_seconds: Optional[int] = None
+    api_key: Optional[SecretStr] = Field(description="API key for VoyageAI", default=None)
+    embedder_model_name: str = Field(
+        default="voyage-3", alias="model_name", description="VoyageAI model name"
+    )
+    max_retries: int = Field(default=0, description="Max retries for embedding requests.")
+    timeout_in_seconds: Optional[int] = Field(
+        default=None, description="Optional timeout in seconds for embedding requests."
+    )
 
     def wrap_error(self, e: Exception) -> Exception:
         if is_internal_error(e=e):
@@ -62,8 +65,9 @@ class VoyageAIEmbeddingConfig(EmbeddingConfig):
         """Creates a VoyageAI python client to embed elements."""
         from voyageai import Client as VoyageAIClient
 
+        api_key = self.api_key.get_secret_value() if self.api_key else None
         client = VoyageAIClient(
-            api_key=self.api_key.get_secret_value(),
+            api_key=api_key,
             max_retries=self.max_retries,
             timeout=self.timeout_in_seconds,
         )
@@ -77,8 +81,9 @@ class VoyageAIEmbeddingConfig(EmbeddingConfig):
         """Creates a VoyageAI python client to embed elements."""
         from voyageai import AsyncClient as AsyncVoyageAIClient
 
+        api_key = self.api_key.get_secret_value() if self.api_key else None
         client = AsyncVoyageAIClient(
-            api_key=self.api_key.get_secret_value(),
+            api_key=api_key,
             max_retries=self.max_retries,
             timeout=self.timeout_in_seconds,
         )

{unstructured_ingest-1.0.15.dist-info → unstructured_ingest-1.0.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.15
+Version: 1.0.17
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0
@@ -19,7 +19,6 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: <3.13,>=3.9
 Requires-Dist: click
-Requires-Dist: dataclasses-json
 Requires-Dist: opentelemetry-sdk
 Requires-Dist: pydantic>=2.7
 Requires-Dist: python-dateutil
@@ -74,7 +73,7 @@ Provides-Extra: duckdb
 Requires-Dist: duckdb; extra == 'duckdb'
 Requires-Dist: pandas; extra == 'duckdb'
 Provides-Extra: elasticsearch
-Requires-Dist: elasticsearch[async]; extra == 'elasticsearch'
+Requires-Dist: elasticsearch[async]<9.0.0; extra == 'elasticsearch'
 Provides-Extra: epub
 Requires-Dist: unstructured[epub]; extra == 'epub'
 Provides-Extra: gcs

{unstructured_ingest-1.0.15.dist-info → unstructured_ingest-1.0.17.dist-info}/RECORD

@@ -1,5 +1,5 @@
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=oGOfVxqR9-1jTBJa12PKKVu0cnuxJyM_nJIiJM33B_A,43
+unstructured_ingest/__version__.py,sha256=HJycVzTiDHeRdW4JUDAnGRPwiviRO2FPCxl56CUWKGY,43
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/errors_v2.py,sha256=9RuRCi7lbDxCguDz07y5RiHoQiFIOWwOD7xqzJ2B3Yw,436
 unstructured_ingest/logger.py,sha256=7e_7UeK6hVOd5BQ6i9NzRUAPCS_DF839Y8TjUDywraY,1428
@@ -22,16 +22,16 @@ unstructured_ingest/data_types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 unstructured_ingest/data_types/entities.py,sha256=ECc6EkZ5_ZUvK7uaALYOynfFmofIrHYIJZfb67hUIxA,371
 unstructured_ingest/data_types/file_data.py,sha256=J0RQa7YXhhxiLVzhPbF5Hl2nzSpxLFK9vrP6RTBWlSg,3833
 unstructured_ingest/embed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-unstructured_ingest/embed/azure_openai.py,sha256=_-I-nwd-wdCiKkSdYBL4UKrTZ2UPWsM_0T69fcObs_I,1707
-unstructured_ingest/embed/bedrock.py,sha256=AjOMSZakPrGvVUEOacnTbhuE1Ka5MdRZgOnK7Qb2lhY,8931
-unstructured_ingest/embed/huggingface.py,sha256=vJeTVeLzPU0mksmZHz-4v9TCtdiNt-d9bsHPbzhpyV8,2253
+unstructured_ingest/embed/azure_openai.py,sha256=nHEkvWH7mETrJB-C9qlci6bEpSEPiW9peG82EUDQ954,1902
+unstructured_ingest/embed/bedrock.py,sha256=dzfCsatB0i8hUp1YnXmoImoxgvUdZ4srKI6eSvn-lYM,9132
+unstructured_ingest/embed/huggingface.py,sha256=6Gx9L3xa3cv9fX4AMuLsePJQF4T_jwkKjovfqF5X1NM,2435
 unstructured_ingest/embed/interfaces.py,sha256=Y3PLhgWnMDmtpugE37hlAiBIbC8izrFFXXkrPVby-HY,5137
-unstructured_ingest/embed/mixedbreadai.py,sha256=_rqH5sZspKbMM1naSU5zYFgEULzZivtc3-5RrgZErpE,4486
-unstructured_ingest/embed/octoai.py,sha256=RWMcoYvp14SPBRoqfPRl0ny_dmt8nVbCW02UGcYetbY,4387
-unstructured_ingest/embed/openai.py,sha256=09Lx9lqlDXTHWXRHNiUerK-g3bhK8hSuW9RLzglHWCA,4493
-unstructured_ingest/embed/togetherai.py,sha256=a1ko06oSgkUgQahYNtPt6pU8v8bmNkRl7WTEK5foBvc,3527
-unstructured_ingest/embed/vertexai.py,sha256=CjSkXYo9ZT590eM3MU3tdDa4p5gHn4HW0bAlm70MjOo,3682
-unstructured_ingest/embed/voyageai.py,sha256=84O49p-vQuQzhNndPIfwCYcCNBoHFIX9QHZvc8m5DdA,4496
+unstructured_ingest/embed/mixedbreadai.py,sha256=pmpGQ0E-bfkkg4rvPvsFxL6Oc7H5f0mJGguHtfL7oLc,4592
+unstructured_ingest/embed/octoai.py,sha256=imuH_vLlmDd3GgAgiA0AaXB1fGjaI9lPpCCBG5HbpU8,4678
+unstructured_ingest/embed/openai.py,sha256=yYqpSZcD8kUZOT36oj39hN8sCFpPKShTHVyV4dI3Bbg,4775
+unstructured_ingest/embed/togetherai.py,sha256=19Le-SdMLp2U1qy5mTk_kO90b-AbOG_-a7Fslp1caJA,3767
+unstructured_ingest/embed/vertexai.py,sha256=jA3Y-AysVVaYwqkVd_OgRKF0JdHLAgZlRgfgddcZV2o,3763
+unstructured_ingest/embed/voyageai.py,sha256=-aKSxZm6G5NcKlloA6je70HmT30WSgcnZWCGEoz9PPo,4826
 unstructured_ingest/interfaces/__init__.py,sha256=QIkWqjsq9INTa89gPuXlMlQL4s3y5TqLmPkuVuTyXcs,795
 unstructured_ingest/interfaces/connector.py,sha256=qUFFJ3qgDMenTCZMtVRjq1DIwsVak6pxNjQOH2eVkMw,1623
 unstructured_ingest/interfaces/downloader.py,sha256=xX0ZzsFRSzZb7SAeoeQph8sIbVq13DRw-3MYkdADrY0,2918
@@ -231,8 +231,8 @@ unstructured_ingest/utils/ndjson.py,sha256=nz8VUOPEgAFdhaDOpuveknvCU4x82fVwqE01q
 unstructured_ingest/utils/pydantic_models.py,sha256=BT_j15e4rX40wQbt8LUXbqfPhA3rJn1PHTI_G_A_EHY,1720
 unstructured_ingest/utils/string_and_date_utils.py,sha256=oXOI6rxXq-8ncbk7EoJK0WCcTXWj75EzKl8pfQMID3U,2522
 unstructured_ingest/utils/table.py,sha256=WZechczgVFvlodUWFcsnCGvBNh1xRm6hr0VbJTPxKAc,3669
-unstructured_ingest-1.0.15.dist-info/METADATA,sha256=FLLtR4NPVBbafV9aTVcwguRub-D_xwmHwQmPWDAGmXU,8720
-unstructured_ingest-1.0.15.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-unstructured_ingest-1.0.15.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-1.0.15.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-1.0.15.dist-info/RECORD,,
+unstructured_ingest-1.0.17.dist-info/METADATA,sha256=XvwbN72yhaJKn5uYKyWePqbH5ZmptqS9I0RP___NbXQ,8694
+unstructured_ingest-1.0.17.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+unstructured_ingest-1.0.17.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-1.0.17.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-1.0.17.dist-info/RECORD,,