unstructured-ingest 0.0.13__py3-none-any.whl → 0.0.15__py3-none-any.whl

unstructured_ingest/v2/processes/embedder.py

@@ -1,3 +1,4 @@
+import json
 from abc import ABC
 from dataclasses import dataclass
 from pathlib import Path
@@ -5,11 +6,10 @@ from typing import TYPE_CHECKING, Any, Literal, Optional
 
 from pydantic import BaseModel, Field, SecretStr
 
-from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 
 if TYPE_CHECKING:
-    from unstructured.embed.interfaces import BaseEmbeddingEncoder
+    from unstructured_ingest.embed.interfaces import BaseEmbeddingEncoder
 
 
 class EmbedderConfig(BaseModel):
@@ -21,6 +21,7 @@ class EmbedderConfig(BaseModel):
             "langchain-vertexai",
             "langchain-voyageai",
             "octoai",
+            "mixedbread-ai",
         ]
     ] = Field(default=None, description="Type of the embedding class to be used.")
     embedding_api_key: Optional[SecretStr] = Field(
@@ -42,30 +43,31 @@ class EmbedderConfig(BaseModel):
         default="us-west-2", description="AWS region used for AWS-based embedders, such as bedrock"
     )
 
-    @requires_dependencies(dependencies=["unstructured"], extras="embed-huggingface")
     def get_huggingface_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
-        from unstructured.embed.huggingface import (
+        from unstructured_ingest.embed.huggingface import (
             HuggingFaceEmbeddingConfig,
             HuggingFaceEmbeddingEncoder,
         )
 
-        return HuggingFaceEmbeddingEncoder(config=HuggingFaceEmbeddingConfig(**embedding_kwargs))
+        return HuggingFaceEmbeddingEncoder(
+            config=HuggingFaceEmbeddingConfig.model_validate(embedding_kwargs)
+        )
 
-    @requires_dependencies(dependencies=["unstructured"], extras="openai")
     def get_openai_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
-        from unstructured.embed.openai import OpenAIEmbeddingConfig, OpenAIEmbeddingEncoder
+        from unstructured_ingest.embed.openai import OpenAIEmbeddingConfig, OpenAIEmbeddingEncoder
 
-        return OpenAIEmbeddingEncoder(config=OpenAIEmbeddingConfig(**embedding_kwargs))
+        return OpenAIEmbeddingEncoder(config=OpenAIEmbeddingConfig.model_validate(embedding_kwargs))
 
-    @requires_dependencies(dependencies=["unstructured"], extras="embed-octoai")
     def get_octoai_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
-        from unstructured.embed.octoai import OctoAiEmbeddingConfig, OctoAIEmbeddingEncoder
+        from unstructured_ingest.embed.octoai import OctoAiEmbeddingConfig, OctoAIEmbeddingEncoder
 
-        return OctoAIEmbeddingEncoder(config=OctoAiEmbeddingConfig(**embedding_kwargs))
+        return OctoAIEmbeddingEncoder(config=OctoAiEmbeddingConfig.model_validate(embedding_kwargs))
 
-    @requires_dependencies(dependencies=["unstructured"], extras="bedrock")
     def get_bedrock_embedder(self) -> "BaseEmbeddingEncoder":
-        from unstructured.embed.bedrock import BedrockEmbeddingConfig, BedrockEmbeddingEncoder
+        from unstructured_ingest.embed.bedrock import (
+            BedrockEmbeddingConfig,
+            BedrockEmbeddingEncoder,
+        )
 
         return BedrockEmbeddingEncoder(
             config=BedrockEmbeddingConfig(
@@ -75,20 +77,35 @@ class EmbedderConfig(BaseModel):
             )
         )
 
-    @requires_dependencies(dependencies=["unstructured"], extras="embed-vertexai")
     def get_vertexai_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
-        from unstructured.embed.vertexai import (
+        from unstructured_ingest.embed.vertexai import (
             VertexAIEmbeddingConfig,
             VertexAIEmbeddingEncoder,
         )
 
-        return VertexAIEmbeddingEncoder(config=VertexAIEmbeddingConfig(**embedding_kwargs))
+        return VertexAIEmbeddingEncoder(
+            config=VertexAIEmbeddingConfig.model_validate(embedding_kwargs)
+        )
 
-    @requires_dependencies(dependencies=["unstructured"], extras="embed-voyageai")
     def get_voyageai_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
-        from unstructured.embed.voyageai import VoyageAIEmbeddingConfig, VoyageAIEmbeddingEncoder
+        from unstructured_ingest.embed.voyageai import (
+            VoyageAIEmbeddingConfig,
+            VoyageAIEmbeddingEncoder,
+        )
+
+        return VoyageAIEmbeddingEncoder(
+            config=VoyageAIEmbeddingConfig.model_validate(embedding_kwargs)
+        )
 
-        return VoyageAIEmbeddingEncoder(config=VoyageAIEmbeddingConfig(**embedding_kwargs))
+    def get_mixedbread_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
+        from unstructured_ingest.embed.mixedbreadai import (
+            MixedbreadAIEmbeddingConfig,
+            MixedbreadAIEmbeddingEncoder,
+        )
+
+        return MixedbreadAIEmbeddingEncoder(
+            config=MixedbreadAIEmbeddingConfig.model_validate(embedding_kwargs)
+        )
 
     def get_embedder(self) -> "BaseEmbeddingEncoder":
         kwargs: dict[str, Any] = {}
@@ -114,6 +131,8 @@ class EmbedderConfig(BaseModel):
 
         if self.embedding_provider == "langchain-voyageai":
             return self.get_voyageai_embedder(embedding_kwargs=kwargs)
+        if self.embedding_provider == "mixedbread-ai":
+            return self.get_mixedbread_embedder(embedding_kwargs=kwargs)
 
         raise ValueError(f"{self.embedding_provider} not a recognized encoder")
 
@@ -122,14 +141,12 @@ class EmbedderConfig(BaseModel):
 class Embedder(BaseProcess, ABC):
     config: EmbedderConfig
 
-    @requires_dependencies(dependencies=["unstructured"])
     def run(self, elements_filepath: Path, **kwargs: Any) -> list[dict]:
-        from unstructured.staging.base import elements_from_json
-
         # TODO update base embedder classes to support async
         embedder = self.config.get_embedder()
-        elements = elements_from_json(filename=str(elements_filepath))
+        with elements_filepath.open("r") as elements_file:
+            elements = json.load(elements_file)
         if not elements:
             return [e.to_dict() for e in elements]
         embedded_elements = embedder.embed_documents(elements=elements)
-        return [e.to_dict() for e in embedded_elements]
+        return embedded_elements

unstructured_ingest/v2/processes/filter.py

@@ -47,7 +47,7 @@ class Filterer(BaseProcess, ABC):
         for pattern in patterns:
             if fnmatch.filter([path], pattern):
                 return True
-        logger.debug(f"The file {path!r} is discarded as it does not match any given glob.")
+        logger.debug(f"the file {path!r} is discarded as it does not match any given glob.")
         return False
 
     def run(self, file_data: FileData, **kwargs: Any) -> Optional[FileData]:

unstructured_ingest/v2/processes/partitioner.py

@@ -145,7 +145,7 @@ class Partitioner(BaseProcess, ABC):
         class FileDataSourceMetadata(DataSourceMetadata):
             filesize_bytes: Optional[int] = None
 
-        logger.debug(f"Using local partition with kwargs: {self.config.to_partition_kwargs()}")
+        logger.debug(f"using local partition with kwargs: {self.config.to_partition_kwargs()}")
         logger.debug(f"partitioning file {filename} with metadata {metadata}")
         elements = partition(
             filename=str(filename.resolve()),
@@ -165,7 +165,7 @@ class Partitioner(BaseProcess, ABC):
 
         partition_request = self.config.to_partition_kwargs()
 
-        # Note(austin): PartitionParameters is a Pydantic model in v0.26.0
+        # NOTE(austin): PartitionParameters is a Pydantic model in v0.26.0
         # Prior to this it was a dataclass which doesn't have .__fields
         try:
             possible_fields = PartitionParameters.__fields__
@@ -182,7 +182,7 @@ class Partitioner(BaseProcess, ABC):
                     ", ".join([v for v in partition_request if v not in filtered_partition_request])
                 )
             )
-        logger.debug(f"Using hosted partitioner with kwargs: {partition_request}")
+        logger.debug(f"using hosted partitioner with kwargs: {partition_request}")
         with open(filename, "rb") as f:
             files = Files(
                 content=f.read(),

unstructured_ingest/v2/utils.py

@@ -20,6 +20,11 @@ def is_secret(value: Any) -> bool:
 def serialize_base_model(model: BaseModel) -> dict:
     # To get the full serialized dict regardless of if values are marked as Secret
     model_dict = model.dict()
+    return serialize_base_dict(model_dict=model_dict)
+
+
+def serialize_base_dict(model_dict: dict) -> dict:
+    model_dict = model_dict.copy()
     for k, v in model_dict.items():
         if isinstance(v, _SecretBase):
             secret_value = v.get_secret_value()
@@ -27,6 +32,8 @@ def serialize_base_model(model: BaseModel) -> dict:
                 model_dict[k] = serialize_base_model(model=secret_value)
             else:
                 model_dict[k] = secret_value
+        if isinstance(v, dict):
+            model_dict[k] = serialize_base_dict(model_dict=v)
 
     return model_dict