unstructured-ingest 0.0.13__py3-none-any.whl → 0.0.14__py3-none-any.whl

unstructured_ingest/interfaces.py

@@ -24,7 +24,8 @@ from unstructured_ingest.utils.data_prep import flatten_dict
 
 if TYPE_CHECKING:
     from unstructured.documents.elements import Element
-    from unstructured.embed.interfaces import BaseEmbeddingEncoder
+
+    from unstructured_ingest.embed.interfaces import BaseEmbeddingEncoder
 
 A = TypeVar("A", bound="DataClassJsonMixin")
 
@@ -204,22 +205,31 @@ class EmbeddingConfig(BaseConfig):
             kwargs["model_name"] = self.model_name
         # TODO make this more dynamic to map to encoder configs
         if self.provider == "langchain-openai":
-            from unstructured.embed.openai import OpenAIEmbeddingConfig, OpenAIEmbeddingEncoder
+            from unstructured_ingest.embed.openai import (
+                OpenAIEmbeddingConfig,
+                OpenAIEmbeddingEncoder,
+            )
 
             return OpenAIEmbeddingEncoder(config=OpenAIEmbeddingConfig(**kwargs))
         elif self.provider == "langchain-huggingface":
-            from unstructured.embed.huggingface import (
+            from unstructured_ingest.embed.huggingface import (
                 HuggingFaceEmbeddingConfig,
                 HuggingFaceEmbeddingEncoder,
             )
 
             return HuggingFaceEmbeddingEncoder(config=HuggingFaceEmbeddingConfig(**kwargs))
         elif self.provider == "octoai":
-            from unstructured.embed.octoai import OctoAiEmbeddingConfig, OctoAIEmbeddingEncoder
+            from unstructured_ingest.embed.octoai import (
+                OctoAiEmbeddingConfig,
+                OctoAIEmbeddingEncoder,
+            )
 
             return OctoAIEmbeddingEncoder(config=OctoAiEmbeddingConfig(**kwargs))
         elif self.provider == "langchain-aws-bedrock":
-            from unstructured.embed.bedrock import BedrockEmbeddingConfig, BedrockEmbeddingEncoder
+            from unstructured_ingest.embed.bedrock import (
+                BedrockEmbeddingConfig,
+                BedrockEmbeddingEncoder,
+            )
 
             return BedrockEmbeddingEncoder(
                 config=BedrockEmbeddingConfig(
@@ -229,14 +239,14 @@ class EmbeddingConfig(BaseConfig):
                 )
             )
         elif self.provider == "langchain-vertexai":
-            from unstructured.embed.vertexai import (
+            from unstructured_ingest.embed.vertexai import (
                 VertexAIEmbeddingConfig,
                 VertexAIEmbeddingEncoder,
             )
 
             return VertexAIEmbeddingEncoder(config=VertexAIEmbeddingConfig(**kwargs))
         elif self.provider == "langchain-voyageai":
-            from unstructured.embed.voyageai import (
+            from unstructured_ingest.embed.voyageai import (
                 VoyageAIEmbeddingConfig,
                 VoyageAIEmbeddingEncoder,
             )

unstructured_ingest/pipeline/reformat/embedding.py

@@ -27,8 +27,6 @@ class Embedder(ReformatNode):
         return hashlib.sha256(json.dumps(hash_dict, sort_keys=True).encode()).hexdigest()[:32]
 
     def run(self, elements_json: str) -> Optional[str]:
-        from unstructured.staging.base import elements_from_json
-
         try:
             elements_json_filename = os.path.basename(elements_json)
             filename_ext = os.path.basename(elements_json_filename)
@@ -48,10 +46,10 @@ class Embedder(ReformatNode):
             ):
                 logger.debug(f"File exists: {json_path}, skipping embedding")
                 return str(json_path)
-            elements = elements_from_json(filename=elements_json)
+            with open(elements_json) as f:
+                elements = json.load(f)
             embedder = self.embedder_config.get_embedder()
-            embedded_elements = embedder.embed_documents(elements=elements)
-            element_dicts = [e.to_dict() for e in embedded_elements]
+            element_dicts = embedder.embed_documents(elements=elements)
             with open(json_path, "w", encoding="utf8") as output_f:
                 logger.info(f"writing embeddings content to {json_path}")
                 json.dump(element_dicts, output_f, ensure_ascii=False, indent=2)

unstructured_ingest/utils/data_prep.py

@@ -1,12 +1,15 @@
 import itertools
 import json
 from datetime import datetime
-from typing import Any, Optional, Sequence, cast
+from typing import Any, Iterable, Optional, Sequence, TypeVar, cast
 
 DATE_FORMATS = ("%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d+%H:%M:%S", "%Y-%m-%dT%H:%M:%S%z")
 
+T = TypeVar("T")
+IterableT = Iterable[T]
 
-def batch_generator(iterable, batch_size=100):
+
+def batch_generator(iterable: IterableT, batch_size: int = 100) -> IterableT:
     """A helper function to break an iterable into batches of size batch_size."""
     it = iter(iterable)
     chunk = tuple(itertools.islice(it, batch_size))
@@ -16,23 +19,28 @@ def batch_generator(iterable, batch_size=100):
 
 
 def generator_batching_wbytes(
-    iterable, batch_size_limit_bytes=15_000_000, max_batch_size: int = 1000
-):
+    iterable: IterableT,
+    batch_size_limit_bytes: Optional[int] = None,
+    max_batch_size: Optional[int] = None,
+) -> IterableT:
+    if not batch_size_limit_bytes and not max_batch_size:
+        return iterable
     """A helper function to break an iterable into chunks of specified bytes."""
     current_batch, current_batch_size = [], 0
 
     for item in iterable:
         item_size_bytes = len(json.dumps(item).encode("utf-8"))
-
-        if (
-            current_batch_size + item_size_bytes <= batch_size_limit_bytes
-            or len(current_batch) == 0  # prevent inifite yielding of empty batch
-        ) and len(current_batch) < max_batch_size:
-            current_batch.append(item)
-            current_batch_size += item_size_bytes
-        else:
+        if batch_size_limit_bytes and current_batch_size + item_size_bytes > batch_size_limit_bytes:
+            yield current_batch
+            current_batch, current_batch_size = [item], item_size_bytes
+            continue
+        if max_batch_size and len(current_batch) + 1 > max_batch_size:
             yield current_batch
             current_batch, current_batch_size = [item], item_size_bytes
+            continue
+
+        current_batch.append(item)
+        current_batch_size += item_size_bytes
 
     if current_batch:
         yield current_batch

unstructured_ingest/v2/cli/base/src.py

@@ -1,5 +1,6 @@
 import logging
 from dataclasses import dataclass, field
+from typing import Any
 
 import click
 from pydantic import BaseModel
@@ -47,7 +48,7 @@ class SrcCmd(BaseCmd):
         options = self.consolidate_options(options=options)
         return options
 
-    def cmd(self, ctx: click.Context, **options) -> None:
+    def cmd(self, ctx: click.Context, **options: dict[str, Any]) -> None:
         if ctx.invoked_subcommand:
             return
 

unstructured_ingest/v2/pipeline/interfaces.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import asyncio
 import logging
 import multiprocessing as mp
@@ -132,7 +134,7 @@ class PipelineStep(ABC):
     async def _run_async(self, fn: Callable, **kwargs: Any) -> Optional[Any]:
         raise NotImplementedError
 
-    def run(self, _fn: Optional[Callable] = None, **kwargs: Any) -> Optional[Any]:
+    def run(self, _fn: Callable[..., Any] | None = None, **kwargs: Any) -> Optional[Any]:
         kwargs = kwargs.copy()
         otel_handler = OtelHandler(otel_endpoint=self.context.otel_endpoint, log_out=logger.debug)
         tracer = otel_handler.get_tracer()

unstructured_ingest/v2/pipeline/pipeline.py

@@ -1,7 +1,9 @@
+from __future__ import annotations
+
 import logging
 import multiprocessing as mp
 from dataclasses import InitVar, dataclass, field
-from typing import Any, Optional, Union
+from typing import Any
 
 from unstructured_ingest.v2.interfaces import ProcessorConfig, Uploader
 from unstructured_ingest.v2.logger import logger, make_default_logger
@@ -48,33 +50,33 @@ class Pipeline:
     partitioner: InitVar[Partitioner]
     partitioner_step: PartitionStep = field(init=False)
 
-    chunker: InitVar[Optional[Chunker]] = None
-    chunker_step: ChunkStep = field(init=False, default=None)
+    chunker: InitVar[Chunker | None] = None
+    chunker_step: ChunkStep | None = field(init=False, default=None)
 
-    embedder: InitVar[Optional[Embedder]] = None
-    embedder_step: EmbedStep = field(init=False, default=None)
+    embedder: InitVar[Embedder | None] = None
+    embedder_step: EmbedStep | None = field(init=False, default=None)
 
-    stager: InitVar[Optional[UploadStager]] = None
-    stager_step: UploadStageStep = field(init=False, default=None)
+    stager: InitVar[UploadStager | None] = None
+    stager_step: UploadStageStep | None = field(init=False, default=None)
 
     uploader: InitVar[Uploader] = field(default=LocalUploader())
-    uploader_step: UploadStep = field(init=False, default=None)
+    uploader_step: UploadStep | None = field(init=False, default=None)
 
-    uncompress_step: UncompressStep = field(init=False, default=None)
+    uncompress_step: UncompressStep | None = field(init=False, default=None)
 
-    filterer: InitVar[Optional[Filterer]] = None
-    filter_step: FilterStep = field(init=False, default=None)
+    filterer: InitVar[Filterer | None] = None
+    filter_step: FilterStep | None = field(init=False, default=None)
 
     def __post_init__(
         self,
         indexer: IndexerT,
         downloader: DownloaderT,
         partitioner: Partitioner,
-        chunker: Chunker = None,
-        embedder: Embedder = None,
-        stager: UploadStager = None,
-        uploader: Uploader = None,
-        filterer: Filterer = None,
+        chunker: Chunker | None = None,
+        embedder: Embedder | None = None,
+        stager: UploadStager | None = None,
+        uploader: Uploader | None = None,
+        filterer: Filterer | None = None,
     ):
         make_default_logger(level=logging.DEBUG if self.context.verbose else logging.INFO)
         otel_handler = OtelHandler(otel_endpoint=self.context.otel_endpoint)
@@ -136,7 +138,7 @@ class Pipeline:
             if self.context.status:
                 raise PipelineError("Pipeline did not run successfully")
 
-    def clean_results(self, results: Optional[list[Union[Any, list[Any]]]]) -> Optional[list[Any]]:
+    def clean_results(self, results: list[Any | list[Any]] | None) -> list[Any] | None:
         if not results:
             return None
         results = [r for r in results if r]
@@ -274,12 +276,12 @@ class Pipeline:
         downloader_config: DownloaderConfigT,
         source_connection_config: ConnectionConfig,
         partitioner_config: PartitionerConfig,
-        filterer_config: FiltererConfig = None,
-        chunker_config: Optional[ChunkerConfig] = None,
-        embedder_config: Optional[EmbedderConfig] = None,
-        destination_connection_config: Optional[ConnectionConfig] = None,
-        stager_config: Optional[UploadStagerConfigT] = None,
-        uploader_config: Optional[UploaderConfigT] = None,
+        filterer_config: FiltererConfig | None = None,
+        chunker_config: ChunkerConfig | None = None,
+        embedder_config: EmbedderConfig | None = None,
+        destination_connection_config: ConnectionConfig | None = None,
+        stager_config: UploadStagerConfigT | None = None,
+        uploader_config: UploaderConfigT | None = None,
     ) -> "Pipeline":
         # Get registry key based on indexer config
         source_entry = {

unstructured_ingest/v2/processes/connectors/chroma.py

@@ -41,9 +41,14 @@ class ChromaAccessConfig(AccessConfig):
     )
 
 
+SecretChromaAccessConfig = Secret[ChromaAccessConfig]
+
+
 class ChromaConnectionConfig(ConnectionConfig):
     collection_name: str = Field(description="The name of the Chroma collection to write into.")
-    access_config: Secret[ChromaAccessConfig]
+    access_config: SecretChromaAccessConfig = Field(
+        default=SecretChromaAccessConfig(secret_value=ChromaAccessConfig())
+    )
     path: Optional[str] = Field(
         default=None, description="Location where Chroma is persisted, if not connecting via http."
     )

unstructured_ingest/v2/processes/connectors/google_drive.py

@@ -28,8 +28,7 @@ from unstructured_ingest.v2.interfaces import (
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
-
-from .utils import conform_string_to_dict
+from unstructured_ingest.v2.processes.connectors.utils import conform_string_to_dict
 
 CONNECTOR_TYPE = "google_drive"
 

unstructured_ingest/v2/processes/connectors/onedrive.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import json
 from dataclasses import dataclass
 from pathlib import Path
@@ -103,7 +105,7 @@ class OnedriveIndexer(Indexer):
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise SourceConnectionError(f"failed to validate connection: {e}")
 
-    def list_objects(self, folder, recursive) -> list["DriveItem"]:
+    def list_objects(self, folder: DriveItem, recursive: bool) -> list["DriveItem"]:
         drive_items = folder.children.get().execute_query()
         files = [d for d in drive_items if d.is_file]
         if not recursive:
@@ -139,12 +141,12 @@ class OnedriveIndexer(Indexer):
         server_path = file_path + "/" + filename
         rel_path = server_path.replace(self.index_config.path, "").lstrip("/")
         date_modified_dt = (
-            parser.parse(drive_item.last_modified_datetime)
+            parser.parse(str(drive_item.last_modified_datetime))
             if drive_item.last_modified_datetime
             else None
         )
         date_created_at = (
-            parser.parse(drive_item.created_datetime) if drive_item.created_datetime else None
+            parser.parse(str(drive_item.created_datetime)) if drive_item.created_datetime else None
         )
         return FileData(
             identifier=drive_item.id,
@@ -156,7 +158,7 @@ class OnedriveIndexer(Indexer):
                 url=drive_item.parent_reference.path + "/" + drive_item.name,
                 version=drive_item.etag,
                 date_modified=str(date_modified_dt.timestamp()) if date_modified_dt else None,
-                date_created=str(date_created_at.timestamp()) if date_modified_dt else None,
+                date_created=str(date_created_at.timestamp()) if date_created_at else None,
                 date_processed=str(time()),
                 record_locator={
                     "user_pname": self.connection_config.user_pname,

unstructured_ingest/v2/processes/connectors/pinecone.py

@@ -27,6 +27,7 @@ if TYPE_CHECKING:
 
 CONNECTOR_TYPE = "pinecone"
 MAX_PAYLOAD_SIZE = 2 * 1024 * 1024  # 2MB
+MAX_POOL_THREADS = 100
 
 
 class PineconeAccessConfig(AccessConfig):
@@ -45,7 +46,7 @@ class PineconeConnectionConfig(ConnectionConfig):
     )
 
     @requires_dependencies(["pinecone"], extras="pinecone")
-    def get_index(self) -> "PineconeIndex":
+    def get_index(self, **index_kwargs) -> "PineconeIndex":
         from pinecone import Pinecone
 
         from unstructured_ingest import __version__ as unstructured_version
@@ -55,7 +56,7 @@ class PineconeConnectionConfig(ConnectionConfig):
             source_tag=f"unstructured_ingest=={unstructured_version}",
         )
 
-        index = pc.Index(self.index_name)
+        index = pc.Index(name=self.index_name, **index_kwargs)
         logger.debug(f"Connected to index: {pc.describe_index(self.index_name)}")
         return index
 
@@ -65,7 +66,13 @@ class PineconeUploadStagerConfig(UploadStagerConfig):
 
 
 class PineconeUploaderConfig(UploaderConfig):
-    batch_size: int = Field(default=100, description="Number of records per batch")
+    batch_size: Optional[int] = Field(
+        default=None,
+        description="Optional number of records per batch. Will otherwise limit by size.",
+    )
+    pool_threads: Optional[int] = Field(
+        default=1, description="Optional limit on number of threads to use for upload"
+    )
 
 
 ALLOWED_FIELDS = (
@@ -149,29 +156,44 @@ class PineconeUploader(Uploader):
             raise DestinationConnectionError(f"failed to validate connection: {e}")
 
     @requires_dependencies(["pinecone"], extras="pinecone")
-    def upsert_batch(self, batch):
+    def upsert_batches_async(self, elements_dict: list[dict]):
         from pinecone.exceptions import PineconeApiException
 
-        try:
-            index = self.connection_config.get_index()
-            response = index.upsert(batch)
-        except PineconeApiException as api_error:
-            raise DestinationConnectionError(f"http error: {api_error}") from api_error
-        logger.debug(f"results: {response}")
+        chunks = list(
+            generator_batching_wbytes(
+                iterable=elements_dict,
+                batch_size_limit_bytes=MAX_PAYLOAD_SIZE - 100,
+                max_batch_size=self.upload_config.batch_size,
+            )
+        )
+        logger.info(f"Split doc with {len(elements_dict)} elements into {len(chunks)} batches")
+
+        max_pool_threads = min(len(chunks), MAX_POOL_THREADS)
+        if self.upload_config.pool_threads:
+            pool_threads = min(self.upload_config.pool_threads, max_pool_threads)
+        else:
+            pool_threads = max_pool_threads
+        index = self.connection_config.get_index(pool_threads=pool_threads)
+        with index:
+            async_results = [index.upsert(vectors=chunk, async_req=True) for chunk in chunks]
+            # Wait for and retrieve responses (this raises in case of error)
+            try:
+                results = [async_result.get() for async_result in async_results]
+            except PineconeApiException as api_error:
+                raise DestinationConnectionError(f"http error: {api_error}") from api_error
+            logger.debug(f"results: {results}")
 
     def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
         with path.open("r") as file:
             elements_dict = json.load(file)
         logger.info(
-            f"writing document batches to destination"
+            f"writing a total of {len(elements_dict)} elements via"
+            f" document batches to destination"
             f" index named {self.connection_config.index_name}"
             f" with batch size {self.upload_config.batch_size}"
         )
 
-        for batch in generator_batching_wbytes(
-            elements_dict, MAX_PAYLOAD_SIZE - 100, self.upload_config.batch_size
-        ):
-            self.upsert_batch(batch=batch)
+        self.upsert_batches_async(elements_dict=elements_dict)
 
 
 pinecone_destination_entry = DestinationRegistryEntry(

unstructured_ingest/v2/processes/connectors/sharepoint.py

@@ -139,7 +139,7 @@ class SharepointConnectionConfig(ConnectionConfig):
 
 class SharepointIndexerConfig(IndexerConfig):
     path: Optional[str] = Field(
-        defaul=None,
+        default=None,
         description="Path from which to start parsing files. If the connector is to \
                 process all sites within the tenant this filter will be applied to \
                 all sites document libraries.",

unstructured_ingest/v2/processes/embedder.py

@@ -1,3 +1,4 @@
+import json
 from abc import ABC
 from dataclasses import dataclass
 from pathlib import Path
@@ -5,11 +6,10 @@ from typing import TYPE_CHECKING, Any, Literal, Optional
 
 from pydantic import BaseModel, Field, SecretStr
 
-from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 
 if TYPE_CHECKING:
-    from unstructured.embed.interfaces import BaseEmbeddingEncoder
+    from unstructured_ingest.embed.interfaces import BaseEmbeddingEncoder
 
 
 class EmbedderConfig(BaseModel):
@@ -21,6 +21,7 @@ class EmbedderConfig(BaseModel):
             "langchain-vertexai",
             "langchain-voyageai",
             "octoai",
+            "mixedbread-ai",
         ]
     ] = Field(default=None, description="Type of the embedding class to be used.")
     embedding_api_key: Optional[SecretStr] = Field(
@@ -42,30 +43,31 @@ class EmbedderConfig(BaseModel):
         default="us-west-2", description="AWS region used for AWS-based embedders, such as bedrock"
     )
 
-    @requires_dependencies(dependencies=["unstructured"], extras="embed-huggingface")
     def get_huggingface_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
-        from unstructured.embed.huggingface import (
+        from unstructured_ingest.embed.huggingface import (
             HuggingFaceEmbeddingConfig,
             HuggingFaceEmbeddingEncoder,
         )
 
-        return HuggingFaceEmbeddingEncoder(config=HuggingFaceEmbeddingConfig(**embedding_kwargs))
+        return HuggingFaceEmbeddingEncoder(
+            config=HuggingFaceEmbeddingConfig.model_validate(embedding_kwargs)
+        )
 
-    @requires_dependencies(dependencies=["unstructured"], extras="openai")
     def get_openai_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
-        from unstructured.embed.openai import OpenAIEmbeddingConfig, OpenAIEmbeddingEncoder
+        from unstructured_ingest.embed.openai import OpenAIEmbeddingConfig, OpenAIEmbeddingEncoder
 
-        return OpenAIEmbeddingEncoder(config=OpenAIEmbeddingConfig(**embedding_kwargs))
+        return OpenAIEmbeddingEncoder(config=OpenAIEmbeddingConfig.model_validate(embedding_kwargs))
 
-    @requires_dependencies(dependencies=["unstructured"], extras="embed-octoai")
     def get_octoai_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
-        from unstructured.embed.octoai import OctoAiEmbeddingConfig, OctoAIEmbeddingEncoder
+        from unstructured_ingest.embed.octoai import OctoAiEmbeddingConfig, OctoAIEmbeddingEncoder
 
-        return OctoAIEmbeddingEncoder(config=OctoAiEmbeddingConfig(**embedding_kwargs))
+        return OctoAIEmbeddingEncoder(config=OctoAiEmbeddingConfig.model_validate(embedding_kwargs))
 
-    @requires_dependencies(dependencies=["unstructured"], extras="bedrock")
     def get_bedrock_embedder(self) -> "BaseEmbeddingEncoder":
-        from unstructured.embed.bedrock import BedrockEmbeddingConfig, BedrockEmbeddingEncoder
+        from unstructured_ingest.embed.bedrock import (
+            BedrockEmbeddingConfig,
+            BedrockEmbeddingEncoder,
+        )
 
         return BedrockEmbeddingEncoder(
             config=BedrockEmbeddingConfig(
@@ -75,20 +77,35 @@ class EmbedderConfig(BaseModel):
             )
         )
 
-    @requires_dependencies(dependencies=["unstructured"], extras="embed-vertexai")
     def get_vertexai_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
-        from unstructured.embed.vertexai import (
+        from unstructured_ingest.embed.vertexai import (
             VertexAIEmbeddingConfig,
             VertexAIEmbeddingEncoder,
         )
 
-        return VertexAIEmbeddingEncoder(config=VertexAIEmbeddingConfig(**embedding_kwargs))
+        return VertexAIEmbeddingEncoder(
+            config=VertexAIEmbeddingConfig.model_validate(embedding_kwargs)
+        )
 
-    @requires_dependencies(dependencies=["unstructured"], extras="embed-voyageai")
     def get_voyageai_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
-        from unstructured.embed.voyageai import VoyageAIEmbeddingConfig, VoyageAIEmbeddingEncoder
+        from unstructured_ingest.embed.voyageai import (
+            VoyageAIEmbeddingConfig,
+            VoyageAIEmbeddingEncoder,
+        )
+
+        return VoyageAIEmbeddingEncoder(
+            config=VoyageAIEmbeddingConfig.model_validate(embedding_kwargs)
+        )
 
-        return VoyageAIEmbeddingEncoder(config=VoyageAIEmbeddingConfig(**embedding_kwargs))
+    def get_mixedbread_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
+        from unstructured_ingest.embed.mixedbreadai import (
+            MixedbreadAIEmbeddingConfig,
+            MixedbreadAIEmbeddingEncoder,
+        )
+
+        return MixedbreadAIEmbeddingEncoder(
+            config=MixedbreadAIEmbeddingConfig.model_validate(embedding_kwargs)
+        )
 
     def get_embedder(self) -> "BaseEmbeddingEncoder":
         kwargs: dict[str, Any] = {}
@@ -114,6 +131,8 @@ class EmbedderConfig(BaseModel):
 
         if self.embedding_provider == "langchain-voyageai":
             return self.get_voyageai_embedder(embedding_kwargs=kwargs)
+        if self.embedding_provider == "mixedbread-ai":
+            return self.get_mixedbread_embedder(embedding_kwargs=kwargs)
 
         raise ValueError(f"{self.embedding_provider} not a recognized encoder")
 
@@ -122,14 +141,12 @@ class EmbedderConfig(BaseModel):
 class Embedder(BaseProcess, ABC):
     config: EmbedderConfig
 
-    @requires_dependencies(dependencies=["unstructured"])
     def run(self, elements_filepath: Path, **kwargs: Any) -> list[dict]:
-        from unstructured.staging.base import elements_from_json
-
         # TODO update base embedder classes to support async
         embedder = self.config.get_embedder()
-        elements = elements_from_json(filename=str(elements_filepath))
+        with elements_filepath.open("r") as elements_file:
+            elements = json.load(elements_file)
         if not elements:
             return [e.to_dict() for e in elements]
         embedded_elements = embedder.embed_documents(elements=elements)
-        return [e.to_dict() for e in embedded_elements]
+        return embedded_elements