unstructured-ingest 1.0.13__tar.gz → 1.0.14__tar.gz

{unstructured_ingest-1.0.13 → unstructured_ingest-1.0.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.13
+Version: 1.0.14
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

unstructured_ingest-1.0.14/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.0.14"  # pragma: no cover

{unstructured_ingest-1.0.13 → unstructured_ingest-1.0.14}/unstructured_ingest/embed/bedrock.py

@@ -26,16 +26,20 @@ from unstructured_ingest.utils.dep_check import requires_dependencies
 if TYPE_CHECKING:
     from botocore.client import BaseClient
 
-    class BedrockClient(BaseClient):
+    class BedrockRuntimeClient(BaseClient):
         def invoke_model(self, body: str, modelId: str, accept: str, contentType: str) -> dict:
             pass
 
-    class AsyncBedrockClient(BaseClient):
+    class AsyncBedrockRuntimeClient(BaseClient):
         async def invoke_model(
             self, body: str, modelId: str, accept: str, contentType: str
         ) -> dict:
             pass
 
+    class BedrockClient(BaseClient):
+        def list_foundation_models(self, byOutputModality: str) -> dict:
+            pass
+
 
 def conform_query(query: str, provider: str) -> dict:
     # replace newlines, which can negatively affect performance.
@@ -87,19 +91,49 @@ class BedrockEmbeddingConfig(EmbeddingConfig):
         logger.error(f"unhandled exception from bedrock: {e}", exc_info=True)
         return e
 
+    def run_precheck(self) -> None:
+        client = self.get_bedrock_client()
+        try:
+            model_info = client.list_foundation_models(byOutputModality="EMBEDDING")
+            summaries = model_info.get("modelSummaries", [])
+            model_ids = [m["modelId"].split(":")[0] for m in summaries]
+            arns = [":".join(m["modelArn"].split(":")[0:-1]) for m in summaries]
+
+            if self.embedder_model_name not in model_ids and self.embedder_model_name not in arns:
+                raise UserError(
+                    "model '{}' not found either : {} or {}".format(
+                        self.embedder_model_name, ", ".join(model_ids), ", ".join(arns)
+                    )
+                )
+        except Exception as e:
+            raise self.wrap_error(e=e)
+
+    def get_client_kwargs(self) -> dict:
+        return {
+            "aws_access_key_id": self.aws_access_key_id.get_secret_value(),
+            "aws_secret_access_key": self.aws_secret_access_key.get_secret_value(),
+            "region_name": self.region_name,
+        }
+
+    @requires_dependencies(
+        ["boto3"],
+        extras="bedrock",
+    )
+    def get_bedrock_client(self) -> "BedrockClient":
+        import boto3
+
+        bedrock_client = boto3.client(service_name="bedrock", **self.get_client_kwargs())
+
+        return bedrock_client
+
     @requires_dependencies(
         ["boto3", "numpy", "botocore"],
         extras="bedrock",
     )
-    def get_client(self) -> "BedrockClient":
+    def get_client(self) -> "BedrockRuntimeClient":
         import boto3
 
-        bedrock_client = boto3.client(
-            service_name="bedrock-runtime",
-            aws_access_key_id=self.aws_access_key_id.get_secret_value(),
-            aws_secret_access_key=self.aws_secret_access_key.get_secret_value(),
-            region_name=self.region_name,
-        )
+        bedrock_client = boto3.client(service_name="bedrock-runtime", **self.get_client_kwargs())
 
         return bedrock_client
 
@@ -108,16 +142,11 @@ class BedrockEmbeddingConfig(EmbeddingConfig):
         extras="bedrock",
     )
     @asynccontextmanager
-    async def get_async_client(self) -> AsyncIterable["AsyncBedrockClient"]:
+    async def get_async_client(self) -> AsyncIterable["AsyncBedrockRuntimeClient"]:
         import aioboto3
 
         session = aioboto3.Session()
-        async with session.client(
-            "bedrock-runtime",
-            aws_access_key_id=self.aws_access_key_id.get_secret_value(),
-            aws_secret_access_key=self.aws_secret_access_key.get_secret_value(),
-            region_name=self.region_name,
-        ) as aws_bedrock:
+        async with session.client("bedrock-runtime", **self.get_client_kwargs()) as aws_bedrock:
             yield aws_bedrock
 
 
@@ -125,6 +154,9 @@ class BedrockEmbeddingConfig(EmbeddingConfig):
 class BedrockEmbeddingEncoder(BaseEmbeddingEncoder):
     config: BedrockEmbeddingConfig
 
+    def precheck(self):
+        self.config.run_precheck()
+
     def wrap_error(self, e: Exception) -> Exception:
         return self.config.wrap_error(e=e)
 
@@ -168,6 +200,9 @@ class BedrockEmbeddingEncoder(BaseEmbeddingEncoder):
 class AsyncBedrockEmbeddingEncoder(AsyncBaseEmbeddingEncoder):
     config: BedrockEmbeddingConfig
 
+    def precheck(self):
+        self.config.run_precheck()
+
     def wrap_error(self, e: Exception) -> Exception:
         return self.config.wrap_error(e=e)
 

{unstructured_ingest-1.0.13 → unstructured_ingest-1.0.14}/unstructured_ingest/embed/huggingface.py

@@ -23,7 +23,7 @@ class HuggingFaceEmbeddingConfig(EmbeddingConfig):
 
     @requires_dependencies(
         ["sentence_transformers"],
-        extras="embed-huggingface",
+        extras="huggingface",
     )
     def get_client(self) -> "SentenceTransformer":
         from sentence_transformers import SentenceTransformer

{unstructured_ingest-1.0.13 → unstructured_ingest-1.0.14}/unstructured_ingest/embed/interfaces.py

@@ -20,6 +20,9 @@ class EmbeddingConfig(BaseModel):
 class BaseEncoder(ABC):
     config: EmbeddingConfig
 
+    def precheck(self):
+        pass
+
     def initialize(self):
         """Initializes the embedding encoder class. Should also validate the instance
         is properly configured: e.g., embed a single a element"""

{unstructured_ingest-1.0.13 → unstructured_ingest-1.0.14}/unstructured_ingest/embed/mixedbreadai.py

@@ -42,7 +42,7 @@ class MixedbreadAIEmbeddingConfig(EmbeddingConfig):
 
     @requires_dependencies(
         ["mixedbread_ai"],
-        extras="embed-mixedbreadai",
+        extras="mixedbreadai",
     )
     def get_client(self) -> "MixedbreadAI":
         """
@@ -59,7 +59,7 @@ class MixedbreadAIEmbeddingConfig(EmbeddingConfig):
 
     @requires_dependencies(
         ["mixedbread_ai"],
-        extras="embed-mixedbreadai",
+        extras="mixedbreadai",
     )
     def get_async_client(self) -> "AsyncMixedbreadAI":
         from mixedbread_ai.client import AsyncMixedbreadAI

{unstructured_ingest-1.0.13 → unstructured_ingest-1.0.14}/unstructured_ingest/embed/octoai.py

@@ -54,9 +54,20 @@ class OctoAiEmbeddingConfig(EmbeddingConfig):
         logger.error(f"unhandled exception from openai: {e}", exc_info=True)
         return e
 
+    def run_precheck(self) -> None:
+        client = self.get_client()
+        try:
+            models = [m.id for m in list(client.models.list())]
+            if self.embedder_model_name not in models:
+                raise UserError(
+                    "model '{}' not found: {}".format(self.embedder_model_name, ", ".join(models))
+                )
+        except Exception as e:
+            raise self.wrap_error(e=e)
+
     @requires_dependencies(
         ["openai", "tiktoken"],
-        extras="embed-octoai",
+        extras="octoai",
     )
     def get_client(self) -> "OpenAI":
         """Creates an OpenAI python client to embed elements. Uses the OpenAI SDK."""
@@ -66,7 +77,7 @@ class OctoAiEmbeddingConfig(EmbeddingConfig):
 
     @requires_dependencies(
         ["openai", "tiktoken"],
-        extras="embed-octoai",
+        extras="octoai",
     )
     def get_async_client(self) -> "AsyncOpenAI":
         """Creates an OpenAI python client to embed elements. Uses the OpenAI SDK."""
@@ -79,6 +90,9 @@ class OctoAiEmbeddingConfig(EmbeddingConfig):
 class OctoAIEmbeddingEncoder(BaseEmbeddingEncoder):
     config: OctoAiEmbeddingConfig
 
+    def precheck(self):
+        self.config.run_precheck()
+
     def wrap_error(self, e: Exception) -> Exception:
         return self.config.wrap_error(e=e)
 
@@ -99,6 +113,9 @@ class OctoAIEmbeddingEncoder(BaseEmbeddingEncoder):
 class AsyncOctoAIEmbeddingEncoder(AsyncBaseEmbeddingEncoder):
     config: OctoAiEmbeddingConfig
 
+    def precheck(self):
+        self.config.run_precheck()
+
     def wrap_error(self, e: Exception) -> Exception:
         return self.config.wrap_error(e=e)
 

{unstructured_ingest-1.0.13 → unstructured_ingest-1.0.14}/unstructured_ingest/embed/openai.py

@@ -28,6 +28,7 @@ class OpenAIEmbeddingConfig(EmbeddingConfig):
     embedder_model_name: str = Field(default="text-embedding-ada-002", alias="model_name")
     base_url: Optional[str] = None
 
+    @requires_dependencies(["openai"], extras="openai")
     def wrap_error(self, e: Exception) -> Exception:
         if is_internal_error(e=e):
             return e
@@ -54,6 +55,33 @@ class OpenAIEmbeddingConfig(EmbeddingConfig):
         logger.error(f"unhandled exception from openai: {e}", exc_info=True)
         return e
 
+    @requires_dependencies(["openai"], extras="openai")
+    def get_models(self) -> Optional[list[str]]:
+        # In case the list model endpoint isn't exposed, don't break
+        from openai import APIStatusError
+
+        client = self.get_client()
+        try:
+            models = [m.id for m in list(client.models.list())]
+            return models
+        except APIStatusError as e:
+            if e.status_code == 404:
+                return None
+        except Exception as e:
+            raise self.wrap_error(e=e)
+
+    def run_precheck(self) -> None:
+        try:
+            models = self.get_models()
+            if models is None:
+                return
+            if self.embedder_model_name not in models:
+                raise UserError(
+                    "model '{}' not found: {}".format(self.embedder_model_name, ", ".join(models))
+                )
+        except Exception as e:
+            raise self.wrap_error(e=e)
+
     @requires_dependencies(["openai"], extras="openai")
     def get_client(self) -> "OpenAI":
         from openai import OpenAI
@@ -71,6 +99,9 @@ class OpenAIEmbeddingConfig(EmbeddingConfig):
 class OpenAIEmbeddingEncoder(BaseEmbeddingEncoder):
     config: OpenAIEmbeddingConfig
 
+    def precheck(self):
+        self.config.run_precheck()
+
     def wrap_error(self, e: Exception) -> Exception:
         return self.config.wrap_error(e=e)
 
@@ -86,6 +117,9 @@ class OpenAIEmbeddingEncoder(BaseEmbeddingEncoder):
 class AsyncOpenAIEmbeddingEncoder(AsyncBaseEmbeddingEncoder):
     config: OpenAIEmbeddingConfig
 
+    def precheck(self):
+        self.config.run_precheck()
+
     def wrap_error(self, e: Exception) -> Exception:
         return self.config.wrap_error(e=e)
 

{unstructured_ingest-1.0.13 → unstructured_ingest-1.0.14}/unstructured_ingest/embed/togetherai.py

@@ -41,6 +41,17 @@ class TogetherAIEmbeddingConfig(EmbeddingConfig):
             return CustomRateLimitError(message)
         return UserError(message)
 
+    def run_precheck(self) -> None:
+        client = self.get_client()
+        try:
+            models = [m.id for m in list(client.models.list())]
+            if self.embedder_model_name not in models:
+                raise UserError(
+                    "model '{}' not found: {}".format(self.embedder_model_name, ", ".join(models))
+                )
+        except Exception as e:
+            raise self.wrap_error(e=e)
+
     @requires_dependencies(["together"], extras="togetherai")
     def get_client(self) -> "Together":
         from together import Together
@@ -58,6 +69,9 @@ class TogetherAIEmbeddingConfig(EmbeddingConfig):
 class TogetherAIEmbeddingEncoder(BaseEmbeddingEncoder):
     config: TogetherAIEmbeddingConfig
 
+    def precheck(self):
+        self.config.run_precheck()
+
     def wrap_error(self, e: Exception) -> Exception:
         return self.config.wrap_error(e=e)
 
@@ -73,6 +87,9 @@ class TogetherAIEmbeddingEncoder(BaseEmbeddingEncoder):
 class AsyncTogetherAIEmbeddingEncoder(AsyncBaseEmbeddingEncoder):
     config: TogetherAIEmbeddingConfig
 
+    def precheck(self):
+        self.config.run_precheck()
+
     def wrap_error(self, e: Exception) -> Exception:
         return self.config.wrap_error(e=e)
 

{unstructured_ingest-1.0.13 → unstructured_ingest-1.0.14}/unstructured_ingest/embed/vertexai.py

@@ -55,7 +55,7 @@ class VertexAIEmbeddingConfig(EmbeddingConfig):
 
     @requires_dependencies(
         ["vertexai"],
-        extras="embed-vertexai",
+        extras="vertexai",
     )
     def get_client(self) -> "TextEmbeddingModel":
         """Creates a VertexAI python client to embed elements."""

{unstructured_ingest-1.0.13 → unstructured_ingest-1.0.14}/unstructured_ingest/processes/connectors/google_drive.py

@@ -38,12 +38,9 @@ CONNECTOR_TYPE = "google_drive"
 
 # Maps Google-native Drive MIME types → export MIME types
 GOOGLE_EXPORT_MIME_MAP = {
-    "application/vnd.google-apps.document": \
-        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-    "application/vnd.google-apps.spreadsheet": \
-        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-    "application/vnd.google-apps.presentation": \
-        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+    "application/vnd.google-apps.document": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",  # noqa: E501
+    "application/vnd.google-apps.spreadsheet": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",  # noqa: E501
+    "application/vnd.google-apps.presentation": "application/vnd.openxmlformats-officedocument.presentationml.presentation",  # noqa: E501
 }
 
 # Maps export MIME types → file extensions
@@ -57,9 +54,9 @@ EXPORT_EXTENSION_MAP = {
 
 
 class GoogleDriveAccessConfig(AccessConfig):
-    service_account_key: Optional[
-        Annotated[dict, BeforeValidator(conform_string_to_dict)]
-    ] = Field(default=None, description="Credentials values to use for authentication")
+    service_account_key: Optional[Annotated[dict, BeforeValidator(conform_string_to_dict)]] = Field(
+        default=None, description="Credentials values to use for authentication"
+    )
     service_account_key_path: Optional[Path] = Field(
         default=None,
         description="File path to credentials values to use for authentication",
@@ -178,14 +175,10 @@ class GoogleDriveIndexer(Indexer):
                     Please enable it in the Google Cloud Console."
                 )
             else:
-                raise SourceConnectionError(
-                    "Google drive API unreachable for an unknown reason!"
-                )
+                raise SourceConnectionError("Google drive API unreachable for an unknown reason!")
 
     @staticmethod
-    def count_files_recursively(
-        files_client, folder_id: str, extensions: list[str] = None
-    ) -> int:
+    def count_files_recursively(files_client, folder_id: str, extensions: list[str] = None) -> int:
         """
         Count non-folder files recursively under the given folder.
         If `extensions` is provided, only count files
@@ -266,9 +259,7 @@ class GoogleDriveIndexer(Indexer):
                         #     that the service account has proper permissions."
                         # )
                     else:
-                        logger.info(
-                            f"Found {file_count} files recursively in the folder."
-                        )
+                        logger.info(f"Found {file_count} files recursively in the folder.")
                 else:
                     # Non-recursive: check for at least one immediate non-folder child.
                     response = client.list(
@@ -319,9 +310,7 @@ class GoogleDriveIndexer(Indexer):
         date_modified_str = root_info.pop("modifiedTime", None)
         parent_path = root_info.pop("parent_path", None)
         parent_root_path = root_info.pop("parent_root_path", None)
-        date_modified_dt = (
-            parser.parse(date_modified_str) if date_modified_str else None
-        )
+        date_modified_dt = parser.parse(date_modified_str) if date_modified_str else None
         if (
             parent_path
             and isinstance(parent_path, str)
@@ -421,9 +410,7 @@ class GoogleDriveIndexer(Indexer):
     ) -> list[FileData]:
         root_info = self.get_root_info(files_client=files_client, object_id=object_id)
         if not self.is_dir(root_info):
-            root_info["permissions"] = self.extract_permissions(
-                root_info.get("permissions")
-            )
+            root_info["permissions"] = self.extract_permissions(root_info.get("permissions"))
             data = [self.map_file_data(root_info)]
         else:
             file_contents = self.get_paginated_results(
@@ -514,9 +501,7 @@ class GoogleDriveDownloader(Downloader):
     )
     connector_type: str = CONNECTOR_TYPE
 
-    def _get_download_url_and_ext(
-        self, file_id: str, mime_type: str
-    ) -> tuple[str, str]:
+    def _get_download_url_and_ext(self, file_id: str, mime_type: str) -> tuple[str, str]:
         """
         Resolves the appropriate download URL and expected file extension for a Google Drive file.
 
@@ -530,9 +515,7 @@ class GoogleDriveDownloader(Downloader):
             SourceConnectionError: If no valid export or download link is available.
         """
         with self.connection_config.get_client() as client:
-            metadata = client.get(
-                fileId=file_id, fields="exportLinks,webContentLink"
-            ).execute()
+            metadata = client.get(fileId=file_id, fields="exportLinks,webContentLink").execute()
 
         export_links = metadata.get("exportLinks", {})
         web_link = metadata.get("webContentLink")
@@ -540,16 +523,12 @@ class GoogleDriveDownloader(Downloader):
         if export_mime := GOOGLE_EXPORT_MIME_MAP.get(mime_type):
             url = export_links.get(export_mime)
             if not url:
-                raise SourceConnectionError(
-                    f"No export link found for {file_id} as {export_mime}"
-                )
+                raise SourceConnectionError(f"No export link found for {file_id} as {export_mime}")
             ext = EXPORT_EXTENSION_MAP.get(export_mime, "")
             return url, ext
 
         if not web_link:
-            raise SourceConnectionError(
-                f"No webContentLink available for file {file_id}"
-            )
+            raise SourceConnectionError(f"No webContentLink available for file {file_id}")
         return web_link, ""
 
     @requires_dependencies(["httpx", "google.auth"], extras="google-drive")
@@ -622,9 +601,7 @@ class GoogleDriveDownloader(Downloader):
         )
         file_data.local_download_path = str(download_path.resolve())
 
-        return self.generate_download_response(
-            file_data=file_data, download_path=download_path
-        )
+        return self.generate_download_response(file_data=file_data, download_path=download_path)
 
 
 google_drive_source_entry = SourceRegistryEntry(

{unstructured_ingest-1.0.13 → unstructured_ingest-1.0.14}/unstructured_ingest/processes/embedder.py

@@ -189,6 +189,10 @@ class Embedder(BaseProcess, ABC):
     def init(self, **kwargs: Any) -> None:
         self.config.get_embedder().initialize()
 
+    def precheck(self) -> None:
+        embedder = self.config.get_embedder()
+        embedder.precheck()
+
     def run(self, elements_filepath: Path, **kwargs: Any) -> list[dict]:
         # TODO update base embedder classes to support async
         embedder = self.config.get_embedder()

unstructured_ingest-1.0.13/unstructured_ingest/__version__.py

@@ -1 +0,0 @@
-__version__ = "1.0.13"  # pragma: no cover