unstructured-ingest 1.2.11__py3-none-any.whl → 1.2.14__py3-none-any.whl

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.2.11"  # pragma: no cover
+__version__ = "1.2.14"  # pragma: no cover

unstructured_ingest/embed/bedrock.py

@@ -58,9 +58,22 @@ def conform_query(query: str, provider: str) -> dict:
 
 
 class BedrockEmbeddingConfig(EmbeddingConfig):
-    aws_access_key_id: SecretStr = Field(description="aws access key id")
-    aws_secret_access_key: SecretStr = Field(description="aws secret access key")
-    region_name: str = Field(description="aws region name", default="us-west-2")
+    aws_access_key_id: SecretStr | None = Field(description="aws access key id", default=None)
+    aws_secret_access_key: SecretStr | None = Field(
+        description="aws secret access key", default=None
+    )
+    region_name: str = Field(
+        description="aws region name", 
+        default_factory=lambda: (
+            os.getenv("BEDROCK_REGION_NAME") or 
+            os.getenv("AWS_DEFAULT_REGION") or 
+            "us-west-2"
+        )
+    )
+    endpoint_url: str | None = Field(description="custom bedrock endpoint url", default=None)
+    access_method: str = Field(
+        description="authentication method", default="credentials"
+    )  # "credentials" or "iam"
     embedder_model_name: str = Field(
         default="amazon.titan-embed-text-v1",
         alias="model_name",
@@ -96,6 +109,20 @@ class BedrockEmbeddingConfig(EmbeddingConfig):
         return e
 
     def run_precheck(self) -> None:
+        # Validate access method and credentials configuration
+        if self.access_method == "credentials":
+            if not (self.aws_access_key_id and self.aws_secret_access_key):
+                raise ValueError(
+                    "Credentials access method requires aws_access_key_id and aws_secret_access_key"
+                )
+        elif self.access_method == "iam":
+            # For IAM, credentials are handled by AWS SDK
+            pass
+        else:
+            raise ValueError(
+                f"Invalid access_method: {self.access_method}. Must be 'credentials' or 'iam'"
+            )
+            
         client = self.get_bedrock_client()
         try:
             model_info = client.list_foundation_models(byOutputModality="EMBEDDING")
@@ -113,11 +140,30 @@ class BedrockEmbeddingConfig(EmbeddingConfig):
             raise self.wrap_error(e=e)
 
     def get_client_kwargs(self) -> dict:
-        return {
-            "aws_access_key_id": self.aws_access_key_id.get_secret_value(),
-            "aws_secret_access_key": self.aws_secret_access_key.get_secret_value(),
+        kwargs = {
             "region_name": self.region_name,
         }
+        
+        if self.endpoint_url:
+            kwargs["endpoint_url"] = self.endpoint_url
+            
+        if self.access_method == "credentials":
+            if self.aws_access_key_id and self.aws_secret_access_key:
+                kwargs["aws_access_key_id"] = self.aws_access_key_id.get_secret_value()
+                kwargs["aws_secret_access_key"] = self.aws_secret_access_key.get_secret_value()
+            else:
+                raise ValueError(
+                    "Credentials access method requires aws_access_key_id and aws_secret_access_key"
+                )
+        elif self.access_method == "iam":
+            # For IAM, boto3 will use default credential chain (IAM roles, environment, etc.)
+            pass
+        else:
+            raise ValueError(
+                f"Invalid access_method: {self.access_method}. Must be 'credentials' or 'iam'"
+            )
+            
+        return kwargs
 
     @requires_dependencies(
         ["boto3"],

unstructured_ingest/processes/connectors/ibm_watsonx/ibm_watsonx_s3.py

@@ -147,6 +147,10 @@ class IbmWatsonxConnectionConfig(ConnectionConfig):
             "s3.access-key-id": self.access_config.get_secret_value().access_key_id,
             "s3.secret-access-key": self.access_config.get_secret_value().secret_access_key,
             "s3.region": self.object_storage_region,
+            # By default this header is set to `vended-credentials`, and default bucket
+            # configuration doesn't allow vending credentials. We need to set it to `None`
+            # in order to use user-provided S3 credentials.
+            "header.X-Iceberg-Access-Delegation": None,
         }
 
     @requires_dependencies(["pyiceberg"], extras="ibm-watsonx-s3")

unstructured_ingest/processes/connectors/milvus.py

@@ -10,7 +10,6 @@ from unstructured_ingest.data_types.file_data import FileData
 from unstructured_ingest.error import (
     DestinationConnectionError,
     KeyError,
-    ValueError,
     WriteError,
 )
 from unstructured_ingest.interfaces import (

unstructured_ingest/processes/connectors/sharepoint.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+import logging
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, AsyncIterator, Optional
 
@@ -210,7 +211,7 @@ class SharepointIndexer(OnedriveIndexer):
 
 
 class SharepointDownloaderConfig(OnedriveDownloaderConfig):
-    pass
+    max_retries: int = 10
 
 
 @dataclass
@@ -219,10 +220,22 @@ class SharepointDownloader(OnedriveDownloader):
     download_config: SharepointDownloaderConfig
     connector_type: str = CONNECTOR_TYPE
 
+    @staticmethod
+    def retry_on_status_code(exc):
+        error_msg = str(exc).lower()
+        return "429" in error_msg or "activitylimitreached" in error_msg or "throttled" in error_msg
+
     @SourceConnectionNetworkError.wrap
     @requires_dependencies(["office365"], extras="sharepoint")
     def _fetch_file(self, file_data: FileData) -> DriveItem:
         from office365.runtime.client_request_exception import ClientRequestException
+        from tenacity import (
+            before_log,
+            retry,
+            retry_if_exception,
+            stop_after_attempt,
+            wait_exponential,
+        )
 
         if file_data.source_identifiers is None or not file_data.source_identifiers.fullpath:
             raise ValueError(
@@ -233,13 +246,27 @@ class SharepointDownloader(OnedriveDownloader):
         server_relative_path = file_data.source_identifiers.fullpath
         client = self.connection_config.get_client()
 
-        try:
-            client_site = client.sites.get_by_url(self.connection_config.site).get().execute_query()
-            site_drive_item = self.connection_config._get_drive_item(client_site)
-        except ClientRequestException:
-            logger.info("Site not found")
-            raise SourceConnectionError(f"Site not found: {self.connection_config.site}")
-        file = site_drive_item.get_by_path(server_relative_path).get().execute_query()
+        @retry(
+            stop=stop_after_attempt(self.download_config.max_retries),
+            wait=wait_exponential(exp_base=2, multiplier=1, min=2, max=10),
+            retry=retry_if_exception(self.retry_on_status_code),
+            before=before_log(logger, logging.DEBUG),
+            reraise=True,
+        )
+        def _get_item_by_path() -> DriveItem:
+            try:
+                client_site = (
+                    client.sites.get_by_url(self.connection_config.site).get().execute_query()
+                )
+                site_drive_item = self.connection_config._get_drive_item(client_site)
+            except ClientRequestException:
+                logger.info(f"Site not found: {self.connection_config.site}")
+                raise SourceConnectionError(f"Site not found: {self.connection_config.site}")
+            file = site_drive_item.get_by_path(server_relative_path).get().execute_query()
+            return file
+
+        # Call the retry-wrapped function
+        file = _get_item_by_path()
 
         if not file:
             raise NotFoundError(f"file not found: {server_relative_path}")

{unstructured_ingest-1.2.11.dist-info → unstructured_ingest-1.2.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.2.11
+Version: 1.2.14
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

{unstructured_ingest-1.2.11.dist-info → unstructured_ingest-1.2.14.dist-info}/RECORD

@@ -1,5 +1,5 @@
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=ah-eP2N3WKTgQpNwUrBqGx6pOEXxVpXyJiiqBeWRLvI,43
+unstructured_ingest/__version__.py,sha256=3NalubdNM3MTB5UJ7yWDPdOo9kbE1-bL_USqp5DEYnk,43
 unstructured_ingest/error.py,sha256=chM7zQSTKjaKaQt_2_QkoZDUwY5XPNeACML7JqOWRLY,4036
 unstructured_ingest/errors_v2.py,sha256=chM7zQSTKjaKaQt_2_QkoZDUwY5XPNeACML7JqOWRLY,4036
 unstructured_ingest/logger.py,sha256=7e_7UeK6hVOd5BQ6i9NzRUAPCS_DF839Y8TjUDywraY,1428
@@ -23,7 +23,7 @@ unstructured_ingest/data_types/entities.py,sha256=ECc6EkZ5_ZUvK7uaALYOynfFmofIrH
 unstructured_ingest/data_types/file_data.py,sha256=J0RQa7YXhhxiLVzhPbF5Hl2nzSpxLFK9vrP6RTBWlSg,3833
 unstructured_ingest/embed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/embed/azure_openai.py,sha256=Q_buBkAcx9FBuTsAqKbRU8vd9vDh8JoDOEth4fFxHbg,2160
-unstructured_ingest/embed/bedrock.py,sha256=EZWGw8f_QKDbYS7dSiFY6seLGIwGd8LmU-HeYT26Stw,9128
+unstructured_ingest/embed/bedrock.py,sha256=BnlKOuL1e4FfR4nV_Ro3A26Fqj3Pq-ZBaSNAgIzUQH0,10955
 unstructured_ingest/embed/huggingface.py,sha256=6Gx9L3xa3cv9fX4AMuLsePJQF4T_jwkKjovfqF5X1NM,2435
 unstructured_ingest/embed/interfaces.py,sha256=VCrCSJiEfIxKB4NL4AHgKb-0vB_SEekb47zMUW6gWf0,5211
 unstructured_ingest/embed/mixedbreadai.py,sha256=uKTqzoi4M_WeYZu-qc_TSxwJONOESzxVbBLUbD1Wbns,3922
@@ -76,7 +76,7 @@ unstructured_ingest/processes/connectors/google_drive.py,sha256=O6RtOQH9dHBUtwKs
 unstructured_ingest/processes/connectors/jira.py,sha256=5BZeYHoWKzcwZKUbJDFsdCUcSHbLDZTErOdNbteDwI0,20290
 unstructured_ingest/processes/connectors/kdbai.py,sha256=XhxYpKSAoFPBsDQWwNuLX03DCxOVr7yquj9VYM55Rtc,5174
 unstructured_ingest/processes/connectors/local.py,sha256=Tsp9d9YSx2zPh4yl2U--P6cMIQSKMzsFAGyNXXtdS-4,7529
-unstructured_ingest/processes/connectors/milvus.py,sha256=vlUxcG1nVSyUQvBJJ0HoJTTli55ho5TulfUxSShDPfE,12118
+unstructured_ingest/processes/connectors/milvus.py,sha256=Bkt4u1zzrKqpO0CZbmuFfbtd824ws5XouiTAnc4I4BM,12102
 unstructured_ingest/processes/connectors/mongodb.py,sha256=zhGWnEJYZnKzjuElyYAEJUT3M7J5m0e48TpVPdiKsBA,15412
 unstructured_ingest/processes/connectors/neo4j.py,sha256=jmnxQmi8EjS22mFKfcdXajZrxoKEkrzHRtrP6QeTuFI,20353
 unstructured_ingest/processes/connectors/onedrive.py,sha256=qhIeFWotFuIxt1Ehg-6IEWXaDu4p-Zhy0u14CfDcnZo,20142
@@ -84,7 +84,7 @@ unstructured_ingest/processes/connectors/outlook.py,sha256=fVW50hRbk1SJUbhEY1ulu
 unstructured_ingest/processes/connectors/pinecone.py,sha256=b6Wot25oiELj9oBhAwlps4pGE0QWkj8n0wdLTcM7osg,15056
 unstructured_ingest/processes/connectors/redisdb.py,sha256=HvqjOjpKdKa5C7i1toH4pPhDhiI9pA1-swozpnuCXZQ,8034
 unstructured_ingest/processes/connectors/salesforce.py,sha256=C2HWiWglZTu0zTibp4eT1lOGYCB-NYnGPr9-Dt635sY,11730
-unstructured_ingest/processes/connectors/sharepoint.py,sha256=2Se6FoHRerBM06UosGcAbD5r5ZtRzAI0rNJxCdVpx34,10668
+unstructured_ingest/processes/connectors/sharepoint.py,sha256=e7X7bMaI7H5T7mC1hytY0m1Kc-QV6r9WGZakqyW-K-g,11636
 unstructured_ingest/processes/connectors/slack.py,sha256=KgDvZGwcKng0FZzElucM0elgJzcVq7PkUQM6PNW30vE,9330
 unstructured_ingest/processes/connectors/utils.py,sha256=TAd0hb1f291N-q7-TUe6JKSCGkhqDyo7Ij8zmliBZUc,2071
 unstructured_ingest/processes/connectors/vectara.py,sha256=fAGiVtqiouJ7GkSFycoJYxf7hzIZpLs3A_q6n6Li37I,12304
@@ -115,7 +115,7 @@ unstructured_ingest/processes/connectors/fsspec/s3.py,sha256=Zng-aV_Z0B52CFILAXf
 unstructured_ingest/processes/connectors/fsspec/sftp.py,sha256=pR_a2SgLjt8ffNkariHrPB1E0HVSTj5h3pt7KxTU3TI,6371
 unstructured_ingest/processes/connectors/fsspec/utils.py,sha256=jec_Qfe2hbfahBuY-u8FnvHuv933AI5HwPFjOL3kEEY,456
 unstructured_ingest/processes/connectors/ibm_watsonx/__init__.py,sha256=kf0UpgdAY2KK1R1FbAB6GEBBAIOeYQ8cZIr3bp660qM,374
-unstructured_ingest/processes/connectors/ibm_watsonx/ibm_watsonx_s3.py,sha256=k_c2PtKcaRA6B9ZFXYCk4-2BWxLJnD_Cfjvluk9hKzs,13876
+unstructured_ingest/processes/connectors/ibm_watsonx/ibm_watsonx_s3.py,sha256=TuQPpm9O7_3PZQC1s4S3HzybUWDKUeZDs-V3ZTzqdjA,14171
 unstructured_ingest/processes/connectors/kafka/__init__.py,sha256=pFN2cWwAStiGTAsQ616GIWKi_hDv0s74ZvNqhJEp1Pc,751
 unstructured_ingest/processes/connectors/kafka/cloud.py,sha256=Ki6iOLoZ86tYWdnLnMWYvb2hUCneKqo4mTJcfXh7YoQ,3432
 unstructured_ingest/processes/connectors/kafka/kafka.py,sha256=VI-e7WTzV48mmSwqhlDsNARSzkjauckbJEFvWjuqt7k,10301
@@ -235,8 +235,8 @@ unstructured_ingest/utils/pydantic_models.py,sha256=BT_j15e4rX40wQbt8LUXbqfPhA3r
 unstructured_ingest/utils/string_and_date_utils.py,sha256=oXOI6rxXq-8ncbk7EoJK0WCcTXWj75EzKl8pfQMID3U,2522
 unstructured_ingest/utils/table.py,sha256=WZechczgVFvlodUWFcsnCGvBNh1xRm6hr0VbJTPxKAc,3669
 unstructured_ingest/utils/tls.py,sha256=Ra8Mii1F4VqErRreg76PBI0eAqPBC009l0sSHa8FdnA,448
-unstructured_ingest-1.2.11.dist-info/METADATA,sha256=WXxL9qEe3d3pG9xwxbuDRArBBGkDH3k2y1fhtNzcHPQ,8827
-unstructured_ingest-1.2.11.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-unstructured_ingest-1.2.11.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-1.2.11.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-1.2.11.dist-info/RECORD,,
+unstructured_ingest-1.2.14.dist-info/METADATA,sha256=GXAxy85oatbdkNDhFN2Yt0g84XuQfdsuAFXeh_U17tI,8827
+unstructured_ingest-1.2.14.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+unstructured_ingest-1.2.14.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-1.2.14.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-1.2.14.dist-info/RECORD,,