unstructured-ingest 1.0.12__py3-none-any.whl → 1.0.13__py3-none-any.whl

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.12"  # pragma: no cover
+__version__ = "1.0.13"  # pragma: no cover

unstructured_ingest/processes/connectors/ibm_watsonx/ibm_watsonx_s3.py

@@ -56,6 +56,12 @@ class IbmWatsonxConnectionConfig(ConnectionConfig):
     object_storage_endpoint: str = Field(description="Cloud Object Storage public endpoint")
     object_storage_region: str = Field(description="Cloud Object Storage region")
     catalog: str = Field(description="Catalog name")
+    max_retries_connection: int = Field(
+        default=10,
+        description="Maximum number of retries in case of a connection error (RESTError)",
+        ge=2,
+        le=100,
+    )
 
     _bearer_token: Optional[dict[str, Any]] = None
 
@@ -145,10 +151,29 @@ class IbmWatsonxConnectionConfig(ConnectionConfig):
     @contextmanager
     def get_catalog(self) -> Generator["RestCatalog", None, None]:
         from pyiceberg.catalog import load_catalog
+        from pyiceberg.exceptions import RESTError
+        from tenacity import (
+            before_log,
+            retry,
+            retry_if_exception_type,
+            stop_after_attempt,
+            wait_exponential,
+        )
+
+        # Retry connection in case of a connection error
+        @retry(
+            stop=stop_after_attempt(self.max_retries_connection),
+            wait=wait_exponential(exp_base=2, multiplier=1, min=2, max=10),
+            retry=retry_if_exception_type(RESTError),
+            before=before_log(logger, logging.DEBUG),
+            reraise=True,
+        )
+        def _get_catalog(catalog_config: dict[str, Any]) -> "RestCatalog":
+            return load_catalog(**catalog_config)
 
         try:
             catalog_config = self.get_catalog_config()
-            catalog = load_catalog(**catalog_config)
+            catalog = _get_catalog(catalog_config)
         except Exception as e:
             logger.error(f"Failed to connect to catalog '{self.catalog}': {e}", exc_info=True)
             raise ProviderError(f"Failed to connect to catalog '{self.catalog}': {e}")
@@ -172,7 +197,10 @@ class IbmWatsonxUploaderConfig(UploaderConfig):
     namespace: str = Field(description="Namespace name")
     table: str = Field(description="Table name")
     max_retries: int = Field(
-        default=5, description="Maximum number of retries to upload data", ge=2, le=500
+        default=50,
+        description="Maximum number of retries to upload data (CommitFailedException)",
+        ge=2,
+        le=500,
     )
     record_id_key: str = Field(
         default=RECORD_ID_LABEL,
@@ -287,15 +315,15 @@ class IbmWatsonxUploader(SQLUploader):
             retry,
             retry_if_exception_type,
             stop_after_attempt,
-            wait_random,
+            wait_exponential,
         )
 
         data_table = self._df_to_arrow_table(df)
 
-        # Retry connection in case of connection error
+        # Retry connection in case of a connection error or token expiration
         @retry(
-            stop=stop_after_attempt(2),
-            wait=wait_random(),
+            stop=stop_after_attempt(self.connection_config.max_retries_connection),
+            wait=wait_exponential(exp_base=2, multiplier=1, min=2, max=10),
             retry=retry_if_exception_type(RESTError),
             before=before_log(logger, logging.DEBUG),
             reraise=True,

unstructured_ingest/processes/connectors/notion/types/database_properties/select.py

@@ -38,6 +38,7 @@ class Select(DBPropertyBase):
     id: str
     name: str
     select: SelectProp
+    description: Optional[str] = None
     type: str = "select"
 
     @classmethod

{unstructured_ingest-1.0.12.dist-info → unstructured_ingest-1.0.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.12
+Version: 1.0.13
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

{unstructured_ingest-1.0.12.dist-info → unstructured_ingest-1.0.13.dist-info}/RECORD

@@ -1,5 +1,5 @@
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=lMbQ0529QKJjjQ_kL1MFTpMu0sLuz_HYCDQvQiNQRtc,43
+unstructured_ingest/__version__.py,sha256=699bwiNA-eOPRKD6w4CsZQiq8SnTP9t_VAkbFCvGi0g,43
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/errors_v2.py,sha256=9RuRCi7lbDxCguDz07y5RiHoQiFIOWwOD7xqzJ2B3Yw,436
 unstructured_ingest/logger.py,sha256=7e_7UeK6hVOd5BQ6i9NzRUAPCS_DF839Y8TjUDywraY,1428
@@ -115,7 +115,7 @@ unstructured_ingest/processes/connectors/fsspec/s3.py,sha256=2ZV6b2E2pIsf_ab1Lty
 unstructured_ingest/processes/connectors/fsspec/sftp.py,sha256=pR_a2SgLjt8ffNkariHrPB1E0HVSTj5h3pt7KxTU3TI,6371
 unstructured_ingest/processes/connectors/fsspec/utils.py,sha256=jec_Qfe2hbfahBuY-u8FnvHuv933AI5HwPFjOL3kEEY,456
 unstructured_ingest/processes/connectors/ibm_watsonx/__init__.py,sha256=kf0UpgdAY2KK1R1FbAB6GEBBAIOeYQ8cZIr3bp660qM,374
-unstructured_ingest/processes/connectors/ibm_watsonx/ibm_watsonx_s3.py,sha256=TOZxk6TUY_iOlitZQbFi-R5LcQmAMyOraWx98E6bzhg,12748
+unstructured_ingest/processes/connectors/ibm_watsonx/ibm_watsonx_s3.py,sha256=NeWLjAAFd7h4GMvVUh92LktbTc6Dnvet42mzju9F0yg,13818
 unstructured_ingest/processes/connectors/kafka/__init__.py,sha256=pFN2cWwAStiGTAsQ616GIWKi_hDv0s74ZvNqhJEp1Pc,751
 unstructured_ingest/processes/connectors/kafka/cloud.py,sha256=Ki6iOLoZ86tYWdnLnMWYvb2hUCneKqo4mTJcfXh7YoQ,3432
 unstructured_ingest/processes/connectors/kafka/kafka.py,sha256=7NMvWijfoliyAgnmz8TM8oJt5x7RDzC-ABPdYAm7J3w,10306
@@ -191,7 +191,7 @@ unstructured_ingest/processes/connectors/notion/types/database_properties/phone_
 unstructured_ingest/processes/connectors/notion/types/database_properties/relation.py,sha256=e4zQmjtP2w1RNfKDaWu2-AOhwjM_QP_TrjyxAQUnpN4,1528
 unstructured_ingest/processes/connectors/notion/types/database_properties/rich_text.py,sha256=EeWvzLx9zYT3crH_eshAOf1L60k5gMqJUqQGiI-is7c,1171
 unstructured_ingest/processes/connectors/notion/types/database_properties/rollup.py,sha256=xyaA2yIvDfe-tu_TXi6ZjCz2jCawFS2DVKvfSr-nxjQ,1277
-unstructured_ingest/processes/connectors/notion/types/database_properties/select.py,sha256=wF-V5JKzi4y3i1-eE6kXQWeSbqYo33D1LXdXJIpTkyM,1756
+unstructured_ingest/processes/connectors/notion/types/database_properties/select.py,sha256=Qy-t9kmJq5cZKohxNX2DbvIftX95CvDX6JMDVWzVYl4,1794
 unstructured_ingest/processes/connectors/notion/types/database_properties/status.py,sha256=GneqbmGaBqOecoApH6I4YWf5flvuVk2m5wMJrRw7Nvo,2044
 unstructured_ingest/processes/connectors/notion/types/database_properties/title.py,sha256=4Tnfxxx_Nr-Wkwd7urotp1xTGO9itWe8qI3OnfooLrE,1023
 unstructured_ingest/processes/connectors/notion/types/database_properties/unique_id.py,sha256=i95msSj2-l3yG8tw02dIChADDEy695MWmNWXJq0jGD4,1173
@@ -231,8 +231,8 @@ unstructured_ingest/utils/ndjson.py,sha256=nz8VUOPEgAFdhaDOpuveknvCU4x82fVwqE01q
 unstructured_ingest/utils/pydantic_models.py,sha256=BT_j15e4rX40wQbt8LUXbqfPhA3rJn1PHTI_G_A_EHY,1720
 unstructured_ingest/utils/string_and_date_utils.py,sha256=oXOI6rxXq-8ncbk7EoJK0WCcTXWj75EzKl8pfQMID3U,2522
 unstructured_ingest/utils/table.py,sha256=WZechczgVFvlodUWFcsnCGvBNh1xRm6hr0VbJTPxKAc,3669
-unstructured_ingest-1.0.12.dist-info/METADATA,sha256=sliUV6p8Zt44PisJ-euZ06l18iPea1V9-UOU6uPGj-4,8720
-unstructured_ingest-1.0.12.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-unstructured_ingest-1.0.12.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-1.0.12.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-1.0.12.dist-info/RECORD,,
+unstructured_ingest-1.0.13.dist-info/METADATA,sha256=hZzFMEM3VU6hSkAk6QN8QviuCh3wjjRcpzYbsvWFL5w,8720
+unstructured_ingest-1.0.13.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+unstructured_ingest-1.0.13.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-1.0.13.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-1.0.13.dist-info/RECORD,,