omnata-plugin-runtime 0.4.0a89__py3-none-any.whl → 0.4.0a91__py3-none-any.whl

omnata_plugin_runtime/omnata_plugin.py

@@ -40,6 +40,7 @@ from snowflake.connector.pandas_tools import write_pandas
 from snowflake.connector.version import VERSION
 from snowflake.snowpark import Session
 from snowflake.snowpark.functions import col
+from tenacity import Retrying, stop_after_attempt, wait_fixed, retry_if_exception_message
 
 from .logging import OmnataPluginLogHandler
 
@@ -690,9 +691,8 @@ class OutboundSyncRequest(SyncRequest):
         logger.info("applying results to table")
         # use a random table name with a random string to avoid collisions
         with self._snowflake_query_lock:
-            attempts_remaining = 12
-            while attempts_remaining > 0:
-                try:
+            for attempt in Retrying(stop=stop_after_attempt(30),wait=wait_fixed(2),reraise=True,retry=retry_if_exception_message(match=".*is being committed.*")):
+                with attempt:
                     success, nchunks, nrows, _ = write_pandas(
                         conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
                         df=self._preprocess_results_dataframe(results_df),
@@ -707,18 +707,6 @@ class OutboundSyncRequest(SyncRequest):
                     logger.info(
                         f"Wrote {nrows} rows and {nchunks} chunks to table {self._full_results_table_name}"
                     )
-                    return
-                except Exception as e:
-                    if 'is being committed' not in str(e):
-                        raise e
-                    logger.error(
-                        f"Transaction clash writing results to table {self._full_results_table_name}: {e}"
-                    )
-                    attempts_remaining -= 1
-                    time.sleep(5)
-            raise ValueError(
-                f"Failed to write results to table {self._full_results_table_name} after {attempts_remaining} attempts"
-            )
 
     def __dataframe_wrapper(
         self, data_frame: pandas.DataFrame, render_jinja: bool = True
@@ -963,8 +951,6 @@ class InboundSyncRequest(SyncRequest):
                 for stream_name in stream_names:
                     self._apply_results[stream_name] = None
                 self._apply_results = {}
-                # update the inbound stream record counts, so we can see progress
-                self.apply_progress_updates()
                 
         # also take care of uploading delete requests
         if hasattr(self,'_apply_results_criteria_deletes') and self._apply_results_criteria_deletes is not None:
@@ -984,6 +970,12 @@ class InboundSyncRequest(SyncRequest):
                     self._apply_criteria_deletes_dataframe(all_dfs)
                     # clear the delete requests
                     self._apply_results_criteria_deletes = {}
+        
+
+        # update the inbound stream record counts, so we can see progress
+        # we do this last, because marking a stream as completed will cause the sync engine to process it
+        # so we need to make sure all the results are applied first
+        self.apply_progress_updates()
     
     def apply_progress_updates(self):
         """
@@ -1323,9 +1315,8 @@ class InboundSyncRequest(SyncRequest):
         """
         if len(results_df) > 0:
             with self._snowflake_query_lock:
-                attempts_remaining = 12
-                while attempts_remaining > 0:
-                    try:
+                for attempt in Retrying(stop=stop_after_attempt(30),wait=wait_fixed(2),reraise=True,retry=retry_if_exception_message(match=".*is being committed.*")):
+                    with attempt:
                         logger.info(
                             f"Applying {len(results_df)} results to {self._full_results_table_name}"
                         )
@@ -1354,18 +1345,6 @@ class InboundSyncRequest(SyncRequest):
                         #                                )
                         for stream_name in stream_names:
                             self._results_exist[stream_name] = True
-                        return
-                    except Exception as e:
-                        if 'is being committed' not in str(e):
-                            raise e
-                        logger.error(
-                            f"Transaction clash writing results to table {self._full_results_table_name}: {e}"
-                        )
-                        attempts_remaining -= 1
-                        time.sleep(5)
-                raise ValueError(
-                    f"Failed to write results to table {self._full_results_table_name} after {attempts_remaining} attempts"
-                )
         else:
             logger.info("Results dataframe is empty, not applying")
 
@@ -1383,9 +1362,8 @@ class InboundSyncRequest(SyncRequest):
         """
         if len(results_df) > 0:
             with self._snowflake_query_lock:
-                attempts_remaining = 12
-                while attempts_remaining > 0:
-                    try:
+                for attempt in Retrying(stop=stop_after_attempt(30),wait=wait_fixed(2),reraise=True,retry=retry_if_exception_message(match=".*is being committed.*")):
+                    with attempt:
                         logger.info(
                             f"Applying {len(results_df)} criteria deletes to {self._criteria_deletes_table_name}"
                         )
@@ -1405,17 +1383,6 @@ class InboundSyncRequest(SyncRequest):
                             f"Wrote {nrows} rows and {nchunks} chunks to table {self._criteria_deletes_table_name}"
                         )
                         return
-                    except Exception as e:
-                        if 'is being committed' not in str(e):
-                            raise e
-                        logger.error(
-                            f"Transaction clash writing results to table {self._full_results_table_name}: {e}"
-                        )
-                        attempts_remaining -= 1
-                        time.sleep(5)
-                raise ValueError(
-                    f"Failed to write results to table {self._full_results_table_name} after {attempts_remaining} attempts"
-                )
         else:
             logger.info("Results dataframe is empty, not applying")
 

{omnata_plugin_runtime-0.4.0a89.dist-info → omnata_plugin_runtime-0.4.0a91.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omnata-plugin-runtime
-Version: 0.4.0a89
+Version: 0.4.0a91
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com
@@ -25,6 +25,7 @@ Requires-Dist: pytz (<=2023.3.post1)
 Requires-Dist: requests (>=2,<=2.31.0)
 Requires-Dist: setuptools (>=68.2.2)
 Requires-Dist: snowflake-snowpark-python (>=1,<2)
+Requires-Dist: tenacity (>=8,<=8.2.2)
 Requires-Dist: tomlkit (>=0.11.1)
 Requires-Dist: urllib3 (<=2.1.0)
 Requires-Dist: wheel (<=0.41.2)

{omnata_plugin_runtime-0.4.0a89.dist-info → omnata_plugin_runtime-0.4.0a91.dist-info}/RECORD

@@ -3,10 +3,10 @@ omnata_plugin_runtime/api.py,sha256=W79CsAcl127Dzy-XVS9CzvzsbS3IigVH4QAhFFDkaXg,
 omnata_plugin_runtime/configuration.py,sha256=7cMekoY8CeZAJHpASU6tCMidF55Hzfr7CD74jtebqIY,35742
 omnata_plugin_runtime/forms.py,sha256=pw_aKVsXSz47EP8PFBI3VDwdSN5IjvZxp8JTjO1V130,18421
 omnata_plugin_runtime/logging.py,sha256=bn7eKoNWvtuyTk7RTwBS9UARMtqkiICtgMtzq3KA2V0,3272
-omnata_plugin_runtime/omnata_plugin.py,sha256=JJtpl8iKcTMgIOymkZh9yKMhZBemXQOGe8s8iJSw5Hk,108276
+omnata_plugin_runtime/omnata_plugin.py,sha256=Jo56GU7R3yUWP0-Go_71P7sxD7XUE2Z0VwNc_eHi33w,107057
 omnata_plugin_runtime/plugin_entrypoints.py,sha256=D2f0Qih7KTJNXIDYkA-E55RTEK_7Jtv9ySqspWxERVA,28272
 omnata_plugin_runtime/rate_limiting.py,sha256=29Hjsr0i1rE8jERVdIFGINfQfp_kI3PDft-IM_ZxvCA,21509
-omnata_plugin_runtime-0.4.0a89.dist-info/LICENSE,sha256=IMF9i4xIpgCADf0U-V1cuf9HBmqWQd3qtI3FSuyW4zE,26526
-omnata_plugin_runtime-0.4.0a89.dist-info/METADATA,sha256=LEkEREX1mBrQoSb6abHniwlf8gZfkBicmmzhbP_Glgw,1603
-omnata_plugin_runtime-0.4.0a89.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-omnata_plugin_runtime-0.4.0a89.dist-info/RECORD,,
+omnata_plugin_runtime-0.4.0a91.dist-info/LICENSE,sha256=IMF9i4xIpgCADf0U-V1cuf9HBmqWQd3qtI3FSuyW4zE,26526
+omnata_plugin_runtime-0.4.0a91.dist-info/METADATA,sha256=YwD_HMJWQW6Zo3ucJyZOVitQr4VaC9n0cf4QVmgTRyk,1641
+omnata_plugin_runtime-0.4.0a91.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+omnata_plugin_runtime-0.4.0a91.dist-info/RECORD,,