omnata-plugin-runtime 0.4.0a89__tar.gz → 0.4.0a91__tar.gz

{omnata_plugin_runtime-0.4.0a89 → omnata_plugin_runtime-0.4.0a91}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omnata-plugin-runtime
-Version: 0.4.0a89
+Version: 0.4.0a91
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com
@@ -25,6 +25,7 @@ Requires-Dist: pytz (<=2023.3.post1)
 Requires-Dist: requests (>=2,<=2.31.0)
 Requires-Dist: setuptools (>=68.2.2)
 Requires-Dist: snowflake-snowpark-python (>=1,<2)
+Requires-Dist: tenacity (>=8,<=8.2.2)
 Requires-Dist: tomlkit (>=0.11.1)
 Requires-Dist: urllib3 (<=2.1.0)
 Requires-Dist: wheel (<=0.41.2)

{omnata_plugin_runtime-0.4.0a89 → omnata_plugin_runtime-0.4.0a91}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "omnata-plugin-runtime"
-version = "0.4.0-a89"
+version = "0.4.0-a91"
 description = "Classes and common runtime components for building and running Omnata Plugins"
 authors = ["James Weakley <james.weakley@omnata.com>"]
 readme = "README.md"
@@ -26,6 +26,7 @@ pytz = "<=2023.3.post1" # latest version available on Snowflake Anaconda
 requests = "^2, <=2.31.0" # latest version available on Snowflake Anaconda
 setuptools = ">=68.2.2" # 68.2.2 was latest version available on Snowflake Anaconda
 tomlkit = ">=0.11.1" # 0.11.1 was latest version available on Snowflake Anaconda
+tenacity = "^8, <=8.2.2" # latest version available on Snowflake Anaconda
 urllib3 = "<=2.1.0" # latest version available on Snowflake Anaconda
 wheel = "<=0.41.2" # latest version available on Snowflake Anaconda
 

{omnata_plugin_runtime-0.4.0a89 → omnata_plugin_runtime-0.4.0a91}/src/omnata_plugin_runtime/omnata_plugin.py

@@ -40,6 +40,7 @@ from snowflake.connector.pandas_tools import write_pandas
 from snowflake.connector.version import VERSION
 from snowflake.snowpark import Session
 from snowflake.snowpark.functions import col
+from tenacity import Retrying, stop_after_attempt, wait_fixed, retry_if_exception_message
 
 from .logging import OmnataPluginLogHandler
 
@@ -690,9 +691,8 @@ class OutboundSyncRequest(SyncRequest):
         logger.info("applying results to table")
         # use a random table name with a random string to avoid collisions
         with self._snowflake_query_lock:
-            attempts_remaining = 12
-            while attempts_remaining > 0:
-                try:
+            for attempt in Retrying(stop=stop_after_attempt(30),wait=wait_fixed(2),reraise=True,retry=retry_if_exception_message(match=".*is being committed.*")):
+                with attempt:
                     success, nchunks, nrows, _ = write_pandas(
                         conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
                         df=self._preprocess_results_dataframe(results_df),
@@ -707,18 +707,6 @@ class OutboundSyncRequest(SyncRequest):
                     logger.info(
                         f"Wrote {nrows} rows and {nchunks} chunks to table {self._full_results_table_name}"
                     )
-                    return
-                except Exception as e:
-                    if 'is being committed' not in str(e):
-                        raise e
-                    logger.error(
-                        f"Transaction clash writing results to table {self._full_results_table_name}: {e}"
-                    )
-                    attempts_remaining -= 1
-                    time.sleep(5)
-            raise ValueError(
-                f"Failed to write results to table {self._full_results_table_name} after {attempts_remaining} attempts"
-            )
 
     def __dataframe_wrapper(
         self, data_frame: pandas.DataFrame, render_jinja: bool = True
@@ -963,8 +951,6 @@ class InboundSyncRequest(SyncRequest):
                 for stream_name in stream_names:
                     self._apply_results[stream_name] = None
                 self._apply_results = {}
-                # update the inbound stream record counts, so we can see progress
-                self.apply_progress_updates()
                 
         # also take care of uploading delete requests
         if hasattr(self,'_apply_results_criteria_deletes') and self._apply_results_criteria_deletes is not None:
@@ -984,6 +970,12 @@ class InboundSyncRequest(SyncRequest):
                     self._apply_criteria_deletes_dataframe(all_dfs)
                     # clear the delete requests
                     self._apply_results_criteria_deletes = {}
+        
+
+        # update the inbound stream record counts, so we can see progress
+        # we do this last, because marking a stream as completed will cause the sync engine to process it
+        # so we need to make sure all the results are applied first
+        self.apply_progress_updates()
     
     def apply_progress_updates(self):
         """
@@ -1323,9 +1315,8 @@ class InboundSyncRequest(SyncRequest):
         """
         if len(results_df) > 0:
             with self._snowflake_query_lock:
-                attempts_remaining = 12
-                while attempts_remaining > 0:
-                    try:
+                for attempt in Retrying(stop=stop_after_attempt(30),wait=wait_fixed(2),reraise=True,retry=retry_if_exception_message(match=".*is being committed.*")):
+                    with attempt:
                         logger.info(
                             f"Applying {len(results_df)} results to {self._full_results_table_name}"
                         )
@@ -1354,18 +1345,6 @@ class InboundSyncRequest(SyncRequest):
                         #                                )
                         for stream_name in stream_names:
                             self._results_exist[stream_name] = True
-                        return
-                    except Exception as e:
-                        if 'is being committed' not in str(e):
-                            raise e
-                        logger.error(
-                            f"Transaction clash writing results to table {self._full_results_table_name}: {e}"
-                        )
-                        attempts_remaining -= 1
-                        time.sleep(5)
-                raise ValueError(
-                    f"Failed to write results to table {self._full_results_table_name} after {attempts_remaining} attempts"
-                )
         else:
             logger.info("Results dataframe is empty, not applying")
 
@@ -1383,9 +1362,8 @@ class InboundSyncRequest(SyncRequest):
         """
         if len(results_df) > 0:
             with self._snowflake_query_lock:
-                attempts_remaining = 12
-                while attempts_remaining > 0:
-                    try:
+                for attempt in Retrying(stop=stop_after_attempt(30),wait=wait_fixed(2),reraise=True,retry=retry_if_exception_message(match=".*is being committed.*")):
+                    with attempt:
                         logger.info(
                             f"Applying {len(results_df)} criteria deletes to {self._criteria_deletes_table_name}"
                         )
@@ -1405,17 +1383,6 @@ class InboundSyncRequest(SyncRequest):
                             f"Wrote {nrows} rows and {nchunks} chunks to table {self._criteria_deletes_table_name}"
                         )
                         return
-                    except Exception as e:
-                        if 'is being committed' not in str(e):
-                            raise e
-                        logger.error(
-                            f"Transaction clash writing results to table {self._full_results_table_name}: {e}"
-                        )
-                        attempts_remaining -= 1
-                        time.sleep(5)
-                raise ValueError(
-                    f"Failed to write results to table {self._full_results_table_name} after {attempts_remaining} attempts"
-                )
         else:
             logger.info("Results dataframe is empty, not applying")