omnata-plugin-runtime 0.4.0a87__tar.gz → 0.4.0a89__tar.gz

{omnata_plugin_runtime-0.4.0a87 → omnata_plugin_runtime-0.4.0a89}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omnata-plugin-runtime
-Version: 0.4.0a87
+Version: 0.4.0a89
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com

{omnata_plugin_runtime-0.4.0a87 → omnata_plugin_runtime-0.4.0a89}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "omnata-plugin-runtime"
-version = "0.4.0-a87"
+version = "0.4.0-a89"
 description = "Classes and common runtime components for building and running Omnata Plugins"
 authors = ["James Weakley <james.weakley@omnata.com>"]
 readme = "README.md"

{omnata_plugin_runtime-0.4.0a87 → omnata_plugin_runtime-0.4.0a89}/src/omnata_plugin_runtime/omnata_plugin.py

@@ -350,12 +350,13 @@ class SyncRequest(ABC):
             update_rate_limit_result = self._plugin_message(
                 PluginMessageRateLimitState(rate_limit_state=self.rate_limit_state_this_sync_and_branch)
             )
-            sync_id:int = update_rate_limit_result["sync_id"]
-            sync_branch_name:str = update_rate_limit_result["sync_branch_name"]
-            latest_state:Dict[int,Dict[str,Dict[str,RateLimitState]]] = parse_obj_as(Dict[int,Dict[str,Dict[str,RateLimitState]]],update_rate_limit_result["latest_state"])
-            (rate_limit_state_all, rate_limit_state_this_branch) = RateLimitState.collapse(latest_state,sync_id, sync_branch_name)
-            self.rate_limit_state_all = rate_limit_state_all
-            self.rate_limit_state_this_sync_and_branch = rate_limit_state_this_branch
+            if update_rate_limit_result is not None:
+                sync_id:int = update_rate_limit_result["sync_id"]
+                sync_branch_name:str = update_rate_limit_result["sync_branch_name"]
+                latest_state:Dict[int,Dict[str,Dict[str,RateLimitState]]] = parse_obj_as(Dict[int,Dict[str,Dict[str,RateLimitState]]],update_rate_limit_result["latest_state"])
+                (rate_limit_state_all, rate_limit_state_this_branch) = RateLimitState.collapse(latest_state,sync_id, sync_branch_name)
+                self.rate_limit_state_all = rate_limit_state_all
+                self.rate_limit_state_this_sync_and_branch = rate_limit_state_this_branch
         else:
             logger.info("No rate limit state to update")
 
@@ -689,19 +690,34 @@ class OutboundSyncRequest(SyncRequest):
         logger.info("applying results to table")
         # use a random table name with a random string to avoid collisions
         with self._snowflake_query_lock:
-            success, nchunks, nrows, _ = write_pandas(
-                conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
-                df=self._preprocess_results_dataframe(results_df),
-                quote_identifiers=False,
-                table_name=self._full_results_table_name,
-                auto_create_table=False
-            )
-            if not success:
-                raise ValueError(
-                    f"Failed to write results to table {self._full_results_table_name}"
-                )
-            logger.info(
-                f"Wrote {nrows} rows and {nchunks} chunks to table {self._full_results_table_name}"
+            attempts_remaining = 12
+            while attempts_remaining > 0:
+                try:
+                    success, nchunks, nrows, _ = write_pandas(
+                        conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
+                        df=self._preprocess_results_dataframe(results_df),
+                        quote_identifiers=False,
+                        table_name=self._full_results_table_name,
+                        auto_create_table=False
+                    )
+                    if not success:
+                        raise ValueError(
+                            f"Failed to write results to table {self._full_results_table_name}"
+                        )
+                    logger.info(
+                        f"Wrote {nrows} rows and {nchunks} chunks to table {self._full_results_table_name}"
+                    )
+                    return
+                except Exception as e:
+                    if 'is being committed' not in str(e):
+                        raise e
+                    logger.error(
+                        f"Transaction clash writing results to table {self._full_results_table_name}: {e}"
+                    )
+                    attempts_remaining -= 1
+                    time.sleep(5)
+            raise ValueError(
+                f"Failed to write results to table {self._full_results_table_name} after {attempts_remaining} attempts"
             )
 
     def __dataframe_wrapper(
@@ -1307,34 +1323,49 @@ class InboundSyncRequest(SyncRequest):
         """
         if len(results_df) > 0:
             with self._snowflake_query_lock:
-                logger.info(
-                    f"Applying {len(results_df)} results to {self._full_results_table_name}"
-                )
-                # try setting parquet engine here, since the engine parameter does not seem to make it through to the write_pandas function
-                success, nchunks, nrows, _ = write_pandas(
-                    conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
-                    df=results_df,
-                    table_name=self._full_results_table_name,
-                    quote_identifiers=False,  # already done in get_temp_table_name
-                    # schema='INBOUND_RAW', # it seems to be ok to provide schema in the table name
-                    table_type="transient"
-                )
-                if not success:
-                    raise ValueError(
-                        f"Failed to write results to table {self._full_results_table_name}"
-                    )
-                logger.info(
-                    f"Wrote {nrows} rows and {nchunks} chunks to table {self._full_results_table_name}"
+                attempts_remaining = 12
+                while attempts_remaining > 0:
+                    try:
+                        logger.info(
+                            f"Applying {len(results_df)} results to {self._full_results_table_name}"
+                        )
+                        # try setting parquet engine here, since the engine parameter does not seem to make it through to the write_pandas function
+                        success, nchunks, nrows, _ = write_pandas(
+                            conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
+                            df=results_df,
+                            table_name=self._full_results_table_name,
+                            quote_identifiers=False,  # already done in get_temp_table_name
+                            # schema='INBOUND_RAW', # it seems to be ok to provide schema in the table name
+                            table_type="transient"
+                        )
+                        if not success:
+                            raise ValueError(
+                                f"Failed to write results to table {self._full_results_table_name}"
+                            )
+                        logger.info(
+                            f"Wrote {nrows} rows and {nchunks} chunks to table {self._full_results_table_name}"
+                        )
+                        # temp tables aren't allowed
+                        # snowflake_df = self._session.create_dataframe(results_df)
+                        # snowflake_df.write.save_as_table(table_name=temp_table,
+                        #                                mode='append',
+                        #                                column_order='index',
+                        #                                #create_temp_table=True
+                        #                                )
+                        for stream_name in stream_names:
+                            self._results_exist[stream_name] = True
+                        return
+                    except Exception as e:
+                        if 'is being committed' not in str(e):
+                            raise e
+                        logger.error(
+                            f"Transaction clash writing results to table {self._full_results_table_name}: {e}"
+                        )
+                        attempts_remaining -= 1
+                        time.sleep(5)
+                raise ValueError(
+                    f"Failed to write results to table {self._full_results_table_name} after {attempts_remaining} attempts"
                 )
-                # temp tables aren't allowed
-                # snowflake_df = self._session.create_dataframe(results_df)
-                # snowflake_df.write.save_as_table(table_name=temp_table,
-                #                                mode='append',
-                #                                column_order='index',
-                #                                #create_temp_table=True
-                #                                )
-                for stream_name in stream_names:
-                    self._results_exist[stream_name] = True
         else:
             logger.info("Results dataframe is empty, not applying")
 
@@ -1352,23 +1383,38 @@ class InboundSyncRequest(SyncRequest):
         """
         if len(results_df) > 0:
             with self._snowflake_query_lock:
-                logger.info(
-                    f"Applying {len(results_df)} criteria deletes to {self._criteria_deletes_table_name}"
-                )
-                # try setting parquet engine here, since the engine parameter does not seem to make it through to the write_pandas function
-                success, nchunks, nrows, _ = write_pandas(
-                    conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
-                    df=results_df,
-                    table_name=self._criteria_deletes_table_name,
-                    quote_identifiers=False,  # already done in get_temp_table_name
-                    table_type="transient"
-                )
-                if not success:
-                    raise ValueError(
-                        f"Failed to write results to table {self._criteria_deletes_table_name}"
-                    )
-                logger.info(
-                    f"Wrote {nrows} rows and {nchunks} chunks to table {self._criteria_deletes_table_name}"
+                attempts_remaining = 12
+                while attempts_remaining > 0:
+                    try:
+                        logger.info(
+                            f"Applying {len(results_df)} criteria deletes to {self._criteria_deletes_table_name}"
+                        )
+                        # try setting parquet engine here, since the engine parameter does not seem to make it through to the write_pandas function
+                        success, nchunks, nrows, _ = write_pandas(
+                            conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
+                            df=results_df,
+                            table_name=self._criteria_deletes_table_name,
+                            quote_identifiers=False,  # already done in get_temp_table_name
+                            table_type="transient"
+                        )
+                        if not success:
+                            raise ValueError(
+                                f"Failed to write results to table {self._criteria_deletes_table_name}"
+                            )
+                        logger.info(
+                            f"Wrote {nrows} rows and {nchunks} chunks to table {self._criteria_deletes_table_name}"
+                        )
+                        return
+                    except Exception as e:
+                        if 'is being committed' not in str(e):
+                            raise e
+                        logger.error(
+                            f"Transaction clash writing results to table {self._full_results_table_name}: {e}"
+                        )
+                        attempts_remaining -= 1
+                        time.sleep(5)
+                raise ValueError(
+                    f"Failed to write results to table {self._full_results_table_name} after {attempts_remaining} attempts"
                 )
         else:
             logger.info("Results dataframe is empty, not applying")