omnata-plugin-runtime 0.4.0a87__py3-none-any.whl → 0.4.0a89__py3-none-any.whl

omnata_plugin_runtime/omnata_plugin.py

@@ -350,12 +350,13 @@ class SyncRequest(ABC):
             update_rate_limit_result = self._plugin_message(
                 PluginMessageRateLimitState(rate_limit_state=self.rate_limit_state_this_sync_and_branch)
             )
-            sync_id:int = update_rate_limit_result["sync_id"]
-            sync_branch_name:str = update_rate_limit_result["sync_branch_name"]
-            latest_state:Dict[int,Dict[str,Dict[str,RateLimitState]]] = parse_obj_as(Dict[int,Dict[str,Dict[str,RateLimitState]]],update_rate_limit_result["latest_state"])
-            (rate_limit_state_all, rate_limit_state_this_branch) = RateLimitState.collapse(latest_state,sync_id, sync_branch_name)
-            self.rate_limit_state_all = rate_limit_state_all
-            self.rate_limit_state_this_sync_and_branch = rate_limit_state_this_branch
+            if update_rate_limit_result is not None:
+                sync_id:int = update_rate_limit_result["sync_id"]
+                sync_branch_name:str = update_rate_limit_result["sync_branch_name"]
+                latest_state:Dict[int,Dict[str,Dict[str,RateLimitState]]] = parse_obj_as(Dict[int,Dict[str,Dict[str,RateLimitState]]],update_rate_limit_result["latest_state"])
+                (rate_limit_state_all, rate_limit_state_this_branch) = RateLimitState.collapse(latest_state,sync_id, sync_branch_name)
+                self.rate_limit_state_all = rate_limit_state_all
+                self.rate_limit_state_this_sync_and_branch = rate_limit_state_this_branch
         else:
             logger.info("No rate limit state to update")
 
@@ -689,19 +690,34 @@ class OutboundSyncRequest(SyncRequest):
         logger.info("applying results to table")
         # use a random table name with a random string to avoid collisions
         with self._snowflake_query_lock:
-            success, nchunks, nrows, _ = write_pandas(
-                conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
-                df=self._preprocess_results_dataframe(results_df),
-                quote_identifiers=False,
-                table_name=self._full_results_table_name,
-                auto_create_table=False
-            )
-            if not success:
-                raise ValueError(
-                    f"Failed to write results to table {self._full_results_table_name}"
-                )
-            logger.info(
-                f"Wrote {nrows} rows and {nchunks} chunks to table {self._full_results_table_name}"
+            attempts_remaining = 12
+            while attempts_remaining > 0:
+                try:
+                    success, nchunks, nrows, _ = write_pandas(
+                        conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
+                        df=self._preprocess_results_dataframe(results_df),
+                        quote_identifiers=False,
+                        table_name=self._full_results_table_name,
+                        auto_create_table=False
+                    )
+                    if not success:
+                        raise ValueError(
+                            f"Failed to write results to table {self._full_results_table_name}"
+                        )
+                    logger.info(
+                        f"Wrote {nrows} rows and {nchunks} chunks to table {self._full_results_table_name}"
+                    )
+                    return
+                except Exception as e:
+                    if 'is being committed' not in str(e):
+                        raise e
+                    logger.error(
+                        f"Transaction clash writing results to table {self._full_results_table_name}: {e}"
+                    )
+                    attempts_remaining -= 1
+                    time.sleep(5)
+            raise ValueError(
+                f"Failed to write results to table {self._full_results_table_name} after {attempts_remaining} attempts"
             )
 
     def __dataframe_wrapper(
@@ -1307,34 +1323,49 @@ class InboundSyncRequest(SyncRequest):
         """
         if len(results_df) > 0:
             with self._snowflake_query_lock:
-                logger.info(
-                    f"Applying {len(results_df)} results to {self._full_results_table_name}"
-                )
-                # try setting parquet engine here, since the engine parameter does not seem to make it through to the write_pandas function
-                success, nchunks, nrows, _ = write_pandas(
-                    conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
-                    df=results_df,
-                    table_name=self._full_results_table_name,
-                    quote_identifiers=False,  # already done in get_temp_table_name
-                    # schema='INBOUND_RAW', # it seems to be ok to provide schema in the table name
-                    table_type="transient"
-                )
-                if not success:
-                    raise ValueError(
-                        f"Failed to write results to table {self._full_results_table_name}"
-                    )
-                logger.info(
-                    f"Wrote {nrows} rows and {nchunks} chunks to table {self._full_results_table_name}"
+                attempts_remaining = 12
+                while attempts_remaining > 0:
+                    try:
+                        logger.info(
+                            f"Applying {len(results_df)} results to {self._full_results_table_name}"
+                        )
+                        # try setting parquet engine here, since the engine parameter does not seem to make it through to the write_pandas function
+                        success, nchunks, nrows, _ = write_pandas(
+                            conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
+                            df=results_df,
+                            table_name=self._full_results_table_name,
+                            quote_identifiers=False,  # already done in get_temp_table_name
+                            # schema='INBOUND_RAW', # it seems to be ok to provide schema in the table name
+                            table_type="transient"
+                        )
+                        if not success:
+                            raise ValueError(
+                                f"Failed to write results to table {self._full_results_table_name}"
+                            )
+                        logger.info(
+                            f"Wrote {nrows} rows and {nchunks} chunks to table {self._full_results_table_name}"
+                        )
+                        # temp tables aren't allowed
+                        # snowflake_df = self._session.create_dataframe(results_df)
+                        # snowflake_df.write.save_as_table(table_name=temp_table,
+                        #                                mode='append',
+                        #                                column_order='index',
+                        #                                #create_temp_table=True
+                        #                                )
+                        for stream_name in stream_names:
+                            self._results_exist[stream_name] = True
+                        return
+                    except Exception as e:
+                        if 'is being committed' not in str(e):
+                            raise e
+                        logger.error(
+                            f"Transaction clash writing results to table {self._full_results_table_name}: {e}"
+                        )
+                        attempts_remaining -= 1
+                        time.sleep(5)
+                raise ValueError(
+                    f"Failed to write results to table {self._full_results_table_name} after {attempts_remaining} attempts"
                 )
-                # temp tables aren't allowed
-                # snowflake_df = self._session.create_dataframe(results_df)
-                # snowflake_df.write.save_as_table(table_name=temp_table,
-                #                                mode='append',
-                #                                column_order='index',
-                #                                #create_temp_table=True
-                #                                )
-                for stream_name in stream_names:
-                    self._results_exist[stream_name] = True
         else:
             logger.info("Results dataframe is empty, not applying")
 
@@ -1352,23 +1383,38 @@ class InboundSyncRequest(SyncRequest):
         """
         if len(results_df) > 0:
             with self._snowflake_query_lock:
-                logger.info(
-                    f"Applying {len(results_df)} criteria deletes to {self._criteria_deletes_table_name}"
-                )
-                # try setting parquet engine here, since the engine parameter does not seem to make it through to the write_pandas function
-                success, nchunks, nrows, _ = write_pandas(
-                    conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
-                    df=results_df,
-                    table_name=self._criteria_deletes_table_name,
-                    quote_identifiers=False,  # already done in get_temp_table_name
-                    table_type="transient"
-                )
-                if not success:
-                    raise ValueError(
-                        f"Failed to write results to table {self._criteria_deletes_table_name}"
-                    )
-                logger.info(
-                    f"Wrote {nrows} rows and {nchunks} chunks to table {self._criteria_deletes_table_name}"
+                attempts_remaining = 12
+                while attempts_remaining > 0:
+                    try:
+                        logger.info(
+                            f"Applying {len(results_df)} criteria deletes to {self._criteria_deletes_table_name}"
+                        )
+                        # try setting parquet engine here, since the engine parameter does not seem to make it through to the write_pandas function
+                        success, nchunks, nrows, _ = write_pandas(
+                            conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
+                            df=results_df,
+                            table_name=self._criteria_deletes_table_name,
+                            quote_identifiers=False,  # already done in get_temp_table_name
+                            table_type="transient"
+                        )
+                        if not success:
+                            raise ValueError(
+                                f"Failed to write results to table {self._criteria_deletes_table_name}"
+                            )
+                        logger.info(
+                            f"Wrote {nrows} rows and {nchunks} chunks to table {self._criteria_deletes_table_name}"
+                        )
+                        return
+                    except Exception as e:
+                        if 'is being committed' not in str(e):
+                            raise e
+                        logger.error(
+                            f"Transaction clash writing results to table {self._full_results_table_name}: {e}"
+                        )
+                        attempts_remaining -= 1
+                        time.sleep(5)
+                raise ValueError(
+                    f"Failed to write results to table {self._full_results_table_name} after {attempts_remaining} attempts"
                 )
         else:
             logger.info("Results dataframe is empty, not applying")

{omnata_plugin_runtime-0.4.0a87.dist-info → omnata_plugin_runtime-0.4.0a89.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omnata-plugin-runtime
-Version: 0.4.0a87
+Version: 0.4.0a89
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com

{omnata_plugin_runtime-0.4.0a87.dist-info → omnata_plugin_runtime-0.4.0a89.dist-info}/RECORD

@@ -3,10 +3,10 @@ omnata_plugin_runtime/api.py,sha256=W79CsAcl127Dzy-XVS9CzvzsbS3IigVH4QAhFFDkaXg,
 omnata_plugin_runtime/configuration.py,sha256=7cMekoY8CeZAJHpASU6tCMidF55Hzfr7CD74jtebqIY,35742
 omnata_plugin_runtime/forms.py,sha256=pw_aKVsXSz47EP8PFBI3VDwdSN5IjvZxp8JTjO1V130,18421
 omnata_plugin_runtime/logging.py,sha256=bn7eKoNWvtuyTk7RTwBS9UARMtqkiICtgMtzq3KA2V0,3272
-omnata_plugin_runtime/omnata_plugin.py,sha256=s9h0zqkCJYeBpKIeUQathy7gOw_0Tw-irhdBW72q2xo,105616
+omnata_plugin_runtime/omnata_plugin.py,sha256=JJtpl8iKcTMgIOymkZh9yKMhZBemXQOGe8s8iJSw5Hk,108276
 omnata_plugin_runtime/plugin_entrypoints.py,sha256=D2f0Qih7KTJNXIDYkA-E55RTEK_7Jtv9ySqspWxERVA,28272
 omnata_plugin_runtime/rate_limiting.py,sha256=29Hjsr0i1rE8jERVdIFGINfQfp_kI3PDft-IM_ZxvCA,21509
-omnata_plugin_runtime-0.4.0a87.dist-info/LICENSE,sha256=IMF9i4xIpgCADf0U-V1cuf9HBmqWQd3qtI3FSuyW4zE,26526
-omnata_plugin_runtime-0.4.0a87.dist-info/METADATA,sha256=-yx21xGaWDk6paSBL0QfVC2JE4IiEyeV6SQiQT7KfMo,1603
-omnata_plugin_runtime-0.4.0a87.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-omnata_plugin_runtime-0.4.0a87.dist-info/RECORD,,
+omnata_plugin_runtime-0.4.0a89.dist-info/LICENSE,sha256=IMF9i4xIpgCADf0U-V1cuf9HBmqWQd3qtI3FSuyW4zE,26526
+omnata_plugin_runtime-0.4.0a89.dist-info/METADATA,sha256=LEkEREX1mBrQoSb6abHniwlf8gZfkBicmmzhbP_Glgw,1603
+omnata_plugin_runtime-0.4.0a89.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+omnata_plugin_runtime-0.4.0a89.dist-info/RECORD,,