omnata-plugin-runtime 0.4.0a88__tar.gz → 0.4.0a90__tar.gz

{omnata_plugin_runtime-0.4.0a88 → omnata_plugin_runtime-0.4.0a90}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omnata-plugin-runtime
-Version: 0.4.0a88
+Version: 0.4.0a90
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com

{omnata_plugin_runtime-0.4.0a88 → omnata_plugin_runtime-0.4.0a90}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "omnata-plugin-runtime"
-version = "0.4.0-a88"
+version = "0.4.0-a90"
 description = "Classes and common runtime components for building and running Omnata Plugins"
 authors = ["James Weakley <james.weakley@omnata.com>"]
 readme = "README.md"

{omnata_plugin_runtime-0.4.0a88 → omnata_plugin_runtime-0.4.0a90}/src/omnata_plugin_runtime/omnata_plugin.py

@@ -690,19 +690,34 @@ class OutboundSyncRequest(SyncRequest):
         logger.info("applying results to table")
         # use a random table name with a random string to avoid collisions
         with self._snowflake_query_lock:
-            success, nchunks, nrows, _ = write_pandas(
-                conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
-                df=self._preprocess_results_dataframe(results_df),
-                quote_identifiers=False,
-                table_name=self._full_results_table_name,
-                auto_create_table=False
-            )
-            if not success:
-                raise ValueError(
-                    f"Failed to write results to table {self._full_results_table_name}"
-                )
-            logger.info(
-                f"Wrote {nrows} rows and {nchunks} chunks to table {self._full_results_table_name}"
+            attempts_remaining = 12
+            while attempts_remaining > 0:
+                try:
+                    success, nchunks, nrows, _ = write_pandas(
+                        conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
+                        df=self._preprocess_results_dataframe(results_df),
+                        quote_identifiers=False,
+                        table_name=self._full_results_table_name,
+                        auto_create_table=False
+                    )
+                    if not success:
+                        raise ValueError(
+                            f"Failed to write results to table {self._full_results_table_name}"
+                        )
+                    logger.info(
+                        f"Wrote {nrows} rows and {nchunks} chunks to table {self._full_results_table_name}"
+                    )
+                    return
+                except Exception as e:
+                    if 'is being committed' not in str(e):
+                        raise e
+                    logger.error(
+                        f"Transaction clash writing results to table {self._full_results_table_name}: {e}"
+                    )
+                    attempts_remaining -= 1
+                    time.sleep(5)
+            raise ValueError(
+                f"Failed to write results to table {self._full_results_table_name} after {attempts_remaining} attempts"
             )
 
     def __dataframe_wrapper(
@@ -948,8 +963,6 @@ class InboundSyncRequest(SyncRequest):
                 for stream_name in stream_names:
                     self._apply_results[stream_name] = None
                 self._apply_results = {}
-                # update the inbound stream record counts, so we can see progress
-                self.apply_progress_updates()
                 
         # also take care of uploading delete requests
         if hasattr(self,'_apply_results_criteria_deletes') and self._apply_results_criteria_deletes is not None:
@@ -969,6 +982,12 @@ class InboundSyncRequest(SyncRequest):
                     self._apply_criteria_deletes_dataframe(all_dfs)
                     # clear the delete requests
                     self._apply_results_criteria_deletes = {}
+        
+
+        # update the inbound stream record counts, so we can see progress
+        # we do this last, because marking a stream as completed will cause the sync engine to process it
+        # so we need to make sure all the results are applied first
+        self.apply_progress_updates()
     
     def apply_progress_updates(self):
         """
@@ -1308,34 +1327,49 @@ class InboundSyncRequest(SyncRequest):
         """
         if len(results_df) > 0:
             with self._snowflake_query_lock:
-                logger.info(
-                    f"Applying {len(results_df)} results to {self._full_results_table_name}"
-                )
-                # try setting parquet engine here, since the engine parameter does not seem to make it through to the write_pandas function
-                success, nchunks, nrows, _ = write_pandas(
-                    conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
-                    df=results_df,
-                    table_name=self._full_results_table_name,
-                    quote_identifiers=False,  # already done in get_temp_table_name
-                    # schema='INBOUND_RAW', # it seems to be ok to provide schema in the table name
-                    table_type="transient"
-                )
-                if not success:
-                    raise ValueError(
-                        f"Failed to write results to table {self._full_results_table_name}"
-                    )
-                logger.info(
-                    f"Wrote {nrows} rows and {nchunks} chunks to table {self._full_results_table_name}"
+                attempts_remaining = 12
+                while attempts_remaining > 0:
+                    try:
+                        logger.info(
+                            f"Applying {len(results_df)} results to {self._full_results_table_name}"
+                        )
+                        # try setting parquet engine here, since the engine parameter does not seem to make it through to the write_pandas function
+                        success, nchunks, nrows, _ = write_pandas(
+                            conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
+                            df=results_df,
+                            table_name=self._full_results_table_name,
+                            quote_identifiers=False,  # already done in get_temp_table_name
+                            # schema='INBOUND_RAW', # it seems to be ok to provide schema in the table name
+                            table_type="transient"
+                        )
+                        if not success:
+                            raise ValueError(
+                                f"Failed to write results to table {self._full_results_table_name}"
+                            )
+                        logger.info(
+                            f"Wrote {nrows} rows and {nchunks} chunks to table {self._full_results_table_name}"
+                        )
+                        # temp tables aren't allowed
+                        # snowflake_df = self._session.create_dataframe(results_df)
+                        # snowflake_df.write.save_as_table(table_name=temp_table,
+                        #                                mode='append',
+                        #                                column_order='index',
+                        #                                #create_temp_table=True
+                        #                                )
+                        for stream_name in stream_names:
+                            self._results_exist[stream_name] = True
+                        return
+                    except Exception as e:
+                        if 'is being committed' not in str(e):
+                            raise e
+                        logger.error(
+                            f"Transaction clash writing results to table {self._full_results_table_name}: {e}"
+                        )
+                        attempts_remaining -= 1
+                        time.sleep(5)
+                raise ValueError(
+                    f"Failed to write results to table {self._full_results_table_name} after {attempts_remaining} attempts"
                 )
-                # temp tables aren't allowed
-                # snowflake_df = self._session.create_dataframe(results_df)
-                # snowflake_df.write.save_as_table(table_name=temp_table,
-                #                                mode='append',
-                #                                column_order='index',
-                #                                #create_temp_table=True
-                #                                )
-                for stream_name in stream_names:
-                    self._results_exist[stream_name] = True
         else:
             logger.info("Results dataframe is empty, not applying")
 
@@ -1353,23 +1387,38 @@ class InboundSyncRequest(SyncRequest):
         """
         if len(results_df) > 0:
             with self._snowflake_query_lock:
-                logger.info(
-                    f"Applying {len(results_df)} criteria deletes to {self._criteria_deletes_table_name}"
-                )
-                # try setting parquet engine here, since the engine parameter does not seem to make it through to the write_pandas function
-                success, nchunks, nrows, _ = write_pandas(
-                    conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
-                    df=results_df,
-                    table_name=self._criteria_deletes_table_name,
-                    quote_identifiers=False,  # already done in get_temp_table_name
-                    table_type="transient"
-                )
-                if not success:
-                    raise ValueError(
-                        f"Failed to write results to table {self._criteria_deletes_table_name}"
-                    )
-                logger.info(
-                    f"Wrote {nrows} rows and {nchunks} chunks to table {self._criteria_deletes_table_name}"
+                attempts_remaining = 12
+                while attempts_remaining > 0:
+                    try:
+                        logger.info(
+                            f"Applying {len(results_df)} criteria deletes to {self._criteria_deletes_table_name}"
+                        )
+                        # try setting parquet engine here, since the engine parameter does not seem to make it through to the write_pandas function
+                        success, nchunks, nrows, _ = write_pandas(
+                            conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
+                            df=results_df,
+                            table_name=self._criteria_deletes_table_name,
+                            quote_identifiers=False,  # already done in get_temp_table_name
+                            table_type="transient"
+                        )
+                        if not success:
+                            raise ValueError(
+                                f"Failed to write results to table {self._criteria_deletes_table_name}"
+                            )
+                        logger.info(
+                            f"Wrote {nrows} rows and {nchunks} chunks to table {self._criteria_deletes_table_name}"
+                        )
+                        return
+                    except Exception as e:
+                        if 'is being committed' not in str(e):
+                            raise e
+                        logger.error(
+                            f"Transaction clash writing results to table {self._full_results_table_name}: {e}"
+                        )
+                        attempts_remaining -= 1
+                        time.sleep(5)
+                raise ValueError(
+                    f"Failed to write results to table {self._full_results_table_name} after {attempts_remaining} attempts"
                 )
         else:
             logger.info("Results dataframe is empty, not applying")