omnata-plugin-runtime 0.4.4__py3-none-any.whl → 0.4.5__py3-none-any.whl

omnata_plugin_runtime/plugin_entrypoints.py

@@ -12,7 +12,7 @@ from pydantic import BaseModel, parse_obj_as  # pylint: disable=no-name-in-modul
 import pydantic.json
 from snowflake.snowpark import Session
 
-from .api import SyncRequestPayload, handle_proc_result
+from .api import PluginMessageStreamProgressUpdate, SyncRequestPayload, handle_proc_result
 from .configuration import (
     ConnectionConfigurationParameters,
     InboundSyncConfigurationParameters,
@@ -218,11 +218,22 @@ class PluginEntrypoint:
                     inbound_sync_request.update_activity("Staging remaining records")
                     logger.info("Calling apply_results_queue")
                     inbound_sync_request.apply_results_queue()
-                    logger.info("Calling apply_rate_limit_state")
-                    inbound_sync_request.apply_rate_limit_state()
-                    logger.info("Calling apply_progress_updates")
-                    # we can't ignore errors here, because the sync engine needs to get the progress updates before we return
-                    inbound_sync_request.apply_progress_updates(ignore_errors=False)
+                    try:
+                        # this is not critical, we wouldn't fail the sync over rate limit usage capture
+                        logger.info("Calling apply_rate_limit_state")
+                        inbound_sync_request.apply_rate_limit_state()
+                    except Exception as e:
+                        logger.error(f"Error applying rate limit state: {str(e)}")
+                    # here we used to do a final inbound_sync_request.apply_progress_updates(ignore_errors=False)
+                    # but it was erroring too much since there was usually a lot of DDL activity on the Snowflake side
+                    # so instead, we'll provide a final progress update via a return value from the proc
+                final_progress_update = PluginMessageStreamProgressUpdate(
+                    stream_total_counts=inbound_sync_request._stream_record_counts,
+                    completed_streams=inbound_sync_request._completed_streams,
+                    stream_errors=omnata_log_handler.stream_global_errors,
+                    total_records_estimate=inbound_sync_request._total_records_estimate
+                )
+                return_dict["final_progress_update"] = final_progress_update.dict()
                 if inbound_sync_request.deadline_reached:
                     # if we actually hit the deadline, this is flagged by the cancellation checking worker and the cancellation
                     # token is set. We throw it here as an error since that's currently how it flows back to the engine with a DELAYED state
@@ -239,11 +250,6 @@ class PluginEntrypoint:
                         inbound_sync_request._rate_limit_update_task.join()  # pylint: disable=protected-access
                 except Exception as e:
                     logger.error(f"Error cleaning up threading: {str(e)}")
-            return_dict["errored_streams"] = list(
-                omnata_log_handler.stream_global_errors.keys()
-            )
-            # we need to calculate counts for:
-            # CHANGED_COUNT by counting up the records in INBOUND_STREAM_RECORD_COUNTS
         logger.info("Finished applying records")
         return return_dict
 

{omnata_plugin_runtime-0.4.4.dist-info → omnata_plugin_runtime-0.4.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omnata-plugin-runtime
-Version: 0.4.4
+Version: 0.4.5
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com

{omnata_plugin_runtime-0.4.4.dist-info → omnata_plugin_runtime-0.4.5.dist-info}/RECORD

@@ -4,9 +4,9 @@ omnata_plugin_runtime/configuration.py,sha256=7cMekoY8CeZAJHpASU6tCMidF55Hzfr7CD
 omnata_plugin_runtime/forms.py,sha256=pw_aKVsXSz47EP8PFBI3VDwdSN5IjvZxp8JTjO1V130,18421
 omnata_plugin_runtime/logging.py,sha256=bn7eKoNWvtuyTk7RTwBS9UARMtqkiICtgMtzq3KA2V0,3272
 omnata_plugin_runtime/omnata_plugin.py,sha256=UJZNTcG6s5APZ7EIOYc5N4bFiyFEq3ekbSP6EK7tCBY,108065
-omnata_plugin_runtime/plugin_entrypoints.py,sha256=GN3JuVFi74TTXICiK2XdWpT7ohJYuxh4R9v_cGdm59A,28549
+omnata_plugin_runtime/plugin_entrypoints.py,sha256=JAGEdVcy9QEXv7TO5zt7co64LTP8nqGusOc0sJG9GtU,29149
 omnata_plugin_runtime/rate_limiting.py,sha256=29Hjsr0i1rE8jERVdIFGINfQfp_kI3PDft-IM_ZxvCA,21509
-omnata_plugin_runtime-0.4.4.dist-info/LICENSE,sha256=IMF9i4xIpgCADf0U-V1cuf9HBmqWQd3qtI3FSuyW4zE,26526
-omnata_plugin_runtime-0.4.4.dist-info/METADATA,sha256=-pZbE3iYcMvI-_e_qGS6JzUdzDwXAEVFf5s9NnnxU5g,1638
-omnata_plugin_runtime-0.4.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-omnata_plugin_runtime-0.4.4.dist-info/RECORD,,
+omnata_plugin_runtime-0.4.5.dist-info/LICENSE,sha256=IMF9i4xIpgCADf0U-V1cuf9HBmqWQd3qtI3FSuyW4zE,26526
+omnata_plugin_runtime-0.4.5.dist-info/METADATA,sha256=0d4xQ6hbq-twZnIkK7UyU_lp1h6lCwFUqldIAyVZDHI,1638
+omnata_plugin_runtime-0.4.5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+omnata_plugin_runtime-0.4.5.dist-info/RECORD,,