omnata-plugin-runtime 0.4.4__py3-none-any.whl → 0.4.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omnata_plugin_runtime/plugin_entrypoints.py +17 -11
- {omnata_plugin_runtime-0.4.4.dist-info → omnata_plugin_runtime-0.4.5.dist-info}/METADATA +1 -1
- {omnata_plugin_runtime-0.4.4.dist-info → omnata_plugin_runtime-0.4.5.dist-info}/RECORD +5 -5
- {omnata_plugin_runtime-0.4.4.dist-info → omnata_plugin_runtime-0.4.5.dist-info}/LICENSE +0 -0
- {omnata_plugin_runtime-0.4.4.dist-info → omnata_plugin_runtime-0.4.5.dist-info}/WHEEL +0 -0
@@ -12,7 +12,7 @@ from pydantic import BaseModel, parse_obj_as # pylint: disable=no-name-in-modul
|
|
12
12
|
import pydantic.json
|
13
13
|
from snowflake.snowpark import Session
|
14
14
|
|
15
|
-
from .api import SyncRequestPayload, handle_proc_result
|
15
|
+
from .api import PluginMessageStreamProgressUpdate, SyncRequestPayload, handle_proc_result
|
16
16
|
from .configuration import (
|
17
17
|
ConnectionConfigurationParameters,
|
18
18
|
InboundSyncConfigurationParameters,
|
@@ -218,11 +218,22 @@ class PluginEntrypoint:
|
|
218
218
|
inbound_sync_request.update_activity("Staging remaining records")
|
219
219
|
logger.info("Calling apply_results_queue")
|
220
220
|
inbound_sync_request.apply_results_queue()
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
221
|
+
try:
|
222
|
+
# this is not critical, we wouldn't fail the sync over rate limit usage capture
|
223
|
+
logger.info("Calling apply_rate_limit_state")
|
224
|
+
inbound_sync_request.apply_rate_limit_state()
|
225
|
+
except Exception as e:
|
226
|
+
logger.error(f"Error applying rate limit state: {str(e)}")
|
227
|
+
# here we used to do a final inbound_sync_request.apply_progress_updates(ignore_errors=False)
|
228
|
+
# but it was erroring too much since there was usually a lot of DDL activity on the Snowflake side
|
229
|
+
# so instead, we'll provide a final progress update via a return value from the proc
|
230
|
+
final_progress_update = PluginMessageStreamProgressUpdate(
|
231
|
+
stream_total_counts=inbound_sync_request._stream_record_counts,
|
232
|
+
completed_streams=inbound_sync_request._completed_streams,
|
233
|
+
stream_errors=omnata_log_handler.stream_global_errors,
|
234
|
+
total_records_estimate=inbound_sync_request._total_records_estimate
|
235
|
+
)
|
236
|
+
return_dict["final_progress_update"] = final_progress_update.dict()
|
226
237
|
if inbound_sync_request.deadline_reached:
|
227
238
|
# if we actually hit the deadline, this is flagged by the cancellation checking worker and the cancellation
|
228
239
|
# token is set. We throw it here as an error since that's currently how it flows back to the engine with a DELAYED state
|
@@ -239,11 +250,6 @@ class PluginEntrypoint:
|
|
239
250
|
inbound_sync_request._rate_limit_update_task.join() # pylint: disable=protected-access
|
240
251
|
except Exception as e:
|
241
252
|
logger.error(f"Error cleaning up threading: {str(e)}")
|
242
|
-
return_dict["errored_streams"] = list(
|
243
|
-
omnata_log_handler.stream_global_errors.keys()
|
244
|
-
)
|
245
|
-
# we need to calculate counts for:
|
246
|
-
# CHANGED_COUNT by counting up the records in INBOUND_STREAM_RECORD_COUNTS
|
247
253
|
logger.info("Finished applying records")
|
248
254
|
return return_dict
|
249
255
|
|
@@ -4,9 +4,9 @@ omnata_plugin_runtime/configuration.py,sha256=7cMekoY8CeZAJHpASU6tCMidF55Hzfr7CD
|
|
4
4
|
omnata_plugin_runtime/forms.py,sha256=pw_aKVsXSz47EP8PFBI3VDwdSN5IjvZxp8JTjO1V130,18421
|
5
5
|
omnata_plugin_runtime/logging.py,sha256=bn7eKoNWvtuyTk7RTwBS9UARMtqkiICtgMtzq3KA2V0,3272
|
6
6
|
omnata_plugin_runtime/omnata_plugin.py,sha256=UJZNTcG6s5APZ7EIOYc5N4bFiyFEq3ekbSP6EK7tCBY,108065
|
7
|
-
omnata_plugin_runtime/plugin_entrypoints.py,sha256=
|
7
|
+
omnata_plugin_runtime/plugin_entrypoints.py,sha256=JAGEdVcy9QEXv7TO5zt7co64LTP8nqGusOc0sJG9GtU,29149
|
8
8
|
omnata_plugin_runtime/rate_limiting.py,sha256=29Hjsr0i1rE8jERVdIFGINfQfp_kI3PDft-IM_ZxvCA,21509
|
9
|
-
omnata_plugin_runtime-0.4.
|
10
|
-
omnata_plugin_runtime-0.4.
|
11
|
-
omnata_plugin_runtime-0.4.
|
12
|
-
omnata_plugin_runtime-0.4.
|
9
|
+
omnata_plugin_runtime-0.4.5.dist-info/LICENSE,sha256=IMF9i4xIpgCADf0U-V1cuf9HBmqWQd3qtI3FSuyW4zE,26526
|
10
|
+
omnata_plugin_runtime-0.4.5.dist-info/METADATA,sha256=0d4xQ6hbq-twZnIkK7UyU_lp1h6lCwFUqldIAyVZDHI,1638
|
11
|
+
omnata_plugin_runtime-0.4.5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
12
|
+
omnata_plugin_runtime-0.4.5.dist-info/RECORD,,
|
File without changes
|
File without changes
|