omnata-plugin-runtime 0.4.1a92__py3-none-any.whl → 0.4.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omnata_plugin_runtime/omnata_plugin.py +23 -22
- omnata_plugin_runtime/plugin_entrypoints.py +2 -0
- {omnata_plugin_runtime-0.4.1a92.dist-info → omnata_plugin_runtime-0.4.2.dist-info}/METADATA +1 -1
- omnata_plugin_runtime-0.4.2.dist-info/RECORD +12 -0
- omnata_plugin_runtime-0.4.1a92.dist-info/RECORD +0 -12
- {omnata_plugin_runtime-0.4.1a92.dist-info → omnata_plugin_runtime-0.4.2.dist-info}/LICENSE +0 -0
- {omnata_plugin_runtime-0.4.1a92.dist-info → omnata_plugin_runtime-0.4.2.dist-info}/WHEEL +0 -0
@@ -875,6 +875,28 @@ class InboundSyncRequest(SyncRequest):
|
|
875
875
|
:param bool test_replay_mode: When enabled, it is safe to assume that HTTP requests are hitting a re-recorded log, so there is no need to wait in between polling
|
876
876
|
:return: nothing
|
877
877
|
"""
|
878
|
+
self.streams = streams
|
879
|
+
self._streams_dict: Dict[str, StoredStreamConfiguration] = {
|
880
|
+
s.stream_name: s for s in streams
|
881
|
+
}
|
882
|
+
self._stream_record_counts: Dict[str, int] = {
|
883
|
+
stream_name: 0 for stream_name in self._streams_dict.keys()
|
884
|
+
}
|
885
|
+
|
886
|
+
# These are similar to the results, but represent requests to delete records by some criteria
|
887
|
+
self._apply_results_criteria_deletes: Dict[str, List[pandas.DataFrame]] = {}
|
888
|
+
self._latest_states: Dict[str, Any] = {}
|
889
|
+
self._temp_tables = {}
|
890
|
+
self._temp_table_lock = threading.Lock()
|
891
|
+
self._results_exist: Dict[
|
892
|
+
str, bool
|
893
|
+
] = {} # track whether or not results exist for stream
|
894
|
+
self._total_records_estimate: Optional[Dict[str,int]] = {}
|
895
|
+
self._stream_change_counts: Dict[str, int] = {
|
896
|
+
stream_name: 0 for stream_name in self._streams_dict.keys()
|
897
|
+
}
|
898
|
+
self._completed_streams: List[str] = []
|
899
|
+
self._omnata_log_handler = omnata_log_handler
|
878
900
|
SyncRequest.__init__(
|
879
901
|
self,
|
880
902
|
run_id=run_id,
|
@@ -890,32 +912,11 @@ class InboundSyncRequest(SyncRequest):
|
|
890
912
|
development_mode=development_mode,
|
891
913
|
test_replay_mode=test_replay_mode
|
892
914
|
)
|
893
|
-
self.streams = streams
|
894
|
-
self._streams_dict: Dict[str, StoredStreamConfiguration] = {
|
895
|
-
s.stream_name: s for s in streams
|
896
|
-
}
|
897
|
-
self._apply_results: Dict[str, List[pandas.DataFrame]] = {}
|
898
915
|
# named by convention, see SyncRunProcessor.enqueue
|
899
916
|
self._criteria_deletes_table_name = (
|
900
917
|
f"{self._source_app_name}.{self._results_schema_name}.{self._results_table_name}_CRITERIA_DELETES"
|
901
918
|
)
|
902
|
-
|
903
|
-
self._apply_results_criteria_deletes: Dict[str, List[pandas.DataFrame]] = {}
|
904
|
-
self._latest_states: Dict[str, Any] = {}
|
905
|
-
self._temp_tables = {}
|
906
|
-
self._temp_table_lock = threading.Lock()
|
907
|
-
self._results_exist: Dict[
|
908
|
-
str, bool
|
909
|
-
] = {} # track whether or not results exist for stream
|
910
|
-
self._stream_record_counts: Dict[str, int] = {
|
911
|
-
stream_name: 0 for stream_name in self._streams_dict.keys()
|
912
|
-
}
|
913
|
-
self._total_records_estimate: Optional[Dict[str,int]] = {}
|
914
|
-
self._stream_change_counts: Dict[str, int] = {
|
915
|
-
stream_name: 0 for stream_name in self._streams_dict.keys()
|
916
|
-
}
|
917
|
-
self._completed_streams: List[str] = []
|
918
|
-
self._omnata_log_handler = omnata_log_handler
|
919
|
+
self._apply_results: Dict[str, List[pandas.DataFrame]] = {}
|
919
920
|
|
920
921
|
def apply_results_queue(self):
|
921
922
|
"""
|
@@ -218,7 +218,9 @@ class PluginEntrypoint:
|
|
218
218
|
inbound_sync_request.update_activity("Staging remaining records")
|
219
219
|
logger.info("Calling apply_results_queue")
|
220
220
|
inbound_sync_request.apply_results_queue()
|
221
|
+
logger.info("Calling apply_rate_limit_state")
|
221
222
|
inbound_sync_request.apply_rate_limit_state()
|
223
|
+
logger.info("Calling apply_progress_updates")
|
222
224
|
inbound_sync_request.apply_progress_updates()
|
223
225
|
if inbound_sync_request.deadline_reached:
|
224
226
|
# if we actually hit the deadline, this is flagged by the cancellation checking worker and the cancellation
|
@@ -0,0 +1,12 @@
|
|
1
|
+
omnata_plugin_runtime/__init__.py,sha256=MS9d1whnfT_B3-ThqZ7l63QeC_8OEKTuaYV5wTwRpBA,1576
|
2
|
+
omnata_plugin_runtime/api.py,sha256=W79CsAcl127Dzy-XVS9CzvzsbS3IigVH4QAhFFDkaXg,6270
|
3
|
+
omnata_plugin_runtime/configuration.py,sha256=7cMekoY8CeZAJHpASU6tCMidF55Hzfr7CD74jtebqIY,35742
|
4
|
+
omnata_plugin_runtime/forms.py,sha256=pw_aKVsXSz47EP8PFBI3VDwdSN5IjvZxp8JTjO1V130,18421
|
5
|
+
omnata_plugin_runtime/logging.py,sha256=bn7eKoNWvtuyTk7RTwBS9UARMtqkiICtgMtzq3KA2V0,3272
|
6
|
+
omnata_plugin_runtime/omnata_plugin.py,sha256=sh19I-l2xIc7aPxO83mv65rijMPfCIDqh5o6sguc1H4,107066
|
7
|
+
omnata_plugin_runtime/plugin_entrypoints.py,sha256=wco1QqubxxE-cMYQiu8D3pUZss3QeiAlrF2UFg5m0GM,28404
|
8
|
+
omnata_plugin_runtime/rate_limiting.py,sha256=29Hjsr0i1rE8jERVdIFGINfQfp_kI3PDft-IM_ZxvCA,21509
|
9
|
+
omnata_plugin_runtime-0.4.2.dist-info/LICENSE,sha256=IMF9i4xIpgCADf0U-V1cuf9HBmqWQd3qtI3FSuyW4zE,26526
|
10
|
+
omnata_plugin_runtime-0.4.2.dist-info/METADATA,sha256=jYFX1QMF1E6nmWEv9zGM_mvRcWPKnzjkrpkojegD8Ek,1638
|
11
|
+
omnata_plugin_runtime-0.4.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
12
|
+
omnata_plugin_runtime-0.4.2.dist-info/RECORD,,
|
@@ -1,12 +0,0 @@
|
|
1
|
-
omnata_plugin_runtime/__init__.py,sha256=MS9d1whnfT_B3-ThqZ7l63QeC_8OEKTuaYV5wTwRpBA,1576
|
2
|
-
omnata_plugin_runtime/api.py,sha256=W79CsAcl127Dzy-XVS9CzvzsbS3IigVH4QAhFFDkaXg,6270
|
3
|
-
omnata_plugin_runtime/configuration.py,sha256=7cMekoY8CeZAJHpASU6tCMidF55Hzfr7CD74jtebqIY,35742
|
4
|
-
omnata_plugin_runtime/forms.py,sha256=pw_aKVsXSz47EP8PFBI3VDwdSN5IjvZxp8JTjO1V130,18421
|
5
|
-
omnata_plugin_runtime/logging.py,sha256=bn7eKoNWvtuyTk7RTwBS9UARMtqkiICtgMtzq3KA2V0,3272
|
6
|
-
omnata_plugin_runtime/omnata_plugin.py,sha256=Jo56GU7R3yUWP0-Go_71P7sxD7XUE2Z0VwNc_eHi33w,107057
|
7
|
-
omnata_plugin_runtime/plugin_entrypoints.py,sha256=D2f0Qih7KTJNXIDYkA-E55RTEK_7Jtv9ySqspWxERVA,28272
|
8
|
-
omnata_plugin_runtime/rate_limiting.py,sha256=29Hjsr0i1rE8jERVdIFGINfQfp_kI3PDft-IM_ZxvCA,21509
|
9
|
-
omnata_plugin_runtime-0.4.1a92.dist-info/LICENSE,sha256=IMF9i4xIpgCADf0U-V1cuf9HBmqWQd3qtI3FSuyW4zE,26526
|
10
|
-
omnata_plugin_runtime-0.4.1a92.dist-info/METADATA,sha256=cjZerqDeUKTk9hgJcVrqv8CAC5wYOW-UYxXHjQ0ppWQ,1641
|
11
|
-
omnata_plugin_runtime-0.4.1a92.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
12
|
-
omnata_plugin_runtime-0.4.1a92.dist-info/RECORD,,
|
File without changes
|
File without changes
|