omnata-plugin-runtime 0.3.28a84__py3-none-any.whl → 0.4.0__py3-none-any.whl

omnata_plugin_runtime/api.py

@@ -66,8 +66,9 @@ class PluginMessageStreamProgressUpdate(BaseModel):
     message_type: Literal["stream_record_counts"] = "stream_record_counts"
     stream_total_counts: Dict[str, int]
     completed_streams: List[str]
-    # older runtime versions didn't have this, so the sync engine can't expect it
+    # older runtime versions didn't have these, so the sync engine can't expect it
     stream_errors: Optional[Dict[str,str]]
+    total_records_estimate: Optional[Dict[str,int]]
 
 
 class PluginMessageCancelledStreams(BaseModel):

omnata_plugin_runtime/omnata_plugin.py

@@ -906,6 +906,7 @@ class InboundSyncRequest(SyncRequest):
         self._stream_record_counts: Dict[str, int] = {
             stream_name: 0 for stream_name in self._streams_dict.keys()
         }
+        self._total_records_estimate: Optional[Dict[str,int]] = {}
         self._stream_change_counts: Dict[str, int] = {
             stream_name: 0 for stream_name in self._streams_dict.keys()
         }
@@ -933,6 +934,7 @@ class InboundSyncRequest(SyncRequest):
                     ] = self._stream_record_counts[stream_name] + total_length
                     results.extend(non_empty_dfs)  # remove any None/empty dataframes
                     stream_names.append(stream_name)
+                self._session.sql("begin").collect()
                 if len(results) > 0:
                     logger.info(
                         f"Applying {len(results)} batches of queued results"
@@ -943,6 +945,7 @@ class InboundSyncRequest(SyncRequest):
                     self._apply_results_dataframe(stream_names, all_dfs)
                 # update the stream state object too
                 self._apply_latest_states()
+                self._session.sql("commit").collect()
                 for stream_name in stream_names:
                     self._apply_results[stream_name] = None
                 self._apply_results = {}
@@ -975,7 +978,8 @@ class InboundSyncRequest(SyncRequest):
         new_progress_update = PluginMessageStreamProgressUpdate(
                 stream_total_counts=self._stream_record_counts,
                 completed_streams=self._completed_streams,
-                stream_errors=self._omnata_log_handler.stream_global_errors
+                stream_errors=self._omnata_log_handler.stream_global_errors,
+                total_records_estimate=self._total_records_estimate
             )
         if self._last_stream_progress_update is None or new_progress_update != self._last_stream_progress_update:
             self._plugin_message(
@@ -1109,6 +1113,12 @@ class InboundSyncRequest(SyncRequest):
         self._completed_streams.append(stream_name)
         # dedup just in case it's called twice
         self._completed_streams = list(set(self._completed_streams))
+    
+    def set_stream_record_count(self, stream_name: str, count: int):
+        """
+        Sets the record count for a stream, used to provide progress updates.
+        """
+        self._stream_record_counts[stream_name] = count
 
     def _enqueue_state(self, stream_name: str, new_state: Any):
         """

omnata_plugin_runtime/rate_limiting.py

@@ -132,19 +132,15 @@ class ApiLimits(SubscriptableBaseModel):
         ):
             longest_wait = rate_limit_state.wait_until
         for request_rate in self.request_rates:
-            print(request_rate)
             if rate_limit_state.previous_request_timestamps is not None and len(rate_limit_state.previous_request_timestamps) > 0:
                 previous_request_timestamps = rate_limit_state.get_relevant_history(request_rate)
-                print(f"previous_request_timestamps: {len(previous_request_timestamps)}")
                 request_index = request_rate.request_count - 1
                 if request_index > len(previous_request_timestamps) - 1:
                     continue # we have not yet made enough requests to hit this rate limit
                 request_index = len(previous_request_timestamps) - 1
                 timestamp_at_horizon = previous_request_timestamps[request_index]
-                print("timestamp_at_horizon: " + str(timestamp_at_horizon))
                 now = datetime.datetime.now().astimezone(datetime.timezone.utc)
                 seconds_since_horizon = (timestamp_at_horizon - now).total_seconds()
-                print("seconds_since_horizon: " + str(seconds_since_horizon))
                 next_allowed_request = timestamp_at_horizon + datetime.timedelta(
                     seconds=request_rate.number_of_seconds()
                 )
@@ -374,7 +370,7 @@ class RateLimitedSession(requests.Session):
             total=max_retries,
             backoff_factor=backoff_factor,
             status_forcelist=statuses_to_include,
-            method_whitelist=["HEAD", "GET", "OPTIONS", "POST", "PUT", "DELETE"]
+            allowed_methods=["HEAD", "GET", "OPTIONS", "POST", "PUT", "DELETE"]
         )
         adapter = HTTPAdapter(max_retries=retry_strategy)
         self.mount("https://", adapter)

{omnata_plugin_runtime-0.3.28a84.dist-info → omnata_plugin_runtime-0.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omnata-plugin-runtime
-Version: 0.3.28a84
+Version: 0.4.0
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com

omnata_plugin_runtime-0.4.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+omnata_plugin_runtime/__init__.py,sha256=MS9d1whnfT_B3-ThqZ7l63QeC_8OEKTuaYV5wTwRpBA,1576
+omnata_plugin_runtime/api.py,sha256=W79CsAcl127Dzy-XVS9CzvzsbS3IigVH4QAhFFDkaXg,6270
+omnata_plugin_runtime/configuration.py,sha256=7cMekoY8CeZAJHpASU6tCMidF55Hzfr7CD74jtebqIY,35742
+omnata_plugin_runtime/forms.py,sha256=pw_aKVsXSz47EP8PFBI3VDwdSN5IjvZxp8JTjO1V130,18421
+omnata_plugin_runtime/logging.py,sha256=bn7eKoNWvtuyTk7RTwBS9UARMtqkiICtgMtzq3KA2V0,3272
+omnata_plugin_runtime/omnata_plugin.py,sha256=y03j6ivm-lHGKQjIhlzSY1DSmb3-OhmFwwU9jusoFVo,105723
+omnata_plugin_runtime/plugin_entrypoints.py,sha256=D2f0Qih7KTJNXIDYkA-E55RTEK_7Jtv9ySqspWxERVA,28272
+omnata_plugin_runtime/rate_limiting.py,sha256=29Hjsr0i1rE8jERVdIFGINfQfp_kI3PDft-IM_ZxvCA,21509
+omnata_plugin_runtime-0.4.0.dist-info/LICENSE,sha256=IMF9i4xIpgCADf0U-V1cuf9HBmqWQd3qtI3FSuyW4zE,26526
+omnata_plugin_runtime-0.4.0.dist-info/METADATA,sha256=fwWghGIXYu3pCJ_urCUoly08C40AkXHrmEGfbY36oEc,1600
+omnata_plugin_runtime-0.4.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+omnata_plugin_runtime-0.4.0.dist-info/RECORD,,

omnata_plugin_runtime-0.3.28a84.dist-info/RECORD

@@ -1,12 +0,0 @@
-omnata_plugin_runtime/__init__.py,sha256=MS9d1whnfT_B3-ThqZ7l63QeC_8OEKTuaYV5wTwRpBA,1576
-omnata_plugin_runtime/api.py,sha256=_N5ok5LN7GDO4J9n3yduXp3tpjmhpySY__U2baiygrs,6217
-omnata_plugin_runtime/configuration.py,sha256=7cMekoY8CeZAJHpASU6tCMidF55Hzfr7CD74jtebqIY,35742
-omnata_plugin_runtime/forms.py,sha256=pw_aKVsXSz47EP8PFBI3VDwdSN5IjvZxp8JTjO1V130,18421
-omnata_plugin_runtime/logging.py,sha256=bn7eKoNWvtuyTk7RTwBS9UARMtqkiICtgMtzq3KA2V0,3272
-omnata_plugin_runtime/omnata_plugin.py,sha256=aXC-Ui9qeYKJM7VCZtZLQsVod9yH8ecEQ6e-IuJuVnY,105248
-omnata_plugin_runtime/plugin_entrypoints.py,sha256=D2f0Qih7KTJNXIDYkA-E55RTEK_7Jtv9ySqspWxERVA,28272
-omnata_plugin_runtime/rate_limiting.py,sha256=n83ANCla6DLwM_oHKmTKHG-JFM99i5-KjsgnkB9pYGo,21786
-omnata_plugin_runtime-0.3.28a84.dist-info/LICENSE,sha256=IMF9i4xIpgCADf0U-V1cuf9HBmqWQd3qtI3FSuyW4zE,26526
-omnata_plugin_runtime-0.3.28a84.dist-info/METADATA,sha256=GQsx3J1EHelIRI53WezceZ8x5BOGwAhFPEeqOUUODnU,1604
-omnata_plugin_runtime-0.3.28a84.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-omnata_plugin_runtime-0.3.28a84.dist-info/RECORD,,