omnata-plugin-runtime 0.2.79__py3-none-any.whl → 0.2.81__py3-none-any.whl

omnata_plugin_runtime/omnata_plugin.py

@@ -1711,7 +1711,9 @@ def managed_inbound_processing(concurrency: int):
                 )
             
             if self._sync_request.development_mode is True:
-                concurrency = 1 # disable concurrency when running in development mode, it interferes with pyvcr
+                concurrency_to_use = 1 # disable concurrency when running in development mode, it interferes with pyvcr
+            else:
+                concurrency_to_use = concurrency
             # if self._sync_request.api_limits is None:
             #    raise ValueError('To use the managed_inbound_processing decorator, API constraints must be defined. These can be provided in the response to the connect method')
             if len(method_args) == 0:
@@ -1734,9 +1736,9 @@ def managed_inbound_processing(concurrency: int):
                 streams_queue.put(stream)
 
             tasks = []
-            logger.info(f"Creating {concurrency} worker(s) for retrieving records")
+            logger.info(f"Creating {concurrency_to_use} worker(s) for retrieving records")
 
-            for i in range(concurrency):
+            for i in range(concurrency_to_use):
                 # the dataframe/generator was put on the queue, so we remove it from the method args
                 task = threading.Thread(
                     target=__managed_inbound_processing_worker,

omnata_plugin_runtime/rate_limiting.py

@@ -132,11 +132,19 @@ class ApiLimits(SubscriptableBaseModel):
         ):
             longest_wait = rate_limit_state.wait_until
         for request_rate in self.request_rates:
+            print(request_rate)
             if rate_limit_state.previous_request_timestamps is not None and len(rate_limit_state.previous_request_timestamps) > 0:
+                previous_request_timestamps = rate_limit_state.get_relevant_history(request_rate)
+                print(f"previous_request_timestamps: {len(previous_request_timestamps)}")
                 request_index = request_rate.request_count - 1
-                if request_index > len(rate_limit_state.previous_request_timestamps) - 1:
-                    request_index = len(rate_limit_state.previous_request_timestamps) - 1
-                timestamp_at_horizon = rate_limit_state.previous_request_timestamps[request_index]
+                if request_index > len(previous_request_timestamps) - 1:
+                    continue # we have not yet made enough requests to hit this rate limit
+                request_index = len(previous_request_timestamps) - 1
+                timestamp_at_horizon = previous_request_timestamps[request_index]
+                print("timestamp_at_horizon: " + str(timestamp_at_horizon))
+                now = datetime.datetime.now().astimezone(datetime.timezone.utc)
+                seconds_since_horizon = (timestamp_at_horizon - now).total_seconds()
+                print("seconds_since_horizon: " + str(seconds_since_horizon))
                 next_allowed_request = timestamp_at_horizon + datetime.timedelta(
                     seconds=request_rate.number_of_seconds()
                 )
@@ -257,6 +265,29 @@ class RateLimitState(SubscriptableBaseModel):
             ts for ts in self.previous_request_timestamps if ts > irrelevance_horizon
         ]
 
+    def get_relevant_history(self, request_rate: RequestRateLimit = None):
+        """
+        Returns the previous requests which are relevant to a specific rate limiting window.
+        First, we prune out timestamps which are older than the time window.
+        We also only need to keep timestamps up until the request count for the rate limit.
+        For example:
+        - Rate limit is 10 requests per 10 seconds
+        - If (somehow, for the sake of argument) there were 20 requests in the last 10 seconds, 
+        we only need to keep the most recent 10 to base our calculation on
+        """
+        if request_rate is None:
+            return self.previous_request_timestamps
+        longest_window_seconds = request_rate.number_of_seconds()
+        irrelevance_horizon = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(
+            seconds=longest_window_seconds
+        )
+        
+        pruned = [ts for ts in self.previous_request_timestamps if ts > irrelevance_horizon]
+        if len(pruned) > request_rate.request_count:
+            return pruned[:request_rate.request_count]
+        else:
+            return pruned
+
 
 class RequestRateLimit(SubscriptableBaseModel):
     """

{omnata_plugin_runtime-0.2.79.dist-info → omnata_plugin_runtime-0.2.81.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omnata-plugin-runtime
-Version: 0.2.79
+Version: 0.2.81
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com

{omnata_plugin_runtime-0.2.79.dist-info → omnata_plugin_runtime-0.2.81.dist-info}/RECORD

@@ -3,10 +3,10 @@ omnata_plugin_runtime/api.py,sha256=vKq7goVPX5cPQ9CVN9l8RmpJDcqDwS5y9v1IWhWjBbk,
 omnata_plugin_runtime/configuration.py,sha256=NICse7qMtReIYY4ZCu8ng4QDJ4rFP0g3mZwc8m1Xl54,32247
 omnata_plugin_runtime/forms.py,sha256=_KqSMQG749wImLKxPZh3B3doTZMbP5jDvF6BhQNkPCM,17375
 omnata_plugin_runtime/logging.py,sha256=Q6eSqrr3SzwfVAg4r4sV1dlxeNS_PzOtZfieoWUEOZQ,3232
-omnata_plugin_runtime/omnata_plugin.py,sha256=gyGZdDBP-qHL3-4WTP4zJwo2w4vdqBtiC2QTnOnbfxU,86008
+omnata_plugin_runtime/omnata_plugin.py,sha256=eshPKK9I4o1kVKXIJoihwNuje-uOnDXA7x5UlzVv_i8,86096
 omnata_plugin_runtime/plugin_entrypoints.py,sha256=dV_JOEWffdkAkofFYJMAsM_-gE9OpM2-paI2dsh5Rzo,24352
-omnata_plugin_runtime/rate_limiting.py,sha256=QO8VB1H8al6a8-ydohUmL0c5JynXG2bulmuPRs2-2-Y,14910
-omnata_plugin_runtime-0.2.79.dist-info/LICENSE,sha256=IMF9i4xIpgCADf0U-V1cuf9HBmqWQd3qtI3FSuyW4zE,26526
-omnata_plugin_runtime-0.2.79.dist-info/METADATA,sha256=kbh70pWeKBiR_3q7ebH0sqQVoE-pIWd9bLXWHv1T4QU,1086
-omnata_plugin_runtime-0.2.79.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-omnata_plugin_runtime-0.2.79.dist-info/RECORD,,
+omnata_plugin_runtime/rate_limiting.py,sha256=Ur_TjtpemAEMQxxoKq4NeR1w0t1OQkyq6ANgbmOGlXQ,16642
+omnata_plugin_runtime-0.2.81.dist-info/LICENSE,sha256=IMF9i4xIpgCADf0U-V1cuf9HBmqWQd3qtI3FSuyW4zE,26526
+omnata_plugin_runtime-0.2.81.dist-info/METADATA,sha256=fskFBfEwVeyauJHnZ2Q8y25R038BrfsLxzBi7mKnmPE,1086
+omnata_plugin_runtime-0.2.81.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+omnata_plugin_runtime-0.2.81.dist-info/RECORD,,