limits 4.7.3__py3-none-any.whl → 5.0.0rc2__py3-none-any.whl

limits/aio/storage/mongodb.py

@@ -169,16 +169,12 @@ class MongoDBStorage(Storage, MovingWindowSupport, SlidingWindowCounterSupport):
 
         return counter and counter["count"] or 0
 
-    async def incr(
-        self, key: str, expiry: int, elastic_expiry: bool = False, amount: int = 1
-    ) -> int:
+    async def incr(self, key: str, expiry: int, amount: int = 1) -> int:
         """
         increments the counter for a given rate limit key
 
         :param key: the key to increment
         :param expiry: amount in seconds for the key to expire in
-        :param elastic_expiry: whether to keep extending the rate limit
-         window every hit.
         :param amount: the number to increment by
         """
         await self.create_indices()
@@ -205,7 +201,7 @@ class MongoDBStorage(Storage, MovingWindowSupport, SlidingWindowCounterSupport):
                             "$cond": {
                                 "if": {"$lt": ["$expireAt", "$$NOW"]},
                                 "then": expiration,
-                                "else": (expiration if elastic_expiry else "$expireAt"),
+                                "else": "$expireAt",
                             }
                         },
                     }
@@ -241,15 +237,16 @@ class MongoDBStorage(Storage, MovingWindowSupport, SlidingWindowCounterSupport):
         :param int expiry: expiry of entry
         :return: (start of window, number of acquired entries)
         """
+
         timestamp = time.time()
-        if result := (
-            await self.database[self.__collection_mapping["windows"]]
+        if (
+            result := await self.database[self.__collection_mapping["windows"]]
             .aggregate(
                 [
                     {"$match": {"_id": key}},
                     {
                         "$project": {
-                            "entries": {
+                            "filteredEntries": {
                                 "$filter": {
                                     "input": "$entries",
                                     "as": "entry",
@@ -258,12 +255,10 @@ class MongoDBStorage(Storage, MovingWindowSupport, SlidingWindowCounterSupport):
                             }
                         }
                     },
-                    {"$unwind": "$entries"},
                     {
-                        "$group": {
-                            "_id": "$_id",
-                            "min": {"$min": "$entries"},
-                            "count": {"$sum": 1},
+                        "$project": {
+                            "min": {"$min": "$filteredEntries"},
+                            "count": {"$size": "$filteredEntries"},
                         }
                     },
                 ]
@@ -337,7 +332,7 @@ class MongoDBStorage(Storage, MovingWindowSupport, SlidingWindowCounterSupport):
                             "$cond": {
                                 "if": {
                                     "$lte": [
-                                        {"$subtract": ["$expiresAt", "$$NOW"]},
+                                        {"$subtract": ["$expireAt", "$$NOW"]},
                                         expiry_ms,
                                     ]
                                 },
@@ -353,7 +348,7 @@ class MongoDBStorage(Storage, MovingWindowSupport, SlidingWindowCounterSupport):
                             "$cond": {
                                 "if": {
                                     "$lte": [
-                                        {"$subtract": ["$expiresAt", "$$NOW"]},
+                                        {"$subtract": ["$expireAt", "$$NOW"]},
                                         expiry_ms,
                                     ]
                                 },
@@ -361,22 +356,22 @@ class MongoDBStorage(Storage, MovingWindowSupport, SlidingWindowCounterSupport):
                                 "else": {"$ifNull": ["$currentCount", 0]},
                             }
                         },
-                        "expiresAt": {
+                        "expireAt": {
                             "$cond": {
                                 "if": {
                                     "$lte": [
-                                        {"$subtract": ["$expiresAt", "$$NOW"]},
+                                        {"$subtract": ["$expireAt", "$$NOW"]},
                                         expiry_ms,
                                     ]
                                 },
                                 "then": {
                                     "$cond": {
-                                        "if": {"$gt": ["$expiresAt", 0]},
-                                        "then": {"$add": ["$expiresAt", expiry_ms]},
+                                        "if": {"$gt": ["$expireAt", 0]},
+                                        "then": {"$add": ["$expireAt", expiry_ms]},
                                         "else": {"$add": ["$$NOW", 2 * expiry_ms]},
                                     }
                                 },
-                                "else": "$expiresAt",
+                                "else": "$expireAt",
                             }
                         },
                     }
@@ -396,7 +391,7 @@ class MongoDBStorage(Storage, MovingWindowSupport, SlidingWindowCounterSupport):
                                                             0,
                                                             {
                                                                 "$subtract": [
-                                                                    "$expiresAt",
+                                                                    "$expireAt",
                                                                     {
                                                                         "$add": [
                                                                             "$$NOW",
@@ -464,7 +459,7 @@ class MongoDBStorage(Storage, MovingWindowSupport, SlidingWindowCounterSupport):
                             "$cond": {
                                 "if": {
                                     "$lte": [
-                                        {"$subtract": ["$expiresAt", "$$NOW"]},
+                                        {"$subtract": ["$expireAt", "$$NOW"]},
                                         expiry_ms,
                                     ]
                                 },
@@ -476,7 +471,7 @@ class MongoDBStorage(Storage, MovingWindowSupport, SlidingWindowCounterSupport):
                             "$cond": {
                                 "if": {
                                     "$lte": [
-                                        {"$subtract": ["$expiresAt", "$$NOW"]},
+                                        {"$subtract": ["$expireAt", "$$NOW"]},
                                         expiry_ms,
                                     ]
                                 },
@@ -484,27 +479,27 @@ class MongoDBStorage(Storage, MovingWindowSupport, SlidingWindowCounterSupport):
                                 "else": {"$ifNull": ["$currentCount", 0]},
                             }
                         },
-                        "expiresAt": {
+                        "expireAt": {
                             "$cond": {
                                 "if": {
                                     "$lte": [
-                                        {"$subtract": ["$expiresAt", "$$NOW"]},
+                                        {"$subtract": ["$expireAt", "$$NOW"]},
                                         expiry_ms,
                                     ]
                                 },
-                                "then": {"$add": ["$expiresAt", expiry_ms]},
-                                "else": "$expiresAt",
+                                "then": {"$add": ["$expireAt", expiry_ms]},
+                                "else": "$expireAt",
                             }
                         },
                     }
                 }
             ],
             return_document=self.proxy_dependency.module.ReturnDocument.AFTER,
-            projection=["currentCount", "previousCount", "expiresAt"],
+            projection=["currentCount", "previousCount", "expireAt"],
         ):
             expires_at = (
-                (result["expiresAt"].replace(tzinfo=datetime.timezone.utc).timestamp())
-                if result.get("expiresAt")
+                (result["expireAt"].replace(tzinfo=datetime.timezone.utc).timestamp())
+                if result.get("expireAt")
                 else time.time()
             )
             current_ttl = max(0, expires_at - time.time())

limits/aio/storage/redis/__init__.py

@@ -139,9 +139,7 @@ class RedisStorage(Storage, MovingWindowSupport, SlidingWindowCounterSupport):
     ) -> type[Exception] | tuple[type[Exception], ...]:  # pragma: no cover
         return self.bridge.base_exceptions
 
-    async def incr(
-        self, key: str, expiry: int, elastic_expiry: bool = False, amount: int = 1
-    ) -> int:
+    async def incr(self, key: str, expiry: int, amount: int = 1) -> int:
         """
         increments the counter for a given rate limit key
 
@@ -150,7 +148,7 @@ class RedisStorage(Storage, MovingWindowSupport, SlidingWindowCounterSupport):
         :param amount: the number to increment by
         """
 
-        return await self.bridge.incr(key, expiry, elastic_expiry, amount)
+        return await self.bridge.incr(key, expiry, amount)
 
     async def get(self, key: str) -> int:
         """

limits/aio/storage/redis/bridge.py

@@ -68,7 +68,6 @@ class RedisBridge(ABC):
         self,
         key: str,
         expiry: int,
-        elastic_expiry: bool = False,
         amount: int = 1,
     ) -> int: ...
 

limits/aio/storage/redis/coredis.py

@@ -112,14 +112,10 @@ class CoredisBridge(RedisBridge):
             self.SCRIPT_ACQUIRE_SLIDING_WINDOW
         )
 
-    async def incr(
-        self, key: str, expiry: int, elastic_expiry: bool = False, amount: int = 1
-    ) -> int:
+    async def incr(self, key: str, expiry: int, amount: int = 1) -> int:
         key = self.prefixed_key(key)
-        value = await self.get_connection().incrby(key, amount)
-        if elastic_expiry or value == amount:
+        if (value := await self.get_connection().incrby(key, amount)) == amount:
             await self.get_connection().expire(key, expiry)
-
         return value
 
     async def get(self, key: str) -> int:

limits/aio/storage/redis/redispy.py

@@ -119,7 +119,6 @@ class RedispyBridge(RedisBridge):
         self,
         key: str,
         expiry: int,
-        elastic_expiry: bool = False,
         amount: int = 1,
     ) -> int:
         """
@@ -131,13 +130,7 @@ class RedispyBridge(RedisBridge):
         :param amount: the number to increment by
         """
         key = self.prefixed_key(key)
-
-        if elastic_expiry:
-            value = await self.get_connection().incrby(key, amount)
-            await self.get_connection().expire(key, expiry)
-            return value
-        else:
-            return cast(int, await self.lua_incr_expire([key], [expiry, amount]))
+        return cast(int, await self.lua_incr_expire([key], [expiry, amount]))
 
     async def get(self, key: str) -> int:
         """

limits/aio/strategies.py

@@ -8,7 +8,7 @@ import time
 from abc import ABC, abstractmethod
 from math import floor, inf
 
-from deprecated.sphinx import deprecated, versionadded
+from deprecated.sphinx import versionadded
 
 from ..limits import RateLimitItem
 from ..storage import StorageTypes
@@ -150,7 +150,6 @@ class FixedWindowRateLimiter(RateLimiter):
             await self.storage.incr(
                 item.key_for(*identifiers),
                 item.get_expiry(),
-                elastic_expiry=False,
                 amount=cost,
             )
             <= item.amount
@@ -304,34 +303,8 @@ class SlidingWindowCounterRateLimiter(RateLimiter):
         return WindowStats(now + min(previous_reset_in, current_reset_in), remaining)
 
 
-@deprecated(version="4.1")
-class FixedWindowElasticExpiryRateLimiter(FixedWindowRateLimiter):
-    """
-    Reference: :ref:`strategies:fixed window with elastic expiry`
-    """
-
-    async def hit(self, item: RateLimitItem, *identifiers: str, cost: int = 1) -> bool:
-        """
-        Consume the rate limit
-
-        :param item: a :class:`limits.limits.RateLimitItem` instance
-        :param identifiers: variable list of strings to uniquely identify the
-         limit
-        :param cost: The cost of this hit, default 1
-        """
-        amount = await self.storage.incr(
-            item.key_for(*identifiers),
-            item.get_expiry(),
-            elastic_expiry=True,
-            amount=cost,
-        )
-
-        return amount <= item.amount
-
-
 STRATEGIES = {
     "sliding-window-counter": SlidingWindowCounterRateLimiter,
     "fixed-window": FixedWindowRateLimiter,
-    "fixed-window-elastic-expiry": FixedWindowElasticExpiryRateLimiter,
     "moving-window": MovingWindowRateLimiter,
 }

limits/resources/redis/lua_scripts/acquire_moving_window.lua

@@ -12,11 +12,14 @@ local entry = redis.call('lindex', KEYS[1], limit - amount)
 if entry and tonumber(entry) >= timestamp - expiry then
     return false
 end
-
+local entries = {}
 for i = 1, amount do
-    redis.call('lpush', KEYS[1], timestamp)
+    entries[i] = timestamp
 end
 
+for i=1,#entries,5000 do
+    redis.call('lpush', KEYS[1], unpack(entries, i, math.min(i+4999, #entries)))
+end
 redis.call('ltrim', KEYS[1], 0, limit - 1)
 redis.call('expire', KEYS[1], expiry)
 

limits/resources/redis/lua_scripts/moving_window.lua

@@ -1,21 +1,30 @@
-local items = redis.call('lrange', KEYS[1], 0, tonumber(ARGV[2]))
+local len = tonumber(ARGV[2])
 local expiry = tonumber(ARGV[1])
-local a = 0
-local oldest = nil
 
-for idx=1,#items do
-    if tonumber(items[idx]) >= expiry then
-        a = a + 1
+-- Binary search to find the oldest valid entry in the window
+local function oldest_entry(high, target)
+    local low = 0
+    local result = nil
 
-        local value = tonumber(items[idx])
-        if oldest == nil or value < oldest then
-            oldest = value
+    while low <= high do
+        local mid = math.floor((low + high) / 2)
+        local val = tonumber(redis.call('lindex', KEYS[1], mid))
+
+        if val and val >= target then
+            result = mid
+            low = mid + 1
+        else
+            high = mid - 1
         end
-    else
-        break
     end
+
+    return result
 end
 
-if oldest then
-    return {tostring(oldest), a}
-end
+local index = oldest_entry(len - 1, expiry)
+
+if index then
+    local count = index + 1
+    local oldest = tonumber(redis.call('lindex', KEYS[1], index))
+    return {tostring(oldest), count}
+end

limits/storage/__init__.py

@@ -12,7 +12,6 @@ import limits  # noqa
 from ..errors import ConfigurationError
 from ..typing import TypeAlias, cast
 from .base import MovingWindowSupport, SlidingWindowCounterSupport, Storage
-from .etcd import EtcdStorage
 from .memcached import MemcachedStorage
 from .memory import MemoryStorage
 from .mongodb import MongoDBStorage, MongoDBStorageBase
@@ -67,7 +66,6 @@ def storage_from_string(
 
 
 __all__ = [
-    "EtcdStorage",
     "MemcachedStorage",
     "MemoryStorage",
     "MongoDBStorage",

limits/storage/base.py

@@ -71,16 +71,12 @@ class Storage(LazyDependency, metaclass=StorageRegistry):
         raise NotImplementedError
 
     @abstractmethod
-    def incr(
-        self, key: str, expiry: int, elastic_expiry: bool = False, amount: int = 1
-    ) -> int:
+    def incr(self, key: str, expiry: int, amount: int = 1) -> int:
         """
         increments the counter for a given rate limit key
 
         :param key: the key to increment
         :param expiry: amount in seconds for the key to expire in
-        :param elastic_expiry: whether to keep extending the rate limit
-         window every hit.
         :param amount: the number to increment by
         """
         raise NotImplementedError

limits/storage/memcached.py

@@ -153,6 +153,8 @@ class MemcachedStorage(Storage, SlidingWindowCounterSupport, TimestampedSlidingW
         Return multiple counters at once
 
         :param keys: the keys to get the counter values for
+
+        :meta private:
         """
         return self.storage.get_many(keys)
 
@@ -166,7 +168,6 @@ class MemcachedStorage(Storage, SlidingWindowCounterSupport, TimestampedSlidingW
         self,
         key: str,
         expiry: float,
-        elastic_expiry: bool = False,
         amount: int = 1,
         set_expiration_key: bool = True,
     ) -> int:
@@ -175,43 +176,21 @@ class MemcachedStorage(Storage, SlidingWindowCounterSupport, TimestampedSlidingW
 
         :param key: the key to increment
         :param expiry: amount in seconds for the key to expire in
-        :param elastic_expiry: whether to keep extending the rate limit
          window every hit.
         :param amount: the number to increment by
         :param set_expiration_key: set the expiration key with the expiration time if needed. If set to False, the key will still expire, but memcached cannot provide the expiration time.
         """
-        value = self.call_memcached_func(self.storage.incr, key, amount, noreply=False)
-        if value is not None:
-            if elastic_expiry:
-                self.call_memcached_func(self.storage.touch, key, ceil(expiry))
-                if set_expiration_key:
-                    self.call_memcached_func(
-                        self.storage.set,
-                        self._expiration_key(key),
-                        expiry + time.time(),
-                        expire=ceil(expiry),
-                        noreply=False,
-                    )
-
+        if (
+            value := self.call_memcached_func(
+                self.storage.incr, key, amount, noreply=False
+            )
+        ) is not None:
             return value
         else:
             if not self.call_memcached_func(
                 self.storage.add, key, amount, ceil(expiry), noreply=False
             ):
-                value = self.storage.incr(key, amount) or amount
-
-                if elastic_expiry:
-                    self.call_memcached_func(self.storage.touch, key, ceil(expiry))
-                    if set_expiration_key:
-                        self.call_memcached_func(
-                            self.storage.set,
-                            self._expiration_key(key),
-                            expiry + time.time(),
-                            expire=ceil(expiry),
-                            noreply=False,
-                        )
-
-                return value
+                return self.storage.incr(key, amount) or amount
             else:
                 if set_expiration_key:
                     self.call_memcached_func(

limits/storage/memory.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import bisect
 import threading
 import time
 from collections import Counter, defaultdict
@@ -25,7 +26,7 @@ class MemoryStorage(
 ):
     """
     rate limit storage using :class:`collections.Counter`
-    as an in memory storage for fixed and elastic window strategies,
+    as an in memory storage for fixed and sliding window strategies,
     and a simple list to implement moving window strategy.
 
     """
@@ -56,9 +57,11 @@ class MemoryStorage(
     def __expire_events(self) -> None:
         for key in list(self.events.keys()):
             with self.locks[key]:
-                for event in list(self.events[key]):
-                    if event.expiry <= time.time() and event in self.events[key]:
-                        self.events[key].remove(event)
+                if events := self.events.get(key, []):
+                    oldest = bisect.bisect_left(
+                        events, -time.time(), key=lambda event: -event.expiry
+                    )
+                    self.events[key] = self.events[key][:oldest]
                 if not self.events.get(key, None):
                     self.locks.pop(key, None)
         for key in list(self.expirations.keys()):
@@ -78,26 +81,20 @@ class MemoryStorage(
     ) -> type[Exception] | tuple[type[Exception], ...]:  # pragma: no cover
         return ValueError
 
-    def incr(
-        self, key: str, expiry: float, elastic_expiry: bool = False, amount: int = 1
-    ) -> int:
+    def incr(self, key: str, expiry: float, amount: int = 1) -> int:
         """
         increments the counter for a given rate limit key
 
         :param key: the key to increment
         :param expiry: amount in seconds for the key to expire in
-        :param elastic_expiry: whether to keep extending the rate limit
-         window every hit.
         :param amount: the number to increment by
         """
         self.get(key)
         self.__schedule_expiry()
         with self.locks[key]:
             self.storage[key] += amount
-
-            if elastic_expiry or self.storage[key] == amount:
+            if self.storage[key] == amount:
                 self.expirations[key] = time.time() + expiry
-
         return self.storage.get(key, 0)
 
     def decr(self, key: str, amount: int = 1) -> int:
@@ -157,7 +154,7 @@ class MemoryStorage(
             if entry and entry.atime >= timestamp - expiry:
                 return False
             else:
-                self.events[key][:0] = [Entry(expiry) for _ in range(amount)]
+                self.events[key][:0] = [Entry(expiry)] * amount
                 return True
 
     def get_expiry(self, key: str) -> float:
@@ -167,21 +164,6 @@ class MemoryStorage(
 
         return self.expirations.get(key, time.time())
 
-    def get_num_acquired(self, key: str, expiry: int) -> int:
-        """
-        returns the number of entries already acquired
-
-        :param key: rate limit key to acquire an entry in
-        :param expiry: expiry of the entry
-        """
-        timestamp = time.time()
-
-        return (
-            len([k for k in self.events.get(key, []) if k.atime >= timestamp - expiry])
-            if self.events.get(key)
-            else 0
-        )
-
     def get_moving_window(self, key: str, limit: int, expiry: int) -> tuple[float, int]:
         """
         returns the starting point and the number of entries in the moving
@@ -192,13 +174,12 @@ class MemoryStorage(
         :return: (start of window, number of acquired entries)
         """
         timestamp = time.time()
-        acquired = self.get_num_acquired(key, expiry)
-
-        for item in self.events.get(key, [])[::-1]:
-            if item.atime >= timestamp - expiry:
-                return item.atime, acquired
-
-        return timestamp, acquired
+        if events := self.events.get(key, []):
+            oldest = bisect.bisect_left(
+                events, -(timestamp - expiry), key=lambda entry: -entry.atime
+            )
+            return events[oldest - 1].atime, oldest
+        return timestamp, 0
 
     def acquire_sliding_window_entry(
         self,