limits 4.1__py3-none-any.whl → 4.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- limits/__init__.py +8 -6
- limits/_version.py +4 -4
- limits/aio/__init__.py +2 -0
- limits/aio/storage/__init__.py +6 -4
- limits/aio/storage/base.py +5 -8
- limits/aio/storage/etcd.py +6 -4
- limits/aio/storage/memcached.py +6 -4
- limits/aio/storage/memory.py +42 -26
- limits/aio/storage/mongodb.py +4 -7
- limits/aio/storage/redis/__init__.py +402 -0
- limits/aio/storage/redis/bridge.py +120 -0
- limits/aio/storage/redis/coredis.py +209 -0
- limits/aio/storage/redis/redispy.py +257 -0
- limits/aio/storage/redis/valkey.py +9 -0
- limits/aio/strategies.py +4 -2
- limits/errors.py +2 -0
- limits/storage/__init__.py +14 -11
- limits/storage/base.py +5 -10
- limits/storage/etcd.py +6 -4
- limits/storage/memcached.py +6 -7
- limits/storage/memory.py +42 -31
- limits/storage/mongodb.py +7 -10
- limits/storage/redis.py +48 -18
- limits/storage/redis_cluster.py +31 -11
- limits/storage/redis_sentinel.py +35 -11
- limits/storage/registry.py +1 -3
- limits/strategies.py +11 -9
- limits/typing.py +45 -42
- limits/util.py +12 -12
- {limits-4.1.dist-info → limits-4.3.dist-info}/METADATA +52 -36
- limits-4.3.dist-info/RECORD +43 -0
- {limits-4.1.dist-info → limits-4.3.dist-info}/WHEEL +1 -1
- limits/aio/storage/redis.py +0 -555
- limits-4.1.dist-info/RECORD +0 -39
- {limits-4.1.dist-info → limits-4.3.dist-info}/LICENSE.txt +0 -0
- {limits-4.1.dist-info → limits-4.3.dist-info}/top_level.txt +0 -0
limits/storage/memcached.py
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import inspect
|
|
2
4
|
import threading
|
|
3
5
|
import time
|
|
@@ -16,11 +18,8 @@ from limits.typing import (
|
|
|
16
18
|
Any,
|
|
17
19
|
Callable,
|
|
18
20
|
MemcachedClientP,
|
|
19
|
-
Optional,
|
|
20
21
|
P,
|
|
21
22
|
R,
|
|
22
|
-
Type,
|
|
23
|
-
Union,
|
|
24
23
|
cast,
|
|
25
24
|
)
|
|
26
25
|
from limits.util import get_dependency
|
|
@@ -41,7 +40,7 @@ class MemcachedStorage(Storage, SlidingWindowCounterSupport, TimestampedSlidingW
|
|
|
41
40
|
self,
|
|
42
41
|
uri: str,
|
|
43
42
|
wrap_exceptions: bool = False,
|
|
44
|
-
**options:
|
|
43
|
+
**options: str | Callable[[], MemcachedClientP],
|
|
45
44
|
) -> None:
|
|
46
45
|
"""
|
|
47
46
|
:param uri: memcached location of the form
|
|
@@ -82,7 +81,7 @@ class MemcachedStorage(Storage, SlidingWindowCounterSupport, TimestampedSlidingW
|
|
|
82
81
|
|
|
83
82
|
if not get_dependency(self.library):
|
|
84
83
|
raise ConfigurationError(
|
|
85
|
-
"memcached prerequisite not available. please install
|
|
84
|
+
f"memcached prerequisite not available. please install {self.library}"
|
|
86
85
|
) # pragma: no cover
|
|
87
86
|
self.local_storage = threading.local()
|
|
88
87
|
self.local_storage.storage = None
|
|
@@ -91,7 +90,7 @@ class MemcachedStorage(Storage, SlidingWindowCounterSupport, TimestampedSlidingW
|
|
|
91
90
|
@property
|
|
92
91
|
def base_exceptions(
|
|
93
92
|
self,
|
|
94
|
-
) ->
|
|
93
|
+
) -> type[Exception] | tuple[type[Exception], ...]: # pragma: no cover
|
|
95
94
|
return self.dependency.MemcacheError # type: ignore[no-any-return]
|
|
96
95
|
|
|
97
96
|
def get_client(
|
|
@@ -253,7 +252,7 @@ class MemcachedStorage(Storage, SlidingWindowCounterSupport, TimestampedSlidingW
|
|
|
253
252
|
except: # noqa
|
|
254
253
|
return False
|
|
255
254
|
|
|
256
|
-
def reset(self) ->
|
|
255
|
+
def reset(self) -> int | None:
|
|
257
256
|
raise NotImplementedError
|
|
258
257
|
|
|
259
258
|
def acquire_sliding_window_entry(
|
limits/storage/memory.py
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import threading
|
|
2
4
|
import time
|
|
3
5
|
from collections import Counter, defaultdict
|
|
@@ -10,14 +12,12 @@ from limits.storage.base import (
|
|
|
10
12
|
Storage,
|
|
11
13
|
TimestampedSlidingWindow,
|
|
12
14
|
)
|
|
13
|
-
from limits.typing import Optional, Type, Union
|
|
14
15
|
|
|
15
16
|
|
|
16
|
-
class
|
|
17
|
+
class Entry:
|
|
17
18
|
def __init__(self, expiry: float) -> None:
|
|
18
19
|
self.atime = time.time()
|
|
19
20
|
self.expiry = self.atime + expiry
|
|
20
|
-
super().__init__()
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
class MemoryStorage(
|
|
@@ -32,30 +32,35 @@ class MemoryStorage(
|
|
|
32
32
|
|
|
33
33
|
STORAGE_SCHEME = ["memory"]
|
|
34
34
|
|
|
35
|
-
def __init__(
|
|
36
|
-
self, uri: Optional[str] = None, wrap_exceptions: bool = False, **_: str
|
|
37
|
-
):
|
|
35
|
+
def __init__(self, uri: str | None = None, wrap_exceptions: bool = False, **_: str):
|
|
38
36
|
self.storage: limits.typing.Counter[str] = Counter()
|
|
39
37
|
self.locks: defaultdict[str, threading.RLock] = defaultdict(threading.RLock)
|
|
40
38
|
self.expirations: dict[str, float] = {}
|
|
41
|
-
self.events: dict[str, list[
|
|
42
|
-
self.timer = threading.Timer(0.01, self.__expire_events)
|
|
39
|
+
self.events: dict[str, list[Entry]] = {}
|
|
40
|
+
self.timer: threading.Timer = threading.Timer(0.01, self.__expire_events)
|
|
43
41
|
self.timer.start()
|
|
44
42
|
super().__init__(uri, wrap_exceptions=wrap_exceptions, **_)
|
|
45
43
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
return
|
|
44
|
+
def __getstate__(self) -> dict[str, limits.typing.Any]: # type: ignore[explicit-any]
|
|
45
|
+
state = self.__dict__.copy()
|
|
46
|
+
del state["timer"]
|
|
47
|
+
del state["locks"]
|
|
48
|
+
return state
|
|
49
|
+
|
|
50
|
+
def __setstate__(self, state: dict[str, limits.typing.Any]) -> None: # type: ignore[explicit-any]
|
|
51
|
+
self.__dict__.update(state)
|
|
52
|
+
self.locks = defaultdict(threading.RLock)
|
|
53
|
+
self.timer = threading.Timer(0.01, self.__expire_events)
|
|
54
|
+
self.timer.start()
|
|
51
55
|
|
|
52
56
|
def __expire_events(self) -> None:
|
|
53
57
|
for key in list(self.events.keys()):
|
|
54
|
-
|
|
55
|
-
|
|
58
|
+
with self.locks[key]:
|
|
59
|
+
for event in list(self.events[key]):
|
|
56
60
|
if event.expiry <= time.time() and event in self.events[key]:
|
|
57
61
|
self.events[key].remove(event)
|
|
58
|
-
|
|
62
|
+
if not self.events.get(key, None):
|
|
63
|
+
self.locks.pop(key, None)
|
|
59
64
|
for key in list(self.expirations.keys()):
|
|
60
65
|
if self.expirations[key] <= time.time():
|
|
61
66
|
self.storage.pop(key, None)
|
|
@@ -67,6 +72,12 @@ class MemoryStorage(
|
|
|
67
72
|
self.timer = threading.Timer(0.01, self.__expire_events)
|
|
68
73
|
self.timer.start()
|
|
69
74
|
|
|
75
|
+
@property
|
|
76
|
+
def base_exceptions(
|
|
77
|
+
self,
|
|
78
|
+
) -> type[Exception] | tuple[type[Exception], ...]: # pragma: no cover
|
|
79
|
+
return ValueError
|
|
80
|
+
|
|
70
81
|
def incr(
|
|
71
82
|
self, key: str, expiry: float, elastic_expiry: bool = False, amount: int = 1
|
|
72
83
|
) -> int:
|
|
@@ -134,19 +145,20 @@ class MemoryStorage(
|
|
|
134
145
|
if amount > limit:
|
|
135
146
|
return False
|
|
136
147
|
|
|
137
|
-
self.events.setdefault(key, [])
|
|
138
148
|
self.__schedule_expiry()
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
149
|
+
with self.locks[key]:
|
|
150
|
+
self.events.setdefault(key, [])
|
|
151
|
+
timestamp = time.time()
|
|
152
|
+
try:
|
|
153
|
+
entry = self.events[key][limit - amount]
|
|
154
|
+
except IndexError:
|
|
155
|
+
entry = None
|
|
156
|
+
|
|
157
|
+
if entry and entry.atime >= timestamp - expiry:
|
|
158
|
+
return False
|
|
159
|
+
else:
|
|
160
|
+
self.events[key][:0] = [Entry(expiry) for _ in range(amount)]
|
|
161
|
+
return True
|
|
150
162
|
|
|
151
163
|
def get_expiry(self, key: str) -> float:
|
|
152
164
|
"""
|
|
@@ -165,7 +177,7 @@ class MemoryStorage(
|
|
|
165
177
|
timestamp = time.time()
|
|
166
178
|
|
|
167
179
|
return (
|
|
168
|
-
len([k for k in self.events[
|
|
180
|
+
len([k for k in self.events.get(key, []) if k.atime >= timestamp - expiry])
|
|
169
181
|
if self.events.get(key)
|
|
170
182
|
else 0
|
|
171
183
|
)
|
|
@@ -218,7 +230,6 @@ class MemoryStorage(
|
|
|
218
230
|
# Limitation: during high concurrency at the end of the window,
|
|
219
231
|
# the counter is shifted and cannot be decremented, so less requests than expected are allowed.
|
|
220
232
|
self.decr(current_key, amount)
|
|
221
|
-
# print("Concurrent call, reverting the counter increase")
|
|
222
233
|
return False
|
|
223
234
|
return True
|
|
224
235
|
|
|
@@ -252,7 +263,7 @@ class MemoryStorage(
|
|
|
252
263
|
|
|
253
264
|
return True
|
|
254
265
|
|
|
255
|
-
def reset(self) ->
|
|
266
|
+
def reset(self) -> int | None:
|
|
256
267
|
num_items = max(len(self.storage), len(self.events))
|
|
257
268
|
self.storage.clear()
|
|
258
269
|
self.expirations.clear()
|
limits/storage/mongodb.py
CHANGED
|
@@ -10,9 +10,6 @@ from limits.typing import (
|
|
|
10
10
|
MongoClient,
|
|
11
11
|
MongoCollection,
|
|
12
12
|
MongoDatabase,
|
|
13
|
-
Optional,
|
|
14
|
-
Type,
|
|
15
|
-
Union,
|
|
16
13
|
cast,
|
|
17
14
|
)
|
|
18
15
|
|
|
@@ -38,7 +35,7 @@ class MongoDBStorageBase(
|
|
|
38
35
|
counter_collection_name: str = "counters",
|
|
39
36
|
window_collection_name: str = "windows",
|
|
40
37
|
wrap_exceptions: bool = False,
|
|
41
|
-
**options:
|
|
38
|
+
**options: int | str | bool,
|
|
42
39
|
) -> None:
|
|
43
40
|
"""
|
|
44
41
|
:param uri: uri of the form ``mongodb://[user:password]@host:port?...``,
|
|
@@ -66,7 +63,7 @@ class MongoDBStorageBase(
|
|
|
66
63
|
self.lib_errors, _ = get_dependency("pymongo.errors")
|
|
67
64
|
self._storage_uri = uri
|
|
68
65
|
self._storage_options = options
|
|
69
|
-
self._storage:
|
|
66
|
+
self._storage: MongoClient | None = None
|
|
70
67
|
|
|
71
68
|
@property
|
|
72
69
|
def storage(self) -> MongoClient:
|
|
@@ -91,21 +88,21 @@ class MongoDBStorageBase(
|
|
|
91
88
|
|
|
92
89
|
@abstractmethod
|
|
93
90
|
def _init_mongo_client(
|
|
94
|
-
self, uri:
|
|
91
|
+
self, uri: str | None, **options: int | str | bool
|
|
95
92
|
) -> MongoClient:
|
|
96
93
|
raise NotImplementedError()
|
|
97
94
|
|
|
98
95
|
@property
|
|
99
96
|
def base_exceptions(
|
|
100
97
|
self,
|
|
101
|
-
) ->
|
|
98
|
+
) -> type[Exception] | tuple[type[Exception], ...]: # pragma: no cover
|
|
102
99
|
return self.lib_errors.PyMongoError # type: ignore
|
|
103
100
|
|
|
104
101
|
def __initialize_database(self) -> None:
|
|
105
102
|
self.counters.create_index("expireAt", expireAfterSeconds=0)
|
|
106
103
|
self.windows.create_index("expireAt", expireAfterSeconds=0)
|
|
107
104
|
|
|
108
|
-
def reset(self) ->
|
|
105
|
+
def reset(self) -> int | None:
|
|
109
106
|
"""
|
|
110
107
|
Delete all rate limit keys in the rate limit collections (counters, windows)
|
|
111
108
|
"""
|
|
@@ -259,7 +256,7 @@ class MongoDBStorageBase(
|
|
|
259
256
|
try:
|
|
260
257
|
updates: dict[
|
|
261
258
|
str,
|
|
262
|
-
dict[str,
|
|
259
|
+
dict[str, datetime.datetime | dict[str, list[float] | int]],
|
|
263
260
|
] = {
|
|
264
261
|
"$push": {
|
|
265
262
|
"entries": {
|
|
@@ -492,6 +489,6 @@ class MongoDBStorage(MongoDBStorageBase):
|
|
|
492
489
|
STORAGE_SCHEME = ["mongodb", "mongodb+srv"]
|
|
493
490
|
|
|
494
491
|
def _init_mongo_client(
|
|
495
|
-
self, uri:
|
|
492
|
+
self, uri: str | None, **options: int | str | bool
|
|
496
493
|
) -> MongoClient:
|
|
497
494
|
return cast(MongoClient, self.lib.MongoClient(uri, **options))
|
limits/storage/redis.py
CHANGED
|
@@ -3,9 +3,10 @@ from __future__ import annotations
|
|
|
3
3
|
import time
|
|
4
4
|
from typing import TYPE_CHECKING, cast
|
|
5
5
|
|
|
6
|
+
from deprecated.sphinx import versionchanged
|
|
6
7
|
from packaging.version import Version
|
|
7
8
|
|
|
8
|
-
from limits.typing import
|
|
9
|
+
from limits.typing import Literal, RedisClient
|
|
9
10
|
|
|
10
11
|
from ..util import get_package_data
|
|
11
12
|
from .base import MovingWindowSupport, SlidingWindowCounterSupport, Storage
|
|
@@ -14,17 +15,32 @@ if TYPE_CHECKING:
|
|
|
14
15
|
import redis
|
|
15
16
|
|
|
16
17
|
|
|
18
|
+
@versionchanged(
|
|
19
|
+
version="4.3",
|
|
20
|
+
reason=(
|
|
21
|
+
"Added support for using the redis client from :pypi:`valkey`"
|
|
22
|
+
" if :paramref:`uri` has the ``valkey://`` schema"
|
|
23
|
+
),
|
|
24
|
+
)
|
|
17
25
|
class RedisStorage(Storage, MovingWindowSupport, SlidingWindowCounterSupport):
|
|
18
26
|
"""
|
|
19
27
|
Rate limit storage with redis as backend.
|
|
20
28
|
|
|
21
|
-
Depends on :pypi:`redis
|
|
29
|
+
Depends on :pypi:`redis` (or :pypi:`valkey` if :paramref:`uri` starts with
|
|
30
|
+
``valkey://``)
|
|
22
31
|
"""
|
|
23
32
|
|
|
24
|
-
STORAGE_SCHEME = [
|
|
33
|
+
STORAGE_SCHEME = [
|
|
34
|
+
"redis",
|
|
35
|
+
"rediss",
|
|
36
|
+
"redis+unix",
|
|
37
|
+
"valkey",
|
|
38
|
+
"valkeys",
|
|
39
|
+
"valkey+unix",
|
|
40
|
+
]
|
|
25
41
|
"""The storage scheme for redis"""
|
|
26
42
|
|
|
27
|
-
DEPENDENCIES = {"redis": Version("3.0")}
|
|
43
|
+
DEPENDENCIES = {"redis": Version("3.0"), "valkey": Version("6.0")}
|
|
28
44
|
|
|
29
45
|
RES_DIR = "resources/redis/lua_scripts"
|
|
30
46
|
|
|
@@ -40,19 +56,20 @@ class RedisStorage(Storage, MovingWindowSupport, SlidingWindowCounterSupport):
|
|
|
40
56
|
f"{RES_DIR}/acquire_sliding_window.lua"
|
|
41
57
|
)
|
|
42
58
|
|
|
43
|
-
lua_moving_window:
|
|
44
|
-
lua_acquire_moving_window:
|
|
45
|
-
lua_sliding_window:
|
|
46
|
-
lua_acquire_sliding_window:
|
|
59
|
+
lua_moving_window: redis.commands.core.Script
|
|
60
|
+
lua_acquire_moving_window: redis.commands.core.Script
|
|
61
|
+
lua_sliding_window: redis.commands.core.Script
|
|
62
|
+
lua_acquire_sliding_window: redis.commands.core.Script
|
|
47
63
|
|
|
48
64
|
PREFIX = "LIMITS"
|
|
65
|
+
target_server: Literal["redis", "valkey"]
|
|
49
66
|
|
|
50
67
|
def __init__(
|
|
51
68
|
self,
|
|
52
69
|
uri: str,
|
|
53
|
-
connection_pool:
|
|
70
|
+
connection_pool: redis.connection.ConnectionPool | None = None,
|
|
54
71
|
wrap_exceptions: bool = False,
|
|
55
|
-
**options:
|
|
72
|
+
**options: float | str | bool,
|
|
56
73
|
) -> None:
|
|
57
74
|
"""
|
|
58
75
|
:param uri: uri of the form ``redis://[:password]@host:port``,
|
|
@@ -60,6 +77,9 @@ class RedisStorage(Storage, MovingWindowSupport, SlidingWindowCounterSupport):
|
|
|
60
77
|
``rediss://[:password]@host:port``, ``redis+unix:///path/to/sock`` etc.
|
|
61
78
|
This uri is passed directly to :func:`redis.from_url` except for the
|
|
62
79
|
case of ``redis+unix://`` where it is replaced with ``unix://``.
|
|
80
|
+
|
|
81
|
+
If the uri scheme is ``valkey`` the implementation used will be from
|
|
82
|
+
:pypi:`valkey`.
|
|
63
83
|
:param connection_pool: if provided, the redis client is initialized with
|
|
64
84
|
the connection pool and any other params passed as :paramref:`options`
|
|
65
85
|
:param wrap_exceptions: Whether to wrap storage exceptions in
|
|
@@ -69,23 +89,33 @@ class RedisStorage(Storage, MovingWindowSupport, SlidingWindowCounterSupport):
|
|
|
69
89
|
:raise ConfigurationError: when the :pypi:`redis` library is not available
|
|
70
90
|
"""
|
|
71
91
|
super().__init__(uri, wrap_exceptions=wrap_exceptions, **options)
|
|
72
|
-
self.
|
|
92
|
+
self.target_server = "valkey" if uri.startswith("valkey") else "redis"
|
|
93
|
+
self.dependency = self.dependencies[self.target_server].module
|
|
73
94
|
|
|
74
|
-
uri = uri.replace("
|
|
95
|
+
uri = uri.replace(f"{self.target_server}+unix", "unix")
|
|
75
96
|
|
|
76
97
|
if not connection_pool:
|
|
77
98
|
self.storage = self.dependency.from_url(uri, **options)
|
|
78
99
|
else:
|
|
79
|
-
self.
|
|
80
|
-
|
|
81
|
-
|
|
100
|
+
if self.target_server == "redis":
|
|
101
|
+
self.storage = self.dependency.Redis(
|
|
102
|
+
connection_pool=connection_pool, **options
|
|
103
|
+
)
|
|
104
|
+
else:
|
|
105
|
+
self.storage = self.dependency.Valkey(
|
|
106
|
+
connection_pool=connection_pool, **options
|
|
107
|
+
)
|
|
82
108
|
self.initialize_storage(uri)
|
|
83
109
|
|
|
84
110
|
@property
|
|
85
111
|
def base_exceptions(
|
|
86
112
|
self,
|
|
87
|
-
) ->
|
|
88
|
-
return
|
|
113
|
+
) -> type[Exception] | tuple[type[Exception], ...]: # pragma: no cover
|
|
114
|
+
return ( # type: ignore[no-any-return]
|
|
115
|
+
self.dependency.RedisError
|
|
116
|
+
if self.target_server == "redis"
|
|
117
|
+
else self.dependency.ValkeyError
|
|
118
|
+
)
|
|
89
119
|
|
|
90
120
|
def initialize_storage(self, _uri: str) -> None:
|
|
91
121
|
self.lua_moving_window = self.get_connection().register_script(
|
|
@@ -268,7 +298,7 @@ class RedisStorage(Storage, MovingWindowSupport, SlidingWindowCounterSupport):
|
|
|
268
298
|
except: # noqa
|
|
269
299
|
return False
|
|
270
300
|
|
|
271
|
-
def reset(self) ->
|
|
301
|
+
def reset(self) -> int | None:
|
|
272
302
|
"""
|
|
273
303
|
This function calls a Lua Script to delete keys prefixed with
|
|
274
304
|
``self.PREFIX`` in blocks of 5000.
|
limits/storage/redis_cluster.py
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import urllib
|
|
2
4
|
|
|
3
5
|
from deprecated.sphinx import versionchanged
|
|
4
6
|
from packaging.version import Version
|
|
5
7
|
|
|
6
8
|
from limits.storage.redis import RedisStorage
|
|
7
|
-
from limits.typing import Optional, Union
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
@versionchanged(
|
|
@@ -24,34 +25,46 @@ however if the version of the package is lower than ``4.2.0`` the implementation
|
|
|
24
25
|
will fallback to trying to use :class:`rediscluster.RedisCluster`.
|
|
25
26
|
""",
|
|
26
27
|
)
|
|
28
|
+
@versionchanged(
|
|
29
|
+
version="4.3",
|
|
30
|
+
reason=(
|
|
31
|
+
"Added support for using the redis client from :pypi:`valkey`"
|
|
32
|
+
" if :paramref:`uri` has the ``valkey+cluster://`` schema"
|
|
33
|
+
),
|
|
34
|
+
)
|
|
27
35
|
class RedisClusterStorage(RedisStorage):
|
|
28
36
|
"""
|
|
29
37
|
Rate limit storage with redis cluster as backend
|
|
30
38
|
|
|
31
|
-
Depends on :pypi:`redis
|
|
39
|
+
Depends on :pypi:`redis` (or :pypi:`valkey` if :paramref:`uri`
|
|
40
|
+
starts with ``valkey+cluster://``).
|
|
32
41
|
"""
|
|
33
42
|
|
|
34
|
-
STORAGE_SCHEME = ["redis+cluster"]
|
|
43
|
+
STORAGE_SCHEME = ["redis+cluster", "valkey+cluster"]
|
|
35
44
|
"""The storage scheme for redis cluster"""
|
|
36
45
|
|
|
37
|
-
DEFAULT_OPTIONS: dict[str,
|
|
46
|
+
DEFAULT_OPTIONS: dict[str, float | str | bool] = {
|
|
38
47
|
"max_connections": 1000,
|
|
39
48
|
}
|
|
40
49
|
"Default options passed to the :class:`~redis.cluster.RedisCluster`"
|
|
41
50
|
|
|
42
51
|
DEPENDENCIES = {
|
|
43
52
|
"redis": Version("4.2.0"),
|
|
53
|
+
"valkey": Version("6.0"),
|
|
44
54
|
}
|
|
45
55
|
|
|
46
56
|
def __init__(
|
|
47
57
|
self,
|
|
48
58
|
uri: str,
|
|
49
59
|
wrap_exceptions: bool = False,
|
|
50
|
-
**options:
|
|
60
|
+
**options: float | str | bool,
|
|
51
61
|
) -> None:
|
|
52
62
|
"""
|
|
53
63
|
:param uri: url of the form
|
|
54
64
|
``redis+cluster://[:password]@host:port,host:port``
|
|
65
|
+
|
|
66
|
+
If the uri scheme is ``valkey+cluster`` the implementation used will be from
|
|
67
|
+
:pypi:`valkey`.
|
|
55
68
|
:param wrap_exceptions: Whether to wrap storage exceptions in
|
|
56
69
|
:exc:`limits.errors.StorageError` before raising it.
|
|
57
70
|
:param options: all remaining keyword arguments are passed
|
|
@@ -60,7 +73,7 @@ class RedisClusterStorage(RedisStorage):
|
|
|
60
73
|
available or if the redis cluster cannot be reached.
|
|
61
74
|
"""
|
|
62
75
|
parsed = urllib.parse.urlparse(uri)
|
|
63
|
-
parsed_auth: dict[str,
|
|
76
|
+
parsed_auth: dict[str, float | str | bool] = {}
|
|
64
77
|
|
|
65
78
|
if parsed.username:
|
|
66
79
|
parsed_auth["username"] = parsed.username
|
|
@@ -74,17 +87,24 @@ class RedisClusterStorage(RedisStorage):
|
|
|
74
87
|
cluster_hosts.append((host, int(port)))
|
|
75
88
|
|
|
76
89
|
self.storage = None
|
|
90
|
+
self.target_server = "valkey" if uri.startswith("valkey") else "redis"
|
|
77
91
|
merged_options = {**self.DEFAULT_OPTIONS, **parsed_auth, **options}
|
|
78
|
-
self.dependency = self.dependencies[
|
|
92
|
+
self.dependency = self.dependencies[self.target_server].module
|
|
79
93
|
startup_nodes = [self.dependency.cluster.ClusterNode(*c) for c in cluster_hosts]
|
|
80
|
-
self.
|
|
81
|
-
|
|
82
|
-
|
|
94
|
+
if self.target_server == "redis":
|
|
95
|
+
self.storage = self.dependency.cluster.RedisCluster(
|
|
96
|
+
startup_nodes=startup_nodes, **merged_options
|
|
97
|
+
)
|
|
98
|
+
else:
|
|
99
|
+
self.storage = self.dependency.cluster.ValkeyCluster(
|
|
100
|
+
startup_nodes=startup_nodes, **merged_options
|
|
101
|
+
)
|
|
102
|
+
|
|
83
103
|
assert self.storage
|
|
84
104
|
self.initialize_storage(uri)
|
|
85
105
|
super(RedisStorage, self).__init__(uri, wrap_exceptions, **options)
|
|
86
106
|
|
|
87
|
-
def reset(self) ->
|
|
107
|
+
def reset(self) -> int | None:
|
|
88
108
|
"""
|
|
89
109
|
Redis Clusters are sharded and deleting across shards
|
|
90
110
|
can't be done atomically. Because of this, this reset loops over all
|
limits/storage/redis_sentinel.py
CHANGED
|
@@ -1,40 +1,59 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import urllib.parse
|
|
2
4
|
from typing import TYPE_CHECKING
|
|
3
5
|
|
|
6
|
+
from deprecated.sphinx import versionchanged
|
|
4
7
|
from packaging.version import Version
|
|
5
8
|
|
|
6
9
|
from limits.errors import ConfigurationError
|
|
7
10
|
from limits.storage.redis import RedisStorage
|
|
8
|
-
from limits.typing import
|
|
11
|
+
from limits.typing import RedisClient
|
|
9
12
|
|
|
10
13
|
if TYPE_CHECKING:
|
|
11
14
|
pass
|
|
12
15
|
|
|
13
16
|
|
|
17
|
+
@versionchanged(
|
|
18
|
+
version="4.3",
|
|
19
|
+
reason=(
|
|
20
|
+
"Added support for using the redis client from :pypi:`valkey`"
|
|
21
|
+
" if :paramref:`uri` has the ``valkey+sentinel://`` schema"
|
|
22
|
+
),
|
|
23
|
+
)
|
|
14
24
|
class RedisSentinelStorage(RedisStorage):
|
|
15
25
|
"""
|
|
16
26
|
Rate limit storage with redis sentinel as backend
|
|
17
27
|
|
|
18
|
-
Depends on :pypi:`redis` package
|
|
28
|
+
Depends on :pypi:`redis` package (or :pypi:`valkey` if :paramref:`uri` starts with
|
|
29
|
+
``valkey+sentinel://``)
|
|
19
30
|
"""
|
|
20
31
|
|
|
21
|
-
STORAGE_SCHEME = ["redis+sentinel"]
|
|
32
|
+
STORAGE_SCHEME = ["redis+sentinel", "valkey+sentinel"]
|
|
22
33
|
"""The storage scheme for redis accessed via a redis sentinel installation"""
|
|
23
34
|
|
|
24
|
-
DEPENDENCIES = {
|
|
35
|
+
DEPENDENCIES = {
|
|
36
|
+
"redis": Version("3.0"),
|
|
37
|
+
"redis.sentinel": Version("3.0"),
|
|
38
|
+
"valkey": Version("6.0"),
|
|
39
|
+
"valkey.sentinel": Version("6.0"),
|
|
40
|
+
}
|
|
25
41
|
|
|
26
42
|
def __init__(
|
|
27
43
|
self,
|
|
28
44
|
uri: str,
|
|
29
|
-
service_name:
|
|
45
|
+
service_name: str | None = None,
|
|
30
46
|
use_replicas: bool = True,
|
|
31
|
-
sentinel_kwargs:
|
|
47
|
+
sentinel_kwargs: dict[str, float | str | bool] | None = None,
|
|
32
48
|
wrap_exceptions: bool = False,
|
|
33
|
-
**options:
|
|
49
|
+
**options: float | str | bool,
|
|
34
50
|
) -> None:
|
|
35
51
|
"""
|
|
36
52
|
:param uri: url of the form
|
|
37
53
|
``redis+sentinel://host:port,host:port/service_name``
|
|
54
|
+
|
|
55
|
+
If the uri scheme is ``valkey+sentinel`` the implementation used will be from
|
|
56
|
+
:pypi:`valkey`.
|
|
38
57
|
:param service_name: sentinel service name
|
|
39
58
|
(if not provided in :attr:`uri`)
|
|
40
59
|
:param use_replicas: Whether to use replicas for read only operations
|
|
@@ -56,7 +75,7 @@ class RedisSentinelStorage(RedisStorage):
|
|
|
56
75
|
sentinel_configuration = []
|
|
57
76
|
sentinel_options = sentinel_kwargs.copy() if sentinel_kwargs else {}
|
|
58
77
|
|
|
59
|
-
parsed_auth: dict[str,
|
|
78
|
+
parsed_auth: dict[str, float | str | bool] = {}
|
|
60
79
|
|
|
61
80
|
if parsed.username:
|
|
62
81
|
parsed_auth["username"] = parsed.username
|
|
@@ -75,7 +94,8 @@ class RedisSentinelStorage(RedisStorage):
|
|
|
75
94
|
if self.service_name is None:
|
|
76
95
|
raise ConfigurationError("'service_name' not provided")
|
|
77
96
|
|
|
78
|
-
|
|
97
|
+
self.target_server = "valkey" if uri.startswith("valkey") else "redis"
|
|
98
|
+
sentinel_dep = self.dependencies[f"{self.target_server}.sentinel"].module
|
|
79
99
|
self.sentinel = sentinel_dep.Sentinel(
|
|
80
100
|
sentinel_configuration,
|
|
81
101
|
sentinel_kwargs={**parsed_auth, **sentinel_options},
|
|
@@ -89,8 +109,12 @@ class RedisSentinelStorage(RedisStorage):
|
|
|
89
109
|
@property
|
|
90
110
|
def base_exceptions(
|
|
91
111
|
self,
|
|
92
|
-
) ->
|
|
93
|
-
return
|
|
112
|
+
) -> type[Exception] | tuple[type[Exception], ...]: # pragma: no cover
|
|
113
|
+
return ( # type: ignore[no-any-return]
|
|
114
|
+
self.dependencies["redis"].module.RedisError
|
|
115
|
+
if self.target_server == "redis"
|
|
116
|
+
else self.dependencies["valkey"].module.ValkeyError
|
|
117
|
+
)
|
|
94
118
|
|
|
95
119
|
def get_connection(self, readonly: bool = False) -> RedisClient:
|
|
96
120
|
return self.storage_slave if (readonly and self.use_replicas) else self.storage
|
limits/storage/registry.py
CHANGED
|
@@ -2,14 +2,12 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
from abc import ABCMeta
|
|
4
4
|
|
|
5
|
-
from limits.typing import Union
|
|
6
|
-
|
|
7
5
|
SCHEMES: dict[str, StorageRegistry] = {}
|
|
8
6
|
|
|
9
7
|
|
|
10
8
|
class StorageRegistry(ABCMeta):
|
|
11
9
|
def __new__(
|
|
12
|
-
mcs, name: str, bases: tuple[type, ...], dct: dict[str,
|
|
10
|
+
mcs, name: str, bases: tuple[type, ...], dct: dict[str, str | list[str]]
|
|
13
11
|
) -> StorageRegistry:
|
|
14
12
|
storage_scheme = dct.get("STORAGE_SCHEME", None)
|
|
15
13
|
cls = super().__new__(mcs, name, bases, dct)
|
limits/strategies.py
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
Rate limiting strategies
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
5
7
|
import time
|
|
6
8
|
from abc import ABCMeta, abstractmethod
|
|
7
9
|
from math import floor, inf
|
|
@@ -12,7 +14,7 @@ from limits.storage.base import SlidingWindowCounterSupport
|
|
|
12
14
|
|
|
13
15
|
from .limits import RateLimitItem
|
|
14
16
|
from .storage import MovingWindowSupport, Storage, StorageTypes
|
|
15
|
-
from .typing import
|
|
17
|
+
from .typing import cast
|
|
16
18
|
from .util import WindowStats
|
|
17
19
|
|
|
18
20
|
|
|
@@ -72,7 +74,7 @@ class MovingWindowRateLimiter(RateLimiter):
|
|
|
72
74
|
):
|
|
73
75
|
raise NotImplementedError(
|
|
74
76
|
"MovingWindowRateLimiting is not implemented for storage "
|
|
75
|
-
"of type
|
|
77
|
+
f"of type {storage.__class__}"
|
|
76
78
|
)
|
|
77
79
|
super().__init__(storage)
|
|
78
80
|
|
|
@@ -191,7 +193,7 @@ class SlidingWindowCounterRateLimiter(RateLimiter):
|
|
|
191
193
|
):
|
|
192
194
|
raise NotImplementedError(
|
|
193
195
|
"SlidingWindowCounterRateLimiting is not implemented for storage "
|
|
194
|
-
"of type
|
|
196
|
+
f"of type {storage.__class__}"
|
|
195
197
|
)
|
|
196
198
|
super().__init__(storage)
|
|
197
199
|
|
|
@@ -311,12 +313,12 @@ class FixedWindowElasticExpiryRateLimiter(FixedWindowRateLimiter):
|
|
|
311
313
|
)
|
|
312
314
|
|
|
313
315
|
|
|
314
|
-
KnownStrategy =
|
|
315
|
-
type[SlidingWindowCounterRateLimiter]
|
|
316
|
-
type[FixedWindowRateLimiter]
|
|
317
|
-
type[FixedWindowElasticExpiryRateLimiter]
|
|
318
|
-
type[MovingWindowRateLimiter]
|
|
319
|
-
|
|
316
|
+
KnownStrategy = (
|
|
317
|
+
type[SlidingWindowCounterRateLimiter]
|
|
318
|
+
| type[FixedWindowRateLimiter]
|
|
319
|
+
| type[FixedWindowElasticExpiryRateLimiter]
|
|
320
|
+
| type[MovingWindowRateLimiter]
|
|
321
|
+
)
|
|
320
322
|
|
|
321
323
|
STRATEGIES: dict[str, KnownStrategy] = {
|
|
322
324
|
"sliding-window-counter": SlidingWindowCounterRateLimiter,
|