limits 3.12.0__tar.gz → 3.14.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {limits-3.12.0 → limits-3.14.0}/HISTORY.rst +27 -0
- {limits-3.12.0 → limits-3.14.0}/PKG-INFO +4 -5
- {limits-3.12.0 → limits-3.14.0}/limits/_version.py +3 -3
- {limits-3.12.0 → limits-3.14.0}/limits/aio/storage/mongodb.py +66 -34
- {limits-3.12.0 → limits-3.14.0}/limits/aio/strategies.py +10 -5
- {limits-3.12.0 → limits-3.14.0}/limits/storage/mongodb.py +16 -3
- limits-3.14.0/limits/storage/redis_cluster.py +105 -0
- {limits-3.12.0 → limits-3.14.0}/limits/storage/redis_sentinel.py +7 -1
- {limits-3.12.0 → limits-3.14.0}/limits/strategies.py +8 -5
- {limits-3.12.0 → limits-3.14.0}/limits/typing.py +2 -2
- {limits-3.12.0 → limits-3.14.0}/limits/util.py +3 -3
- {limits-3.12.0 → limits-3.14.0}/limits.egg-info/PKG-INFO +4 -5
- {limits-3.12.0 → limits-3.14.0}/limits.egg-info/requires.txt +2 -3
- {limits-3.12.0 → limits-3.14.0}/requirements/docs.txt +3 -3
- {limits-3.12.0 → limits-3.14.0}/requirements/main.txt +0 -1
- limits-3.14.0/requirements/storage/async-memcached.txt +2 -0
- {limits-3.12.0 → limits-3.14.0}/requirements/test.txt +2 -2
- {limits-3.12.0 → limits-3.14.0}/setup.py +1 -1
- {limits-3.12.0 → limits-3.14.0}/tests/test_storage.py +14 -13
- {limits-3.12.0 → limits-3.14.0}/tests/test_strategy.py +5 -3
- limits-3.12.0/limits/storage/redis_cluster.py +0 -139
- limits-3.12.0/requirements/storage/async-memcached.txt +0 -2
- {limits-3.12.0 → limits-3.14.0}/CLASSIFIERS +0 -0
- {limits-3.12.0 → limits-3.14.0}/CONTRIBUTIONS.rst +0 -0
- {limits-3.12.0 → limits-3.14.0}/LICENSE.txt +0 -0
- {limits-3.12.0 → limits-3.14.0}/MANIFEST.in +0 -0
- {limits-3.12.0 → limits-3.14.0}/README.rst +0 -0
- {limits-3.12.0 → limits-3.14.0}/doc/Makefile +0 -0
- {limits-3.12.0 → limits-3.14.0}/doc/source/_static/custom.css +0 -0
- {limits-3.12.0 → limits-3.14.0}/doc/source/api.rst +0 -0
- {limits-3.12.0 → limits-3.14.0}/doc/source/async.rst +0 -0
- {limits-3.12.0 → limits-3.14.0}/doc/source/changelog.rst +0 -0
- {limits-3.12.0 → limits-3.14.0}/doc/source/conf.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/doc/source/custom-storage.rst +0 -0
- {limits-3.12.0 → limits-3.14.0}/doc/source/index.rst +0 -0
- {limits-3.12.0 → limits-3.14.0}/doc/source/installation.rst +0 -0
- {limits-3.12.0 → limits-3.14.0}/doc/source/quickstart.rst +0 -0
- {limits-3.12.0 → limits-3.14.0}/doc/source/storage.rst +0 -0
- {limits-3.12.0 → limits-3.14.0}/doc/source/strategies.rst +0 -0
- {limits-3.12.0 → limits-3.14.0}/doc/source/theme_config.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/__init__.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/aio/__init__.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/aio/storage/__init__.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/aio/storage/base.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/aio/storage/etcd.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/aio/storage/memcached.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/aio/storage/memory.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/aio/storage/redis.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/errors.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/limits.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/py.typed +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/resources/redis/lua_scripts/acquire_moving_window.lua +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/resources/redis/lua_scripts/clear_keys.lua +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/resources/redis/lua_scripts/incr_expire.lua +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/resources/redis/lua_scripts/moving_window.lua +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/storage/__init__.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/storage/base.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/storage/etcd.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/storage/memcached.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/storage/memory.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/storage/redis.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/storage/registry.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits/version.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits.egg-info/SOURCES.txt +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits.egg-info/dependency_links.txt +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits.egg-info/not-zip-safe +0 -0
- {limits-3.12.0 → limits-3.14.0}/limits.egg-info/top_level.txt +0 -0
- {limits-3.12.0 → limits-3.14.0}/pyproject.toml +0 -0
- {limits-3.12.0 → limits-3.14.0}/requirements/ci.txt +0 -0
- {limits-3.12.0 → limits-3.14.0}/requirements/dev.txt +0 -0
- {limits-3.12.0 → limits-3.14.0}/requirements/storage/async-etcd.txt +0 -0
- {limits-3.12.0 → limits-3.14.0}/requirements/storage/async-mongodb.txt +0 -0
- {limits-3.12.0 → limits-3.14.0}/requirements/storage/async-redis.txt +0 -0
- {limits-3.12.0 → limits-3.14.0}/requirements/storage/etcd.txt +0 -0
- {limits-3.12.0 → limits-3.14.0}/requirements/storage/memcached.txt +0 -0
- {limits-3.12.0 → limits-3.14.0}/requirements/storage/mongodb.txt +0 -0
- {limits-3.12.0 → limits-3.14.0}/requirements/storage/redis.txt +0 -0
- {limits-3.12.0 → limits-3.14.0}/requirements/storage/rediscluster.txt +0 -0
- {limits-3.12.0 → limits-3.14.0}/setup.cfg +0 -0
- {limits-3.12.0 → limits-3.14.0}/tests/test_limit_granularities.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/tests/test_limits.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/tests/test_ratelimit_parser.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/tests/test_utils.py +0 -0
- {limits-3.12.0 → limits-3.14.0}/versioneer.py +0 -0
|
@@ -3,6 +3,31 @@
|
|
|
3
3
|
Changelog
|
|
4
4
|
=========
|
|
5
5
|
|
|
6
|
+
v3.14.0
|
|
7
|
+
-------
|
|
8
|
+
Release Date: 2024-11-29
|
|
9
|
+
|
|
10
|
+
* Feature
|
|
11
|
+
|
|
12
|
+
* Allow custom collection names in mongodb storage
|
|
13
|
+
|
|
14
|
+
* Compatibility
|
|
15
|
+
|
|
16
|
+
* Add support for python 3.13
|
|
17
|
+
* Drop support for python 3.8
|
|
18
|
+
|
|
19
|
+
* Deprecations
|
|
20
|
+
|
|
21
|
+
* Remove fallback support to use redis-py-cluster
|
|
22
|
+
|
|
23
|
+
v3.13.0
|
|
24
|
+
-------
|
|
25
|
+
Release Date: 2024-06-22
|
|
26
|
+
|
|
27
|
+
* Feature
|
|
28
|
+
|
|
29
|
+
* Add ``cost`` parameter to ``test`` methods in strategies.
|
|
30
|
+
|
|
6
31
|
v3.12.0
|
|
7
32
|
-------
|
|
8
33
|
Release Date: 2024-05-12
|
|
@@ -706,6 +731,8 @@ Release Date: 2015-01-08
|
|
|
706
731
|
|
|
707
732
|
|
|
708
733
|
|
|
734
|
+
|
|
735
|
+
|
|
709
736
|
|
|
710
737
|
|
|
711
738
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: limits
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.14.0
|
|
4
4
|
Summary: Rate limiting utilities
|
|
5
5
|
Home-page: https://limits.readthedocs.org
|
|
6
6
|
Author: Ali-Akber Saifee
|
|
@@ -20,10 +20,9 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.11
|
|
21
21
|
Classifier: Programming Language :: Python :: 3.12
|
|
22
22
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
23
|
-
Requires-Python: >=3.
|
|
23
|
+
Requires-Python: >=3.9
|
|
24
24
|
License-File: LICENSE.txt
|
|
25
25
|
Requires-Dist: deprecated>=1.2
|
|
26
|
-
Requires-Dist: importlib_resources>=1.3
|
|
27
26
|
Requires-Dist: packaging<25,>=21
|
|
28
27
|
Requires-Dist: typing_extensions
|
|
29
28
|
Provides-Extra: redis
|
|
@@ -40,7 +39,7 @@ Provides-Extra: async-redis
|
|
|
40
39
|
Requires-Dist: coredis<5,>=3.4.0; extra == "async-redis"
|
|
41
40
|
Provides-Extra: async-memcached
|
|
42
41
|
Requires-Dist: emcache>=0.6.1; python_version < "3.11" and extra == "async-memcached"
|
|
43
|
-
Requires-Dist: emcache>=1; python_version >= "3.11" and extra == "async-memcached"
|
|
42
|
+
Requires-Dist: emcache>=1; (python_version >= "3.11" and python_version < "3.13.0") and extra == "async-memcached"
|
|
44
43
|
Provides-Extra: async-mongodb
|
|
45
44
|
Requires-Dist: motor<4,>=3; extra == "async-mongodb"
|
|
46
45
|
Provides-Extra: async-etcd
|
|
@@ -53,7 +52,7 @@ Requires-Dist: pymongo<5,>4.1; extra == "all"
|
|
|
53
52
|
Requires-Dist: etcd3; extra == "all"
|
|
54
53
|
Requires-Dist: coredis<5,>=3.4.0; extra == "all"
|
|
55
54
|
Requires-Dist: emcache>=0.6.1; python_version < "3.11" and extra == "all"
|
|
56
|
-
Requires-Dist: emcache>=1; python_version >= "3.11" and extra == "all"
|
|
55
|
+
Requires-Dist: emcache>=1; (python_version >= "3.11" and python_version < "3.13.0") and extra == "all"
|
|
57
56
|
Requires-Dist: motor<4,>=3; extra == "all"
|
|
58
57
|
Requires-Dist: aetcd; extra == "all"
|
|
59
58
|
|
|
@@ -8,11 +8,11 @@ import json
|
|
|
8
8
|
|
|
9
9
|
version_json = '''
|
|
10
10
|
{
|
|
11
|
-
"date": "2024-
|
|
11
|
+
"date": "2024-11-29T18:35:52-0800",
|
|
12
12
|
"dirty": false,
|
|
13
13
|
"error": null,
|
|
14
|
-
"full-revisionid": "
|
|
15
|
-
"version": "3.
|
|
14
|
+
"full-revisionid": "68e30b16c1b081d837e0050929de63f38ea3ca38",
|
|
15
|
+
"version": "3.14.0"
|
|
16
16
|
}
|
|
17
17
|
''' # END VERSION_JSON
|
|
18
18
|
|
|
@@ -6,7 +6,7 @@ import datetime
|
|
|
6
6
|
import time
|
|
7
7
|
from typing import Any, cast
|
|
8
8
|
|
|
9
|
-
from deprecated.sphinx import versionadded
|
|
9
|
+
from deprecated.sphinx import versionadded, versionchanged
|
|
10
10
|
|
|
11
11
|
from limits.aio.storage.base import MovingWindowSupport, Storage
|
|
12
12
|
from limits.typing import Dict, Optional, ParamSpec, Tuple, Type, TypeVar, Union
|
|
@@ -17,6 +17,10 @@ R = TypeVar("R")
|
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
@versionadded(version="2.1")
|
|
20
|
+
@versionchanged(
|
|
21
|
+
version="3.14.0",
|
|
22
|
+
reason="Added option to select custom collection names for windows & counters",
|
|
23
|
+
)
|
|
20
24
|
class MongoDBStorage(Storage, MovingWindowSupport):
|
|
21
25
|
"""
|
|
22
26
|
Rate limit storage with MongoDB as backend.
|
|
@@ -35,6 +39,8 @@ class MongoDBStorage(Storage, MovingWindowSupport):
|
|
|
35
39
|
self,
|
|
36
40
|
uri: str,
|
|
37
41
|
database_name: str = "limits",
|
|
42
|
+
counter_collection_name: str = "counters",
|
|
43
|
+
window_collection_name: str = "windows",
|
|
38
44
|
wrap_exceptions: bool = False,
|
|
39
45
|
**options: Union[float, str, bool],
|
|
40
46
|
) -> None:
|
|
@@ -43,6 +49,9 @@ class MongoDBStorage(Storage, MovingWindowSupport):
|
|
|
43
49
|
This uri is passed directly to :class:`~motor.motor_asyncio.AsyncIOMotorClient`
|
|
44
50
|
:param database_name: The database to use for storing the rate limit
|
|
45
51
|
collections.
|
|
52
|
+
:param counter_collection_name: The collection name to use for individual counters
|
|
53
|
+
used in fixed window strategies
|
|
54
|
+
:param window_collection_name: The collection name to use for moving window storage
|
|
46
55
|
:param wrap_exceptions: Whether to wrap storage exceptions in
|
|
47
56
|
:exc:`limits.errors.StorageError` before raising it.
|
|
48
57
|
:param options: all remaining keyword arguments are passed
|
|
@@ -66,6 +75,10 @@ class MongoDBStorage(Storage, MovingWindowSupport):
|
|
|
66
75
|
self.storage.get_io_loop = asyncio.get_running_loop
|
|
67
76
|
|
|
68
77
|
self.__database_name = database_name
|
|
78
|
+
self.__collection_mapping = {
|
|
79
|
+
"counters": counter_collection_name,
|
|
80
|
+
"windows": window_collection_name,
|
|
81
|
+
}
|
|
69
82
|
self.__indices_created = False
|
|
70
83
|
|
|
71
84
|
@property
|
|
@@ -81,8 +94,12 @@ class MongoDBStorage(Storage, MovingWindowSupport):
|
|
|
81
94
|
async def create_indices(self) -> None:
|
|
82
95
|
if not self.__indices_created:
|
|
83
96
|
await asyncio.gather(
|
|
84
|
-
self.database.counters.create_index(
|
|
85
|
-
|
|
97
|
+
self.database[self.__collection_mapping["counters"]].create_index(
|
|
98
|
+
"expireAt", expireAfterSeconds=0
|
|
99
|
+
),
|
|
100
|
+
self.database[self.__collection_mapping["windows"]].create_index(
|
|
101
|
+
"expireAt", expireAfterSeconds=0
|
|
102
|
+
),
|
|
86
103
|
)
|
|
87
104
|
self.__indices_created = True
|
|
88
105
|
|
|
@@ -92,12 +109,15 @@ class MongoDBStorage(Storage, MovingWindowSupport):
|
|
|
92
109
|
"""
|
|
93
110
|
num_keys = sum(
|
|
94
111
|
await asyncio.gather(
|
|
95
|
-
self.database.counters.count_documents(
|
|
96
|
-
|
|
112
|
+
self.database[self.__collection_mapping["counters"]].count_documents(
|
|
113
|
+
{}
|
|
114
|
+
),
|
|
115
|
+
self.database[self.__collection_mapping["windows"]].count_documents({}),
|
|
97
116
|
)
|
|
98
117
|
)
|
|
99
118
|
await asyncio.gather(
|
|
100
|
-
self.database.counters.drop(),
|
|
119
|
+
self.database[self.__collection_mapping["counters"]].drop(),
|
|
120
|
+
self.database[self.__collection_mapping["windows"]].drop(),
|
|
101
121
|
)
|
|
102
122
|
|
|
103
123
|
return cast(int, num_keys)
|
|
@@ -107,15 +127,21 @@ class MongoDBStorage(Storage, MovingWindowSupport):
|
|
|
107
127
|
:param key: the key to clear rate limits for
|
|
108
128
|
"""
|
|
109
129
|
await asyncio.gather(
|
|
110
|
-
self.database.counters.find_one_and_delete(
|
|
111
|
-
|
|
130
|
+
self.database[self.__collection_mapping["counters"]].find_one_and_delete(
|
|
131
|
+
{"_id": key}
|
|
132
|
+
),
|
|
133
|
+
self.database[self.__collection_mapping["windows"]].find_one_and_delete(
|
|
134
|
+
{"_id": key}
|
|
135
|
+
),
|
|
112
136
|
)
|
|
113
137
|
|
|
114
138
|
async def get_expiry(self, key: str) -> int:
|
|
115
139
|
"""
|
|
116
140
|
:param key: the key to get the expiry for
|
|
117
141
|
"""
|
|
118
|
-
counter = await self.database.counters.find_one(
|
|
142
|
+
counter = await self.database[self.__collection_mapping["counters"]].find_one(
|
|
143
|
+
{"_id": key}
|
|
144
|
+
)
|
|
119
145
|
expiry = (
|
|
120
146
|
counter["expireAt"]
|
|
121
147
|
if counter
|
|
@@ -128,7 +154,7 @@ class MongoDBStorage(Storage, MovingWindowSupport):
|
|
|
128
154
|
"""
|
|
129
155
|
:param key: the key to get the counter value for
|
|
130
156
|
"""
|
|
131
|
-
counter = await self.database.counters.find_one(
|
|
157
|
+
counter = await self.database[self.__collection_mapping["counters"]].find_one(
|
|
132
158
|
{
|
|
133
159
|
"_id": key,
|
|
134
160
|
"expireAt": {"$gte": datetime.datetime.now(datetime.timezone.utc)},
|
|
@@ -156,7 +182,9 @@ class MongoDBStorage(Storage, MovingWindowSupport):
|
|
|
156
182
|
seconds=expiry
|
|
157
183
|
)
|
|
158
184
|
|
|
159
|
-
response = await self.database
|
|
185
|
+
response = await self.database[
|
|
186
|
+
self.__collection_mapping["counters"]
|
|
187
|
+
].find_one_and_update(
|
|
160
188
|
{"_id": key},
|
|
161
189
|
[
|
|
162
190
|
{
|
|
@@ -209,30 +237,34 @@ class MongoDBStorage(Storage, MovingWindowSupport):
|
|
|
209
237
|
:return: (start of window, number of acquired entries)
|
|
210
238
|
"""
|
|
211
239
|
timestamp = time.time()
|
|
212
|
-
result =
|
|
213
|
-
[
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
"$
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
"
|
|
221
|
-
|
|
240
|
+
result = (
|
|
241
|
+
await self.database[self.__collection_mapping["windows"]]
|
|
242
|
+
.aggregate(
|
|
243
|
+
[
|
|
244
|
+
{"$match": {"_id": key}},
|
|
245
|
+
{
|
|
246
|
+
"$project": {
|
|
247
|
+
"entries": {
|
|
248
|
+
"$filter": {
|
|
249
|
+
"input": "$entries",
|
|
250
|
+
"as": "entry",
|
|
251
|
+
"cond": {"$gte": ["$$entry", timestamp - expiry]},
|
|
252
|
+
}
|
|
222
253
|
}
|
|
223
254
|
}
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
255
|
+
},
|
|
256
|
+
{"$unwind": "$entries"},
|
|
257
|
+
{
|
|
258
|
+
"$group": {
|
|
259
|
+
"_id": "$_id",
|
|
260
|
+
"min": {"$min": "$entries"},
|
|
261
|
+
"count": {"$sum": 1},
|
|
262
|
+
}
|
|
263
|
+
},
|
|
264
|
+
]
|
|
265
|
+
)
|
|
266
|
+
.to_list(length=1)
|
|
267
|
+
)
|
|
236
268
|
|
|
237
269
|
if result:
|
|
238
270
|
return (int(result[0]["min"]), result[0]["count"])
|
|
@@ -266,7 +298,7 @@ class MongoDBStorage(Storage, MovingWindowSupport):
|
|
|
266
298
|
)
|
|
267
299
|
}
|
|
268
300
|
updates["$push"]["entries"]["$each"] = [timestamp] * amount
|
|
269
|
-
await self.database.windows.update_one(
|
|
301
|
+
await self.database[self.__collection_mapping["windows"]].update_one(
|
|
270
302
|
{
|
|
271
303
|
"_id": key,
|
|
272
304
|
"entries.%d" % (limit - amount): {
|
|
@@ -29,13 +29,14 @@ class RateLimiter(ABC):
|
|
|
29
29
|
raise NotImplementedError
|
|
30
30
|
|
|
31
31
|
@abstractmethod
|
|
32
|
-
async def test(self, item: RateLimitItem, *identifiers: str) -> bool:
|
|
32
|
+
async def test(self, item: RateLimitItem, *identifiers: str, cost: int = 1) -> bool:
|
|
33
33
|
"""
|
|
34
34
|
Check if the rate limit can be consumed
|
|
35
35
|
|
|
36
36
|
:param item: the rate limit item
|
|
37
37
|
:param identifiers: variable list of strings to uniquely identify the
|
|
38
38
|
limit
|
|
39
|
+
:param cost: The expected cost to be consumed, default 1
|
|
39
40
|
"""
|
|
40
41
|
raise NotImplementedError
|
|
41
42
|
|
|
@@ -86,13 +87,14 @@ class MovingWindowRateLimiter(RateLimiter):
|
|
|
86
87
|
item.key_for(*identifiers), item.amount, item.get_expiry(), amount=cost
|
|
87
88
|
)
|
|
88
89
|
|
|
89
|
-
async def test(self, item: RateLimitItem, *identifiers: str) -> bool:
|
|
90
|
+
async def test(self, item: RateLimitItem, *identifiers: str, cost: int = 1) -> bool:
|
|
90
91
|
"""
|
|
91
92
|
Check if the rate limit can be consumed
|
|
92
93
|
|
|
93
94
|
:param item: the rate limit item
|
|
94
95
|
:param identifiers: variable list of strings to uniquely identify the
|
|
95
96
|
limit
|
|
97
|
+
:param cost: The expected cost to be consumed, default 1
|
|
96
98
|
"""
|
|
97
99
|
res = await cast(MovingWindowSupport, self.storage).get_moving_window(
|
|
98
100
|
item.key_for(*identifiers),
|
|
@@ -101,7 +103,7 @@ class MovingWindowRateLimiter(RateLimiter):
|
|
|
101
103
|
)
|
|
102
104
|
amount = res[1]
|
|
103
105
|
|
|
104
|
-
return amount
|
|
106
|
+
return amount <= item.amount - cost
|
|
105
107
|
|
|
106
108
|
async def get_window_stats(
|
|
107
109
|
self, item: RateLimitItem, *identifiers: str
|
|
@@ -147,16 +149,19 @@ class FixedWindowRateLimiter(RateLimiter):
|
|
|
147
149
|
<= item.amount
|
|
148
150
|
)
|
|
149
151
|
|
|
150
|
-
async def test(self, item: RateLimitItem, *identifiers: str) -> bool:
|
|
152
|
+
async def test(self, item: RateLimitItem, *identifiers: str, cost: int = 1) -> bool:
|
|
151
153
|
"""
|
|
152
154
|
Check if the rate limit can be consumed
|
|
153
155
|
|
|
154
156
|
:param item: the rate limit item
|
|
155
157
|
:param identifiers: variable list of strings to uniquely identify the
|
|
156
158
|
limit
|
|
159
|
+
:param cost: The expected cost to be consumed, default 1
|
|
157
160
|
"""
|
|
158
161
|
|
|
159
|
-
return
|
|
162
|
+
return (
|
|
163
|
+
await self.storage.get(item.key_for(*identifiers)) < item.amount - cost + 1
|
|
164
|
+
)
|
|
160
165
|
|
|
161
166
|
async def get_window_stats(
|
|
162
167
|
self, item: RateLimitItem, *identifiers: str
|
|
@@ -6,7 +6,7 @@ import time
|
|
|
6
6
|
from abc import ABC, abstractmethod
|
|
7
7
|
from typing import Any, cast
|
|
8
8
|
|
|
9
|
-
from deprecated.sphinx import versionadded
|
|
9
|
+
from deprecated.sphinx import versionadded, versionchanged
|
|
10
10
|
|
|
11
11
|
from limits.typing import (
|
|
12
12
|
Dict,
|
|
@@ -36,6 +36,8 @@ class MongoDBStorageBase(Storage, MovingWindowSupport, ABC):
|
|
|
36
36
|
self,
|
|
37
37
|
uri: str,
|
|
38
38
|
database_name: str = "limits",
|
|
39
|
+
counter_collection_name: str = "counters",
|
|
40
|
+
window_collection_name: str = "windows",
|
|
39
41
|
wrap_exceptions: bool = False,
|
|
40
42
|
**options: Union[int, str, bool],
|
|
41
43
|
) -> None:
|
|
@@ -44,6 +46,9 @@ class MongoDBStorageBase(Storage, MovingWindowSupport, ABC):
|
|
|
44
46
|
This uri is passed directly to :class:`~pymongo.mongo_client.MongoClient`
|
|
45
47
|
:param database_name: The database to use for storing the rate limit
|
|
46
48
|
collections.
|
|
49
|
+
:param counter_collection_name: The collection name to use for individual counters
|
|
50
|
+
used in fixed window strategies
|
|
51
|
+
:param window_collection_name: The collection name to use for moving window storage
|
|
47
52
|
:param wrap_exceptions: Whether to wrap storage exceptions in
|
|
48
53
|
:exc:`limits.errors.StorageError` before raising it.
|
|
49
54
|
:param options: all remaining keyword arguments are passed to the
|
|
@@ -53,6 +58,10 @@ class MongoDBStorageBase(Storage, MovingWindowSupport, ABC):
|
|
|
53
58
|
|
|
54
59
|
super().__init__(uri, wrap_exceptions=wrap_exceptions, **options)
|
|
55
60
|
self._database_name = database_name
|
|
61
|
+
self._collection_mapping = {
|
|
62
|
+
"counters": counter_collection_name,
|
|
63
|
+
"windows": window_collection_name,
|
|
64
|
+
}
|
|
56
65
|
self.lib = self.dependencies["pymongo"].module
|
|
57
66
|
self.lib_errors, _ = get_dependency("pymongo.errors")
|
|
58
67
|
self._storage_uri = uri
|
|
@@ -74,11 +83,11 @@ class MongoDBStorageBase(Storage, MovingWindowSupport, ABC):
|
|
|
74
83
|
|
|
75
84
|
@property
|
|
76
85
|
def counters(self) -> MongoCollection:
|
|
77
|
-
return self._database["counters"]
|
|
86
|
+
return self._database[self._collection_mapping["counters"]]
|
|
78
87
|
|
|
79
88
|
@property
|
|
80
89
|
def windows(self) -> MongoCollection:
|
|
81
|
-
return self._database["windows"]
|
|
90
|
+
return self._database[self._collection_mapping["windows"]]
|
|
82
91
|
|
|
83
92
|
@abstractmethod
|
|
84
93
|
def _init_mongo_client(
|
|
@@ -278,6 +287,10 @@ class MongoDBStorageBase(Storage, MovingWindowSupport, ABC):
|
|
|
278
287
|
|
|
279
288
|
|
|
280
289
|
@versionadded(version="2.1")
|
|
290
|
+
@versionchanged(
|
|
291
|
+
version="3.14.0",
|
|
292
|
+
reason="Added option to select custom collection names for windows & counters",
|
|
293
|
+
)
|
|
281
294
|
class MongoDBStorage(MongoDBStorageBase):
|
|
282
295
|
STORAGE_SCHEME = ["mongodb", "mongodb+srv"]
|
|
283
296
|
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import urllib
|
|
2
|
+
|
|
3
|
+
from deprecated.sphinx import versionchanged
|
|
4
|
+
from packaging.version import Version
|
|
5
|
+
|
|
6
|
+
from limits.storage.redis import RedisStorage
|
|
7
|
+
from limits.typing import Dict, Optional, Union
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@versionchanged(
|
|
11
|
+
version="3.14.0",
|
|
12
|
+
reason="""
|
|
13
|
+
Dropped support for the :pypi:`redis-py-cluster` library
|
|
14
|
+
which has been abandoned/deprecated.
|
|
15
|
+
""",
|
|
16
|
+
)
|
|
17
|
+
@versionchanged(
|
|
18
|
+
version="2.5.0",
|
|
19
|
+
reason="""
|
|
20
|
+
Cluster support was provided by the :pypi:`redis-py-cluster` library
|
|
21
|
+
which has been absorbed into the official :pypi:`redis` client. By
|
|
22
|
+
default the :class:`redis.cluster.RedisCluster` client will be used
|
|
23
|
+
however if the version of the package is lower than ``4.2.0`` the implementation
|
|
24
|
+
will fallback to trying to use :class:`rediscluster.RedisCluster`.
|
|
25
|
+
""",
|
|
26
|
+
)
|
|
27
|
+
class RedisClusterStorage(RedisStorage):
|
|
28
|
+
"""
|
|
29
|
+
Rate limit storage with redis cluster as backend
|
|
30
|
+
|
|
31
|
+
Depends on :pypi:`redis`.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
STORAGE_SCHEME = ["redis+cluster"]
|
|
35
|
+
"""The storage scheme for redis cluster"""
|
|
36
|
+
|
|
37
|
+
DEFAULT_OPTIONS: Dict[str, Union[float, str, bool]] = {
|
|
38
|
+
"max_connections": 1000,
|
|
39
|
+
}
|
|
40
|
+
"Default options passed to the :class:`~redis.cluster.RedisCluster`"
|
|
41
|
+
|
|
42
|
+
DEPENDENCIES = {
|
|
43
|
+
"redis": Version("4.2.0"),
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
def __init__(
|
|
47
|
+
self,
|
|
48
|
+
uri: str,
|
|
49
|
+
wrap_exceptions: bool = False,
|
|
50
|
+
**options: Union[float, str, bool],
|
|
51
|
+
) -> None:
|
|
52
|
+
"""
|
|
53
|
+
:param uri: url of the form
|
|
54
|
+
``redis+cluster://[:password]@host:port,host:port``
|
|
55
|
+
:param wrap_exceptions: Whether to wrap storage exceptions in
|
|
56
|
+
:exc:`limits.errors.StorageError` before raising it.
|
|
57
|
+
:param options: all remaining keyword arguments are passed
|
|
58
|
+
directly to the constructor of :class:`redis.cluster.RedisCluster`
|
|
59
|
+
:raise ConfigurationError: when the :pypi:`redis` library is not
|
|
60
|
+
available or if the redis cluster cannot be reached.
|
|
61
|
+
"""
|
|
62
|
+
parsed = urllib.parse.urlparse(uri)
|
|
63
|
+
parsed_auth: Dict[str, Union[float, str, bool]] = {}
|
|
64
|
+
|
|
65
|
+
if parsed.username:
|
|
66
|
+
parsed_auth["username"] = parsed.username
|
|
67
|
+
if parsed.password:
|
|
68
|
+
parsed_auth["password"] = parsed.password
|
|
69
|
+
|
|
70
|
+
sep = parsed.netloc.find("@") + 1
|
|
71
|
+
cluster_hosts = []
|
|
72
|
+
for loc in parsed.netloc[sep:].split(","):
|
|
73
|
+
host, port = loc.split(":")
|
|
74
|
+
cluster_hosts.append((host, int(port)))
|
|
75
|
+
|
|
76
|
+
self.storage = None
|
|
77
|
+
merged_options = {**self.DEFAULT_OPTIONS, **parsed_auth, **options}
|
|
78
|
+
self.dependency = self.dependencies["redis"].module
|
|
79
|
+
startup_nodes = [self.dependency.cluster.ClusterNode(*c) for c in cluster_hosts]
|
|
80
|
+
self.storage = self.dependency.cluster.RedisCluster(
|
|
81
|
+
startup_nodes=startup_nodes, **merged_options
|
|
82
|
+
)
|
|
83
|
+
assert self.storage
|
|
84
|
+
self.initialize_storage(uri)
|
|
85
|
+
super(RedisStorage, self).__init__(uri, wrap_exceptions, **options)
|
|
86
|
+
|
|
87
|
+
def reset(self) -> Optional[int]:
|
|
88
|
+
"""
|
|
89
|
+
Redis Clusters are sharded and deleting across shards
|
|
90
|
+
can't be done atomically. Because of this, this reset loops over all
|
|
91
|
+
keys that are prefixed with ``self.PREFIX`` and calls delete on them,
|
|
92
|
+
one at a time.
|
|
93
|
+
|
|
94
|
+
.. warning::
|
|
95
|
+
This operation was not tested with extremely large data sets.
|
|
96
|
+
On a large production based system, care should be taken with its
|
|
97
|
+
usage as it could be slow on very large data sets"""
|
|
98
|
+
|
|
99
|
+
prefix = self.prefixed_key("*")
|
|
100
|
+
count = 0
|
|
101
|
+
for primary in self.storage.get_primaries():
|
|
102
|
+
node = self.storage.get_redis_connection(primary)
|
|
103
|
+
keys = node.keys(prefix)
|
|
104
|
+
count += sum([node.delete(k.decode("utf-8")) for k in keys])
|
|
105
|
+
return count
|
|
@@ -5,7 +5,7 @@ from packaging.version import Version
|
|
|
5
5
|
|
|
6
6
|
from limits.errors import ConfigurationError
|
|
7
7
|
from limits.storage.redis import RedisStorage
|
|
8
|
-
from limits.typing import Dict, Optional, Union
|
|
8
|
+
from limits.typing import Dict, Optional, Tuple, Type, Union
|
|
9
9
|
|
|
10
10
|
if TYPE_CHECKING:
|
|
11
11
|
import redis.sentinel
|
|
@@ -86,6 +86,12 @@ class RedisSentinelStorage(RedisStorage):
|
|
|
86
86
|
self.use_replicas = use_replicas
|
|
87
87
|
self.initialize_storage(uri)
|
|
88
88
|
|
|
89
|
+
@property
|
|
90
|
+
def base_exceptions(
|
|
91
|
+
self,
|
|
92
|
+
) -> Union[Type[Exception], Tuple[Type[Exception], ...]]: # pragma: no cover
|
|
93
|
+
return self.dependencies["redis"].RedisError # type: ignore[no-any-return, attr-defined]
|
|
94
|
+
|
|
89
95
|
def get(self, key: str) -> int:
|
|
90
96
|
"""
|
|
91
97
|
:param key: the key to get the counter value for
|
|
@@ -28,13 +28,14 @@ class RateLimiter(metaclass=ABCMeta):
|
|
|
28
28
|
raise NotImplementedError
|
|
29
29
|
|
|
30
30
|
@abstractmethod
|
|
31
|
-
def test(self, item: RateLimitItem, *identifiers: str) -> bool:
|
|
31
|
+
def test(self, item: RateLimitItem, *identifiers: str, cost: int = 1) -> bool:
|
|
32
32
|
"""
|
|
33
33
|
Check the rate limit without consuming from it.
|
|
34
34
|
|
|
35
35
|
:param item: The rate limit item
|
|
36
36
|
:param identifiers: variable list of strings to uniquely identify this
|
|
37
37
|
instance of the limit
|
|
38
|
+
:param cost: The expected cost to be consumed, default 1
|
|
38
39
|
"""
|
|
39
40
|
raise NotImplementedError
|
|
40
41
|
|
|
@@ -84,13 +85,14 @@ class MovingWindowRateLimiter(RateLimiter):
|
|
|
84
85
|
item.key_for(*identifiers), item.amount, item.get_expiry(), amount=cost
|
|
85
86
|
)
|
|
86
87
|
|
|
87
|
-
def test(self, item: RateLimitItem, *identifiers: str) -> bool:
|
|
88
|
+
def test(self, item: RateLimitItem, *identifiers: str, cost: int = 1) -> bool:
|
|
88
89
|
"""
|
|
89
90
|
Check if the rate limit can be consumed
|
|
90
91
|
|
|
91
92
|
:param item: The rate limit item
|
|
92
93
|
:param identifiers: variable list of strings to uniquely identify this
|
|
93
94
|
instance of the limit
|
|
95
|
+
:param cost: The expected cost to be consumed, default 1
|
|
94
96
|
"""
|
|
95
97
|
|
|
96
98
|
return (
|
|
@@ -99,7 +101,7 @@ class MovingWindowRateLimiter(RateLimiter):
|
|
|
99
101
|
item.amount,
|
|
100
102
|
item.get_expiry(),
|
|
101
103
|
)[1]
|
|
102
|
-
|
|
104
|
+
<= item.amount - cost
|
|
103
105
|
)
|
|
104
106
|
|
|
105
107
|
def get_window_stats(self, item: RateLimitItem, *identifiers: str) -> WindowStats:
|
|
@@ -144,16 +146,17 @@ class FixedWindowRateLimiter(RateLimiter):
|
|
|
144
146
|
<= item.amount
|
|
145
147
|
)
|
|
146
148
|
|
|
147
|
-
def test(self, item: RateLimitItem, *identifiers: str) -> bool:
|
|
149
|
+
def test(self, item: RateLimitItem, *identifiers: str, cost: int = 1) -> bool:
|
|
148
150
|
"""
|
|
149
151
|
Check if the rate limit can be consumed
|
|
150
152
|
|
|
151
153
|
:param item: The rate limit item
|
|
152
154
|
:param identifiers: variable list of strings to uniquely identify this
|
|
153
155
|
instance of the limit
|
|
156
|
+
:param cost: The expected cost to be consumed, default 1
|
|
154
157
|
"""
|
|
155
158
|
|
|
156
|
-
return self.storage.get(item.key_for(*identifiers)) < item.amount
|
|
159
|
+
return self.storage.get(item.key_for(*identifiers)) < item.amount - cost + 1
|
|
157
160
|
|
|
158
161
|
def get_window_stats(self, item: RateLimitItem, *identifiers: str) -> WindowStats:
|
|
159
162
|
"""
|
|
@@ -110,8 +110,8 @@ class ScriptP(Protocol[R_co]):
|
|
|
110
110
|
|
|
111
111
|
|
|
112
112
|
MongoClient: TypeAlias = "pymongo.MongoClient[Dict[str, Any]]" # type:ignore[misc]
|
|
113
|
-
MongoDatabase: TypeAlias = "pymongo.database.Database[Dict[str, Any]]" # type:ignore
|
|
114
|
-
MongoCollection: TypeAlias = "pymongo.collection.Collection[Dict[str, Any]]" # type:ignore
|
|
113
|
+
MongoDatabase: TypeAlias = "pymongo.database.Database[Dict[str, Any]]" # type:ignore
|
|
114
|
+
MongoCollection: TypeAlias = "pymongo.collection.Collection[Dict[str, Any]]" # type:ignore
|
|
115
115
|
|
|
116
116
|
__all__ = [
|
|
117
117
|
"AsyncRedisClient",
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
""" """
|
|
2
2
|
|
|
3
3
|
import dataclasses
|
|
4
|
+
import importlib.resources
|
|
4
5
|
import re
|
|
5
6
|
import sys
|
|
6
7
|
from collections import UserDict
|
|
7
8
|
from types import ModuleType
|
|
8
|
-
from typing import TYPE_CHECKING
|
|
9
|
+
from typing import TYPE_CHECKING
|
|
9
10
|
|
|
10
|
-
import importlib_resources
|
|
11
11
|
from packaging.version import Version
|
|
12
12
|
|
|
13
13
|
from limits.typing import Dict, List, NamedTuple, Optional, Tuple, Type, Union
|
|
@@ -142,7 +142,7 @@ def get_dependency(module_path: str) -> Tuple[Optional[ModuleType], Optional[Ver
|
|
|
142
142
|
|
|
143
143
|
|
|
144
144
|
def get_package_data(path: str) -> bytes:
|
|
145
|
-
return
|
|
145
|
+
return importlib.resources.files("limits").joinpath(path).read_bytes()
|
|
146
146
|
|
|
147
147
|
|
|
148
148
|
def parse_many(limit_string: str) -> List[RateLimitItem]:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: limits
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.14.0
|
|
4
4
|
Summary: Rate limiting utilities
|
|
5
5
|
Home-page: https://limits.readthedocs.org
|
|
6
6
|
Author: Ali-Akber Saifee
|
|
@@ -20,10 +20,9 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.11
|
|
21
21
|
Classifier: Programming Language :: Python :: 3.12
|
|
22
22
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
23
|
-
Requires-Python: >=3.
|
|
23
|
+
Requires-Python: >=3.9
|
|
24
24
|
License-File: LICENSE.txt
|
|
25
25
|
Requires-Dist: deprecated>=1.2
|
|
26
|
-
Requires-Dist: importlib_resources>=1.3
|
|
27
26
|
Requires-Dist: packaging<25,>=21
|
|
28
27
|
Requires-Dist: typing_extensions
|
|
29
28
|
Provides-Extra: redis
|
|
@@ -40,7 +39,7 @@ Provides-Extra: async-redis
|
|
|
40
39
|
Requires-Dist: coredis<5,>=3.4.0; extra == "async-redis"
|
|
41
40
|
Provides-Extra: async-memcached
|
|
42
41
|
Requires-Dist: emcache>=0.6.1; python_version < "3.11" and extra == "async-memcached"
|
|
43
|
-
Requires-Dist: emcache>=1; python_version >= "3.11" and extra == "async-memcached"
|
|
42
|
+
Requires-Dist: emcache>=1; (python_version >= "3.11" and python_version < "3.13.0") and extra == "async-memcached"
|
|
44
43
|
Provides-Extra: async-mongodb
|
|
45
44
|
Requires-Dist: motor<4,>=3; extra == "async-mongodb"
|
|
46
45
|
Provides-Extra: async-etcd
|
|
@@ -53,7 +52,7 @@ Requires-Dist: pymongo<5,>4.1; extra == "all"
|
|
|
53
52
|
Requires-Dist: etcd3; extra == "all"
|
|
54
53
|
Requires-Dist: coredis<5,>=3.4.0; extra == "all"
|
|
55
54
|
Requires-Dist: emcache>=0.6.1; python_version < "3.11" and extra == "all"
|
|
56
|
-
Requires-Dist: emcache>=1; python_version >= "3.11" and extra == "all"
|
|
55
|
+
Requires-Dist: emcache>=1; (python_version >= "3.11" and python_version < "3.13.0") and extra == "all"
|
|
57
56
|
Requires-Dist: motor<4,>=3; extra == "all"
|
|
58
57
|
Requires-Dist: aetcd; extra == "all"
|
|
59
58
|
|