funboost 45.0__py3-none-any.whl → 45.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of funboost might be problematic. Click here for more details.
- funboost/__init__.py +1 -1
- funboost/__init__old.py +1 -1
- funboost/constant.py +3 -0
- funboost/consumers/base_consumer.py +10 -2
- funboost/core/booster.py +6 -5
- funboost/core/current_task.py +1 -1
- funboost/core/func_params_model.py +5 -6
- funboost/core/helper_funs.py +1 -1
- funboost/function_result_web/__pycache__/app.cpython-37.pyc +0 -0
- funboost/function_result_web/__pycache__/functions.cpython-37.pyc +0 -0
- funboost/publishers/base_publisher.py +17 -7
- funboost/set_frame_config.py +1 -1
- funboost/utils/dependency_packages_in_pythonpath/__pycache__/__init__.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/__pycache__/__init__.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/__pycache__/__init__.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/__pycache__/add_to_pythonpath.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/__pycache__/add_to_pythonpath.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/__pycache__/add_to_pythonpath.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__init__.py +59 -59
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/__init__.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/__init__.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/__init__.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/client.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/client.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/client.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/compat.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/compat.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/compat.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/connection.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/connection.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/connection.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/exceptions.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/exceptions.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/exceptions.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/lock.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/lock.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/lock.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/utils.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/utils.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/utils.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/client.py +4804 -4804
- funboost/utils/dependency_packages_in_pythonpath/aioredis/compat.py +8 -8
- funboost/utils/dependency_packages_in_pythonpath/aioredis/connection.py +1668 -1668
- funboost/utils/dependency_packages_in_pythonpath/aioredis/exceptions.py +96 -96
- funboost/utils/dependency_packages_in_pythonpath/aioredis/lock.py +306 -306
- funboost/utils/dependency_packages_in_pythonpath/aioredis/log.py +15 -15
- funboost/utils/dependency_packages_in_pythonpath/aioredis/sentinel.py +329 -329
- funboost/utils/dependency_packages_in_pythonpath/aioredis/utils.py +61 -61
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__init__.py +16 -16
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/StoppableThread.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/StoppableThread.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/StoppableThread.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/__init__.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/__init__.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/__init__.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/dafunc.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/dafunc.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/dafunc.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/exceptions.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/exceptions.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/exceptions.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/py3_raise.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/py3_raise.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/py3_raise.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/exceptions.py +98 -98
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/py2_raise.py +7 -7
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/py3_raise.py +7 -7
- funboost/utils/times/__init__.py +85 -85
- funboost/utils/times/version.py +1 -1
- {funboost-45.0.dist-info → funboost-45.3.dist-info}/METADATA +7 -6
- {funboost-45.0.dist-info → funboost-45.3.dist-info}/RECORD +75 -47
- {funboost-45.0.dist-info → funboost-45.3.dist-info}/entry_points.txt +1 -0
- funboost/core/funboost_current_task_context_thread.py +0 -12
- funboost/function_result_web/__pycache__/functions.cpython-39.pyc +0 -0
- {funboost-45.0.dist-info → funboost-45.3.dist-info}/LICENSE +0 -0
- {funboost-45.0.dist-info → funboost-45.3.dist-info}/WHEEL +0 -0
- {funboost-45.0.dist-info → funboost-45.3.dist-info}/top_level.txt +0 -0
|
@@ -1,306 +1,306 @@
|
|
|
1
|
-
import asyncio
|
|
2
|
-
import threading
|
|
3
|
-
import uuid
|
|
4
|
-
from types import SimpleNamespace
|
|
5
|
-
from typing import TYPE_CHECKING, Awaitable, NoReturn, Optional, Union
|
|
6
|
-
|
|
7
|
-
from aioredis.exceptions import LockError, LockNotOwnedError
|
|
8
|
-
|
|
9
|
-
if TYPE_CHECKING:
|
|
10
|
-
from aioredis import Redis
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
class Lock:
|
|
14
|
-
"""
|
|
15
|
-
A shared, distributed Lock. Using Redis for locking allows the Lock
|
|
16
|
-
to be shared across processes and/or machines.
|
|
17
|
-
|
|
18
|
-
It's left to the user to resolve deadlock issues and make sure
|
|
19
|
-
multiple clients play nicely together.
|
|
20
|
-
"""
|
|
21
|
-
|
|
22
|
-
lua_release = None
|
|
23
|
-
lua_extend = None
|
|
24
|
-
lua_reacquire = None
|
|
25
|
-
|
|
26
|
-
# KEYS[1] - lock name
|
|
27
|
-
# ARGV[1] - token
|
|
28
|
-
# return 1 if the lock was released, otherwise 0
|
|
29
|
-
LUA_RELEASE_SCRIPT = """
|
|
30
|
-
local token = redis.call('get', KEYS[1])
|
|
31
|
-
if not token or token ~= ARGV[1] then
|
|
32
|
-
return 0
|
|
33
|
-
end
|
|
34
|
-
redis.call('del', KEYS[1])
|
|
35
|
-
return 1
|
|
36
|
-
"""
|
|
37
|
-
|
|
38
|
-
# KEYS[1] - lock name
|
|
39
|
-
# ARGV[1] - token
|
|
40
|
-
# ARGV[2] - additional milliseconds
|
|
41
|
-
# ARGV[3] - "0" if the additional time should be added to the lock's
|
|
42
|
-
# existing ttl or "1" if the existing ttl should be replaced
|
|
43
|
-
# return 1 if the locks time was extended, otherwise 0
|
|
44
|
-
LUA_EXTEND_SCRIPT = """
|
|
45
|
-
local token = redis.call('get', KEYS[1])
|
|
46
|
-
if not token or token ~= ARGV[1] then
|
|
47
|
-
return 0
|
|
48
|
-
end
|
|
49
|
-
local expiration = redis.call('pttl', KEYS[1])
|
|
50
|
-
if not expiration then
|
|
51
|
-
expiration = 0
|
|
52
|
-
end
|
|
53
|
-
if expiration < 0 then
|
|
54
|
-
return 0
|
|
55
|
-
end
|
|
56
|
-
|
|
57
|
-
local newttl = ARGV[2]
|
|
58
|
-
if ARGV[3] == "0" then
|
|
59
|
-
newttl = ARGV[2] + expiration
|
|
60
|
-
end
|
|
61
|
-
redis.call('pexpire', KEYS[1], newttl)
|
|
62
|
-
return 1
|
|
63
|
-
"""
|
|
64
|
-
|
|
65
|
-
# KEYS[1] - lock name
|
|
66
|
-
# ARGV[1] - token
|
|
67
|
-
# ARGV[2] - milliseconds
|
|
68
|
-
# return 1 if the locks time was reacquired, otherwise 0
|
|
69
|
-
LUA_REACQUIRE_SCRIPT = """
|
|
70
|
-
local token = redis.call('get', KEYS[1])
|
|
71
|
-
if not token or token ~= ARGV[1] then
|
|
72
|
-
return 0
|
|
73
|
-
end
|
|
74
|
-
redis.call('pexpire', KEYS[1], ARGV[2])
|
|
75
|
-
return 1
|
|
76
|
-
"""
|
|
77
|
-
|
|
78
|
-
def __init__(
|
|
79
|
-
self,
|
|
80
|
-
redis: "Redis",
|
|
81
|
-
name: Union[str, bytes, memoryview],
|
|
82
|
-
timeout: Optional[float] = None,
|
|
83
|
-
sleep: float = 0.1,
|
|
84
|
-
blocking: bool = True,
|
|
85
|
-
blocking_timeout: Optional[float] = None,
|
|
86
|
-
thread_local: bool = True,
|
|
87
|
-
):
|
|
88
|
-
"""
|
|
89
|
-
Create a new Lock instance named ``name`` using the Redis client
|
|
90
|
-
supplied by ``redis``.
|
|
91
|
-
|
|
92
|
-
``timeout`` indicates a maximum life for the lock.
|
|
93
|
-
By default, it will remain locked until release() is called.
|
|
94
|
-
``timeout`` can be specified as a float or integer, both representing
|
|
95
|
-
the number of seconds to wait.
|
|
96
|
-
|
|
97
|
-
``sleep`` indicates the amount of time to sleep per loop iteration
|
|
98
|
-
when the lock is in blocking mode and another client is currently
|
|
99
|
-
holding the lock.
|
|
100
|
-
|
|
101
|
-
``blocking`` indicates whether calling ``acquire`` should block until
|
|
102
|
-
the lock has been acquired or to fail immediately, causing ``acquire``
|
|
103
|
-
to return False and the lock not being acquired. Defaults to True.
|
|
104
|
-
Note this value can be overridden by passing a ``blocking``
|
|
105
|
-
argument to ``acquire``.
|
|
106
|
-
|
|
107
|
-
``blocking_timeout`` indicates the maximum amount of time in seconds to
|
|
108
|
-
spend trying to acquire the lock. A value of ``None`` indicates
|
|
109
|
-
continue trying forever. ``blocking_timeout`` can be specified as a
|
|
110
|
-
float or integer, both representing the number of seconds to wait.
|
|
111
|
-
|
|
112
|
-
``thread_local`` indicates whether the lock token is placed in
|
|
113
|
-
thread-local storage. By default, the token is placed in thread local
|
|
114
|
-
storage so that a thread only sees its token, not a token set by
|
|
115
|
-
another thread. Consider the following timeline:
|
|
116
|
-
|
|
117
|
-
time: 0, thread-1 acquires `my-lock`, with a timeout of 5 seconds.
|
|
118
|
-
thread-1 sets the token to "abc"
|
|
119
|
-
time: 1, thread-2 blocks trying to acquire `my-lock` using the
|
|
120
|
-
Lock instance.
|
|
121
|
-
time: 5, thread-1 has not yet completed. redis expires the lock
|
|
122
|
-
key.
|
|
123
|
-
time: 5, thread-2 acquired `my-lock` now that it's available.
|
|
124
|
-
thread-2 sets the token to "xyz"
|
|
125
|
-
time: 6, thread-1 finishes its work and calls release(). if the
|
|
126
|
-
token is *not* stored in thread local storage, then
|
|
127
|
-
thread-1 would see the token value as "xyz" and would be
|
|
128
|
-
able to successfully release the thread-2's lock.
|
|
129
|
-
|
|
130
|
-
In some use cases it's necessary to disable thread local storage. For
|
|
131
|
-
example, if you have code where one thread acquires a lock and passes
|
|
132
|
-
that lock instance to a worker thread to release later. If thread
|
|
133
|
-
local storage isn't disabled in this case, the worker thread won't see
|
|
134
|
-
the token set by the thread that acquired the lock. Our assumption
|
|
135
|
-
is that these cases aren't common and as such default to using
|
|
136
|
-
thread local storage.
|
|
137
|
-
"""
|
|
138
|
-
self.redis = redis
|
|
139
|
-
self.name = name
|
|
140
|
-
self.timeout = timeout
|
|
141
|
-
self.sleep = sleep
|
|
142
|
-
self.blocking = blocking
|
|
143
|
-
self.blocking_timeout = blocking_timeout
|
|
144
|
-
self.thread_local = bool(thread_local)
|
|
145
|
-
self.local = threading.local() if self.thread_local else SimpleNamespace()
|
|
146
|
-
self.local.token = None
|
|
147
|
-
self.register_scripts()
|
|
148
|
-
|
|
149
|
-
def register_scripts(self):
|
|
150
|
-
cls = self.__class__
|
|
151
|
-
client = self.redis
|
|
152
|
-
if cls.lua_release is None:
|
|
153
|
-
cls.lua_release = client.register_script(cls.LUA_RELEASE_SCRIPT)
|
|
154
|
-
if cls.lua_extend is None:
|
|
155
|
-
cls.lua_extend = client.register_script(cls.LUA_EXTEND_SCRIPT)
|
|
156
|
-
if cls.lua_reacquire is None:
|
|
157
|
-
cls.lua_reacquire = client.register_script(cls.LUA_REACQUIRE_SCRIPT)
|
|
158
|
-
|
|
159
|
-
async def __aenter__(self):
|
|
160
|
-
if await self.acquire():
|
|
161
|
-
return self
|
|
162
|
-
raise LockError("Unable to acquire lock within the time specified")
|
|
163
|
-
|
|
164
|
-
async def __aexit__(self, exc_type, exc_value, traceback):
|
|
165
|
-
await self.release()
|
|
166
|
-
|
|
167
|
-
async def acquire(
|
|
168
|
-
self,
|
|
169
|
-
blocking: Optional[bool] = None,
|
|
170
|
-
blocking_timeout: Optional[float] = None,
|
|
171
|
-
token: Optional[Union[str, bytes]] = None,
|
|
172
|
-
):
|
|
173
|
-
"""
|
|
174
|
-
Use Redis to hold a shared, distributed lock named ``name``.
|
|
175
|
-
Returns True once the lock is acquired.
|
|
176
|
-
|
|
177
|
-
If ``blocking`` is False, always return immediately. If the lock
|
|
178
|
-
was acquired, return True, otherwise return False.
|
|
179
|
-
|
|
180
|
-
``blocking_timeout`` specifies the maximum number of seconds to
|
|
181
|
-
wait trying to acquire the lock.
|
|
182
|
-
|
|
183
|
-
``token`` specifies the token value to be used. If provided, token
|
|
184
|
-
must be a bytes object or a string that can be encoded to a bytes
|
|
185
|
-
object with the default encoding. If a token isn't specified, a UUID
|
|
186
|
-
will be generated.
|
|
187
|
-
"""
|
|
188
|
-
loop = asyncio.get_event_loop()
|
|
189
|
-
sleep = self.sleep
|
|
190
|
-
if token is None:
|
|
191
|
-
token = uuid.uuid1().hex.encode()
|
|
192
|
-
else:
|
|
193
|
-
encoder = self.redis.connection_pool.get_encoder()
|
|
194
|
-
token = encoder.encode(token)
|
|
195
|
-
if blocking is None:
|
|
196
|
-
blocking = self.blocking
|
|
197
|
-
if blocking_timeout is None:
|
|
198
|
-
blocking_timeout = self.blocking_timeout
|
|
199
|
-
stop_trying_at = None
|
|
200
|
-
if blocking_timeout is not None:
|
|
201
|
-
stop_trying_at = loop.time() + blocking_timeout
|
|
202
|
-
while True:
|
|
203
|
-
if await self.do_acquire(token):
|
|
204
|
-
self.local.token = token
|
|
205
|
-
return True
|
|
206
|
-
if not blocking:
|
|
207
|
-
return False
|
|
208
|
-
next_try_at = loop.time() + sleep
|
|
209
|
-
if stop_trying_at is not None and next_try_at > stop_trying_at:
|
|
210
|
-
return False
|
|
211
|
-
await asyncio.sleep(sleep)
|
|
212
|
-
|
|
213
|
-
async def do_acquire(self, token: Union[str, bytes]) -> bool:
|
|
214
|
-
if self.timeout:
|
|
215
|
-
# convert to milliseconds
|
|
216
|
-
timeout = int(self.timeout * 1000)
|
|
217
|
-
else:
|
|
218
|
-
timeout = None
|
|
219
|
-
if await self.redis.set(self.name, token, nx=True, px=timeout):
|
|
220
|
-
return True
|
|
221
|
-
return False
|
|
222
|
-
|
|
223
|
-
async def locked(self) -> bool:
|
|
224
|
-
"""
|
|
225
|
-
Returns True if this key is locked by any process, otherwise False.
|
|
226
|
-
"""
|
|
227
|
-
return await self.redis.get(self.name) is not None
|
|
228
|
-
|
|
229
|
-
async def owned(self) -> bool:
|
|
230
|
-
"""
|
|
231
|
-
Returns True if this key is locked by this lock, otherwise False.
|
|
232
|
-
"""
|
|
233
|
-
stored_token = await self.redis.get(self.name)
|
|
234
|
-
# need to always compare bytes to bytes
|
|
235
|
-
# TODO: this can be simplified when the context manager is finished
|
|
236
|
-
if stored_token and not isinstance(stored_token, bytes):
|
|
237
|
-
encoder = self.redis.connection_pool.get_encoder()
|
|
238
|
-
stored_token = encoder.encode(stored_token)
|
|
239
|
-
return self.local.token is not None and stored_token == self.local.token
|
|
240
|
-
|
|
241
|
-
def release(self) -> Awaitable[NoReturn]:
|
|
242
|
-
"""Releases the already acquired lock"""
|
|
243
|
-
expected_token = self.local.token
|
|
244
|
-
if expected_token is None:
|
|
245
|
-
raise LockError("Cannot release an unlocked lock")
|
|
246
|
-
self.local.token = None
|
|
247
|
-
return self.do_release(expected_token)
|
|
248
|
-
|
|
249
|
-
async def do_release(self, expected_token: bytes):
|
|
250
|
-
if not bool(
|
|
251
|
-
await self.lua_release(
|
|
252
|
-
keys=[self.name], args=[expected_token], client=self.redis
|
|
253
|
-
)
|
|
254
|
-
):
|
|
255
|
-
raise LockNotOwnedError("Cannot release a lock" " that's no longer owned")
|
|
256
|
-
|
|
257
|
-
def extend(
|
|
258
|
-
self, additional_time: float, replace_ttl: bool = False
|
|
259
|
-
) -> Awaitable[bool]:
|
|
260
|
-
"""
|
|
261
|
-
Adds more time to an already acquired lock.
|
|
262
|
-
|
|
263
|
-
``additional_time`` can be specified as an integer or a float, both
|
|
264
|
-
representing the number of seconds to add.
|
|
265
|
-
|
|
266
|
-
``replace_ttl`` if False (the default), add `additional_time` to
|
|
267
|
-
the lock's existing ttl. If True, replace the lock's ttl with
|
|
268
|
-
`additional_time`.
|
|
269
|
-
"""
|
|
270
|
-
if self.local.token is None:
|
|
271
|
-
raise LockError("Cannot extend an unlocked lock")
|
|
272
|
-
if self.timeout is None:
|
|
273
|
-
raise LockError("Cannot extend a lock with no timeout")
|
|
274
|
-
return self.do_extend(additional_time, replace_ttl)
|
|
275
|
-
|
|
276
|
-
async def do_extend(self, additional_time, replace_ttl) -> bool:
|
|
277
|
-
additional_time = int(additional_time * 1000)
|
|
278
|
-
if not bool(
|
|
279
|
-
await self.lua_extend(
|
|
280
|
-
keys=[self.name],
|
|
281
|
-
args=[self.local.token, additional_time, replace_ttl and "1" or "0"],
|
|
282
|
-
client=self.redis,
|
|
283
|
-
)
|
|
284
|
-
):
|
|
285
|
-
raise LockNotOwnedError("Cannot extend a lock that's" " no longer owned")
|
|
286
|
-
return True
|
|
287
|
-
|
|
288
|
-
def reacquire(self) -> Awaitable[bool]:
|
|
289
|
-
"""
|
|
290
|
-
Resets a TTL of an already acquired lock back to a timeout value.
|
|
291
|
-
"""
|
|
292
|
-
if self.local.token is None:
|
|
293
|
-
raise LockError("Cannot reacquire an unlocked lock")
|
|
294
|
-
if self.timeout is None:
|
|
295
|
-
raise LockError("Cannot reacquire a lock with no timeout")
|
|
296
|
-
return self.do_reacquire()
|
|
297
|
-
|
|
298
|
-
async def do_reacquire(self) -> bool:
|
|
299
|
-
timeout = int(self.timeout * 1000)
|
|
300
|
-
if not bool(
|
|
301
|
-
await self.lua_reacquire(
|
|
302
|
-
keys=[self.name], args=[self.local.token, timeout], client=self.redis
|
|
303
|
-
)
|
|
304
|
-
):
|
|
305
|
-
raise LockNotOwnedError("Cannot reacquire a lock that's" " no longer owned")
|
|
306
|
-
return True
|
|
1
|
+
import asyncio
|
|
2
|
+
import threading
|
|
3
|
+
import uuid
|
|
4
|
+
from types import SimpleNamespace
|
|
5
|
+
from typing import TYPE_CHECKING, Awaitable, NoReturn, Optional, Union
|
|
6
|
+
|
|
7
|
+
from aioredis.exceptions import LockError, LockNotOwnedError
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from aioredis import Redis
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Lock:
|
|
14
|
+
"""
|
|
15
|
+
A shared, distributed Lock. Using Redis for locking allows the Lock
|
|
16
|
+
to be shared across processes and/or machines.
|
|
17
|
+
|
|
18
|
+
It's left to the user to resolve deadlock issues and make sure
|
|
19
|
+
multiple clients play nicely together.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
lua_release = None
|
|
23
|
+
lua_extend = None
|
|
24
|
+
lua_reacquire = None
|
|
25
|
+
|
|
26
|
+
# KEYS[1] - lock name
|
|
27
|
+
# ARGV[1] - token
|
|
28
|
+
# return 1 if the lock was released, otherwise 0
|
|
29
|
+
LUA_RELEASE_SCRIPT = """
|
|
30
|
+
local token = redis.call('get', KEYS[1])
|
|
31
|
+
if not token or token ~= ARGV[1] then
|
|
32
|
+
return 0
|
|
33
|
+
end
|
|
34
|
+
redis.call('del', KEYS[1])
|
|
35
|
+
return 1
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
# KEYS[1] - lock name
|
|
39
|
+
# ARGV[1] - token
|
|
40
|
+
# ARGV[2] - additional milliseconds
|
|
41
|
+
# ARGV[3] - "0" if the additional time should be added to the lock's
|
|
42
|
+
# existing ttl or "1" if the existing ttl should be replaced
|
|
43
|
+
# return 1 if the locks time was extended, otherwise 0
|
|
44
|
+
LUA_EXTEND_SCRIPT = """
|
|
45
|
+
local token = redis.call('get', KEYS[1])
|
|
46
|
+
if not token or token ~= ARGV[1] then
|
|
47
|
+
return 0
|
|
48
|
+
end
|
|
49
|
+
local expiration = redis.call('pttl', KEYS[1])
|
|
50
|
+
if not expiration then
|
|
51
|
+
expiration = 0
|
|
52
|
+
end
|
|
53
|
+
if expiration < 0 then
|
|
54
|
+
return 0
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
local newttl = ARGV[2]
|
|
58
|
+
if ARGV[3] == "0" then
|
|
59
|
+
newttl = ARGV[2] + expiration
|
|
60
|
+
end
|
|
61
|
+
redis.call('pexpire', KEYS[1], newttl)
|
|
62
|
+
return 1
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
# KEYS[1] - lock name
|
|
66
|
+
# ARGV[1] - token
|
|
67
|
+
# ARGV[2] - milliseconds
|
|
68
|
+
# return 1 if the locks time was reacquired, otherwise 0
|
|
69
|
+
LUA_REACQUIRE_SCRIPT = """
|
|
70
|
+
local token = redis.call('get', KEYS[1])
|
|
71
|
+
if not token or token ~= ARGV[1] then
|
|
72
|
+
return 0
|
|
73
|
+
end
|
|
74
|
+
redis.call('pexpire', KEYS[1], ARGV[2])
|
|
75
|
+
return 1
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
def __init__(
|
|
79
|
+
self,
|
|
80
|
+
redis: "Redis",
|
|
81
|
+
name: Union[str, bytes, memoryview],
|
|
82
|
+
timeout: Optional[float] = None,
|
|
83
|
+
sleep: float = 0.1,
|
|
84
|
+
blocking: bool = True,
|
|
85
|
+
blocking_timeout: Optional[float] = None,
|
|
86
|
+
thread_local: bool = True,
|
|
87
|
+
):
|
|
88
|
+
"""
|
|
89
|
+
Create a new Lock instance named ``name`` using the Redis client
|
|
90
|
+
supplied by ``redis``.
|
|
91
|
+
|
|
92
|
+
``timeout`` indicates a maximum life for the lock.
|
|
93
|
+
By default, it will remain locked until release() is called.
|
|
94
|
+
``timeout`` can be specified as a float or integer, both representing
|
|
95
|
+
the number of seconds to wait.
|
|
96
|
+
|
|
97
|
+
``sleep`` indicates the amount of time to sleep per loop iteration
|
|
98
|
+
when the lock is in blocking mode and another client is currently
|
|
99
|
+
holding the lock.
|
|
100
|
+
|
|
101
|
+
``blocking`` indicates whether calling ``acquire`` should block until
|
|
102
|
+
the lock has been acquired or to fail immediately, causing ``acquire``
|
|
103
|
+
to return False and the lock not being acquired. Defaults to True.
|
|
104
|
+
Note this value can be overridden by passing a ``blocking``
|
|
105
|
+
argument to ``acquire``.
|
|
106
|
+
|
|
107
|
+
``blocking_timeout`` indicates the maximum amount of time in seconds to
|
|
108
|
+
spend trying to acquire the lock. A value of ``None`` indicates
|
|
109
|
+
continue trying forever. ``blocking_timeout`` can be specified as a
|
|
110
|
+
float or integer, both representing the number of seconds to wait.
|
|
111
|
+
|
|
112
|
+
``thread_local`` indicates whether the lock token is placed in
|
|
113
|
+
thread-local storage. By default, the token is placed in thread local
|
|
114
|
+
storage so that a thread only sees its token, not a token set by
|
|
115
|
+
another thread. Consider the following timeline:
|
|
116
|
+
|
|
117
|
+
time: 0, thread-1 acquires `my-lock`, with a timeout of 5 seconds.
|
|
118
|
+
thread-1 sets the token to "abc"
|
|
119
|
+
time: 1, thread-2 blocks trying to acquire `my-lock` using the
|
|
120
|
+
Lock instance.
|
|
121
|
+
time: 5, thread-1 has not yet completed. redis expires the lock
|
|
122
|
+
key.
|
|
123
|
+
time: 5, thread-2 acquired `my-lock` now that it's available.
|
|
124
|
+
thread-2 sets the token to "xyz"
|
|
125
|
+
time: 6, thread-1 finishes its work and calls release(). if the
|
|
126
|
+
token is *not* stored in thread local storage, then
|
|
127
|
+
thread-1 would see the token value as "xyz" and would be
|
|
128
|
+
able to successfully release the thread-2's lock.
|
|
129
|
+
|
|
130
|
+
In some use cases it's necessary to disable thread local storage. For
|
|
131
|
+
example, if you have code where one thread acquires a lock and passes
|
|
132
|
+
that lock instance to a worker thread to release later. If thread
|
|
133
|
+
local storage isn't disabled in this case, the worker thread won't see
|
|
134
|
+
the token set by the thread that acquired the lock. Our assumption
|
|
135
|
+
is that these cases aren't common and as such default to using
|
|
136
|
+
thread local storage.
|
|
137
|
+
"""
|
|
138
|
+
self.redis = redis
|
|
139
|
+
self.name = name
|
|
140
|
+
self.timeout = timeout
|
|
141
|
+
self.sleep = sleep
|
|
142
|
+
self.blocking = blocking
|
|
143
|
+
self.blocking_timeout = blocking_timeout
|
|
144
|
+
self.thread_local = bool(thread_local)
|
|
145
|
+
self.local = threading.local() if self.thread_local else SimpleNamespace()
|
|
146
|
+
self.local.token = None
|
|
147
|
+
self.register_scripts()
|
|
148
|
+
|
|
149
|
+
def register_scripts(self):
|
|
150
|
+
cls = self.__class__
|
|
151
|
+
client = self.redis
|
|
152
|
+
if cls.lua_release is None:
|
|
153
|
+
cls.lua_release = client.register_script(cls.LUA_RELEASE_SCRIPT)
|
|
154
|
+
if cls.lua_extend is None:
|
|
155
|
+
cls.lua_extend = client.register_script(cls.LUA_EXTEND_SCRIPT)
|
|
156
|
+
if cls.lua_reacquire is None:
|
|
157
|
+
cls.lua_reacquire = client.register_script(cls.LUA_REACQUIRE_SCRIPT)
|
|
158
|
+
|
|
159
|
+
async def __aenter__(self):
|
|
160
|
+
if await self.acquire():
|
|
161
|
+
return self
|
|
162
|
+
raise LockError("Unable to acquire lock within the time specified")
|
|
163
|
+
|
|
164
|
+
async def __aexit__(self, exc_type, exc_value, traceback):
|
|
165
|
+
await self.release()
|
|
166
|
+
|
|
167
|
+
async def acquire(
|
|
168
|
+
self,
|
|
169
|
+
blocking: Optional[bool] = None,
|
|
170
|
+
blocking_timeout: Optional[float] = None,
|
|
171
|
+
token: Optional[Union[str, bytes]] = None,
|
|
172
|
+
):
|
|
173
|
+
"""
|
|
174
|
+
Use Redis to hold a shared, distributed lock named ``name``.
|
|
175
|
+
Returns True once the lock is acquired.
|
|
176
|
+
|
|
177
|
+
If ``blocking`` is False, always return immediately. If the lock
|
|
178
|
+
was acquired, return True, otherwise return False.
|
|
179
|
+
|
|
180
|
+
``blocking_timeout`` specifies the maximum number of seconds to
|
|
181
|
+
wait trying to acquire the lock.
|
|
182
|
+
|
|
183
|
+
``token`` specifies the token value to be used. If provided, token
|
|
184
|
+
must be a bytes object or a string that can be encoded to a bytes
|
|
185
|
+
object with the default encoding. If a token isn't specified, a UUID
|
|
186
|
+
will be generated.
|
|
187
|
+
"""
|
|
188
|
+
loop = asyncio.get_event_loop()
|
|
189
|
+
sleep = self.sleep
|
|
190
|
+
if token is None:
|
|
191
|
+
token = uuid.uuid1().hex.encode()
|
|
192
|
+
else:
|
|
193
|
+
encoder = self.redis.connection_pool.get_encoder()
|
|
194
|
+
token = encoder.encode(token)
|
|
195
|
+
if blocking is None:
|
|
196
|
+
blocking = self.blocking
|
|
197
|
+
if blocking_timeout is None:
|
|
198
|
+
blocking_timeout = self.blocking_timeout
|
|
199
|
+
stop_trying_at = None
|
|
200
|
+
if blocking_timeout is not None:
|
|
201
|
+
stop_trying_at = loop.time() + blocking_timeout
|
|
202
|
+
while True:
|
|
203
|
+
if await self.do_acquire(token):
|
|
204
|
+
self.local.token = token
|
|
205
|
+
return True
|
|
206
|
+
if not blocking:
|
|
207
|
+
return False
|
|
208
|
+
next_try_at = loop.time() + sleep
|
|
209
|
+
if stop_trying_at is not None and next_try_at > stop_trying_at:
|
|
210
|
+
return False
|
|
211
|
+
await asyncio.sleep(sleep)
|
|
212
|
+
|
|
213
|
+
async def do_acquire(self, token: Union[str, bytes]) -> bool:
|
|
214
|
+
if self.timeout:
|
|
215
|
+
# convert to milliseconds
|
|
216
|
+
timeout = int(self.timeout * 1000)
|
|
217
|
+
else:
|
|
218
|
+
timeout = None
|
|
219
|
+
if await self.redis.set(self.name, token, nx=True, px=timeout):
|
|
220
|
+
return True
|
|
221
|
+
return False
|
|
222
|
+
|
|
223
|
+
async def locked(self) -> bool:
|
|
224
|
+
"""
|
|
225
|
+
Returns True if this key is locked by any process, otherwise False.
|
|
226
|
+
"""
|
|
227
|
+
return await self.redis.get(self.name) is not None
|
|
228
|
+
|
|
229
|
+
async def owned(self) -> bool:
|
|
230
|
+
"""
|
|
231
|
+
Returns True if this key is locked by this lock, otherwise False.
|
|
232
|
+
"""
|
|
233
|
+
stored_token = await self.redis.get(self.name)
|
|
234
|
+
# need to always compare bytes to bytes
|
|
235
|
+
# TODO: this can be simplified when the context manager is finished
|
|
236
|
+
if stored_token and not isinstance(stored_token, bytes):
|
|
237
|
+
encoder = self.redis.connection_pool.get_encoder()
|
|
238
|
+
stored_token = encoder.encode(stored_token)
|
|
239
|
+
return self.local.token is not None and stored_token == self.local.token
|
|
240
|
+
|
|
241
|
+
def release(self) -> Awaitable[NoReturn]:
|
|
242
|
+
"""Releases the already acquired lock"""
|
|
243
|
+
expected_token = self.local.token
|
|
244
|
+
if expected_token is None:
|
|
245
|
+
raise LockError("Cannot release an unlocked lock")
|
|
246
|
+
self.local.token = None
|
|
247
|
+
return self.do_release(expected_token)
|
|
248
|
+
|
|
249
|
+
async def do_release(self, expected_token: bytes):
|
|
250
|
+
if not bool(
|
|
251
|
+
await self.lua_release(
|
|
252
|
+
keys=[self.name], args=[expected_token], client=self.redis
|
|
253
|
+
)
|
|
254
|
+
):
|
|
255
|
+
raise LockNotOwnedError("Cannot release a lock" " that's no longer owned")
|
|
256
|
+
|
|
257
|
+
def extend(
|
|
258
|
+
self, additional_time: float, replace_ttl: bool = False
|
|
259
|
+
) -> Awaitable[bool]:
|
|
260
|
+
"""
|
|
261
|
+
Adds more time to an already acquired lock.
|
|
262
|
+
|
|
263
|
+
``additional_time`` can be specified as an integer or a float, both
|
|
264
|
+
representing the number of seconds to add.
|
|
265
|
+
|
|
266
|
+
``replace_ttl`` if False (the default), add `additional_time` to
|
|
267
|
+
the lock's existing ttl. If True, replace the lock's ttl with
|
|
268
|
+
`additional_time`.
|
|
269
|
+
"""
|
|
270
|
+
if self.local.token is None:
|
|
271
|
+
raise LockError("Cannot extend an unlocked lock")
|
|
272
|
+
if self.timeout is None:
|
|
273
|
+
raise LockError("Cannot extend a lock with no timeout")
|
|
274
|
+
return self.do_extend(additional_time, replace_ttl)
|
|
275
|
+
|
|
276
|
+
async def do_extend(self, additional_time, replace_ttl) -> bool:
|
|
277
|
+
additional_time = int(additional_time * 1000)
|
|
278
|
+
if not bool(
|
|
279
|
+
await self.lua_extend(
|
|
280
|
+
keys=[self.name],
|
|
281
|
+
args=[self.local.token, additional_time, replace_ttl and "1" or "0"],
|
|
282
|
+
client=self.redis,
|
|
283
|
+
)
|
|
284
|
+
):
|
|
285
|
+
raise LockNotOwnedError("Cannot extend a lock that's" " no longer owned")
|
|
286
|
+
return True
|
|
287
|
+
|
|
288
|
+
def reacquire(self) -> Awaitable[bool]:
|
|
289
|
+
"""
|
|
290
|
+
Resets a TTL of an already acquired lock back to a timeout value.
|
|
291
|
+
"""
|
|
292
|
+
if self.local.token is None:
|
|
293
|
+
raise LockError("Cannot reacquire an unlocked lock")
|
|
294
|
+
if self.timeout is None:
|
|
295
|
+
raise LockError("Cannot reacquire a lock with no timeout")
|
|
296
|
+
return self.do_reacquire()
|
|
297
|
+
|
|
298
|
+
async def do_reacquire(self) -> bool:
|
|
299
|
+
timeout = int(self.timeout * 1000)
|
|
300
|
+
if not bool(
|
|
301
|
+
await self.lua_reacquire(
|
|
302
|
+
keys=[self.name], args=[self.local.token, timeout], client=self.redis
|
|
303
|
+
)
|
|
304
|
+
):
|
|
305
|
+
raise LockNotOwnedError("Cannot reacquire a lock that's" " no longer owned")
|
|
306
|
+
return True
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import logging
|
|
2
|
-
import os
|
|
3
|
-
import sys
|
|
4
|
-
|
|
5
|
-
logger = logging.getLogger("aioredis")
|
|
6
|
-
sentinel_logger = logger.getChild("sentinel")
|
|
7
|
-
|
|
8
|
-
if os.environ.get("AIOREDIS_DEBUG"):
|
|
9
|
-
logger.setLevel(logging.DEBUG)
|
|
10
|
-
handler = logging.StreamHandler(stream=sys.stderr)
|
|
11
|
-
handler.setFormatter(
|
|
12
|
-
logging.Formatter("%(asctime)s %(name)s %(levelname)s %(message)s")
|
|
13
|
-
)
|
|
14
|
-
logger.addHandler(handler)
|
|
15
|
-
os.environ["AIOREDIS_DEBUG"] = ""
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
import sys
|
|
4
|
+
|
|
5
|
+
logger = logging.getLogger("aioredis")
|
|
6
|
+
sentinel_logger = logger.getChild("sentinel")
|
|
7
|
+
|
|
8
|
+
if os.environ.get("AIOREDIS_DEBUG"):
|
|
9
|
+
logger.setLevel(logging.DEBUG)
|
|
10
|
+
handler = logging.StreamHandler(stream=sys.stderr)
|
|
11
|
+
handler.setFormatter(
|
|
12
|
+
logging.Formatter("%(asctime)s %(name)s %(levelname)s %(message)s")
|
|
13
|
+
)
|
|
14
|
+
logger.addHandler(handler)
|
|
15
|
+
os.environ["AIOREDIS_DEBUG"] = ""
|