stouputils 1.16.3__py3-none-any.whl → 1.17.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stouputils/__init__.py +1 -0
- stouputils/__init__.pyi +1 -0
- stouputils/all_doctests.py +1 -1
- stouputils/decorators.py +1 -1
- stouputils/image.py +1 -1
- stouputils/image.pyi +1 -1
- stouputils/io.py +1 -1
- stouputils/io.pyi +1 -1
- stouputils/lock/__init__.py +36 -0
- stouputils/lock/__init__.pyi +5 -0
- stouputils/lock/base.py +536 -0
- stouputils/lock/base.pyi +169 -0
- stouputils/lock/queue.py +377 -0
- stouputils/lock/queue.pyi +131 -0
- stouputils/lock/re_entrant.py +115 -0
- stouputils/lock/re_entrant.pyi +81 -0
- stouputils/lock/redis_fifo.py +299 -0
- stouputils/lock/redis_fifo.pyi +123 -0
- stouputils/lock/shared.py +30 -0
- stouputils/lock/shared.pyi +16 -0
- {stouputils-1.16.3.dist-info → stouputils-1.17.0.dist-info}/METADATA +3 -1
- {stouputils-1.16.3.dist-info → stouputils-1.17.0.dist-info}/RECORD +24 -12
- {stouputils-1.16.3.dist-info → stouputils-1.17.0.dist-info}/WHEEL +0 -0
- {stouputils-1.16.3.dist-info → stouputils-1.17.0.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
|
|
2
|
+
# Imports
|
|
3
|
+
import os
|
|
4
|
+
from typing import ClassVar
|
|
5
|
+
|
|
6
|
+
from .base import LockFifo
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class RLockFifo(LockFifo):
|
|
10
|
+
""" A re-entrant cross-process lock backed by a file.
|
|
11
|
+
|
|
12
|
+
This lock is re-entrant for the same owner, where owner identity is the
|
|
13
|
+
tuple ``(path, pid, thread_id)``. Repeated calls to :meth:`acquire` by the
|
|
14
|
+
same owner increment an internal counter; only the final :meth:`release`
|
|
15
|
+
will release the underlying file lock managed by :class:`LockFifo`.
|
|
16
|
+
|
|
17
|
+
Key behaviour:
|
|
18
|
+
- Owner identity: (path, pid, thread_id)
|
|
19
|
+
- Reentrancy applies only within the same thread of the same process.
|
|
20
|
+
Other threads or processes will block (or raise ``LockTimeoutError``)
|
|
21
|
+
according to the lock's timeout and blocking parameters.
|
|
22
|
+
- Implemented on top of :class:`LockFifo` and shares its constructor
|
|
23
|
+
parameters and error semantics.
|
|
24
|
+
|
|
25
|
+
Args:
|
|
26
|
+
name (str): Lock filename or path. If a simple name is given,
|
|
27
|
+
it is created in the system temporary directory.
|
|
28
|
+
timeout (float | None): Seconds to wait for the lock. ``None`` means block indefinitely.
|
|
29
|
+
blocking (bool): Whether to block until acquired (subject to ``timeout``).
|
|
30
|
+
check_interval (float): Interval between lock attempts, in seconds.
|
|
31
|
+
fifo (bool): Whether to enforce Fifo ordering (default: True).
|
|
32
|
+
fifo_stale_timeout (float | None): Seconds after which a ticket is considered stale; if ``None`` the lock's ``timeout`` value will be used.
|
|
33
|
+
|
|
34
|
+
Raises:
|
|
35
|
+
LockTimeoutError: If the lock could not be acquired within the timeout (LockError & TimeoutError subclass)
|
|
36
|
+
LockError: On unexpected locking errors. (RunTimeError subclass)
|
|
37
|
+
|
|
38
|
+
Examples:
|
|
39
|
+
>>> with RLockFifo("my.lock", timeout=5):
|
|
40
|
+
... # critical section
|
|
41
|
+
... pass
|
|
42
|
+
|
|
43
|
+
>>> lock = RLockFifo("my.lock")
|
|
44
|
+
>>> lock.acquire()
|
|
45
|
+
>>> lock.acquire() # re-entrant acquire by same thread/process
|
|
46
|
+
>>> lock.release()
|
|
47
|
+
>>> lock.release() # underlying lock released here
|
|
48
|
+
|
|
49
|
+
>>> # Reentrancy with Fifo enabled should not create multiple tickets
|
|
50
|
+
>>> lock = RLockFifo("my_r.lock", fifo=True, timeout=1)
|
|
51
|
+
>>> lock.acquire()
|
|
52
|
+
>>> lock.acquire()
|
|
53
|
+
>>> lock.release()
|
|
54
|
+
>>> lock.release()
|
|
55
|
+
|
|
56
|
+
>>> # Cleanup behaviour: after closing a re-entrant lock the queue should be removed when empty
|
|
57
|
+
>>> import tempfile, os
|
|
58
|
+
>>> tmp = tempfile.mkdtemp()
|
|
59
|
+
>>> p = tmp + "/rlock"
|
|
60
|
+
>>> r = RLockFifo(p, fifo=True, timeout=1)
|
|
61
|
+
>>> r.acquire(); r.acquire(); r.release(); r.release(); r.close()
|
|
62
|
+
>>> os.path.exists(p + ".queue")
|
|
63
|
+
False
|
|
64
|
+
"""
|
|
65
|
+
owners: ClassVar[dict[tuple[str, int, int], int]] = {}
|
|
66
|
+
""" Mapping of owner keys to re-entrant acquisition counts. """
|
|
67
|
+
|
|
68
|
+
def __init__(
|
|
69
|
+
self,
|
|
70
|
+
name: str,
|
|
71
|
+
timeout: float | None = None,
|
|
72
|
+
blocking: bool = True,
|
|
73
|
+
check_interval: float = 0.05,
|
|
74
|
+
fifo: bool = True,
|
|
75
|
+
fifo_stale_timeout: float | None = None
|
|
76
|
+
) -> None:
|
|
77
|
+
""" Initialize the re-entrant lock and compute owner key.
|
|
78
|
+
|
|
79
|
+
The attribute ``self.key`` is a tuple ``(path, pid, thread_id)`` used to
|
|
80
|
+
track ownership and re-entrant acquisition counts in the
|
|
81
|
+
:class:`owners` mapping.
|
|
82
|
+
"""
|
|
83
|
+
super().__init__(name, timeout=timeout, blocking=blocking, check_interval=check_interval, fifo=fifo, fifo_stale_timeout=fifo_stale_timeout)
|
|
84
|
+
self.key: tuple[str, int, int] = (self.path, os.getpid(), __import__("threading").get_ident())
|
|
85
|
+
|
|
86
|
+
def acquire(self, timeout: float | None = None, blocking: bool | None = None, check_interval: float | None = None) -> None:
|
|
87
|
+
""" Acquire the lock with re-entrancy for the same owner.
|
|
88
|
+
|
|
89
|
+
If the current owner (same ``self.key``) already holds the lock, the
|
|
90
|
+
internal counter is incremented and the underlying file lock is not
|
|
91
|
+
re-acquired. Otherwise this delegates to :meth:`LockFifo.acquire`.
|
|
92
|
+
"""
|
|
93
|
+
cnt: int = self.owners.get(self.key, 0)
|
|
94
|
+
if cnt > 0:
|
|
95
|
+
self.owners[self.key] = cnt + 1
|
|
96
|
+
return
|
|
97
|
+
super().acquire(timeout=timeout, blocking=blocking, check_interval=check_interval)
|
|
98
|
+
self.owners[self.key] = 1
|
|
99
|
+
|
|
100
|
+
def release(self) -> None:
|
|
101
|
+
""" Release the lock for this owner.
|
|
102
|
+
|
|
103
|
+
Decrements the re-entrant counter for the current owner and only when
|
|
104
|
+
the counter reaches zero the underlying :class:`LockFifo` is released.
|
|
105
|
+
"""
|
|
106
|
+
cnt: int = self.owners.get(self.key, 0)
|
|
107
|
+
if cnt <= 1:
|
|
108
|
+
# last release: release underlying lock
|
|
109
|
+
try:
|
|
110
|
+
super().release()
|
|
111
|
+
finally:
|
|
112
|
+
self.owners.pop(self.key, None)
|
|
113
|
+
else:
|
|
114
|
+
self.owners[self.key] = cnt - 1
|
|
115
|
+
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
from .base import LockFifo as LockFifo
|
|
2
|
+
from typing import ClassVar
|
|
3
|
+
|
|
4
|
+
class RLockFifo(LockFifo):
|
|
5
|
+
''' A re-entrant cross-process lock backed by a file.
|
|
6
|
+
|
|
7
|
+
This lock is re-entrant for the same owner, where owner identity is the
|
|
8
|
+
tuple ``(path, pid, thread_id)``. Repeated calls to :meth:`acquire` by the
|
|
9
|
+
same owner increment an internal counter; only the final :meth:`release`
|
|
10
|
+
will release the underlying file lock managed by :class:`LockFifo`.
|
|
11
|
+
|
|
12
|
+
Key behaviour:
|
|
13
|
+
- Owner identity: (path, pid, thread_id)
|
|
14
|
+
- Reentrancy applies only within the same thread of the same process.
|
|
15
|
+
Other threads or processes will block (or raise ``LockTimeoutError``)
|
|
16
|
+
according to the lock\'s timeout and blocking parameters.
|
|
17
|
+
- Implemented on top of :class:`LockFifo` and shares its constructor
|
|
18
|
+
parameters and error semantics.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
name (str): Lock filename or path. If a simple name is given,
|
|
22
|
+
it is created in the system temporary directory.
|
|
23
|
+
timeout (float | None): Seconds to wait for the lock. ``None`` means block indefinitely.
|
|
24
|
+
blocking (bool): Whether to block until acquired (subject to ``timeout``).
|
|
25
|
+
check_interval (float): Interval between lock attempts, in seconds.
|
|
26
|
+
fifo (bool): Whether to enforce Fifo ordering (default: True).
|
|
27
|
+
fifo_stale_timeout (float | None): Seconds after which a ticket is considered stale; if ``None`` the lock\'s ``timeout`` value will be used.
|
|
28
|
+
|
|
29
|
+
Raises:
|
|
30
|
+
LockTimeoutError: If the lock could not be acquired within the timeout (LockError & TimeoutError subclass)
|
|
31
|
+
LockError: On unexpected locking errors. (RunTimeError subclass)
|
|
32
|
+
|
|
33
|
+
Examples:
|
|
34
|
+
>>> with RLockFifo("my.lock", timeout=5):
|
|
35
|
+
... # critical section
|
|
36
|
+
... pass
|
|
37
|
+
|
|
38
|
+
>>> lock = RLockFifo("my.lock")
|
|
39
|
+
>>> lock.acquire()
|
|
40
|
+
>>> lock.acquire() # re-entrant acquire by same thread/process
|
|
41
|
+
>>> lock.release()
|
|
42
|
+
>>> lock.release() # underlying lock released here
|
|
43
|
+
|
|
44
|
+
>>> # Reentrancy with Fifo enabled should not create multiple tickets
|
|
45
|
+
>>> lock = RLockFifo("my_r.lock", fifo=True, timeout=1)
|
|
46
|
+
>>> lock.acquire()
|
|
47
|
+
>>> lock.acquire()
|
|
48
|
+
>>> lock.release()
|
|
49
|
+
>>> lock.release()
|
|
50
|
+
|
|
51
|
+
>>> # Cleanup behaviour: after closing a re-entrant lock the queue should be removed when empty
|
|
52
|
+
>>> import tempfile, os
|
|
53
|
+
>>> tmp = tempfile.mkdtemp()
|
|
54
|
+
>>> p = tmp + "/rlock"
|
|
55
|
+
>>> r = RLockFifo(p, fifo=True, timeout=1)
|
|
56
|
+
>>> r.acquire(); r.acquire(); r.release(); r.release(); r.close()
|
|
57
|
+
>>> os.path.exists(p + ".queue")
|
|
58
|
+
False
|
|
59
|
+
'''
|
|
60
|
+
owners: ClassVar[dict[tuple[str, int, int], int]]
|
|
61
|
+
key: tuple[str, int, int]
|
|
62
|
+
def __init__(self, name: str, timeout: float | None = None, blocking: bool = True, check_interval: float = 0.05, fifo: bool = True, fifo_stale_timeout: float | None = None) -> None:
|
|
63
|
+
""" Initialize the re-entrant lock and compute owner key.
|
|
64
|
+
|
|
65
|
+
The attribute ``self.key`` is a tuple ``(path, pid, thread_id)`` used to
|
|
66
|
+
track ownership and re-entrant acquisition counts in the
|
|
67
|
+
:class:`owners` mapping.
|
|
68
|
+
"""
|
|
69
|
+
def acquire(self, timeout: float | None = None, blocking: bool | None = None, check_interval: float | None = None) -> None:
|
|
70
|
+
""" Acquire the lock with re-entrancy for the same owner.
|
|
71
|
+
|
|
72
|
+
If the current owner (same ``self.key``) already holds the lock, the
|
|
73
|
+
internal counter is incremented and the underlying file lock is not
|
|
74
|
+
re-acquired. Otherwise this delegates to :meth:`LockFifo.acquire`.
|
|
75
|
+
"""
|
|
76
|
+
def release(self) -> None:
|
|
77
|
+
""" Release the lock for this owner.
|
|
78
|
+
|
|
79
|
+
Decrements the re-entrant counter for the current owner and only when
|
|
80
|
+
the counter reaches zero the underlying :class:`LockFifo` is released.
|
|
81
|
+
"""
|
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
|
|
2
|
+
# Imports
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
import uuid
|
|
7
|
+
from collections.abc import Awaitable
|
|
8
|
+
from contextlib import AbstractContextManager
|
|
9
|
+
from typing import TYPE_CHECKING, Any
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
import redis
|
|
13
|
+
|
|
14
|
+
from .shared import LockError, LockTimeoutError
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class RedisLockFifo(AbstractContextManager["RedisLockFifo"]):
|
|
18
|
+
""" A Redis-backed inter-process lock (requires `redis`).
|
|
19
|
+
|
|
20
|
+
This lock provides optional Fifo fairness (enabled by default) and is
|
|
21
|
+
implemented using atomic Redis primitives. Acquisition of the underlying
|
|
22
|
+
lock uses an owner token and `SET NX` (with optional PX expiry when a
|
|
23
|
+
timeout/TTL is specified). When Fifo is enabled the implementation uses
|
|
24
|
+
a small ticket queue using `INCR` + `ZADD` and only the queue head attempts
|
|
25
|
+
to `SET NX`. Release uses an atomic Lua script to ensure only the token
|
|
26
|
+
owner can delete the lock key.
|
|
27
|
+
|
|
28
|
+
Notes:
|
|
29
|
+
- The lock stores a locally-generated random token; releasing without the
|
|
30
|
+
correct token has no effect on the remote key.
|
|
31
|
+
- When Fifo is enabled, queue entries are removed when the client acquires
|
|
32
|
+
the lock; stale queue entries (from crashed clients) are removed lazily
|
|
33
|
+
when their age exceeds ``fifo_stale_timeout`` (defaults to ``timeout`` if
|
|
34
|
+
``None``).
|
|
35
|
+
- This class raises ``ImportError`` if the ``redis`` package is not
|
|
36
|
+
installed and raises ``LockTimeoutError`` / ``LockError`` for runtime
|
|
37
|
+
acquisition errors.
|
|
38
|
+
|
|
39
|
+
Args:
|
|
40
|
+
name (str): Redis key name used for the lock.
|
|
41
|
+
redis_client (redis.Redis | None): Optional Redis client. A client is created lazily if not provided.
|
|
42
|
+
timeout (float | None): Maximum time to wait for the lock and (when provided) the lock TTL used by ``SET PX`` in seconds. ``None`` means block indefinitely and no automatic expiry.
|
|
43
|
+
blocking (bool): Whether to block until acquired (subject to ``timeout``).
|
|
44
|
+
check_interval (float): Poll interval while waiting for the lock, in seconds.
|
|
45
|
+
fifo (bool): Whether to enforce Fifo ordering using a ZSET queue (default: True).
|
|
46
|
+
fifo_stale_timeout (float | None): Seconds after which a queue entry is considered stale; if ``None`` the lock's ``timeout`` value will be used; if both are ``None``, no stale cleanup is performed.
|
|
47
|
+
|
|
48
|
+
Raises:
|
|
49
|
+
ImportError: If the ``redis`` package is not installed.
|
|
50
|
+
LockTimeoutError: If the lock cannot be acquired within ``timeout``.
|
|
51
|
+
LockError: On unexpected redis errors.
|
|
52
|
+
|
|
53
|
+
Examples:
|
|
54
|
+
>>> # Redis-backed examples; run only on non-Windows environments
|
|
55
|
+
>>> def _redis_doctest():
|
|
56
|
+
... import redis, time
|
|
57
|
+
... client = redis.Redis()
|
|
58
|
+
...
|
|
59
|
+
... # Simple usage (assumes redis is available in the test environment)
|
|
60
|
+
... with RedisLockFifo('test:lock', timeout=1):
|
|
61
|
+
... pass
|
|
62
|
+
...
|
|
63
|
+
... # Non-Fifo usage example
|
|
64
|
+
... with RedisLockFifo('test:lock', fifo=False, timeout=1):
|
|
65
|
+
... pass
|
|
66
|
+
...
|
|
67
|
+
... # Fifo stale-ticket behaviour (requires a local redis server)
|
|
68
|
+
... # Inject a stale head entry
|
|
69
|
+
... name = 'doctest:lock:stale'
|
|
70
|
+
... _ = client.delete(f"{name}:queue")
|
|
71
|
+
... _ = client.delete(f"{name}:seq")
|
|
72
|
+
... _ = client.delete(name)
|
|
73
|
+
... old_ts = int((time.time() - 10) * 1000)
|
|
74
|
+
... _ = client.zadd(f"{name}:queue", {f"1:stale:{old_ts}": 1})
|
|
75
|
+
... # Now acquire with small stale timeout which should remove head then succeed
|
|
76
|
+
... with RedisLockFifo(name, fifo=True, fifo_stale_timeout=0.01, timeout=1):
|
|
77
|
+
... print('acquired')
|
|
78
|
+
... _ = client.delete(f"{name}:queue")
|
|
79
|
+
... _ = client.delete(f"{name}:seq")
|
|
80
|
+
... _ = client.delete(name)
|
|
81
|
+
... # After using the lock, the queue keys should be removed when empty
|
|
82
|
+
... with RedisLockFifo(name, timeout=1):
|
|
83
|
+
... pass
|
|
84
|
+
... print(client.exists(f"{name}:queue") == 0 and client.exists(f"{name}:seq") == 0)
|
|
85
|
+
...
|
|
86
|
+
... # Non-Fifo acquisition should not create queue keys
|
|
87
|
+
... name2 = 'doctest:lock:nonfifo'
|
|
88
|
+
... _ = client.delete(f"{name2}:queue"); _ = client.delete(f"{name2}:seq")
|
|
89
|
+
... with RedisLockFifo(name2, fifo=False, timeout=1):
|
|
90
|
+
... pass
|
|
91
|
+
... print(client.exists(f"{name2}:queue") == 0 and client.exists(f"{name2}:seq") == 0)
|
|
92
|
+
...
|
|
93
|
+
>>> import os
|
|
94
|
+
>>> if os.name != 'nt':
|
|
95
|
+
... _redis_doctest()
|
|
96
|
+
... else:
|
|
97
|
+
... print("acquired\\nTrue\\nTrue")
|
|
98
|
+
acquired
|
|
99
|
+
True
|
|
100
|
+
True
|
|
101
|
+
""" # noqa: E501
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
RELEASE_SCRIPT: str = """
|
|
105
|
+
if redis.call('get', KEYS[1]) == ARGV[1] then
|
|
106
|
+
return redis.call('del', KEYS[1])
|
|
107
|
+
else
|
|
108
|
+
return 0
|
|
109
|
+
end
|
|
110
|
+
"""
|
|
111
|
+
|
|
112
|
+
def __init__(
|
|
113
|
+
self,
|
|
114
|
+
name: str,
|
|
115
|
+
redis_client: redis.Redis | None = None,
|
|
116
|
+
timeout: float | None = None,
|
|
117
|
+
blocking: bool = True,
|
|
118
|
+
check_interval: float = 0.05,
|
|
119
|
+
fifo: bool = True,
|
|
120
|
+
fifo_stale_timeout: float | None = None
|
|
121
|
+
) -> None:
|
|
122
|
+
try:
|
|
123
|
+
import redis # type: ignore # noqa: F401
|
|
124
|
+
except (ImportError, ModuleNotFoundError) as e:
|
|
125
|
+
raise ImportError("`redis` package is not installed; Please install it to use RedisLockFifo.") from e
|
|
126
|
+
self.name: str = name
|
|
127
|
+
self.client: redis.Redis | None = redis_client
|
|
128
|
+
self.timeout: float | None = timeout
|
|
129
|
+
self.blocking: bool = blocking
|
|
130
|
+
self.check_interval: float = check_interval
|
|
131
|
+
self.fifo: bool = fifo
|
|
132
|
+
self.fifo_stale_timeout: float | None = fifo_stale_timeout
|
|
133
|
+
self.token: str | None = None
|
|
134
|
+
self.queue_member: str | None = None
|
|
135
|
+
# Lazy queue backend; created on first Fifo acquisition
|
|
136
|
+
self.queue = None
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def ensure_client(self) -> redis.Redis:
|
|
140
|
+
""" Ensure a ``redis.Redis`` client is available (lazy creation). """
|
|
141
|
+
if self.client is None:
|
|
142
|
+
import redis
|
|
143
|
+
self.client = redis.Redis()
|
|
144
|
+
return self.client
|
|
145
|
+
|
|
146
|
+
def _cleanup_stalequeue(self) -> None:
|
|
147
|
+
""" Remove a stale head member from the queue if it exceeds the stale timeout. """
|
|
148
|
+
if not self.fifo:
|
|
149
|
+
return
|
|
150
|
+
# Determine effective stale timeout (seconds)
|
|
151
|
+
stale: float | None = self.fifo_stale_timeout if self.fifo_stale_timeout is not None else self.timeout
|
|
152
|
+
if stale is None:
|
|
153
|
+
return
|
|
154
|
+
client: redis.Redis = self.ensure_client()
|
|
155
|
+
try:
|
|
156
|
+
head: Awaitable[Any] | Any = client.zrange(f"{self.name}:queue", 0, 0) # type: ignore
|
|
157
|
+
if not head:
|
|
158
|
+
return
|
|
159
|
+
head_member = str(head[0].decode()) # type: ignore
|
|
160
|
+
# member format: ticket:token:ts_ms
|
|
161
|
+
parts: list[str] = head_member.split(":")
|
|
162
|
+
if len(parts) < 3:
|
|
163
|
+
return
|
|
164
|
+
ts_ms: int = int(parts[2])
|
|
165
|
+
age: float = (time.monotonic() * 1000) - ts_ms
|
|
166
|
+
if age >= (stale * 1000):
|
|
167
|
+
try:
|
|
168
|
+
client.zrem(f"{self.name}:queue", head_member)
|
|
169
|
+
except Exception:
|
|
170
|
+
pass
|
|
171
|
+
except Exception:
|
|
172
|
+
pass
|
|
173
|
+
|
|
174
|
+
def acquire(self, timeout: float | None = None, blocking: bool | None = None, check_interval: float | None = None) -> None:
|
|
175
|
+
""" Acquire the Redis lock.
|
|
176
|
+
|
|
177
|
+
When Fifo is enabled (default), this function obtains a ticket via INCR
|
|
178
|
+
and registers it in a ZSET. The client waits until its ticket is the
|
|
179
|
+
head of the queue and then attempts to SET NX the lock key.
|
|
180
|
+
"""
|
|
181
|
+
# Use instance defaults if parameters not provided
|
|
182
|
+
if blocking is None:
|
|
183
|
+
blocking = self.blocking
|
|
184
|
+
if timeout is None:
|
|
185
|
+
timeout = self.timeout
|
|
186
|
+
if check_interval is None:
|
|
187
|
+
check_interval = self.check_interval
|
|
188
|
+
deadline: float | None = None if timeout is None else (time.monotonic() + timeout)
|
|
189
|
+
self.client = self.ensure_client()
|
|
190
|
+
token: str = uuid.uuid4().hex
|
|
191
|
+
|
|
192
|
+
# Non-Fifo fast path
|
|
193
|
+
if not self.fifo:
|
|
194
|
+
while True:
|
|
195
|
+
px: int | None = None if timeout is None else int((timeout or 0) * 1000)
|
|
196
|
+
try:
|
|
197
|
+
ok: Any = self.client.set(self.name, token, nx=True, px=px)
|
|
198
|
+
except Exception as exc:
|
|
199
|
+
raise LockError(str(exc)) from exc
|
|
200
|
+
if ok:
|
|
201
|
+
self.token = token
|
|
202
|
+
return
|
|
203
|
+
if not blocking:
|
|
204
|
+
raise LockTimeoutError("Lock is already held and blocking is False")
|
|
205
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
206
|
+
raise LockTimeoutError(f"Timeout while waiting for redis lock '{self.name}'")
|
|
207
|
+
time.sleep(check_interval)
|
|
208
|
+
|
|
209
|
+
# Fifo path using RedisTicketQueue backend
|
|
210
|
+
try:
|
|
211
|
+
if self.queue is None:
|
|
212
|
+
from .queue import RedisTicketQueue
|
|
213
|
+
self.queue = RedisTicketQueue(self.name, self.client, stale_timeout=(self.fifo_stale_timeout if self.fifo_stale_timeout is not None else self.timeout))
|
|
214
|
+
ticket, member = self.queue.register()
|
|
215
|
+
|
|
216
|
+
while True:
|
|
217
|
+
self.queue.cleanup_stale()
|
|
218
|
+
if not self.queue.is_head(ticket):
|
|
219
|
+
if not blocking:
|
|
220
|
+
raise LockTimeoutError("Lock is already held and blocking is False")
|
|
221
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
222
|
+
raise LockTimeoutError(f"Timeout while waiting for redis lock '{self.name}'")
|
|
223
|
+
time.sleep(check_interval)
|
|
224
|
+
continue
|
|
225
|
+
# We're head; attempt to SET NX
|
|
226
|
+
px = None if timeout is None else int((timeout or 0) * 1000)
|
|
227
|
+
try:
|
|
228
|
+
ok: Any = self.client.set(self.name, token, nx=True, px=px)
|
|
229
|
+
except Exception as exc:
|
|
230
|
+
raise LockError(str(exc)) from exc
|
|
231
|
+
if ok:
|
|
232
|
+
self.token = token
|
|
233
|
+
try:
|
|
234
|
+
self.queue.remove(member)
|
|
235
|
+
self.queue_member = None
|
|
236
|
+
except Exception:
|
|
237
|
+
pass
|
|
238
|
+
return
|
|
239
|
+
if not blocking:
|
|
240
|
+
raise LockTimeoutError("Lock is already held and blocking is False")
|
|
241
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
242
|
+
raise LockTimeoutError(f"Timeout while waiting for redis lock '{self.name}'")
|
|
243
|
+
time.sleep(check_interval)
|
|
244
|
+
except Exception:
|
|
245
|
+
# On error, ensure we remove our queue entry if present
|
|
246
|
+
try:
|
|
247
|
+
if hasattr(self, "queue") and self.queue is not None and self.queue_member is not None:
|
|
248
|
+
self.queue.remove(self.queue_member)
|
|
249
|
+
self.queue_member = None
|
|
250
|
+
except Exception:
|
|
251
|
+
pass
|
|
252
|
+
raise
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def release(self) -> None:
|
|
256
|
+
""" Release the lock if currently owned by this instance.
|
|
257
|
+
|
|
258
|
+
Uses an atomic Lua script to check that the stored token matches the
|
|
259
|
+
key value and deletes it only when owned. Additionally removes any
|
|
260
|
+
lingering queue entry for this client.
|
|
261
|
+
"""
|
|
262
|
+
if not self.token:
|
|
263
|
+
return
|
|
264
|
+
self.client = self.ensure_client()
|
|
265
|
+
|
|
266
|
+
try:
|
|
267
|
+
# Use eval to run atomic check-and-del
|
|
268
|
+
self.client.eval(self.RELEASE_SCRIPT, 1, self.name, self.token)
|
|
269
|
+
finally:
|
|
270
|
+
# Ensure local state cleared and remove any queue entry we may have left
|
|
271
|
+
try:
|
|
272
|
+
if self.queue_member is not None:
|
|
273
|
+
self.client.zrem(f"{self.name}:queue", self.queue_member)
|
|
274
|
+
except Exception:
|
|
275
|
+
pass
|
|
276
|
+
self.queue_member = None
|
|
277
|
+
self.token = None
|
|
278
|
+
|
|
279
|
+
# Best-effort cleanup of the queue keys when empty
|
|
280
|
+
try:
|
|
281
|
+
if hasattr(self, "queue") and self.queue is not None:
|
|
282
|
+
try:
|
|
283
|
+
self.queue.cleanup_stale()
|
|
284
|
+
except Exception:
|
|
285
|
+
pass
|
|
286
|
+
try:
|
|
287
|
+
self.queue.maybe_cleanup()
|
|
288
|
+
except Exception:
|
|
289
|
+
pass
|
|
290
|
+
except Exception:
|
|
291
|
+
pass
|
|
292
|
+
|
|
293
|
+
def __enter__(self) -> RedisLockFifo:
|
|
294
|
+
self.acquire()
|
|
295
|
+
return self
|
|
296
|
+
|
|
297
|
+
def __exit__(self, exc_type: type | None, exc: BaseException | None, tb: Any | None) -> None:
|
|
298
|
+
self.release()
|
|
299
|
+
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import redis
|
|
2
|
+
from .shared import LockError as LockError, LockTimeoutError as LockTimeoutError
|
|
3
|
+
from _typeshed import Incomplete
|
|
4
|
+
from contextlib import AbstractContextManager
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
class RedisLockFifo(AbstractContextManager['RedisLockFifo']):
|
|
8
|
+
''' A Redis-backed inter-process lock (requires `redis`).
|
|
9
|
+
|
|
10
|
+
This lock provides optional Fifo fairness (enabled by default) and is
|
|
11
|
+
implemented using atomic Redis primitives. Acquisition of the underlying
|
|
12
|
+
lock uses an owner token and `SET NX` (with optional PX expiry when a
|
|
13
|
+
timeout/TTL is specified). When Fifo is enabled the implementation uses
|
|
14
|
+
a small ticket queue using `INCR` + `ZADD` and only the queue head attempts
|
|
15
|
+
to `SET NX`. Release uses an atomic Lua script to ensure only the token
|
|
16
|
+
owner can delete the lock key.
|
|
17
|
+
|
|
18
|
+
Notes:
|
|
19
|
+
- The lock stores a locally-generated random token; releasing without the
|
|
20
|
+
correct token has no effect on the remote key.
|
|
21
|
+
- When Fifo is enabled, queue entries are removed when the client acquires
|
|
22
|
+
the lock; stale queue entries (from crashed clients) are removed lazily
|
|
23
|
+
when their age exceeds ``fifo_stale_timeout`` (defaults to ``timeout`` if
|
|
24
|
+
``None``).
|
|
25
|
+
- This class raises ``ImportError`` if the ``redis`` package is not
|
|
26
|
+
installed and raises ``LockTimeoutError`` / ``LockError`` for runtime
|
|
27
|
+
acquisition errors.
|
|
28
|
+
|
|
29
|
+
Args:
|
|
30
|
+
name (str): Redis key name used for the lock.
|
|
31
|
+
redis_client (redis.Redis | None): Optional Redis client. A client is created lazily if not provided.
|
|
32
|
+
timeout (float | None): Maximum time to wait for the lock and (when provided) the lock TTL used by ``SET PX`` in seconds. ``None`` means block indefinitely and no automatic expiry.
|
|
33
|
+
blocking (bool): Whether to block until acquired (subject to ``timeout``).
|
|
34
|
+
check_interval (float): Poll interval while waiting for the lock, in seconds.
|
|
35
|
+
fifo (bool): Whether to enforce Fifo ordering using a ZSET queue (default: True).
|
|
36
|
+
fifo_stale_timeout (float | None): Seconds after which a queue entry is considered stale; if ``None`` the lock\'s ``timeout`` value will be used; if both are ``None``, no stale cleanup is performed.
|
|
37
|
+
|
|
38
|
+
Raises:
|
|
39
|
+
ImportError: If the ``redis`` package is not installed.
|
|
40
|
+
LockTimeoutError: If the lock cannot be acquired within ``timeout``.
|
|
41
|
+
LockError: On unexpected redis errors.
|
|
42
|
+
|
|
43
|
+
Examples:
|
|
44
|
+
>>> # Redis-backed examples; run only on non-Windows environments
|
|
45
|
+
>>> def _redis_doctest():
|
|
46
|
+
... import redis, time
|
|
47
|
+
... client = redis.Redis()
|
|
48
|
+
...
|
|
49
|
+
... # Simple usage (assumes redis is available in the test environment)
|
|
50
|
+
... with RedisLockFifo(\'test:lock\', timeout=1):
|
|
51
|
+
... pass
|
|
52
|
+
...
|
|
53
|
+
... # Non-Fifo usage example
|
|
54
|
+
... with RedisLockFifo(\'test:lock\', fifo=False, timeout=1):
|
|
55
|
+
... pass
|
|
56
|
+
...
|
|
57
|
+
... # Fifo stale-ticket behaviour (requires a local redis server)
|
|
58
|
+
... # Inject a stale head entry
|
|
59
|
+
... name = \'doctest:lock:stale\'
|
|
60
|
+
... _ = client.delete(f"{name}:queue")
|
|
61
|
+
... _ = client.delete(f"{name}:seq")
|
|
62
|
+
... _ = client.delete(name)
|
|
63
|
+
... old_ts = int((time.time() - 10) * 1000)
|
|
64
|
+
... _ = client.zadd(f"{name}:queue", {f"1:stale:{old_ts}": 1})
|
|
65
|
+
... # Now acquire with small stale timeout which should remove head then succeed
|
|
66
|
+
... with RedisLockFifo(name, fifo=True, fifo_stale_timeout=0.01, timeout=1):
|
|
67
|
+
... print(\'acquired\')
|
|
68
|
+
... _ = client.delete(f"{name}:queue")
|
|
69
|
+
... _ = client.delete(f"{name}:seq")
|
|
70
|
+
... _ = client.delete(name)
|
|
71
|
+
... # After using the lock, the queue keys should be removed when empty
|
|
72
|
+
... with RedisLockFifo(name, timeout=1):
|
|
73
|
+
... pass
|
|
74
|
+
... print(client.exists(f"{name}:queue") == 0 and client.exists(f"{name}:seq") == 0)
|
|
75
|
+
...
|
|
76
|
+
... # Non-Fifo acquisition should not create queue keys
|
|
77
|
+
... name2 = \'doctest:lock:nonfifo\'
|
|
78
|
+
... _ = client.delete(f"{name2}:queue"); _ = client.delete(f"{name2}:seq")
|
|
79
|
+
... with RedisLockFifo(name2, fifo=False, timeout=1):
|
|
80
|
+
... pass
|
|
81
|
+
... print(client.exists(f"{name2}:queue") == 0 and client.exists(f"{name2}:seq") == 0)
|
|
82
|
+
...
|
|
83
|
+
>>> import os
|
|
84
|
+
>>> if os.name != \'nt\':
|
|
85
|
+
... _redis_doctest()
|
|
86
|
+
... else:
|
|
87
|
+
... print("acquired\\nTrue\\nTrue")
|
|
88
|
+
acquired
|
|
89
|
+
True
|
|
90
|
+
True
|
|
91
|
+
'''
|
|
92
|
+
RELEASE_SCRIPT: str
|
|
93
|
+
name: str
|
|
94
|
+
client: redis.Redis | None
|
|
95
|
+
timeout: float | None
|
|
96
|
+
blocking: bool
|
|
97
|
+
check_interval: float
|
|
98
|
+
fifo: bool
|
|
99
|
+
fifo_stale_timeout: float | None
|
|
100
|
+
token: str | None
|
|
101
|
+
queue_member: str | None
|
|
102
|
+
queue: Incomplete
|
|
103
|
+
def __init__(self, name: str, redis_client: redis.Redis | None = None, timeout: float | None = None, blocking: bool = True, check_interval: float = 0.05, fifo: bool = True, fifo_stale_timeout: float | None = None) -> None: ...
|
|
104
|
+
def ensure_client(self) -> redis.Redis:
|
|
105
|
+
""" Ensure a ``redis.Redis`` client is available (lazy creation). """
|
|
106
|
+
def _cleanup_stalequeue(self) -> None:
|
|
107
|
+
""" Remove a stale head member from the queue if it exceeds the stale timeout. """
|
|
108
|
+
def acquire(self, timeout: float | None = None, blocking: bool | None = None, check_interval: float | None = None) -> None:
|
|
109
|
+
""" Acquire the Redis lock.
|
|
110
|
+
|
|
111
|
+
When Fifo is enabled (default), this function obtains a ticket via INCR
|
|
112
|
+
and registers it in a ZSET. The client waits until its ticket is the
|
|
113
|
+
head of the queue and then attempts to SET NX the lock key.
|
|
114
|
+
"""
|
|
115
|
+
def release(self) -> None:
|
|
116
|
+
""" Release the lock if currently owned by this instance.
|
|
117
|
+
|
|
118
|
+
Uses an atomic Lua script to check that the stored token matches the
|
|
119
|
+
key value and deletes it only when owned. Additionally removes any
|
|
120
|
+
lingering queue entry for this client.
|
|
121
|
+
"""
|
|
122
|
+
def __enter__(self) -> RedisLockFifo: ...
|
|
123
|
+
def __exit__(self, exc_type: type | None, exc: BaseException | None, tb: Any | None) -> None: ...
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
|
|
2
|
+
# Imports
|
|
3
|
+
import os
|
|
4
|
+
import tempfile
|
|
5
|
+
|
|
6
|
+
from ..io import clean_path
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class LockError(RuntimeError):
|
|
10
|
+
""" Base lock error. """
|
|
11
|
+
class LockTimeoutError(TimeoutError, LockError):
|
|
12
|
+
""" Raised when a lock could not be acquired within ``timeout`` seconds. """
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def resolve_path(path: str) -> str:
|
|
16
|
+
""" Resolve a lock file path, placing it in the system temporary directory if only a name is given.
|
|
17
|
+
|
|
18
|
+
Examples:
|
|
19
|
+
>>> import os, tempfile
|
|
20
|
+
>>> p = resolve_path('foo.lock')
|
|
21
|
+
>>> os.path.basename(p) == 'foo.lock'
|
|
22
|
+
True
|
|
23
|
+
"""
|
|
24
|
+
path = clean_path(path)
|
|
25
|
+
name = os.path.basename(path)
|
|
26
|
+
if name == path:
|
|
27
|
+
path = f"{tempfile.gettempdir()}/{name}"
|
|
28
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
29
|
+
return path
|
|
30
|
+
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from ..io import clean_path as clean_path
|
|
2
|
+
|
|
3
|
+
class LockError(RuntimeError):
|
|
4
|
+
""" Base lock error. """
|
|
5
|
+
class LockTimeoutError(TimeoutError, LockError):
|
|
6
|
+
""" Raised when a lock could not be acquired within ``timeout`` seconds. """
|
|
7
|
+
|
|
8
|
+
def resolve_path(path: str) -> str:
|
|
9
|
+
""" Resolve a lock file path, placing it in the system temporary directory if only a name is given.
|
|
10
|
+
|
|
11
|
+
Examples:
|
|
12
|
+
>>> import os, tempfile
|
|
13
|
+
>>> p = resolve_path('foo.lock')
|
|
14
|
+
>>> os.path.basename(p) == 'foo.lock'
|
|
15
|
+
True
|
|
16
|
+
"""
|