stouputils 1.16.3__py3-none-any.whl → 1.17.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stouputils/__init__.py +1 -0
- stouputils/__init__.pyi +1 -0
- stouputils/all_doctests.py +1 -1
- stouputils/decorators.py +1 -1
- stouputils/image.py +1 -1
- stouputils/image.pyi +1 -1
- stouputils/io.py +1 -1
- stouputils/io.pyi +1 -1
- stouputils/lock/__init__.py +36 -0
- stouputils/lock/__init__.pyi +5 -0
- stouputils/lock/base.py +536 -0
- stouputils/lock/base.pyi +169 -0
- stouputils/lock/queue.py +377 -0
- stouputils/lock/queue.pyi +131 -0
- stouputils/lock/re_entrant.py +115 -0
- stouputils/lock/re_entrant.pyi +81 -0
- stouputils/lock/redis_fifo.py +299 -0
- stouputils/lock/redis_fifo.pyi +123 -0
- stouputils/lock/shared.py +30 -0
- stouputils/lock/shared.pyi +16 -0
- {stouputils-1.16.3.dist-info → stouputils-1.17.0.dist-info}/METADATA +3 -1
- {stouputils-1.16.3.dist-info → stouputils-1.17.0.dist-info}/RECORD +24 -12
- {stouputils-1.16.3.dist-info → stouputils-1.17.0.dist-info}/WHEEL +0 -0
- {stouputils-1.16.3.dist-info → stouputils-1.17.0.dist-info}/entry_points.txt +0 -0
stouputils/lock/base.pyi
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
from .shared import LockError as LockError, LockTimeoutError as LockTimeoutError, resolve_path as resolve_path
|
|
2
|
+
from _typeshed import Incomplete
|
|
3
|
+
from contextlib import AbstractContextManager
|
|
4
|
+
from typing import Any, IO
|
|
5
|
+
|
|
6
|
+
def _lock_fd(fd: int, blocking: bool, timeout: float | None) -> None:
|
|
7
|
+
"""Try to acquire an exclusive lock on an open file descriptor.
|
|
8
|
+
|
|
9
|
+
This helper attempts POSIX `fcntl` first, then Windows `msvcrt`.
|
|
10
|
+
It raises BlockingIOError when the lock is busy, ImportError if neither
|
|
11
|
+
backend is available, or OSError for unexpected errors.
|
|
12
|
+
"""
|
|
13
|
+
def _unlock_fd(fd: int | None) -> None:
|
|
14
|
+
"""Unlock an open file descriptor using the available backend."""
|
|
15
|
+
def _remove_file_if_unlocked(path: str) -> None:
|
|
16
|
+
"""Attempt to remove a file only if we can confirm nobody holds the lock.
|
|
17
|
+
|
|
18
|
+
Uses a non-blocking lock test via fcntl or msvcrt. This is best-effort and
|
|
19
|
+
will not raise on failure.
|
|
20
|
+
"""
|
|
21
|
+
def _worker(lp: str, op: str, idx: int) -> None:
|
|
22
|
+
""" Module-level helper used by doctests as a multiprocessing target. """
|
|
23
|
+
def _hold(path: str) -> None:
|
|
24
|
+
""" Module-level helper used by doctests as a multiprocessing target.
|
|
25
|
+
|
|
26
|
+
This creates a small readiness marker file while holding the lock so
|
|
27
|
+
doctests can reliably detect when the child process has acquired it
|
|
28
|
+
(useful on Windows spawn semantics).
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
class LockFifo(AbstractContextManager['LockFifo']):
|
|
32
|
+
''' A simple cross-platform inter-process lock backed by a file.
|
|
33
|
+
|
|
34
|
+
This implementation supports optional Fifo ordering via a small ticket queue
|
|
35
|
+
stored alongside the lock file. Fifo is enabled by default to avoid
|
|
36
|
+
starvation. Fifo behaviour is implemented with a small sequence file and
|
|
37
|
+
per-ticket files in ``<lockpath>.queue/``. On platforms without fcntl the
|
|
38
|
+
implementation falls back to a timestamp-based ticket.
|
|
39
|
+
|
|
40
|
+
Args:
|
|
41
|
+
name (str): Lock filename or path. If a simple name is given,
|
|
42
|
+
it is created in the system temporary directory.
|
|
43
|
+
timeout (float | None): Seconds to wait for the lock. ``None`` means block indefinitely.
|
|
44
|
+
blocking (bool): Whether to block until acquired (subject to ``timeout``).
|
|
45
|
+
check_interval (float): Interval between lock attempts, in seconds.
|
|
46
|
+
fifo (bool): Whether to enforce Fifo ordering (default: True).
|
|
47
|
+
fifo_stale_timeout (float | None): Seconds after which a ticket is considered stale; if ``None`` the lock\'s ``timeout`` value will be used.
|
|
48
|
+
|
|
49
|
+
Raises:
|
|
50
|
+
LockTimeoutError: If the lock could not be acquired within the timeout (LockError & TimeoutError subclass)
|
|
51
|
+
LockError: On unexpected locking errors. (RunTimeError subclass)
|
|
52
|
+
|
|
53
|
+
Examples:
|
|
54
|
+
>>> # Basic context-manager usage (Fifo enabled by default)
|
|
55
|
+
>>> with LockFifo("my.lock", timeout=1):
|
|
56
|
+
... pass
|
|
57
|
+
|
|
58
|
+
>>> # Explicit acquire/release
|
|
59
|
+
>>> lock = LockFifo("my.lock", timeout=1)
|
|
60
|
+
>>> lock.acquire()
|
|
61
|
+
>>> lock.release()
|
|
62
|
+
|
|
63
|
+
>>> # Doctest: simple multi-process Fifo check (fast and deterministic)
|
|
64
|
+
>>> import tempfile, multiprocessing, time
|
|
65
|
+
>>> tmpdir = tempfile.mkdtemp()
|
|
66
|
+
>>> lockpath = tmpdir + "/t.lock"
|
|
67
|
+
>>> out = tmpdir + "/out.txt"
|
|
68
|
+
>>> # Worker function is module-level: `_worker`
|
|
69
|
+
>>> # (Defined at module scope so it can be pickled on Windows)
|
|
70
|
+
>>> procs = []
|
|
71
|
+
>>> for i in range(3):
|
|
72
|
+
... p = multiprocessing.Process(target=_worker, args=(lockpath, out, i))
|
|
73
|
+
... p.start(); procs.append(p); time.sleep(0.05)
|
|
74
|
+
>>> for p in procs: p.join(1)
|
|
75
|
+
>>> with open(out) as f: print([int(x) for x in f.read().splitlines()])
|
|
76
|
+
[0, 1, 2]
|
|
77
|
+
|
|
78
|
+
>>> # Doctest: cleanup of artifacts on close
|
|
79
|
+
>>> import tempfile, os
|
|
80
|
+
>>> tmp = tempfile.mkdtemp()
|
|
81
|
+
>>> p = tmp + "/tlock"
|
|
82
|
+
>>> l = LockFifo(p, timeout=1)
|
|
83
|
+
>>> l.acquire(); l.release(); l.close()
|
|
84
|
+
>>> import os
|
|
85
|
+
>>> # The lock file should not remain on any platform after close()
|
|
86
|
+
>>> assert not os.path.exists(p)
|
|
87
|
+
>>> assert not os.path.exists(p + ".queue")
|
|
88
|
+
|
|
89
|
+
>>> # Non-Fifo fast-path should not create a queue directory
|
|
90
|
+
>>> tmp2 = tempfile.mkdtemp()
|
|
91
|
+
>>> p2 = tmp2 + "/tlock2"
|
|
92
|
+
>>> l2 = LockFifo(p2, fifo=False, timeout=1)
|
|
93
|
+
>>> l2.acquire(); l2.release(); l2.close()
|
|
94
|
+
>>> os.path.exists(p2 + ".queue")
|
|
95
|
+
False
|
|
96
|
+
|
|
97
|
+
>>> # Attempting a non-blocking acquire while another process holds the lock raises LockTimeoutError
|
|
98
|
+
>>> import multiprocessing, time
|
|
99
|
+
>>> # Hold function is module-level: `_hold`
|
|
100
|
+
>>> # (Defined at module scope so it can be pickled on Windows)
|
|
101
|
+
>>> p = multiprocessing.Process(target=_hold, args=(p2,))
|
|
102
|
+
>>> p.start()
|
|
103
|
+
>>> import time, os
|
|
104
|
+
>>> deadline = time.time() + 1.0
|
|
105
|
+
>>> while not os.path.exists(p2 + ".held") and time.time() < deadline:
|
|
106
|
+
... time.sleep(0.01)
|
|
107
|
+
>>> l3 = LockFifo(p2, timeout=1)
|
|
108
|
+
>>> try:
|
|
109
|
+
... l3.acquire(blocking=False)
|
|
110
|
+
... except LockTimeoutError:
|
|
111
|
+
... print("timeout")
|
|
112
|
+
... finally:
|
|
113
|
+
... p.terminate(); p.join()
|
|
114
|
+
timeout
|
|
115
|
+
'''
|
|
116
|
+
path: str
|
|
117
|
+
timeout: float | None
|
|
118
|
+
blocking: bool
|
|
119
|
+
check_interval: float
|
|
120
|
+
file: IO[bytes] | None
|
|
121
|
+
fd: int | None
|
|
122
|
+
is_locked: bool
|
|
123
|
+
fifo: bool
|
|
124
|
+
fifo_stale_timeout: float | None
|
|
125
|
+
queue_dir: str
|
|
126
|
+
queue: Incomplete
|
|
127
|
+
def __init__(self, name: str, timeout: float | None = None, blocking: bool = True, check_interval: float = 0.05, fifo: bool = True, fifo_stale_timeout: float | None = None) -> None: ...
|
|
128
|
+
def _get_ticket(self) -> int:
|
|
129
|
+
""" Obtain a monotonically increasing ticket number.
|
|
130
|
+
|
|
131
|
+
Uses a small sequence file protected by an exclusive lock (fcntl) when
|
|
132
|
+
available. When fcntl is not available, falls back to a timestamp-based
|
|
133
|
+
ticket (still monotonic enough on typical systems).
|
|
134
|
+
"""
|
|
135
|
+
def _cleanup_stale_tickets(self) -> None:
|
|
136
|
+
""" Remove stale ticket files from the queue directory.
|
|
137
|
+
|
|
138
|
+
A ticket is considered stale when its mtime is older than the effective
|
|
139
|
+
stale timeout. If ``self.fifo_stale_timeout`` is ``None``, the lock's
|
|
140
|
+
``timeout`` value is used; if that is also ``None``, no cleanup is
|
|
141
|
+
performed.
|
|
142
|
+
"""
|
|
143
|
+
def perform_lock(self, blocking: bool, timeout: float | None, check_interval: float) -> None:
|
|
144
|
+
""" Core platform-specific lock acquisition. This contains the original
|
|
145
|
+
flock-based implementation and is used both by Fifo and non-Fifo
|
|
146
|
+
paths.
|
|
147
|
+
"""
|
|
148
|
+
def acquire(self, timeout: float | None = None, blocking: bool | None = None, check_interval: float | None = None) -> None:
|
|
149
|
+
""" Acquire the lock, optionally using Fifo ordering.
|
|
150
|
+
|
|
151
|
+
When Fifo is enabled (default), a ticket file is created and the caller
|
|
152
|
+
waits until its ticket becomes head of the queue before attempting the
|
|
153
|
+
actual underlying lock. This avoids starvation by ensuring waiters are
|
|
154
|
+
served in arrival order.
|
|
155
|
+
"""
|
|
156
|
+
def release(self) -> None:
|
|
157
|
+
""" Release the lock. """
|
|
158
|
+
def __enter__(self) -> LockFifo: ...
|
|
159
|
+
def __exit__(self, exc_type: type | None, exc: BaseException | None, tb: Any | None) -> None: ...
|
|
160
|
+
def close(self) -> None:
|
|
161
|
+
""" Release and close underlying file descriptor.
|
|
162
|
+
|
|
163
|
+
Also attempts best-effort cleanup of queue artifacts and the lock file
|
|
164
|
+
itself when it is safe to do so (no waiting clients and the lock is not
|
|
165
|
+
held). This avoids leaving behind ``<lock>.queue/`` and ``<lock>``
|
|
166
|
+
files when they are no longer in use.
|
|
167
|
+
"""
|
|
168
|
+
def __del__(self) -> None: ...
|
|
169
|
+
def __repr__(self) -> str: ...
|
stouputils/lock/queue.py
ADDED
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
|
|
2
|
+
# Imports
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import time
|
|
7
|
+
import uuid
|
|
8
|
+
from typing import IO, TYPE_CHECKING, Any, cast
|
|
9
|
+
|
|
10
|
+
from ..decorators import abstract
|
|
11
|
+
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
import redis
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class BaseTicketQueue:
|
|
17
|
+
""" Base API for ticket queues. """
|
|
18
|
+
|
|
19
|
+
@abstract
|
|
20
|
+
def register(self) -> tuple[int, str]:
|
|
21
|
+
raise NotImplementedError
|
|
22
|
+
|
|
23
|
+
@abstract
|
|
24
|
+
def is_head(self, ticket: int) -> bool:
|
|
25
|
+
raise NotImplementedError
|
|
26
|
+
|
|
27
|
+
@abstract
|
|
28
|
+
def remove(self, member: str) -> None:
|
|
29
|
+
raise NotImplementedError
|
|
30
|
+
|
|
31
|
+
@abstract
|
|
32
|
+
def cleanup_stale(self) -> None:
|
|
33
|
+
raise NotImplementedError
|
|
34
|
+
|
|
35
|
+
@abstract
|
|
36
|
+
def is_empty(self) -> bool:
|
|
37
|
+
""" Return True if the queue currently has no waiting members.
|
|
38
|
+
|
|
39
|
+
Implementations should consider the concrete storage details (e.g. on
|
|
40
|
+
filesystem the "seq" file is not considered a queue member).
|
|
41
|
+
"""
|
|
42
|
+
raise NotImplementedError
|
|
43
|
+
|
|
44
|
+
@abstract
|
|
45
|
+
def maybe_cleanup(self) -> None:
|
|
46
|
+
""" Attempt to remove any on-disk or remote artifacts when the queue is empty.
|
|
47
|
+
|
|
48
|
+
This should be a best-effort no-op if other clients are concurrently
|
|
49
|
+
active. Implementations should handle errors internally and not raise.
|
|
50
|
+
"""
|
|
51
|
+
raise NotImplementedError
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class FileTicketQueue(BaseTicketQueue):
|
|
55
|
+
""" File-system backed ticket queue.
|
|
56
|
+
|
|
57
|
+
Tickets are assigned using a small ``seq`` file protected by an exclusive
|
|
58
|
+
lock (via ``fcntl`` on POSIX). Each waiter creates a ticket file named
|
|
59
|
+
``{ticket:020d}.{pid}.{uuid}`` in the queue directory. The head of the
|
|
60
|
+
sorted directory listing is considered the current owner.
|
|
61
|
+
|
|
62
|
+
Examples:
|
|
63
|
+
>>> # Basic filesystem queue behaviour and cleanup
|
|
64
|
+
>>> import tempfile, os, time
|
|
65
|
+
>>> tmp = tempfile.mkdtemp()
|
|
66
|
+
>>> qd = tmp + "/q"
|
|
67
|
+
>>> q = FileTicketQueue(qd, stale_timeout=0.01)
|
|
68
|
+
>>> t1, m1 = q.register()
|
|
69
|
+
>>> t2, m2 = q.register()
|
|
70
|
+
>>> q.is_head(t1)
|
|
71
|
+
True
|
|
72
|
+
>>> q.remove(m1)
|
|
73
|
+
>>> q.is_head(t2)
|
|
74
|
+
True
|
|
75
|
+
>>> # Make the remaining ticket appear stale and cleanup
|
|
76
|
+
>>> p = os.path.join(qd, m2)
|
|
77
|
+
>>> os.utime(p, (0, 0))
|
|
78
|
+
>>> q.cleanup_stale()
|
|
79
|
+
>>> q.is_empty()
|
|
80
|
+
True
|
|
81
|
+
>>> q.maybe_cleanup()
|
|
82
|
+
>>> os.path.exists(qd)
|
|
83
|
+
False
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
def __init__(self, queue_dir: str, stale_timeout: float | None = None) -> None:
|
|
87
|
+
self.queue_dir: str = queue_dir
|
|
88
|
+
self.stale_timeout: float | None = stale_timeout
|
|
89
|
+
os.makedirs(queue_dir, exist_ok=True)
|
|
90
|
+
|
|
91
|
+
def _get_ticket(self) -> int:
|
|
92
|
+
seq_path: str = os.path.join(self.queue_dir, "seq")
|
|
93
|
+
# Ensure queue directory exists
|
|
94
|
+
os.makedirs(self.queue_dir, exist_ok=True)
|
|
95
|
+
|
|
96
|
+
def _inc_seq_in_file(f: IO[Any]) -> int:
|
|
97
|
+
"""Read, increment and persist the sequence in the given open file."""
|
|
98
|
+
f.seek(0)
|
|
99
|
+
data: str = f.read().decode().strip()
|
|
100
|
+
seq: int = int(data) if data else 0
|
|
101
|
+
seq += 1
|
|
102
|
+
f.seek(0)
|
|
103
|
+
f.truncate(0)
|
|
104
|
+
f.write(str(seq).encode())
|
|
105
|
+
f.flush()
|
|
106
|
+
return seq
|
|
107
|
+
|
|
108
|
+
# Try POSIX advisory lock via fcntl when available
|
|
109
|
+
try:
|
|
110
|
+
import fcntl
|
|
111
|
+
with open(seq_path, "a+b") as f:
|
|
112
|
+
fcntl.flock(f, fcntl.LOCK_EX) # type: ignore
|
|
113
|
+
try:
|
|
114
|
+
seq = _inc_seq_in_file(f)
|
|
115
|
+
finally:
|
|
116
|
+
try:
|
|
117
|
+
fcntl.flock(f, fcntl.LOCK_UN) # type: ignore
|
|
118
|
+
except Exception:
|
|
119
|
+
pass
|
|
120
|
+
return seq
|
|
121
|
+
except Exception:
|
|
122
|
+
# fallthrough to try Windows locking
|
|
123
|
+
pass
|
|
124
|
+
|
|
125
|
+
# Try Windows locking via msvcrt
|
|
126
|
+
try:
|
|
127
|
+
import msvcrt
|
|
128
|
+
with open(seq_path, "a+b") as f:
|
|
129
|
+
fd = f.fileno()
|
|
130
|
+
# Lock first byte of the file (blocking)
|
|
131
|
+
locked = False
|
|
132
|
+
try:
|
|
133
|
+
msvcrt.locking(fd, msvcrt.LK_LOCK, 1) # type: ignore
|
|
134
|
+
locked = True
|
|
135
|
+
except Exception:
|
|
136
|
+
# Fallback to non-blocking lock if needed
|
|
137
|
+
try:
|
|
138
|
+
msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) # type: ignore
|
|
139
|
+
locked = True
|
|
140
|
+
except Exception:
|
|
141
|
+
locked = False
|
|
142
|
+
try:
|
|
143
|
+
if locked:
|
|
144
|
+
seq = _inc_seq_in_file(f)
|
|
145
|
+
else:
|
|
146
|
+
# If locking failed, still attempt a best-effort increment
|
|
147
|
+
seq = _inc_seq_in_file(f)
|
|
148
|
+
finally:
|
|
149
|
+
try:
|
|
150
|
+
if locked:
|
|
151
|
+
msvcrt.locking(fd, msvcrt.LK_UNLCK, 1) # type: ignore
|
|
152
|
+
except Exception:
|
|
153
|
+
pass
|
|
154
|
+
return seq
|
|
155
|
+
except Exception:
|
|
156
|
+
# Fallback to timestamp + random suffix to reduce collisions
|
|
157
|
+
import random
|
|
158
|
+
return int(time.time() * 1e6) * 1000000 + random.getrandbits(48)
|
|
159
|
+
|
|
160
|
+
def register(self) -> tuple[int, str]:
|
|
161
|
+
ticket: int = self._get_ticket()
|
|
162
|
+
fname: str = f"{ticket:020d}.{os.getpid()}.{uuid.uuid4().hex}"
|
|
163
|
+
p: str = os.path.join(self.queue_dir, fname)
|
|
164
|
+
# Create our ticket file
|
|
165
|
+
with open(p, "w") as f:
|
|
166
|
+
f.write(str(time.time()))
|
|
167
|
+
return ticket, fname
|
|
168
|
+
|
|
169
|
+
def is_head(self, ticket: int) -> bool:
|
|
170
|
+
try:
|
|
171
|
+
files: list[str] = sorted(os.listdir(self.queue_dir))
|
|
172
|
+
except FileNotFoundError:
|
|
173
|
+
return False
|
|
174
|
+
if not files:
|
|
175
|
+
return False
|
|
176
|
+
try:
|
|
177
|
+
head_ticket: int = int(files[0].split(".")[0])
|
|
178
|
+
except Exception:
|
|
179
|
+
return False
|
|
180
|
+
return head_ticket == ticket
|
|
181
|
+
|
|
182
|
+
def remove(self, member: str) -> None:
|
|
183
|
+
try:
|
|
184
|
+
p: str = os.path.join(self.queue_dir, member)
|
|
185
|
+
if os.path.exists(p):
|
|
186
|
+
os.remove(p)
|
|
187
|
+
except Exception:
|
|
188
|
+
pass
|
|
189
|
+
|
|
190
|
+
def cleanup_stale(self) -> None:
|
|
191
|
+
""" Remove stale head ticket if its mtime exceeds the stale timeout. """
|
|
192
|
+
stale: float | None = self.stale_timeout
|
|
193
|
+
if stale is None:
|
|
194
|
+
return
|
|
195
|
+
try:
|
|
196
|
+
files: list[str] = sorted(os.listdir(self.queue_dir))
|
|
197
|
+
if not files:
|
|
198
|
+
return
|
|
199
|
+
head: str = files[0]
|
|
200
|
+
p: str = os.path.join(self.queue_dir, head)
|
|
201
|
+
try:
|
|
202
|
+
mtime: float = os.path.getmtime(p)
|
|
203
|
+
except Exception:
|
|
204
|
+
return
|
|
205
|
+
if time.time() - mtime >= stale:
|
|
206
|
+
try:
|
|
207
|
+
os.remove(p)
|
|
208
|
+
except Exception:
|
|
209
|
+
pass
|
|
210
|
+
except Exception:
|
|
211
|
+
pass
|
|
212
|
+
|
|
213
|
+
def is_empty(self) -> bool:
|
|
214
|
+
"""Return True if the queue directory contains no ticket files.
|
|
215
|
+
|
|
216
|
+
The sequence file ``seq`` is ignored when determining emptiness.
|
|
217
|
+
"""
|
|
218
|
+
try:
|
|
219
|
+
files: list[str] = sorted(os.listdir(self.queue_dir))
|
|
220
|
+
except Exception:
|
|
221
|
+
return True
|
|
222
|
+
# Exclude the seq file which is used to allocate tickets
|
|
223
|
+
members = [f for f in files if f != "seq"]
|
|
224
|
+
return len(members) == 0
|
|
225
|
+
|
|
226
|
+
def maybe_cleanup(self) -> None:
|
|
227
|
+
""" Try to remove sequence file and queue dir if the queue is empty.
|
|
228
|
+
|
|
229
|
+
This is a best-effort operation: if other clients are active or a
|
|
230
|
+
race occurs, the function simply returns without raising.
|
|
231
|
+
"""
|
|
232
|
+
try:
|
|
233
|
+
if not self.is_empty():
|
|
234
|
+
return
|
|
235
|
+
# Remove seq file if present
|
|
236
|
+
seq_path: str = os.path.join(self.queue_dir, "seq")
|
|
237
|
+
try:
|
|
238
|
+
if os.path.exists(seq_path):
|
|
239
|
+
os.remove(seq_path)
|
|
240
|
+
except Exception:
|
|
241
|
+
pass
|
|
242
|
+
# Attempt to remove directory if empty
|
|
243
|
+
try:
|
|
244
|
+
os.rmdir(self.queue_dir)
|
|
245
|
+
except Exception:
|
|
246
|
+
pass
|
|
247
|
+
except Exception:
|
|
248
|
+
pass
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
class RedisTicketQueue(BaseTicketQueue):
|
|
252
|
+
""" Redis-backed ticket queue using INCR + ZADD.
|
|
253
|
+
|
|
254
|
+
Member format: ``{ticket}:{token}:{ts_ms}`` where ``ts_ms`` is the
|
|
255
|
+
insertion timestamp in milliseconds. The ZSET score is the ticket number
|
|
256
|
+
which provides ordering. This class performs stale head cleanup based on
|
|
257
|
+
the provided stale timeout.
|
|
258
|
+
|
|
259
|
+
Examples:
|
|
260
|
+
>>> # Redis queue examples; run only on non-Windows environments
|
|
261
|
+
>>> def _redis_ticket_queue_doctest():
|
|
262
|
+
... import time, redis
|
|
263
|
+
... client = redis.Redis()
|
|
264
|
+
... name = "doctest:rq"
|
|
265
|
+
... # Ensure clean start
|
|
266
|
+
... _ = client.delete(f"{name}:queue")
|
|
267
|
+
... _ = client.delete(f"{name}:seq")
|
|
268
|
+
... q = RedisTicketQueue(name, client, stale_timeout=0.01)
|
|
269
|
+
... t1, m1 = q.register()
|
|
270
|
+
... t2, m2 = q.register()
|
|
271
|
+
... q.is_head(t1)
|
|
272
|
+
... True
|
|
273
|
+
... q.remove(m1)
|
|
274
|
+
... q.is_head(t2)
|
|
275
|
+
... True
|
|
276
|
+
... q.remove(m2)
|
|
277
|
+
... q.maybe_cleanup()
|
|
278
|
+
... print(client.exists(f"{name}:queue") == 0 and client.exists(f"{name}:seq") == 0)
|
|
279
|
+
>>> import os
|
|
280
|
+
>>> if os.name != 'nt':
|
|
281
|
+
... _redis_ticket_queue_doctest()
|
|
282
|
+
... else:
|
|
283
|
+
... print("True")
|
|
284
|
+
True
|
|
285
|
+
"""
|
|
286
|
+
|
|
287
|
+
def __init__(self, name: str, client: redis.Redis | None = None, stale_timeout: float | None = None) -> None:
|
|
288
|
+
self.name: str = name
|
|
289
|
+
self.client: redis.Redis | None = client
|
|
290
|
+
self.stale_timeout: float | None = stale_timeout
|
|
291
|
+
|
|
292
|
+
def ensure_client(self) -> redis.Redis:
|
|
293
|
+
if self.client is None:
|
|
294
|
+
import redis
|
|
295
|
+
self.client = redis.Redis()
|
|
296
|
+
return self.client
|
|
297
|
+
|
|
298
|
+
def register(self) -> tuple[int, str]:
|
|
299
|
+
client: redis.Redis = self.ensure_client()
|
|
300
|
+
# redis-py may have a partly unknown return type; cast to int for Pylance
|
|
301
|
+
ticket: int = cast(int, client.incr(f"{self.name}:seq"))
|
|
302
|
+
ts_ms: int = int(time.monotonic() * 1000)
|
|
303
|
+
token: str = uuid.uuid4().hex
|
|
304
|
+
member: str = f"{ticket}:{token}:{ts_ms}"
|
|
305
|
+
client.zadd(f"{self.name}:queue", {member: ticket})
|
|
306
|
+
return ticket, member
|
|
307
|
+
|
|
308
|
+
def is_head(self, ticket: int) -> bool:
|
|
309
|
+
client: redis.Redis = self.ensure_client()
|
|
310
|
+
# zrange may return an Awaitable or a list of bytes; cast to list[bytes]
|
|
311
|
+
head = cast(list[bytes], client.zrange(f"{self.name}:queue", 0, 0)) # type: ignore[reportUnknownMemberType]
|
|
312
|
+
if not head:
|
|
313
|
+
return False
|
|
314
|
+
head_member: str = head[0].decode()
|
|
315
|
+
try:
|
|
316
|
+
head_ticket: int = int(head_member.split(":")[0])
|
|
317
|
+
except Exception:
|
|
318
|
+
return False
|
|
319
|
+
return head_ticket == ticket
|
|
320
|
+
|
|
321
|
+
def remove(self, member: str) -> None:
|
|
322
|
+
try:
|
|
323
|
+
client: redis.Redis = self.ensure_client()
|
|
324
|
+
client.zrem(f"{self.name}:queue", member)
|
|
325
|
+
except Exception:
|
|
326
|
+
pass
|
|
327
|
+
|
|
328
|
+
def cleanup_stale(self) -> None:
|
|
329
|
+
stale: float | None = self.stale_timeout
|
|
330
|
+
if stale is None:
|
|
331
|
+
return
|
|
332
|
+
try:
|
|
333
|
+
client: redis.Redis = self.ensure_client()
|
|
334
|
+
# zrange may return an Awaitable or a list of bytes; cast to list[bytes]
|
|
335
|
+
head = cast(list[bytes], client.zrange(f"{self.name}:queue", 0, 0)) # type: ignore[reportUnknownMemberType]
|
|
336
|
+
if not head:
|
|
337
|
+
return
|
|
338
|
+
head_member: str = head[0].decode()
|
|
339
|
+
parts: list[str] = head_member.split(":" )
|
|
340
|
+
if len(parts) < 3:
|
|
341
|
+
return
|
|
342
|
+
ts_ms: int = int(parts[2])
|
|
343
|
+
age: float = (time.monotonic() * 1000) - ts_ms
|
|
344
|
+
if age >= (stale * 1000):
|
|
345
|
+
try:
|
|
346
|
+
client.zrem(f"{self.name}:queue", head_member)
|
|
347
|
+
except Exception:
|
|
348
|
+
pass
|
|
349
|
+
except Exception:
|
|
350
|
+
pass
|
|
351
|
+
|
|
352
|
+
def is_empty(self) -> bool:
|
|
353
|
+
try:
|
|
354
|
+
client: redis.Redis = self.ensure_client()
|
|
355
|
+
cnt = cast(int, client.zcard(f"{self.name}:queue"))
|
|
356
|
+
return cnt == 0
|
|
357
|
+
except Exception:
|
|
358
|
+
# On error assume non-empty to avoid aggressive cleanup
|
|
359
|
+
return False
|
|
360
|
+
|
|
361
|
+
def maybe_cleanup(self) -> None:
|
|
362
|
+
"""Attempt to remove Redis keys used by the queue when it is empty.
|
|
363
|
+
|
|
364
|
+
This is best effort: if concurrent clients are active the operation may
|
|
365
|
+
be a no-op.
|
|
366
|
+
"""
|
|
367
|
+
try:
|
|
368
|
+
if not self.is_empty():
|
|
369
|
+
return
|
|
370
|
+
client: redis.Redis = self.ensure_client()
|
|
371
|
+
try:
|
|
372
|
+
client.delete(f"{self.name}:queue")
|
|
373
|
+
client.delete(f"{self.name}:seq")
|
|
374
|
+
except Exception:
|
|
375
|
+
pass
|
|
376
|
+
except Exception:
|
|
377
|
+
pass
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import redis
|
|
2
|
+
from ..decorators import abstract as abstract
|
|
3
|
+
|
|
4
|
+
class BaseTicketQueue:
|
|
5
|
+
""" Base API for ticket queues. """
|
|
6
|
+
@abstract
|
|
7
|
+
def register(self) -> tuple[int, str]: ...
|
|
8
|
+
@abstract
|
|
9
|
+
def is_head(self, ticket: int) -> bool: ...
|
|
10
|
+
@abstract
|
|
11
|
+
def remove(self, member: str) -> None: ...
|
|
12
|
+
@abstract
|
|
13
|
+
def cleanup_stale(self) -> None: ...
|
|
14
|
+
@abstract
|
|
15
|
+
def is_empty(self) -> bool:
|
|
16
|
+
''' Return True if the queue currently has no waiting members.
|
|
17
|
+
|
|
18
|
+
Implementations should consider the concrete storage details (e.g. on
|
|
19
|
+
filesystem the "seq" file is not considered a queue member).
|
|
20
|
+
'''
|
|
21
|
+
@abstract
|
|
22
|
+
def maybe_cleanup(self) -> None:
|
|
23
|
+
""" Attempt to remove any on-disk or remote artifacts when the queue is empty.
|
|
24
|
+
|
|
25
|
+
This should be a best-effort no-op if other clients are concurrently
|
|
26
|
+
active. Implementations should handle errors internally and not raise.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
class FileTicketQueue(BaseTicketQueue):
|
|
30
|
+
''' File-system backed ticket queue.
|
|
31
|
+
|
|
32
|
+
Tickets are assigned using a small ``seq`` file protected by an exclusive
|
|
33
|
+
lock (via ``fcntl`` on POSIX). Each waiter creates a ticket file named
|
|
34
|
+
``{ticket:020d}.{pid}.{uuid}`` in the queue directory. The head of the
|
|
35
|
+
sorted directory listing is considered the current owner.
|
|
36
|
+
|
|
37
|
+
Examples:
|
|
38
|
+
>>> # Basic filesystem queue behaviour and cleanup
|
|
39
|
+
>>> import tempfile, os, time
|
|
40
|
+
>>> tmp = tempfile.mkdtemp()
|
|
41
|
+
>>> qd = tmp + "/q"
|
|
42
|
+
>>> q = FileTicketQueue(qd, stale_timeout=0.01)
|
|
43
|
+
>>> t1, m1 = q.register()
|
|
44
|
+
>>> t2, m2 = q.register()
|
|
45
|
+
>>> q.is_head(t1)
|
|
46
|
+
True
|
|
47
|
+
>>> q.remove(m1)
|
|
48
|
+
>>> q.is_head(t2)
|
|
49
|
+
True
|
|
50
|
+
>>> # Make the remaining ticket appear stale and cleanup
|
|
51
|
+
>>> p = os.path.join(qd, m2)
|
|
52
|
+
>>> os.utime(p, (0, 0))
|
|
53
|
+
>>> q.cleanup_stale()
|
|
54
|
+
>>> q.is_empty()
|
|
55
|
+
True
|
|
56
|
+
>>> q.maybe_cleanup()
|
|
57
|
+
>>> os.path.exists(qd)
|
|
58
|
+
False
|
|
59
|
+
'''
|
|
60
|
+
queue_dir: str
|
|
61
|
+
stale_timeout: float | None
|
|
62
|
+
def __init__(self, queue_dir: str, stale_timeout: float | None = None) -> None: ...
|
|
63
|
+
def _get_ticket(self) -> int: ...
|
|
64
|
+
def register(self) -> tuple[int, str]: ...
|
|
65
|
+
def is_head(self, ticket: int) -> bool: ...
|
|
66
|
+
def remove(self, member: str) -> None: ...
|
|
67
|
+
def cleanup_stale(self) -> None:
|
|
68
|
+
""" Remove stale head ticket if its mtime exceeds the stale timeout. """
|
|
69
|
+
def is_empty(self) -> bool:
|
|
70
|
+
"""Return True if the queue directory contains no ticket files.
|
|
71
|
+
|
|
72
|
+
The sequence file ``seq`` is ignored when determining emptiness.
|
|
73
|
+
"""
|
|
74
|
+
def maybe_cleanup(self) -> None:
|
|
75
|
+
""" Try to remove sequence file and queue dir if the queue is empty.
|
|
76
|
+
|
|
77
|
+
This is a best-effort operation: if other clients are active or a
|
|
78
|
+
race occurs, the function simply returns without raising.
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
class RedisTicketQueue(BaseTicketQueue):
|
|
82
|
+
''' Redis-backed ticket queue using INCR + ZADD.
|
|
83
|
+
|
|
84
|
+
Member format: ``{ticket}:{token}:{ts_ms}`` where ``ts_ms`` is the
|
|
85
|
+
insertion timestamp in milliseconds. The ZSET score is the ticket number
|
|
86
|
+
which provides ordering. This class performs stale head cleanup based on
|
|
87
|
+
the provided stale timeout.
|
|
88
|
+
|
|
89
|
+
Examples:
|
|
90
|
+
>>> # Redis queue examples; run only on non-Windows environments
|
|
91
|
+
>>> def _redis_ticket_queue_doctest():
|
|
92
|
+
... import time, redis
|
|
93
|
+
... client = redis.Redis()
|
|
94
|
+
... name = "doctest:rq"
|
|
95
|
+
... # Ensure clean start
|
|
96
|
+
... _ = client.delete(f"{name}:queue")
|
|
97
|
+
... _ = client.delete(f"{name}:seq")
|
|
98
|
+
... q = RedisTicketQueue(name, client, stale_timeout=0.01)
|
|
99
|
+
... t1, m1 = q.register()
|
|
100
|
+
... t2, m2 = q.register()
|
|
101
|
+
... q.is_head(t1)
|
|
102
|
+
... True
|
|
103
|
+
... q.remove(m1)
|
|
104
|
+
... q.is_head(t2)
|
|
105
|
+
... True
|
|
106
|
+
... q.remove(m2)
|
|
107
|
+
... q.maybe_cleanup()
|
|
108
|
+
... print(client.exists(f"{name}:queue") == 0 and client.exists(f"{name}:seq") == 0)
|
|
109
|
+
>>> import os
|
|
110
|
+
>>> if os.name != \'nt\':
|
|
111
|
+
... _redis_ticket_queue_doctest()
|
|
112
|
+
... else:
|
|
113
|
+
... print("True")
|
|
114
|
+
True
|
|
115
|
+
'''
|
|
116
|
+
name: str
|
|
117
|
+
client: redis.Redis | None
|
|
118
|
+
stale_timeout: float | None
|
|
119
|
+
def __init__(self, name: str, client: redis.Redis | None = None, stale_timeout: float | None = None) -> None: ...
|
|
120
|
+
def ensure_client(self) -> redis.Redis: ...
|
|
121
|
+
def register(self) -> tuple[int, str]: ...
|
|
122
|
+
def is_head(self, ticket: int) -> bool: ...
|
|
123
|
+
def remove(self, member: str) -> None: ...
|
|
124
|
+
def cleanup_stale(self) -> None: ...
|
|
125
|
+
def is_empty(self) -> bool: ...
|
|
126
|
+
def maybe_cleanup(self) -> None:
|
|
127
|
+
"""Attempt to remove Redis keys used by the queue when it is empty.
|
|
128
|
+
|
|
129
|
+
This is best effort: if concurrent clients are active the operation may
|
|
130
|
+
be a no-op.
|
|
131
|
+
"""
|