taskwire 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- taskwire/__init__.py +135 -0
- taskwire/ambient.py +161 -0
- taskwire/conformance.py +474 -0
- taskwire/contrib/__init__.py +17 -0
- taskwire/contrib/asgi.py +91 -0
- taskwire/contrib/celery.py +352 -0
- taskwire/contrib/fastapi.py +82 -0
- taskwire/contrib/muxws.py +370 -0
- taskwire/contrib/redis_store.py +575 -0
- taskwire/contrib/schema.py +204 -0
- taskwire/contrib/threads.py +43 -0
- taskwire/contrib/viewsets.py +292 -0
- taskwire/decorators.py +88 -0
- taskwire/delivery.py +123 -0
- taskwire/headers.py +36 -0
- taskwire/models.py +683 -0
- taskwire/py.typed +0 -0
- taskwire/reader.py +120 -0
- taskwire/register.py +182 -0
- taskwire/reporter.py +1162 -0
- taskwire/rest.py +312 -0
- taskwire/settings.py +126 -0
- taskwire/store.py +647 -0
- taskwire/transport.py +95 -0
- taskwire-0.1.0.dist-info/METADATA +138 -0
- taskwire-0.1.0.dist-info/RECORD +28 -0
- taskwire-0.1.0.dist-info/WHEEL +4 -0
- taskwire-0.1.0.dist-info/licenses/LICENSE +21 -0
taskwire/store.py
ADDED
|
@@ -0,0 +1,647 @@
|
|
|
1
|
+
"""The store port (§5.3) and the in-process implementation.
|
|
2
|
+
|
|
3
|
+
The store is the truth. Everything else in the library is an accelerator over it, and the two rules
|
|
4
|
+
that make that hold are worth stating before any code:
|
|
5
|
+
|
|
6
|
+
**No read writes anything** (TW-STORE-009, TW-INV-013). `snapshot()` and `list_operations()` leave
|
|
7
|
+
every key's TTL and content byte-identical. This is why expiry is a stored deadline compared on
|
|
8
|
+
read rather than a background sweeper - a sweeper is a write, and a store that writes on read grows
|
|
9
|
+
a `:seen` key eventually.
|
|
10
|
+
|
|
11
|
+
**`rev` and the document it labels are written together** (TW-REV-003). `commit` returns the same
|
|
12
|
+
`rev` a concurrent `snapshot()` reads alongside that same body, so a client can never see a `rev`
|
|
13
|
+
that belongs to a document it did not receive.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import asyncio
|
|
19
|
+
import contextlib
|
|
20
|
+
import time
|
|
21
|
+
|
|
22
|
+
from abc import ABC, abstractmethod
|
|
23
|
+
from collections.abc import Callable
|
|
24
|
+
from dataclasses import dataclass, replace
|
|
25
|
+
from enum import Enum
|
|
26
|
+
|
|
27
|
+
from .models import (
|
|
28
|
+
DialogReply,
|
|
29
|
+
DialogRequest,
|
|
30
|
+
DialogState,
|
|
31
|
+
Envelope,
|
|
32
|
+
Progress,
|
|
33
|
+
ProgressState,
|
|
34
|
+
Result,
|
|
35
|
+
Snapshot,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def key(ns: str, token: str) -> str:
|
|
40
|
+
"""TW-SEC-001 / TW-INV-007: `f"{ns}:{token}"`, always.
|
|
41
|
+
|
|
42
|
+
There is no code path that reaches the store without a namespace. The namespace is the only
|
|
43
|
+
authorization scope (TW-SEC-003); the token is an address inside it, never a grant.
|
|
44
|
+
"""
|
|
45
|
+
return f"{ns}:{token}"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def split_key(k: str) -> tuple[str, str]:
|
|
49
|
+
"""Recover `(ns, token)` from a key.
|
|
50
|
+
|
|
51
|
+
Split on the **last** colon: a token is a UUID and contains none, but an application's namespace
|
|
52
|
+
may well contain one (`tenant:42`), and splitting on the first would silently mis-attribute
|
|
53
|
+
every one of that tenant's operations.
|
|
54
|
+
"""
|
|
55
|
+
ns, _, token = k.rpartition(":")
|
|
56
|
+
return ns, token
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass(frozen=True, slots=True)
|
|
60
|
+
class Commit:
|
|
61
|
+
"""What one commit reports back: the new `rev`, and the cancel flag that write saw.
|
|
62
|
+
|
|
63
|
+
Both in one return because both are read on every commit and a networked store must not be
|
|
64
|
+
asked twice for one write's outcome (TW-STORE-014). The flag is the sticky one
|
|
65
|
+
(TW-CANCEL-001) as it stood when the document was written, which is what `Reporter.cancelled`
|
|
66
|
+
then answers for free.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
rev: int
|
|
70
|
+
cancelled: bool
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class DialogResolution(Enum):
|
|
74
|
+
"""The only signal any caller may trust about who won a dialog race (TW-STORE-003)."""
|
|
75
|
+
|
|
76
|
+
ACCEPTED = "accepted"
|
|
77
|
+
NOT_FOUND = "not_found"
|
|
78
|
+
ALREADY_ANSWERED = "already_answered"
|
|
79
|
+
CANCELLED = "cancelled"
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
CANCELLED_BUTTON = "__taskwire_cancelled__"
|
|
83
|
+
"""The reserved button a withdrawal pushes onto a dialog's reply channel (TW-CANCEL-007).
|
|
84
|
+
|
|
85
|
+
A dialog has no deadline (TW-DLG-005), so a sentinel on the reply channel is the ONLY thing that can
|
|
86
|
+
wake a worker blocked in `ask()`. It is spelled with a reserved name rather than by a second channel
|
|
87
|
+
or a separate flag so that the waking path and the answering path are the same code - a second path
|
|
88
|
+
is a second place to forget to wake somebody.
|
|
89
|
+
|
|
90
|
+
`ask()` recognises it and raises `OperationCancelled` whatever `raise_on_cancel` says (TW-DLG-012);
|
|
91
|
+
it is never returned to an application as an answer.
|
|
92
|
+
"""
|
|
93
|
+
|
|
94
|
+
_CANCELLED_SENTINEL = DialogReply(button=CANCELLED_BUTTON, values={})
|
|
95
|
+
|
|
96
|
+
_AWAIT_POLL_SECONDS = 0.05
|
|
97
|
+
"""How long `await_dialog` blocks before re-reading (TW-BP-008).
|
|
98
|
+
|
|
99
|
+
A liveness device inside the store, never a deadline offered to `ask()` - the loop it bounds has no
|
|
100
|
+
overall deadline at all. A lost wakeup costs this much latency instead of hanging forever.
|
|
101
|
+
"""
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class DialogVanished(Exception): # noqa: N818 - the name is specified (TW-DLG-008)
|
|
105
|
+
"""The dialog document no longer exists.
|
|
106
|
+
|
|
107
|
+
A distinguishable fault, never an answer nobody gave (TW-DLG-008).
|
|
108
|
+
"""
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class TaskwireStore(ABC):
|
|
112
|
+
"""TW-STORE-001: exactly these methods.
|
|
113
|
+
|
|
114
|
+
Per-operation methods take an already-namespaced `key`; `list_operations` and `publish` take a
|
|
115
|
+
bare `ns`. That asymmetry is deliberate and is the shape of the authorization boundary.
|
|
116
|
+
"""
|
|
117
|
+
|
|
118
|
+
@abstractmethod
|
|
119
|
+
async def commit(self, key: str, progress: Progress, ttl: float) -> Commit:
|
|
120
|
+
"""Write the document and bump `rev` in one indivisible step.
|
|
121
|
+
|
|
122
|
+
Returns the new `rev` and the cancel flag the same write saw, so a caller that refreshes
|
|
123
|
+
`cancelled` after every commit needs no second round trip (TW-STORE-014).
|
|
124
|
+
"""
|
|
125
|
+
|
|
126
|
+
@abstractmethod
|
|
127
|
+
async def snapshot(self, key: str) -> Snapshot | None:
|
|
128
|
+
"""Read. Writes nothing, including TTL (TW-STORE-009)."""
|
|
129
|
+
|
|
130
|
+
@abstractmethod
|
|
131
|
+
async def touch(self, key: str, ttl: float) -> bool:
|
|
132
|
+
"""Refresh the expiry. Returns False when the key is already gone."""
|
|
133
|
+
|
|
134
|
+
@abstractmethod
|
|
135
|
+
async def list_operations(self, ns: str) -> list[Snapshot]:
|
|
136
|
+
"""Every live **shared** operation of `ns`, plus anything of `ns` holding an open dialog.
|
|
137
|
+
|
|
138
|
+
Unfiltered, with no parameter by which a caller could ask for less (TW-STORE-006).
|
|
139
|
+
"""
|
|
140
|
+
|
|
141
|
+
@abstractmethod
|
|
142
|
+
async def put_dialog(self, key: str, dialog: DialogRequest, ttl: float) -> int:
|
|
143
|
+
"""Write a dialog and bump `rev` atomically."""
|
|
144
|
+
|
|
145
|
+
@abstractmethod
|
|
146
|
+
async def resolve_dialog(self, key: str, did: str, reply: DialogReply) -> DialogResolution:
|
|
147
|
+
"""The arbiter of first-answer-wins (TW-STORE-003)."""
|
|
148
|
+
|
|
149
|
+
@abstractmethod
|
|
150
|
+
async def withdraw_dialogs(self, key: str) -> list[str]:
|
|
151
|
+
"""Flip every still-open dialog to `cancelled`; return the ids flipped."""
|
|
152
|
+
|
|
153
|
+
@abstractmethod
|
|
154
|
+
async def await_dialog(self, key: str, did: str) -> DialogReply:
|
|
155
|
+
"""Block until resolved. Takes **no deadline** - the caller has none to give (TW-DLG-005)."""
|
|
156
|
+
|
|
157
|
+
@abstractmethod
|
|
158
|
+
async def release_result(self, key: str) -> Result | None:
|
|
159
|
+
"""The single arbiter of the three release paths (TW-STORE-016)."""
|
|
160
|
+
|
|
161
|
+
@abstractmethod
|
|
162
|
+
async def request_cancel(self, key: str) -> int:
|
|
163
|
+
"""Set the sticky cancel flag and return the new `rev`. The flag is never cleared."""
|
|
164
|
+
|
|
165
|
+
@abstractmethod
|
|
166
|
+
async def is_cancelled(self, key: str) -> bool:
|
|
167
|
+
"""Read the sticky cancel flag."""
|
|
168
|
+
|
|
169
|
+
@abstractmethod
|
|
170
|
+
async def publish(self, ns: str, envelope: Envelope) -> None:
|
|
171
|
+
"""Drive the reader. MUST NOT raise (TW-STORE-008)."""
|
|
172
|
+
|
|
173
|
+
@abstractmethod
|
|
174
|
+
async def drop(self, key: str) -> None:
|
|
175
|
+
"""Remove the operation from the documents, the index and the asking set. Idempotent."""
|
|
176
|
+
|
|
177
|
+
@abstractmethod
|
|
178
|
+
async def migrate(self, from_ns: str, to_ns: str) -> int:
|
|
179
|
+
"""Re-key every operation of `from_ns` onto `to_ns`. Idempotent; publishes nothing."""
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class _Entry:
|
|
183
|
+
"""One operation's stored state. Not a wire shape - the store's own bookkeeping."""
|
|
184
|
+
|
|
185
|
+
__slots__ = ("progress", "rev", "expires_at", "dialogs", "cancel_requested", "replies")
|
|
186
|
+
|
|
187
|
+
def __init__(self, progress: Progress, rev: int, expires_at: float) -> None:
|
|
188
|
+
self.progress = progress
|
|
189
|
+
self.rev = rev
|
|
190
|
+
self.expires_at = expires_at
|
|
191
|
+
self.dialogs: dict[str, DialogRequest] = {}
|
|
192
|
+
self.cancel_requested = False
|
|
193
|
+
# One queue per dialog id: what wakes a worker blocked in `ask()`. `drop` clears it with the
|
|
194
|
+
# rest of the entry.
|
|
195
|
+
self.replies: dict[str, asyncio.Queue] = {}
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
class MemoryStore(TaskwireStore):
|
|
199
|
+
"""Single-process store. Ships in core (TW-STORE-011); the conformance suite runs against it.
|
|
200
|
+
|
|
201
|
+
A single lock rather than one per key: this store is single-process by definition, the critical
|
|
202
|
+
sections are microseconds long, and one lock makes `commit`'s read-modify-write provably atomic
|
|
203
|
+
against `list_operations`' pruning without a lock-ordering argument.
|
|
204
|
+
"""
|
|
205
|
+
|
|
206
|
+
def __init__(self, *, clock: Callable[[], float] | None = None) -> None:
|
|
207
|
+
self._clock = clock or time.time
|
|
208
|
+
self._lock = asyncio.Lock()
|
|
209
|
+
self._entries: dict[str, _Entry] = {}
|
|
210
|
+
# TW-KEY-003: shared operations only. A private operation is addressable but unlisted, and
|
|
211
|
+
# that is the whole of its privacy in the store (TW-PRIV-002).
|
|
212
|
+
self._index: dict[str, dict[str, str]] = {}
|
|
213
|
+
# TW-KEY-005: tokens currently holding at least one open dialog, private ones included.
|
|
214
|
+
self._asking: dict[str, set[str]] = {}
|
|
215
|
+
|
|
216
|
+
# ---------------------------------------------------------------- internals
|
|
217
|
+
|
|
218
|
+
def _now_ms(self) -> int:
|
|
219
|
+
return int(self._clock() * 1000)
|
|
220
|
+
|
|
221
|
+
def _live(self, k: str) -> _Entry | None:
|
|
222
|
+
"""The entry if it has not expired. Reads the deadline; never writes one."""
|
|
223
|
+
entry = self._entries.get(k)
|
|
224
|
+
if entry is None:
|
|
225
|
+
return None
|
|
226
|
+
if entry.expires_at <= self._clock():
|
|
227
|
+
return None
|
|
228
|
+
return entry
|
|
229
|
+
|
|
230
|
+
def _prune(self, ns: str) -> None:
|
|
231
|
+
"""Lazy pruning of the two index structures (TW-KEY-003, TW-KEY-005).
|
|
232
|
+
|
|
233
|
+
This is not a read side effect on the documents - it drops index members whose document has
|
|
234
|
+
already expired, which is exactly what TW-STORE-006 requires of a listing read. The
|
|
235
|
+
documents themselves are untouched.
|
|
236
|
+
"""
|
|
237
|
+
index = self._index.get(ns)
|
|
238
|
+
if index:
|
|
239
|
+
for token in [tok for tok in index if self._live(key(ns, tok)) is None]:
|
|
240
|
+
index.pop(token, None)
|
|
241
|
+
asking = self._asking.get(ns)
|
|
242
|
+
if asking:
|
|
243
|
+
for token in [tok for tok in asking if self._live(key(ns, tok)) is None]:
|
|
244
|
+
asking.discard(token)
|
|
245
|
+
|
|
246
|
+
def _build_snapshot(self, k: str, entry: _Entry) -> Snapshot:
|
|
247
|
+
_ns, token = split_key(k)
|
|
248
|
+
return Snapshot(
|
|
249
|
+
token=token,
|
|
250
|
+
rev=entry.rev,
|
|
251
|
+
progress=self._derived_progress(entry),
|
|
252
|
+
dialogs=list(entry.dialogs.values()),
|
|
253
|
+
cancel_requested=entry.cancel_requested,
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
@staticmethod
|
|
257
|
+
def _derived_progress(entry: _Entry) -> Progress:
|
|
258
|
+
"""Present `waiting_input` while a dialog is open, without ever having written it.
|
|
259
|
+
|
|
260
|
+
TW-DLG-009 says the state is `waiting_input` while a dialog is open and `running` again on
|
|
261
|
+
resolution. TW-PROG-010 and TW-PROG-012 say the state must be **derived** and that no caller
|
|
262
|
+
may write it. Both hold at once only if the derivation happens here, on the way out.
|
|
263
|
+
|
|
264
|
+
The copy is not tidiness. The reporter reads a snapshot, mutates the `Progress` it finds and
|
|
265
|
+
commits it back; handing it the same object with a derived state would persist that state,
|
|
266
|
+
and an operation would stay `waiting_input` after its last question closed - parked forever,
|
|
267
|
+
with nothing to release it (TW-INV-006).
|
|
268
|
+
"""
|
|
269
|
+
progress = entry.progress
|
|
270
|
+
if progress.is_terminal:
|
|
271
|
+
return progress
|
|
272
|
+
# TW-PROG-013: an open dialog holds a worker; an uncollected result holds nothing at all.
|
|
273
|
+
# Both derive the same state and neither is written by a caller, and any code that reasons
|
|
274
|
+
# "waiting_input implies a pinned worker" is wrong about the second one.
|
|
275
|
+
has_open_dialog = any(DialogState(d.state) is DialogState.OPEN for d in entry.dialogs.values())
|
|
276
|
+
if not has_open_dialog and progress.result is None:
|
|
277
|
+
return progress
|
|
278
|
+
return replace(progress, state=ProgressState.WAITING_INPUT)
|
|
279
|
+
|
|
280
|
+
# ---------------------------------------------------------------- writes
|
|
281
|
+
|
|
282
|
+
async def commit(self, key: str, progress: Progress, ttl: float) -> Commit: # noqa: A002
|
|
283
|
+
"""TW-STORE-002: bump rev and write the document in one indivisible step.
|
|
284
|
+
|
|
285
|
+
Three rules meet here and each is easy to lose:
|
|
286
|
+
|
|
287
|
+
TW-STORE-010 freezes `state` once terminal - a later commit may still update `label`, `data`
|
|
288
|
+
and `updated_at`, and the store enforces that rather than trusting the caller, so a cleanup
|
|
289
|
+
`set()` cannot flip a cancelled operation back to `running` (TW-INV-014).
|
|
290
|
+
|
|
291
|
+
TW-PROG-006 freezes `result_kind`, `origin_session` and `origin_connection` at the opening
|
|
292
|
+
write. There is no promotion rule: an operation is private or shared from start to finish.
|
|
293
|
+
|
|
294
|
+
TW-REV-002 seeds a new document's `rev` from `max(now_ms, previous + 1)`. Seeding from zero
|
|
295
|
+
would let a key that expired under a live operation resurrect below the client's last-seen
|
|
296
|
+
value, and every subsequent envelope would be dropped by client dedup - a bar frozen at 97 %
|
|
297
|
+
with no error anywhere.
|
|
298
|
+
"""
|
|
299
|
+
async with self._lock:
|
|
300
|
+
k = key
|
|
301
|
+
ns, token = split_key(k)
|
|
302
|
+
existing = self._live(k)
|
|
303
|
+
previous_rev = existing.rev if existing is not None else (self._entries[k].rev if k in self._entries else 0)
|
|
304
|
+
|
|
305
|
+
written = Progress(
|
|
306
|
+
state=progress.state,
|
|
307
|
+
percent=progress.percent,
|
|
308
|
+
title=progress.title,
|
|
309
|
+
label=progress.label,
|
|
310
|
+
icon=progress.icon,
|
|
311
|
+
data=progress.data,
|
|
312
|
+
error=progress.error,
|
|
313
|
+
result=progress.result,
|
|
314
|
+
result_kind=progress.result_kind,
|
|
315
|
+
origin_session=progress.origin_session,
|
|
316
|
+
origin_connection=progress.origin_connection,
|
|
317
|
+
created_at=progress.created_at,
|
|
318
|
+
updated_at=progress.updated_at,
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
if existing is not None:
|
|
322
|
+
if existing.progress.is_terminal:
|
|
323
|
+
# Terminal is final. Display fields may still be corrected; the state may not.
|
|
324
|
+
written.state = existing.progress.state
|
|
325
|
+
written.error = existing.progress.error
|
|
326
|
+
written.result = existing.progress.result
|
|
327
|
+
# Write-once fields keep whatever the opening write put there.
|
|
328
|
+
written.result_kind = existing.progress.result_kind
|
|
329
|
+
written.origin_session = existing.progress.origin_session
|
|
330
|
+
written.origin_connection = existing.progress.origin_connection
|
|
331
|
+
written.created_at = existing.progress.created_at
|
|
332
|
+
|
|
333
|
+
rev = max(self._now_ms(), previous_rev + 1)
|
|
334
|
+
entry = existing or _Entry(written, rev, 0.0)
|
|
335
|
+
entry.progress = written
|
|
336
|
+
entry.rev = rev
|
|
337
|
+
entry.expires_at = self._clock() + ttl
|
|
338
|
+
self._entries[k] = entry
|
|
339
|
+
|
|
340
|
+
# TW-KEY-003: indexed only if shared. The privacy decision was made at the opening
|
|
341
|
+
# write and is read back off the stored document, never off this call's argument.
|
|
342
|
+
#
|
|
343
|
+
# A terminal document is never in the register (TW-REG-007), and the store enforces that
|
|
344
|
+
# rather than trusting the caller to follow a write with a de-registration: the two would
|
|
345
|
+
# be two steps, and a reader between them would see a `failed` row.
|
|
346
|
+
if written.is_terminal:
|
|
347
|
+
self._index.get(ns, {}).pop(token, None)
|
|
348
|
+
elif written.is_shared:
|
|
349
|
+
self._index.setdefault(ns, {})[token] = written.created_at
|
|
350
|
+
return Commit(rev=rev, cancelled=entry.cancel_requested)
|
|
351
|
+
|
|
352
|
+
async def touch(self, key: str, ttl: float) -> bool: # noqa: A002
|
|
353
|
+
"""TW-RET-002's half of the keepalive. Refreshing an expiry is a write, and is not a read."""
|
|
354
|
+
async with self._lock:
|
|
355
|
+
entry = self._live(key)
|
|
356
|
+
if entry is None:
|
|
357
|
+
return False
|
|
358
|
+
entry.expires_at = self._clock() + ttl
|
|
359
|
+
return True
|
|
360
|
+
|
|
361
|
+
async def request_cancel(self, key: str) -> int: # noqa: A002
|
|
362
|
+
"""TW-CANCEL-001: sticky, and never cleared. Setting it does not by itself change `state`."""
|
|
363
|
+
async with self._lock:
|
|
364
|
+
entry = self._live(key)
|
|
365
|
+
if entry is None:
|
|
366
|
+
return 0
|
|
367
|
+
entry.cancel_requested = True
|
|
368
|
+
entry.rev = max(self._now_ms(), entry.rev + 1)
|
|
369
|
+
|
|
370
|
+
# TW-CANCEL-007: a worker blocked in `ask()` must wake IMMEDIATELY. A dialog has no
|
|
371
|
+
# deadline (TW-DLG-005), so a sentinel on the reply channel is the only thing that can
|
|
372
|
+
# wake it - otherwise a cancelled operation would sit on a question until Celery's
|
|
373
|
+
# execution timeout killed the task, which is minutes of a pinned worker for a click.
|
|
374
|
+
for did, dialog in entry.dialogs.items():
|
|
375
|
+
if DialogState(dialog.state) is DialogState.OPEN:
|
|
376
|
+
dialog.state = DialogState.CANCELLED
|
|
377
|
+
self._wake(entry, did, _CANCELLED_SENTINEL)
|
|
378
|
+
self._release_asking_if_quiet(key, entry)
|
|
379
|
+
return entry.rev
|
|
380
|
+
|
|
381
|
+
async def drop(self, key: str) -> None: # noqa: A002
|
|
382
|
+
"""TW-STORE-007: idempotent, and leaves every other operation of the namespace untouched."""
|
|
383
|
+
async with self._lock:
|
|
384
|
+
ns, token = split_key(key)
|
|
385
|
+
self._entries.pop(key, None)
|
|
386
|
+
if ns in self._index:
|
|
387
|
+
self._index[ns].pop(token, None)
|
|
388
|
+
if ns in self._asking:
|
|
389
|
+
self._asking[ns].discard(token)
|
|
390
|
+
|
|
391
|
+
async def migrate(self, from_ns: str, to_ns: str) -> int:
|
|
392
|
+
"""TW-STORE-015. Idempotent by construction: a token already under `to_ns` is left alone.
|
|
393
|
+
|
|
394
|
+
That skip is the whole of the idempotence, and it is also what makes the call unable to
|
|
395
|
+
clobber the destination - which matters because the one caller is an application merging an
|
|
396
|
+
anonymous session onto a freshly established account (TW-AMB-011, TW-SEC-008).
|
|
397
|
+
|
|
398
|
+
Publishes nothing. A migration is a bulk re-keying, not a state change any client is waiting
|
|
399
|
+
to hear about; the client's next register read under the new namespace is the recovery
|
|
400
|
+
(TW-AMB-012).
|
|
401
|
+
"""
|
|
402
|
+
async with self._lock:
|
|
403
|
+
prefix = f"{from_ns}:"
|
|
404
|
+
moved = 0
|
|
405
|
+
for source in [k for k in self._entries if k.startswith(prefix)]:
|
|
406
|
+
_ns, token = split_key(source)
|
|
407
|
+
destination = key(to_ns, token)
|
|
408
|
+
if destination in self._entries:
|
|
409
|
+
# Destination wins; the source is discarded rather than merged.
|
|
410
|
+
self._entries.pop(source, None)
|
|
411
|
+
continue
|
|
412
|
+
entry = self._entries.pop(source)
|
|
413
|
+
self._entries[destination] = entry
|
|
414
|
+
if token in self._index.get(from_ns, {}):
|
|
415
|
+
created = self._index[from_ns].pop(token)
|
|
416
|
+
self._index.setdefault(to_ns, {})[token] = created
|
|
417
|
+
if token in self._asking.get(from_ns, set()):
|
|
418
|
+
self._asking[from_ns].discard(token)
|
|
419
|
+
self._asking.setdefault(to_ns, set()).add(token)
|
|
420
|
+
moved += 1
|
|
421
|
+
return moved
|
|
422
|
+
|
|
423
|
+
async def publish(self, ns: str, envelope: Envelope) -> None:
|
|
424
|
+
"""TW-CORE-004: drives the reader, synchronously and in process.
|
|
425
|
+
|
|
426
|
+
TW-STORE-008: MUST NOT raise. A publish is a notification that state already changed; the
|
|
427
|
+
state is safely written by the time we get here, so a failure to announce it is a latency
|
|
428
|
+
problem and never a correctness one (TW-CORE-002).
|
|
429
|
+
"""
|
|
430
|
+
from . import reader # local import: reader imports models, and this avoids a cycle
|
|
431
|
+
|
|
432
|
+
with contextlib.suppress(Exception):
|
|
433
|
+
await reader.dispatch(ns, envelope)
|
|
434
|
+
|
|
435
|
+
# ---------------------------------------------------------------- reads
|
|
436
|
+
|
|
437
|
+
async def snapshot(self, key: str) -> Snapshot | None: # noqa: A002
|
|
438
|
+
"""TW-STORE-009: leaves TTL and content byte-identical."""
|
|
439
|
+
entry = self._live(key)
|
|
440
|
+
if entry is None:
|
|
441
|
+
return None
|
|
442
|
+
return self._build_snapshot(key, entry)
|
|
443
|
+
|
|
444
|
+
async def is_cancelled(self, key: str) -> bool: # noqa: A002
|
|
445
|
+
entry = self._live(key)
|
|
446
|
+
return bool(entry and entry.cancel_requested)
|
|
447
|
+
|
|
448
|
+
async def list_operations(self, ns: str) -> list[Snapshot]:
|
|
449
|
+
"""TW-STORE-006: the shared index UNION anything asking.
|
|
450
|
+
|
|
451
|
+
The index holds shared operations only, and a private operation holding an open dialog is
|
|
452
|
+
listed for exactly as long as it asks (TW-PRIV-004).
|
|
453
|
+
"""
|
|
454
|
+
self._prune(ns)
|
|
455
|
+
tokens = list(self._index.get(ns, {}).keys())
|
|
456
|
+
for token in self._asking.get(ns, set()):
|
|
457
|
+
if token not in tokens:
|
|
458
|
+
tokens.append(token)
|
|
459
|
+
|
|
460
|
+
snapshots: list[Snapshot] = []
|
|
461
|
+
for token in tokens:
|
|
462
|
+
k = key(ns, token)
|
|
463
|
+
entry = self._live(k)
|
|
464
|
+
if entry is None:
|
|
465
|
+
# Pruned above, but a document can expire between the prune and here. Silently drop
|
|
466
|
+
# it rather than returning a hole.
|
|
467
|
+
continue
|
|
468
|
+
snapshots.append(self._build_snapshot(k, entry))
|
|
469
|
+
return snapshots
|
|
470
|
+
|
|
471
|
+
# ------------------------------------------------------- dialogs and results
|
|
472
|
+
|
|
473
|
+
async def put_dialog(self, key: str, dialog: DialogRequest, ttl: float) -> int: # noqa: A002
|
|
474
|
+
"""Write a dialog and bump `rev` atomically (TW-STORE-002).
|
|
475
|
+
|
|
476
|
+
Adds the token to the asking set, which is the ONLY thing that set is for (TW-KEY-005): it
|
|
477
|
+
answers TW-STORE-006's second term, so a private operation is listed for exactly as long as
|
|
478
|
+
it holds a question and no longer. It is not a subscription, not a watcher list, and never an
|
|
479
|
+
authorization scope.
|
|
480
|
+
"""
|
|
481
|
+
async with self._lock:
|
|
482
|
+
ns, token = split_key(key)
|
|
483
|
+
entry = self._live(key)
|
|
484
|
+
if entry is None:
|
|
485
|
+
return 0
|
|
486
|
+
entry.dialogs[dialog.id] = dialog
|
|
487
|
+
entry.replies.setdefault(dialog.id, asyncio.Queue())
|
|
488
|
+
entry.rev = max(self._now_ms(), entry.rev + 1)
|
|
489
|
+
entry.expires_at = self._clock() + ttl
|
|
490
|
+
# TW-PRIV-004: privacy never narrows the dialog family. A private operation asking a
|
|
491
|
+
# question joins the register for as long as it asks, because a question nobody can see
|
|
492
|
+
# is a worker nobody can free (TW-INV-005).
|
|
493
|
+
self._asking.setdefault(ns, set()).add(token)
|
|
494
|
+
return entry.rev
|
|
495
|
+
|
|
496
|
+
async def resolve_dialog(self, key: str, did: str, reply: DialogReply) -> DialogResolution: # noqa: A002
|
|
497
|
+
"""**The single arbiter of first-answer-wins** (TW-STORE-003).
|
|
498
|
+
|
|
499
|
+
Atomically: verify the dialog is still `open`, record the reply, flip it to `answered`, bump
|
|
500
|
+
`rev`, wake any waiter, return ACCEPTED. Otherwise return the reason and change NOTHING.
|
|
501
|
+
|
|
502
|
+
Its return value is the ONLY signal any caller may trust about who won. A caller that read
|
|
503
|
+
the document instead would be reading it after the race, not during it, and two tabs
|
|
504
|
+
answering at once would both believe they had.
|
|
505
|
+
"""
|
|
506
|
+
async with self._lock:
|
|
507
|
+
entry = self._live(key)
|
|
508
|
+
if entry is None:
|
|
509
|
+
return DialogResolution.NOT_FOUND
|
|
510
|
+
dialog = entry.dialogs.get(did)
|
|
511
|
+
if dialog is None:
|
|
512
|
+
return DialogResolution.NOT_FOUND
|
|
513
|
+
|
|
514
|
+
state = DialogState(dialog.state)
|
|
515
|
+
if state is DialogState.ANSWERED:
|
|
516
|
+
return DialogResolution.ALREADY_ANSWERED
|
|
517
|
+
if state is DialogState.CANCELLED:
|
|
518
|
+
return DialogResolution.CANCELLED
|
|
519
|
+
|
|
520
|
+
dialog.reply = reply
|
|
521
|
+
dialog.state = DialogState.ANSWERED
|
|
522
|
+
entry.rev = max(self._now_ms(), entry.rev + 1)
|
|
523
|
+
self._wake(entry, did, reply)
|
|
524
|
+
self._release_asking_if_quiet(key, entry)
|
|
525
|
+
return DialogResolution.ACCEPTED
|
|
526
|
+
|
|
527
|
+
async def withdraw_dialogs(self, key: str) -> list[str]: # noqa: A002
|
|
528
|
+
"""Flip every still-open dialog to `cancelled`, through the same arbiter (TW-STORE-004).
|
|
529
|
+
|
|
530
|
+
A dialog a reply reached first is left `answered` - the two paths race and exactly one wins,
|
|
531
|
+
which is why they share the lock rather than each having their own idea of the state.
|
|
532
|
+
|
|
533
|
+
Called when an operation reaches a terminal state (TW-CANCEL-011) and when a cancel is
|
|
534
|
+
observed, so a dialog can never outlive the operation that asked it.
|
|
535
|
+
"""
|
|
536
|
+
async with self._lock:
|
|
537
|
+
entry = self._live(key)
|
|
538
|
+
if entry is None:
|
|
539
|
+
return []
|
|
540
|
+
withdrawn: list[str] = []
|
|
541
|
+
for did, dialog in entry.dialogs.items():
|
|
542
|
+
if DialogState(dialog.state) is not DialogState.OPEN:
|
|
543
|
+
continue
|
|
544
|
+
dialog.state = DialogState.CANCELLED
|
|
545
|
+
withdrawn.append(did)
|
|
546
|
+
self._wake(entry, did, _CANCELLED_SENTINEL)
|
|
547
|
+
if withdrawn:
|
|
548
|
+
# One bump for the whole withdrawal, not one per dialog: it is a single change of
|
|
549
|
+
# the operation's situation, and `rev` labels the document, not the dialogs.
|
|
550
|
+
entry.rev = max(self._now_ms(), entry.rev + 1)
|
|
551
|
+
self._release_asking_if_quiet(key, entry)
|
|
552
|
+
return withdrawn
|
|
553
|
+
|
|
554
|
+
async def await_dialog(self, key: str, did: str) -> DialogReply: # noqa: A002
|
|
555
|
+
"""Block until the dialog is resolved. **Takes no deadline** (TW-DLG-005, TW-STORE-005).
|
|
556
|
+
|
|
557
|
+
The caller has none to give: `ask()` has no timeout, no default answer and no
|
|
558
|
+
`DialogTimeout`, and their absence is the design rather than an omission. What ends this wait
|
|
559
|
+
is a reply, a withdrawal, or the document disappearing.
|
|
560
|
+
|
|
561
|
+
The wait is a bounded loop that RE-READS the dialog on every wakeup (TW-BP-008), so a lost
|
|
562
|
+
notification degrades to poll latency instead of hanging forever. The bound is a liveness
|
|
563
|
+
device inside the store; it is never a deadline offered to `ask()`.
|
|
564
|
+
"""
|
|
565
|
+
while True:
|
|
566
|
+
async with self._lock:
|
|
567
|
+
entry = self._live(key)
|
|
568
|
+
if entry is None or did not in entry.dialogs:
|
|
569
|
+
raise DialogVanished(f"taskwire: dialog {did} no longer exists")
|
|
570
|
+
dialog = entry.dialogs[did]
|
|
571
|
+
state = DialogState(dialog.state)
|
|
572
|
+
if state is DialogState.ANSWERED and dialog.reply is not None:
|
|
573
|
+
return dialog.reply
|
|
574
|
+
if state is DialogState.CANCELLED:
|
|
575
|
+
return _CANCELLED_SENTINEL
|
|
576
|
+
queue = entry.replies.setdefault(did, asyncio.Queue())
|
|
577
|
+
|
|
578
|
+
try:
|
|
579
|
+
return await asyncio.wait_for(queue.get(), timeout=_AWAIT_POLL_SECONDS)
|
|
580
|
+
except TimeoutError:
|
|
581
|
+
# Re-read and wait again. No overall deadline is consumed by this.
|
|
582
|
+
continue
|
|
583
|
+
|
|
584
|
+
def _wake(self, entry: _Entry, did: str, reply: DialogReply) -> None:
|
|
585
|
+
"""Push onto the dialog's reply channel so a blocked worker wakes immediately."""
|
|
586
|
+
queue = entry.replies.setdefault(did, asyncio.Queue())
|
|
587
|
+
queue.put_nowait(reply)
|
|
588
|
+
|
|
589
|
+
def _release_asking_if_quiet(self, key: str, entry: _Entry) -> None:
|
|
590
|
+
"""Leave the asking set once no dialog under this token is still `open` (TW-KEY-005).
|
|
591
|
+
|
|
592
|
+
Membership is a fact about a token's *dialogs*, so it never makes a private operation shared:
|
|
593
|
+
the moment its last question closes it leaves the set and the entry leaves the register,
|
|
594
|
+
while `result_kind` stays what it was at start (TW-PROG-006).
|
|
595
|
+
"""
|
|
596
|
+
if any(DialogState(d.state) is DialogState.OPEN for d in entry.dialogs.values()):
|
|
597
|
+
return
|
|
598
|
+
ns, token = split_key(key)
|
|
599
|
+
if ns in self._asking:
|
|
600
|
+
self._asking[ns].discard(token)
|
|
601
|
+
|
|
602
|
+
async def release_result(self, key: str) -> Result | None: # noqa: A002
|
|
603
|
+
"""**The single arbiter of the three release paths** (TW-STORE-016, TW-RES-010).
|
|
604
|
+
|
|
605
|
+
Collection, dismissal and the `result_ttl` backstop all come through here, so a collect
|
|
606
|
+
racing a dismiss releases exactly once. That matters more than it looks: each release fires
|
|
607
|
+
`result_released`, and an application that deletes its own artefact in that hook would
|
|
608
|
+
otherwise delete it twice.
|
|
609
|
+
|
|
610
|
+
Atomically: verify the operation is in `waiting_input` holding an uncollected result, write
|
|
611
|
+
the terminal `done`, bump `rev`, and return the released `Result`. Otherwise change nothing
|
|
612
|
+
and return `None`.
|
|
613
|
+
"""
|
|
614
|
+
async with self._lock:
|
|
615
|
+
entry = self._live(key)
|
|
616
|
+
if entry is None:
|
|
617
|
+
return None
|
|
618
|
+
progress = entry.progress
|
|
619
|
+
if progress.result is None or progress.is_terminal:
|
|
620
|
+
return None
|
|
621
|
+
if any(DialogState(d.state) is DialogState.OPEN for d in entry.dialogs.values()):
|
|
622
|
+
# A blocked worker is not a collectable result (TW-REST-008). The operation is in
|
|
623
|
+
# `waiting_input` for the other reason, and releasing here would end a job that is
|
|
624
|
+
# still running.
|
|
625
|
+
return None
|
|
626
|
+
|
|
627
|
+
released = progress.result
|
|
628
|
+
progress.result = None
|
|
629
|
+
progress.state = ProgressState.DONE
|
|
630
|
+
entry.rev = max(self._now_ms(), entry.rev + 1)
|
|
631
|
+
# The write that ends an operation takes it out of the register, wherever that write
|
|
632
|
+
# happens (TW-REG-007). This one ends it as surely as a terminal `commit` does.
|
|
633
|
+
ns, token = split_key(key)
|
|
634
|
+
self._index.get(ns, {}).pop(token, None)
|
|
635
|
+
return released
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
__all__ = [
|
|
639
|
+
"DialogResolution",
|
|
640
|
+
"DialogState",
|
|
641
|
+
"DialogVanished",
|
|
642
|
+
"MemoryStore",
|
|
643
|
+
"ProgressState",
|
|
644
|
+
"TaskwireStore",
|
|
645
|
+
"key",
|
|
646
|
+
"split_key",
|
|
647
|
+
]
|