flo-python 0.1.6__py3-none-any.whl → 0.1.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flo/__init__.py +31 -1
- flo/kv_body.py +264 -1
- flo/queue.py +67 -72
- flo/retry.py +13 -0
- flo/streams.py +186 -0
- flo/types.py +209 -24
- flo/wire.py +7 -3
- flo/worker.py +35 -5
- {flo_python-0.1.6.dist-info → flo_python-0.1.7.dist-info}/METADATA +90 -7
- flo_python-0.1.7.dist-info/RECORD +20 -0
- flo_python-0.1.6.dist-info/RECORD +0 -20
- {flo_python-0.1.6.dist-info → flo_python-0.1.7.dist-info}/WHEEL +0 -0
- {flo_python-0.1.6.dist-info → flo_python-0.1.7.dist-info}/licenses/LICENSE +0 -0
flo/__init__.py
CHANGED
|
@@ -91,8 +91,12 @@ from .types import (
|
|
|
91
91
|
DeleteOptions,
|
|
92
92
|
DequeueOptions,
|
|
93
93
|
DequeueResult,
|
|
94
|
+
DlqCursor,
|
|
95
|
+
DlqDeleteOptions,
|
|
94
96
|
DlqListOptions,
|
|
95
97
|
DlqPage,
|
|
98
|
+
DlqRequeueOptions,
|
|
99
|
+
DlqResult,
|
|
96
100
|
EnqueueOptions,
|
|
97
101
|
ExistsResult,
|
|
98
102
|
FloQLPoint,
|
|
@@ -100,11 +104,19 @@ from .types import (
|
|
|
100
104
|
GetJsonResult,
|
|
101
105
|
GetOptions,
|
|
102
106
|
GetResult,
|
|
107
|
+
GroupDeadLetter,
|
|
108
|
+
GroupDlqCursor,
|
|
109
|
+
GroupDlqDeleteOptions,
|
|
110
|
+
GroupDlqListOptions,
|
|
111
|
+
GroupDlqPage,
|
|
112
|
+
GroupDlqRequeueOptions,
|
|
113
|
+
GroupDlqResult,
|
|
103
114
|
GroupMemberState,
|
|
104
115
|
GroupReadResult,
|
|
105
116
|
GroupRecord,
|
|
106
117
|
GroupRefusal,
|
|
107
118
|
GroupReportResult,
|
|
119
|
+
GroupStealResult,
|
|
108
120
|
GroupTouchResult,
|
|
109
121
|
HistoryOptions,
|
|
110
122
|
HistoryResult,
|
|
@@ -153,8 +165,10 @@ from .types import (
|
|
|
153
165
|
ScanResult,
|
|
154
166
|
StatusCode,
|
|
155
167
|
StorageTier,
|
|
168
|
+
StreamAlterOptions,
|
|
156
169
|
StreamAppendOptions,
|
|
157
170
|
StreamAppendResult,
|
|
171
|
+
StreamCreateOptions,
|
|
158
172
|
StreamDeleteOptions,
|
|
159
173
|
StreamGroupAckOptions,
|
|
160
174
|
StreamGroupCreateOptions,
|
|
@@ -165,6 +179,7 @@ from .types import (
|
|
|
165
179
|
StreamGroupMembersResult,
|
|
166
180
|
StreamGroupNackOptions,
|
|
167
181
|
StreamGroupReadOptions,
|
|
182
|
+
StreamGroupStealOptions,
|
|
168
183
|
StreamID,
|
|
169
184
|
StreamInfo,
|
|
170
185
|
StreamInfoOptions,
|
|
@@ -231,7 +246,7 @@ from .worker import (
|
|
|
231
246
|
)
|
|
232
247
|
from .workflows import WorkflowOperations
|
|
233
248
|
|
|
234
|
-
__version__ = "0.1.
|
|
249
|
+
__version__ = "0.1.7"
|
|
235
250
|
|
|
236
251
|
__all__ = [
|
|
237
252
|
"Transaction",
|
|
@@ -325,6 +340,21 @@ __all__ = [
|
|
|
325
340
|
"HistoryOptions",
|
|
326
341
|
"EnqueueOptions",
|
|
327
342
|
"DequeueOptions",
|
|
343
|
+
"DlqCursor",
|
|
344
|
+
"DlqResult",
|
|
345
|
+
"DlqRequeueOptions",
|
|
346
|
+
"DlqDeleteOptions",
|
|
347
|
+
"GroupStealResult",
|
|
348
|
+
"GroupDeadLetter",
|
|
349
|
+
"GroupDlqCursor",
|
|
350
|
+
"GroupDlqPage",
|
|
351
|
+
"GroupDlqResult",
|
|
352
|
+
"StreamCreateOptions",
|
|
353
|
+
"StreamAlterOptions",
|
|
354
|
+
"StreamGroupStealOptions",
|
|
355
|
+
"GroupDlqListOptions",
|
|
356
|
+
"GroupDlqRequeueOptions",
|
|
357
|
+
"GroupDlqDeleteOptions",
|
|
328
358
|
"DlqListOptions",
|
|
329
359
|
"DlqPage",
|
|
330
360
|
"PeekOptions",
|
flo/kv_body.py
CHANGED
|
@@ -22,11 +22,22 @@ from .types import (
|
|
|
22
22
|
ActionListEntry,
|
|
23
23
|
ActionRunStatus,
|
|
24
24
|
Condition,
|
|
25
|
+
DeadLetter,
|
|
25
26
|
DequeueResult,
|
|
27
|
+
DlqCursor,
|
|
28
|
+
DlqPage,
|
|
29
|
+
DlqResult,
|
|
26
30
|
ExistsResult,
|
|
27
31
|
GetOptions,
|
|
28
32
|
GetResult,
|
|
33
|
+
GroupDeadLetter,
|
|
34
|
+
GroupDlqCursor,
|
|
35
|
+
GroupDlqPage,
|
|
36
|
+
GroupDlqResult,
|
|
29
37
|
GroupMemberState,
|
|
38
|
+
GroupRecord,
|
|
39
|
+
GroupRefusal,
|
|
40
|
+
GroupStealResult,
|
|
30
41
|
Message,
|
|
31
42
|
MGetEntry,
|
|
32
43
|
OpCode,
|
|
@@ -44,6 +55,7 @@ from .types import (
|
|
|
44
55
|
StreamGroupListResult,
|
|
45
56
|
StreamGroupMember,
|
|
46
57
|
StreamGroupMembersResult,
|
|
58
|
+
StreamID,
|
|
47
59
|
TaskAssignment,
|
|
48
60
|
VersionEntry,
|
|
49
61
|
WaitFor,
|
|
@@ -85,6 +97,12 @@ MAX_CONSUMER = 4082
|
|
|
85
97
|
MIN_LEASE_MS = 5_000
|
|
86
98
|
MAX_LEASE_MS = 43_200_000
|
|
87
99
|
MAX_QUEUE_ERROR = 1024
|
|
100
|
+
# The most records or messages a steal, or a dead letter requeue or delete,
|
|
101
|
+
# names, and the most dead letters one page lists.
|
|
102
|
+
MAX_DEAD_BATCH = 100
|
|
103
|
+
MAX_DLQ_PAGE = 100
|
|
104
|
+
# A steal's idle time: up to the longest lease.
|
|
105
|
+
MAX_MIN_IDLE_MS = 43_200_000
|
|
88
106
|
# The longest TTL whose nanoseconds fit a u64.
|
|
89
107
|
MAX_TTL_MS = 18_446_744_073_709
|
|
90
108
|
_U64_MAX = (1 << 64) - 1
|
|
@@ -118,11 +136,18 @@ _TABLE_OPS = frozenset(
|
|
|
118
136
|
OpCode.WORKER_INFO,
|
|
119
137
|
OpCode.STREAM_GROUP_LIST,
|
|
120
138
|
OpCode.STREAM_GROUP_MEMBERS,
|
|
139
|
+
OpCode.STREAM_GROUP_STEAL,
|
|
140
|
+
OpCode.STREAM_GROUP_DLQ_LIST,
|
|
141
|
+
OpCode.STREAM_GROUP_DLQ_REQUEUE,
|
|
142
|
+
OpCode.STREAM_GROUP_DLQ_DELETE,
|
|
121
143
|
OpCode.QUEUE_PEEK,
|
|
122
144
|
OpCode.QUEUE_DEQUEUE,
|
|
123
145
|
OpCode.QUEUE_COMPLETE,
|
|
124
146
|
OpCode.QUEUE_FAIL,
|
|
125
147
|
OpCode.QUEUE_TOUCH,
|
|
148
|
+
OpCode.QUEUE_DLQ_LIST,
|
|
149
|
+
OpCode.QUEUE_DLQ_REQUEUE,
|
|
150
|
+
OpCode.QUEUE_DLQ_DELETE,
|
|
126
151
|
OpCode.TS_READ,
|
|
127
152
|
OpCode.TS_INFO,
|
|
128
153
|
OpCode.PROCESSING_REMOVE,
|
|
@@ -505,6 +530,124 @@ def encode_peek_body(limit: int) -> bytes:
|
|
|
505
530
|
return _limit("peek limit", limit, MAX_PEEK)
|
|
506
531
|
|
|
507
532
|
|
|
533
|
+
def _dead_pick(names: str, count: int, all_: bool, limit: int) -> None:
|
|
534
|
+
"""A dead letter requeue or delete names some, or takes all: not both,
|
|
535
|
+
nor neither."""
|
|
536
|
+
if all_ == (count > 0):
|
|
537
|
+
raise ValidationError(
|
|
538
|
+
f"name the dead letters ({names}) or give all, not both"
|
|
539
|
+
if all_
|
|
540
|
+
else f"name the dead letters ({names}), or give all"
|
|
541
|
+
)
|
|
542
|
+
if count > MAX_DEAD_BATCH:
|
|
543
|
+
raise ValidationError(f"a batch names at most {MAX_DEAD_BATCH} {names}, got {count}")
|
|
544
|
+
_limit("limit", limit, MAX_DEAD_BATCH)
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def encode_queue_dlq_list_body(limit: int, after: DlqCursor | None) -> bytes:
|
|
548
|
+
"""queue.dlq_list's request: limit, after?."""
|
|
549
|
+
body = _limit("dlq list limit", limit, MAX_DLQ_PAGE)
|
|
550
|
+
if after is None:
|
|
551
|
+
return body + b"\x00"
|
|
552
|
+
return body + b"\x01" + _u64("cursor", after.dead_ns) + _u64("cursor", after.seq)
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
def encode_queue_dlq_body(
|
|
556
|
+
seqs: Sequence[int] | None, all_: bool, limit: int, dry_run: bool | None = None
|
|
557
|
+
) -> bytes:
|
|
558
|
+
"""queue.dlq_requeue's request: messages?, all, limit; queue.dlq_delete's
|
|
559
|
+
adds dry_run (``dry_run`` not None)."""
|
|
560
|
+
picked = list(seqs or [])
|
|
561
|
+
_dead_pick("messages", len(picked), all_, limit)
|
|
562
|
+
body = (
|
|
563
|
+
(b"\x01" + bytes([len(picked)]) + b"".join(_u64("seq", q) for q in picked))
|
|
564
|
+
if picked
|
|
565
|
+
else b"\x00"
|
|
566
|
+
)
|
|
567
|
+
body += b"\x01" + _flag(all_) + _limit("limit", limit, MAX_DEAD_BATCH)
|
|
568
|
+
if dry_run is not None:
|
|
569
|
+
body += b"\x01" + _flag(dry_run)
|
|
570
|
+
return body
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
def _record_id(r: StreamID) -> bytes:
|
|
574
|
+
return _u64("record ms", r.timestamp_ms) + _u64("record seq", r.sequence)
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
def encode_group_steal_body(
|
|
578
|
+
group: str,
|
|
579
|
+
consumer: str,
|
|
580
|
+
min_idle_ms: int,
|
|
581
|
+
lease_ms: int | None,
|
|
582
|
+
records: Sequence[tuple[StreamID, int]],
|
|
583
|
+
) -> bytes:
|
|
584
|
+
"""stream.group.steal's request: group, consumer, min_idle_ms, lease_ms?,
|
|
585
|
+
records (each id and the deliveries its holder has)."""
|
|
586
|
+
if not consumer:
|
|
587
|
+
raise ValidationError("a steal names the consumer that takes the records")
|
|
588
|
+
if (
|
|
589
|
+
not isinstance(min_idle_ms, int)
|
|
590
|
+
or isinstance(min_idle_ms, bool)
|
|
591
|
+
or not 1 <= min_idle_ms <= MAX_MIN_IDLE_MS
|
|
592
|
+
):
|
|
593
|
+
raise ValidationError(f"min_idle_ms must be 1..{MAX_MIN_IDLE_MS}, got {min_idle_ms!r}")
|
|
594
|
+
if lease_ms is not None and (
|
|
595
|
+
not isinstance(lease_ms, int)
|
|
596
|
+
or isinstance(lease_ms, bool)
|
|
597
|
+
or not MIN_LEASE_MS <= lease_ms <= MAX_LEASE_MS
|
|
598
|
+
):
|
|
599
|
+
raise ValidationError(f"lease_ms must be {MIN_LEASE_MS}..{MAX_LEASE_MS}, got {lease_ms!r}")
|
|
600
|
+
if not 1 <= len(records) <= MAX_DEAD_BATCH:
|
|
601
|
+
raise ValidationError(f"a steal names 1..{MAX_DEAD_BATCH} records, got {len(records)}")
|
|
602
|
+
return (
|
|
603
|
+
_bytes16("group", group.encode("utf-8"), _MAX_KEY)
|
|
604
|
+
+ _bytes16("consumer", consumer.encode("utf-8"), _MAX_KEY)
|
|
605
|
+
+ _u64("min_idle_ms", min_idle_ms)
|
|
606
|
+
+ _opt_u64("lease_ms", lease_ms)
|
|
607
|
+
+ bytes([len(records)])
|
|
608
|
+
+ b"".join(_record_id(rid) + _u32("deliveries", d) for rid, d in records)
|
|
609
|
+
)
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
def encode_group_dlq_list_body(group: str, limit: int, after: GroupDlqCursor | None) -> bytes:
|
|
613
|
+
"""stream.group.dlq_list's request: group, limit, after?."""
|
|
614
|
+
body = _bytes16("group", group.encode("utf-8"), _MAX_KEY) + _limit(
|
|
615
|
+
"dlq list limit", limit, MAX_DLQ_PAGE
|
|
616
|
+
)
|
|
617
|
+
if after is None:
|
|
618
|
+
return body + b"\x00"
|
|
619
|
+
return (
|
|
620
|
+
body
|
|
621
|
+
+ b"\x01"
|
|
622
|
+
+ _u64("cursor", after.dead_ns)
|
|
623
|
+
+ _u64("cursor", after.ms)
|
|
624
|
+
+ _u64("cursor", after.seq)
|
|
625
|
+
)
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def encode_group_dlq_body(
|
|
629
|
+
group: str,
|
|
630
|
+
records: Sequence[StreamID] | None,
|
|
631
|
+
all_: bool,
|
|
632
|
+
limit: int,
|
|
633
|
+
dry_run: bool | None = None,
|
|
634
|
+
) -> bytes:
|
|
635
|
+
"""stream.group.dlq_requeue's request: group, records?, all, limit;
|
|
636
|
+
stream.group.dlq_delete's adds dry_run (``dry_run`` not None)."""
|
|
637
|
+
picked = list(records or [])
|
|
638
|
+
_dead_pick("records", len(picked), all_, limit)
|
|
639
|
+
body = _bytes16("group", group.encode("utf-8"), _MAX_KEY)
|
|
640
|
+
body += (
|
|
641
|
+
(b"\x01" + bytes([len(picked)]) + b"".join(_record_id(r) for r in picked))
|
|
642
|
+
if picked
|
|
643
|
+
else b"\x00"
|
|
644
|
+
)
|
|
645
|
+
body += b"\x01" + _flag(all_) + _limit("limit", limit, MAX_DEAD_BATCH)
|
|
646
|
+
if dry_run is not None:
|
|
647
|
+
body += b"\x01" + _flag(dry_run)
|
|
648
|
+
return body
|
|
649
|
+
|
|
650
|
+
|
|
508
651
|
# ── Answers ───────────────────────────────────────────────────────────────
|
|
509
652
|
|
|
510
653
|
|
|
@@ -713,7 +856,8 @@ def _text(a: _Answer, op: str, raw: bytes) -> str:
|
|
|
713
856
|
|
|
714
857
|
def decode_action_status_answer(run_id: str, data: bytes) -> ActionRunStatus:
|
|
715
858
|
"""action.status's answer: status, created_at, started_at?, completed_at?,
|
|
716
|
-
attempt, output?, error_message?, worker_id?, outcome
|
|
859
|
+
attempt, output?, error_message?, worker_id?, outcome? (u16 lengths),
|
|
860
|
+
held? (a u8 length)."""
|
|
717
861
|
a = _Answer("action.status", data)
|
|
718
862
|
code = a.u8()
|
|
719
863
|
try:
|
|
@@ -730,6 +874,7 @@ def decode_action_status_answer(run_id: str, data: bytes) -> ActionRunStatus:
|
|
|
730
874
|
error = a.bytes16() if a.present() else None
|
|
731
875
|
worker_id = a.bytes16() if a.present() else None
|
|
732
876
|
outcome = a.bytes16() if a.present() else None
|
|
877
|
+
held = a.bytes8() if a.present() else None
|
|
733
878
|
a.done()
|
|
734
879
|
return ActionRunStatus(
|
|
735
880
|
run_id=run_id,
|
|
@@ -742,6 +887,7 @@ def decode_action_status_answer(run_id: str, data: bytes) -> ActionRunStatus:
|
|
|
742
887
|
error_message=error.decode("utf-8", errors="replace") if error is not None else None,
|
|
743
888
|
worker_id=worker_id.decode("utf-8", errors="replace") if worker_id is not None else None,
|
|
744
889
|
outcome=outcome.decode("utf-8", errors="replace") if outcome is not None else None,
|
|
890
|
+
held=held.decode("utf-8", errors="replace") if held is not None else None,
|
|
745
891
|
)
|
|
746
892
|
|
|
747
893
|
|
|
@@ -1064,3 +1210,120 @@ def decode_queue_touch_answer(data: bytes) -> QueueTouchResult:
|
|
|
1064
1210
|
refused = _refusals(a, op)
|
|
1065
1211
|
a.done()
|
|
1066
1212
|
return QueueTouchResult(expires_at_ms=expires_at_ms, applied=applied, refused=refused)
|
|
1213
|
+
|
|
1214
|
+
|
|
1215
|
+
def _why(a: _Answer, op: str) -> QueueRefusalReason:
|
|
1216
|
+
code = a.u8()
|
|
1217
|
+
try:
|
|
1218
|
+
return QueueRefusalReason(code)
|
|
1219
|
+
except ValueError:
|
|
1220
|
+
raise IncompleteResponseError(f"{op} answer has an unknown refusal {code}") from None
|
|
1221
|
+
|
|
1222
|
+
|
|
1223
|
+
def _group_refusals(a: _Answer, op: str) -> list[GroupRefusal]:
|
|
1224
|
+
refused = []
|
|
1225
|
+
for _ in range(a.u8()):
|
|
1226
|
+
ms, seq = a.u64(), a.u64()
|
|
1227
|
+
why = _why(a, op)
|
|
1228
|
+
refused.append(
|
|
1229
|
+
GroupRefusal(
|
|
1230
|
+
id=StreamID(timestamp_ms=ms, sequence=seq),
|
|
1231
|
+
why=why,
|
|
1232
|
+
reason=_text(a, op, a.bytes8()),
|
|
1233
|
+
)
|
|
1234
|
+
)
|
|
1235
|
+
return refused
|
|
1236
|
+
|
|
1237
|
+
|
|
1238
|
+
def decode_group_steal_answer(data: bytes, consumer: str) -> GroupStealResult:
|
|
1239
|
+
"""stream.group.steal's answer: term, index, expires_at_ms, stolen (each
|
|
1240
|
+
id and new deliveries), refused."""
|
|
1241
|
+
op = "stream.group.steal"
|
|
1242
|
+
a = _Answer(op, data)
|
|
1243
|
+
term, index, expires_at_ms = a.u64(), a.u64(), a.i64()
|
|
1244
|
+
stolen = []
|
|
1245
|
+
for _ in range(a.u8()):
|
|
1246
|
+
ms, seq, deliveries = a.u64(), a.u64(), a.u32()
|
|
1247
|
+
stolen.append(
|
|
1248
|
+
GroupRecord(id=StreamID(timestamp_ms=ms, sequence=seq), deliveries=deliveries)
|
|
1249
|
+
)
|
|
1250
|
+
refused = _group_refusals(a, op)
|
|
1251
|
+
a.done()
|
|
1252
|
+
return GroupStealResult(
|
|
1253
|
+
records=stolen,
|
|
1254
|
+
term=term,
|
|
1255
|
+
index=index,
|
|
1256
|
+
expires_at_ms=expires_at_ms,
|
|
1257
|
+
consumer=consumer,
|
|
1258
|
+
refused=refused,
|
|
1259
|
+
)
|
|
1260
|
+
|
|
1261
|
+
|
|
1262
|
+
def decode_group_dlq_list_answer(data: bytes) -> GroupDlqPage:
|
|
1263
|
+
"""stream.group.dlq_list's answer: dead, cursor?."""
|
|
1264
|
+
op = "stream.group.dlq_list"
|
|
1265
|
+
a = _Answer(op, data)
|
|
1266
|
+
dead = []
|
|
1267
|
+
for _ in range(a.u8()):
|
|
1268
|
+
ms, seq, deliveries, failures, dead_at_ms = a.u64(), a.u64(), a.u32(), a.u32(), a.i64()
|
|
1269
|
+
error = a.bytes16().decode("utf-8", errors="replace")
|
|
1270
|
+
dead.append(
|
|
1271
|
+
GroupDeadLetter(
|
|
1272
|
+
id=StreamID(timestamp_ms=ms, sequence=seq),
|
|
1273
|
+
deliveries=deliveries,
|
|
1274
|
+
failures=failures,
|
|
1275
|
+
dead_at_ms=dead_at_ms,
|
|
1276
|
+
error=error,
|
|
1277
|
+
)
|
|
1278
|
+
)
|
|
1279
|
+
cursor = GroupDlqCursor(dead_ns=a.u64(), ms=a.u64(), seq=a.u64()) if a.present() else None
|
|
1280
|
+
a.done()
|
|
1281
|
+
return GroupDlqPage(dead=dead, cursor=cursor)
|
|
1282
|
+
|
|
1283
|
+
|
|
1284
|
+
def decode_group_dlq_answer(op: str, data: bytes) -> GroupDlqResult:
|
|
1285
|
+
"""stream.group.dlq_requeue's and dlq_delete's answer: applied, refused, more."""
|
|
1286
|
+
a = _Answer(op, data)
|
|
1287
|
+
applied = a.u32()
|
|
1288
|
+
refused = _group_refusals(a, op)
|
|
1289
|
+
more = a.flag()
|
|
1290
|
+
a.done()
|
|
1291
|
+
return GroupDlqResult(applied=applied, refused=refused, more=more)
|
|
1292
|
+
|
|
1293
|
+
|
|
1294
|
+
def decode_queue_dlq_list_answer(data: bytes) -> DlqPage:
|
|
1295
|
+
"""queue.dlq_list's answer: dead, cursor?."""
|
|
1296
|
+
op = "queue.dlq_list"
|
|
1297
|
+
a = _Answer(op, data)
|
|
1298
|
+
messages = []
|
|
1299
|
+
for _ in range(a.u8()):
|
|
1300
|
+
seq, priority, deliveries, failures = a.u64(), a.u32(), a.u32(), a.u32()
|
|
1301
|
+
enqueued_at_ms, dead_at_ms, size = a.i64(), a.i64(), a.u32()
|
|
1302
|
+
payload = a.bytes16()
|
|
1303
|
+
error = a.bytes16().decode("utf-8", errors="replace")
|
|
1304
|
+
messages.append(
|
|
1305
|
+
DeadLetter(
|
|
1306
|
+
seq=seq,
|
|
1307
|
+
priority=priority,
|
|
1308
|
+
deliveries=deliveries,
|
|
1309
|
+
failures=failures,
|
|
1310
|
+
enqueued_at_ms=enqueued_at_ms,
|
|
1311
|
+
dead_at_ms=dead_at_ms,
|
|
1312
|
+
size=size,
|
|
1313
|
+
payload=payload,
|
|
1314
|
+
error=error,
|
|
1315
|
+
)
|
|
1316
|
+
)
|
|
1317
|
+
cursor = DlqCursor(dead_ns=a.u64(), seq=a.u64()) if a.present() else None
|
|
1318
|
+
a.done()
|
|
1319
|
+
return DlqPage(messages=messages, cursor=cursor)
|
|
1320
|
+
|
|
1321
|
+
|
|
1322
|
+
def decode_queue_dlq_answer(op: str, data: bytes) -> DlqResult:
|
|
1323
|
+
"""queue.dlq_requeue's and dlq_delete's answer: applied, refused, more."""
|
|
1324
|
+
a = _Answer(op, data)
|
|
1325
|
+
applied = a.u32()
|
|
1326
|
+
refused = _refusals(a, op)
|
|
1327
|
+
more = a.flag()
|
|
1328
|
+
a.done()
|
|
1329
|
+
return DlqResult(applied=applied, refused=refused, more=more)
|
flo/queue.py
CHANGED
|
@@ -4,6 +4,7 @@ Queue operations for Flo client.
|
|
|
4
4
|
"""
|
|
5
5
|
|
|
6
6
|
import struct
|
|
7
|
+
from collections.abc import Sequence
|
|
7
8
|
from typing import TYPE_CHECKING
|
|
8
9
|
|
|
9
10
|
from .exceptions import IncompleteResponseError, ValidationError
|
|
@@ -11,19 +12,25 @@ from .kv_body import (
|
|
|
11
12
|
MAX_PEEK,
|
|
12
13
|
decode_dequeue_answer,
|
|
13
14
|
decode_peek_answer,
|
|
15
|
+
decode_queue_dlq_answer,
|
|
16
|
+
decode_queue_dlq_list_answer,
|
|
14
17
|
decode_queue_report_answer,
|
|
15
18
|
decode_queue_touch_answer,
|
|
16
19
|
encode_dequeue_body,
|
|
17
20
|
encode_peek_body,
|
|
18
21
|
encode_queue_complete_body,
|
|
22
|
+
encode_queue_dlq_body,
|
|
23
|
+
encode_queue_dlq_list_body,
|
|
19
24
|
encode_queue_fail_body,
|
|
20
25
|
)
|
|
21
26
|
from .types import (
|
|
22
|
-
DeadLetter,
|
|
23
27
|
DequeueOptions,
|
|
24
28
|
DequeueResult,
|
|
29
|
+
DlqDeleteOptions,
|
|
25
30
|
DlqListOptions,
|
|
26
31
|
DlqPage,
|
|
32
|
+
DlqRequeueOptions,
|
|
33
|
+
DlqResult,
|
|
27
34
|
EnqueueOptions,
|
|
28
35
|
Message,
|
|
29
36
|
OpCode,
|
|
@@ -40,8 +47,6 @@ from .wire import OptionsBuilder, parse_enqueue_response
|
|
|
40
47
|
|
|
41
48
|
# The longest an enqueued message waits before it's ready: a day.
|
|
42
49
|
MAX_ENQUEUE_DELAY_MS = 86_400_000
|
|
43
|
-
# The most dead letters one page lists.
|
|
44
|
-
MAX_DLQ_PAGE = 100
|
|
45
50
|
|
|
46
51
|
if TYPE_CHECKING:
|
|
47
52
|
from .client import FloClient
|
|
@@ -225,48 +230,81 @@ class QueueOperations:
|
|
|
225
230
|
queue: str,
|
|
226
231
|
options: DlqListOptions | None = None,
|
|
227
232
|
) -> DlqPage:
|
|
228
|
-
"""
|
|
233
|
+
"""A page of a queue's dead letters, oldest dead first, each with up
|
|
234
|
+
to 4 KiB of its payload, its failures, when it died and its last
|
|
235
|
+
error.
|
|
229
236
|
|
|
230
237
|
A message lands here once failed or expired five times, or failed
|
|
231
|
-
with ``dead``; it stays until
|
|
238
|
+
with ``dead``; it stays until requeued, deleted or purged.
|
|
232
239
|
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
options: The page's limit (1..100) and ``after``, the seq that
|
|
236
|
-
ended the page before (``last_seq``); None starts at the oldest.
|
|
237
|
-
|
|
238
|
-
Returns:
|
|
239
|
-
DlqPage with the page's dead letters, how many there are in all,
|
|
240
|
-
and whether more follow.
|
|
240
|
+
Raises:
|
|
241
|
+
NotFoundError: No such queue.
|
|
241
242
|
|
|
242
243
|
Example:
|
|
243
244
|
page = await client.queue.dlq_list("tasks")
|
|
244
245
|
while True:
|
|
245
246
|
for msg in page.messages:
|
|
246
|
-
print(f"
|
|
247
|
-
if
|
|
247
|
+
print(f"Dead message {msg.seq}: {msg.error}")
|
|
248
|
+
if page.cursor is None:
|
|
248
249
|
break
|
|
249
|
-
page = await client.queue.dlq_list(
|
|
250
|
-
"tasks", DlqListOptions(after=page.last_seq)
|
|
251
|
-
)
|
|
250
|
+
page = await client.queue.dlq_list("tasks", DlqListOptions(after=page.cursor))
|
|
252
251
|
"""
|
|
253
252
|
opts = options or DlqListOptions()
|
|
254
|
-
namespace = self._client.get_namespace(opts.namespace)
|
|
255
|
-
if not 1 <= opts.limit <= MAX_DLQ_PAGE:
|
|
256
|
-
raise ValidationError(f"dlq list limit must be 1..{MAX_DLQ_PAGE}, got {opts.limit}")
|
|
257
|
-
|
|
258
|
-
# The page goes in the value: [limit:u32][after:u64]?
|
|
259
|
-
value = struct.pack("<I", opts.limit)
|
|
260
|
-
if opts.after is not None:
|
|
261
|
-
value += struct.pack("<Q", opts.after)
|
|
262
253
|
response = await self._client._send_and_check(
|
|
263
254
|
OpCode.QUEUE_DLQ_LIST,
|
|
264
|
-
namespace,
|
|
255
|
+
self._client.get_namespace(opts.namespace),
|
|
256
|
+
queue.encode("utf-8"),
|
|
257
|
+
encode_queue_dlq_list_body(opts.limit, opts.after),
|
|
258
|
+
)
|
|
259
|
+
return decode_queue_dlq_list_answer(response.data)
|
|
260
|
+
|
|
261
|
+
async def dlq_requeue(
|
|
262
|
+
self,
|
|
263
|
+
queue: str,
|
|
264
|
+
messages: Sequence[int] | None = None,
|
|
265
|
+
options: DlqRequeueOptions | None = None,
|
|
266
|
+
) -> DlqResult:
|
|
267
|
+
"""Make dead letters ready again at once, at their priority, their
|
|
268
|
+
failures reset to 0 (their deliveries and last error stay).
|
|
269
|
+
|
|
270
|
+
Name up to 100 by seq, or set ``all`` to take the oldest dead, a
|
|
271
|
+
batch of up to ``limit``: ``more`` says others remain. One that isn't
|
|
272
|
+
dead is refused ``NOT_DEAD``; one that isn't there, ``GONE``.
|
|
273
|
+
|
|
274
|
+
Raises:
|
|
275
|
+
NotFoundError: No such queue.
|
|
276
|
+
"""
|
|
277
|
+
opts = options or DlqRequeueOptions()
|
|
278
|
+
response = await self._client._send_and_check(
|
|
279
|
+
OpCode.QUEUE_DLQ_REQUEUE,
|
|
280
|
+
self._client.get_namespace(opts.namespace),
|
|
265
281
|
queue.encode("utf-8"),
|
|
266
|
-
|
|
282
|
+
encode_queue_dlq_body(messages, opts.all, opts.limit),
|
|
267
283
|
)
|
|
284
|
+
return decode_queue_dlq_answer("queue.dlq_requeue", response.data)
|
|
268
285
|
|
|
269
|
-
|
|
286
|
+
async def dlq_delete(
|
|
287
|
+
self,
|
|
288
|
+
queue: str,
|
|
289
|
+
messages: Sequence[int] | None = None,
|
|
290
|
+
options: DlqDeleteOptions | None = None,
|
|
291
|
+
) -> DlqResult:
|
|
292
|
+
"""Delete dead letters for good: up to 100 named by seq, or with
|
|
293
|
+
``all`` the oldest dead, a batch of up to ``limit`` (``more`` says
|
|
294
|
+
others remain). With ``dry_run``, ``applied`` is how many it would
|
|
295
|
+
delete, and nothing is deleted.
|
|
296
|
+
|
|
297
|
+
Raises:
|
|
298
|
+
NotFoundError: No such queue.
|
|
299
|
+
"""
|
|
300
|
+
opts = options or DlqDeleteOptions()
|
|
301
|
+
response = await self._client._send_and_check(
|
|
302
|
+
OpCode.QUEUE_DLQ_DELETE,
|
|
303
|
+
self._client.get_namespace(opts.namespace),
|
|
304
|
+
queue.encode("utf-8"),
|
|
305
|
+
encode_queue_dlq_body(messages, opts.all, opts.limit, opts.dry_run),
|
|
306
|
+
)
|
|
307
|
+
return decode_queue_dlq_answer("queue.dlq_delete", response.data)
|
|
270
308
|
|
|
271
309
|
async def peek(
|
|
272
310
|
self,
|
|
@@ -292,46 +330,3 @@ class QueueOperations:
|
|
|
292
330
|
encode_peek_body(limit),
|
|
293
331
|
)
|
|
294
332
|
return decode_peek_answer(response.data)
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
def parse_dlq_page(data: bytes) -> DlqPage:
|
|
298
|
-
"""A dead letter page: [count:u32]([seq:u64][size:u32][len:u32][payload]
|
|
299
|
-
[enqueued_at_ms:i64][deliveries:u32][priority:u8][moved_at_ms:i64])*
|
|
300
|
-
[total:u64][has_more:u8][last_seq:u64]."""
|
|
301
|
-
pos = 0
|
|
302
|
-
|
|
303
|
-
def take(n: int) -> bytes:
|
|
304
|
-
nonlocal pos
|
|
305
|
-
if pos + n > len(data):
|
|
306
|
-
raise IncompleteResponseError("queue.dlq_list answer is cut short")
|
|
307
|
-
b = data[pos : pos + n]
|
|
308
|
-
pos += n
|
|
309
|
-
return b
|
|
310
|
-
|
|
311
|
-
(count,) = struct.unpack("<I", take(4))
|
|
312
|
-
if count > MAX_DLQ_PAGE:
|
|
313
|
-
raise IncompleteResponseError(f"queue.dlq_list answer lists {count} dead letters")
|
|
314
|
-
messages = []
|
|
315
|
-
for _ in range(count):
|
|
316
|
-
seq, size, n = struct.unpack("<QII", take(16))
|
|
317
|
-
payload = take(n)
|
|
318
|
-
enqueued_at_ms, deliveries, priority, moved_at_ms = struct.unpack("<qIBq", take(21))
|
|
319
|
-
messages.append(
|
|
320
|
-
DeadLetter(
|
|
321
|
-
seq=seq,
|
|
322
|
-
size=size,
|
|
323
|
-
payload=payload,
|
|
324
|
-
enqueued_at_ms=enqueued_at_ms,
|
|
325
|
-
deliveries=deliveries,
|
|
326
|
-
priority=priority,
|
|
327
|
-
moved_at_ms=moved_at_ms,
|
|
328
|
-
)
|
|
329
|
-
)
|
|
330
|
-
total, has_more, last_seq = struct.unpack("<QBQ", take(17))
|
|
331
|
-
if has_more > 1:
|
|
332
|
-
raise IncompleteResponseError(f"queue.dlq_list answer has a has_more of {has_more}")
|
|
333
|
-
if pos != len(data):
|
|
334
|
-
raise IncompleteResponseError(
|
|
335
|
-
f"queue.dlq_list answer has {len(data) - pos} bytes left over"
|
|
336
|
-
)
|
|
337
|
-
return DlqPage(messages=messages, total=total, has_more=bool(has_more), last_seq=last_seq)
|
flo/retry.py
CHANGED
|
@@ -96,6 +96,19 @@ def is_refused_unrun(exc: BaseException) -> bool:
|
|
|
96
96
|
)
|
|
97
97
|
|
|
98
98
|
|
|
99
|
+
def _is_overloaded_unrun(exc: BaseException) -> bool:
|
|
100
|
+
"""Whether exc is a worker result refused overloaded and not run, so the
|
|
101
|
+
same result can go again. Not unavailable: a shard that halted while
|
|
102
|
+
proposing one answers it that way and stays so until an operator repairs
|
|
103
|
+
it. Nor a full group's (see is_group_held_full): that is a read's refusal."""
|
|
104
|
+
return (
|
|
105
|
+
isinstance(exc, ServerError)
|
|
106
|
+
and exc.status_code == StatusCode.OVERLOADED
|
|
107
|
+
and exc.ran == Ran.NO
|
|
108
|
+
and exc.reason != Reason.GROUP_HELD_FULL
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
|
|
99
112
|
def is_group_held_full(exc: BaseException) -> bool:
|
|
100
113
|
"""Whether exc is a group read that took nothing new because the group
|
|
101
114
|
(records out and dead together) or its shard's queues and groups are at
|