flo-python 0.1.0.dev4__py3-none-any.whl → 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flo/__init__.py +60 -15
- flo/actions.py +20 -27
- flo/client.py +193 -46
- flo/exceptions.py +83 -23
- flo/kv.py +373 -165
- flo/kv_body.py +554 -0
- flo/kv_txn.py +237 -0
- flo/processing.py +3 -67
- flo/queue.py +25 -101
- flo/retry.py +124 -0
- flo/streams.py +146 -31
- flo/types.py +431 -221
- flo/wire.py +263 -208
- flo/worker.py +684 -250
- {flo_python-0.1.0.dev4.dist-info → flo_python-0.1.2.dist-info}/METADATA +165 -98
- flo_python-0.1.2.dist-info/RECORD +19 -0
- {flo_python-0.1.0.dev4.dist-info → flo_python-0.1.2.dist-info}/WHEEL +1 -1
- flo_python-0.1.2.dist-info/licenses/LICENSE +202 -0
- flo_python-0.1.0.dev4.dist-info/RECORD +0 -16
- flo_python-0.1.0.dev4.dist-info/licenses/LICENSE +0 -21
flo/wire.py
CHANGED
|
@@ -4,33 +4,39 @@ Binary serialization/deserialization for the Flo protocol.
|
|
|
4
4
|
Header: 32 bytes, little-endian, CRC32 validated.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
+
import json
|
|
7
8
|
import struct
|
|
8
9
|
from binascii import crc32
|
|
9
10
|
from dataclasses import dataclass
|
|
11
|
+
from typing import Any
|
|
10
12
|
|
|
11
13
|
from . import types as types
|
|
12
14
|
from .exceptions import (
|
|
15
|
+
BlockTooLongError,
|
|
13
16
|
IncompleteResponseError,
|
|
14
17
|
InvalidChecksumError,
|
|
15
18
|
InvalidMagicError,
|
|
16
19
|
KeyTooLargeError,
|
|
17
20
|
NamespaceTooLargeError,
|
|
18
|
-
|
|
21
|
+
TableMismatchError,
|
|
22
|
+
ValidationError,
|
|
19
23
|
ValueTooLargeError,
|
|
20
24
|
)
|
|
25
|
+
from .kv_body import is_table_op
|
|
21
26
|
from .types import (
|
|
22
27
|
HEADER_SIZE,
|
|
23
28
|
MAGIC,
|
|
29
|
+
MAX_BLOCK_MS,
|
|
24
30
|
MAX_KEY_SIZE,
|
|
25
31
|
MAX_NAMESPACE_SIZE,
|
|
26
32
|
MAX_VALUE_SIZE,
|
|
33
|
+
TABLE_HASH,
|
|
27
34
|
VERSION,
|
|
28
35
|
DequeueResult,
|
|
29
|
-
KVEntry,
|
|
30
36
|
Message,
|
|
31
37
|
OpCode,
|
|
32
38
|
OptionTag,
|
|
33
|
-
|
|
39
|
+
PendingEntry,
|
|
34
40
|
StatusCode,
|
|
35
41
|
StorageTier,
|
|
36
42
|
StreamAppendResult,
|
|
@@ -38,18 +44,17 @@ from .types import (
|
|
|
38
44
|
StreamInfo,
|
|
39
45
|
StreamReadResult,
|
|
40
46
|
StreamRecord,
|
|
41
|
-
VersionEntry,
|
|
42
47
|
)
|
|
43
48
|
|
|
44
49
|
# Request header (32 bytes):
|
|
45
50
|
# magic(u32) + payload_len(u32) + request_id(u64)
|
|
46
|
-
# + crc32(u32) + op_code(u16) + version(u8) + flags(u8) +
|
|
47
|
-
REQUEST_HEADER_FORMAT = "<
|
|
51
|
+
# + crc32(u32) + op_code(u16) + version(u8) + flags(u8) + table_hash(u64)
|
|
52
|
+
REQUEST_HEADER_FORMAT = "<IIQIHBBQ"
|
|
48
53
|
|
|
49
54
|
# Response header (32 bytes):
|
|
50
55
|
# magic(u32) + data_len(u32) + request_id(u64)
|
|
51
|
-
# + crc32(u32) + version(u8) + status(u8) + flags(u8) + _pad(u8) +
|
|
52
|
-
RESPONSE_HEADER_FORMAT = "<
|
|
56
|
+
# + crc32(u32) + version(u8) + status(u8) + flags(u8) + _pad(u8) + table_hash(u64)
|
|
57
|
+
RESPONSE_HEADER_FORMAT = "<IIQIBBBBQ"
|
|
53
58
|
|
|
54
59
|
|
|
55
60
|
# =============================================================================
|
|
@@ -57,6 +62,18 @@ RESPONSE_HEADER_FORMAT = "<IIQIBBBB8s"
|
|
|
57
62
|
# =============================================================================
|
|
58
63
|
|
|
59
64
|
|
|
65
|
+
def _validate_block_ms(block_ms: int) -> None:
|
|
66
|
+
"""Refuse a block_ms the server would refuse or u32 cannot encode, before the round trip."""
|
|
67
|
+
if not isinstance(block_ms, int) or isinstance(block_ms, bool):
|
|
68
|
+
raise ValidationError(f"block_ms must be an int, got {block_ms!r}")
|
|
69
|
+
if block_ms < 0:
|
|
70
|
+
raise ValidationError(f"block_ms must not be negative, got {block_ms}")
|
|
71
|
+
if block_ms > MAX_BLOCK_MS:
|
|
72
|
+
raise BlockTooLongError(
|
|
73
|
+
f"a blocking wait (block_ms) is at most {MAX_BLOCK_MS} ms (5 minutes), got {block_ms}"
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
60
77
|
class OptionsBuilder:
|
|
61
78
|
"""Helper for building TLV-encoded options."""
|
|
62
79
|
|
|
@@ -70,6 +87,8 @@ class OptionsBuilder:
|
|
|
70
87
|
|
|
71
88
|
def add_u32(self, tag: OptionTag, value: int) -> "OptionsBuilder":
|
|
72
89
|
"""Add a u32 option."""
|
|
90
|
+
if tag == OptionTag.BLOCK_MS:
|
|
91
|
+
_validate_block_ms(value)
|
|
73
92
|
self._buffer.extend(bytes([tag, 4]))
|
|
74
93
|
self._buffer.extend(struct.pack("<I", value))
|
|
75
94
|
return self
|
|
@@ -178,7 +197,7 @@ def compute_crc32(header_bytes: bytes, payload: bytes) -> int:
|
|
|
178
197
|
|
|
179
198
|
The CRC32 is computed over:
|
|
180
199
|
- Header bytes 0-15 (magic, payload_length, request_id)
|
|
181
|
-
- Header bytes 20-31 (op_code/version/flags/
|
|
200
|
+
- Header bytes 20-31 (op_code/version/flags/table_hash or version/status/flags/pad/table_hash)
|
|
182
201
|
- Payload bytes
|
|
183
202
|
"""
|
|
184
203
|
# Hash bytes 0-15 and 20-31 of header (skip crc32 field at 16-19)
|
|
@@ -233,13 +252,19 @@ def serialize_request(
|
|
|
233
252
|
payload.extend(struct.pack("<H", len(key)))
|
|
234
253
|
payload.extend(key)
|
|
235
254
|
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
255
|
+
if is_table_op(op_code):
|
|
256
|
+
# A table op's body, coded by its schema, is `value`; it takes no options.
|
|
257
|
+
if options:
|
|
258
|
+
raise ValidationError(f"op {op_code} takes a body, not options")
|
|
259
|
+
payload.extend(value)
|
|
260
|
+
else:
|
|
261
|
+
# Value: [len:u32][data]
|
|
262
|
+
payload.extend(struct.pack("<I", len(value)))
|
|
263
|
+
payload.extend(value)
|
|
239
264
|
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
265
|
+
# Options: [len:u16][data]
|
|
266
|
+
payload.extend(struct.pack("<H", len(options)))
|
|
267
|
+
payload.extend(options)
|
|
243
268
|
|
|
244
269
|
payload_bytes = bytes(payload)
|
|
245
270
|
|
|
@@ -253,7 +278,7 @@ def serialize_request(
|
|
|
253
278
|
op_code,
|
|
254
279
|
VERSION,
|
|
255
280
|
0, # flags
|
|
256
|
-
|
|
281
|
+
TABLE_HASH,
|
|
257
282
|
)
|
|
258
283
|
|
|
259
284
|
# Compute CRC32
|
|
@@ -269,7 +294,7 @@ def serialize_request(
|
|
|
269
294
|
op_code,
|
|
270
295
|
VERSION,
|
|
271
296
|
0, # flags
|
|
272
|
-
|
|
297
|
+
TABLE_HASH,
|
|
273
298
|
)
|
|
274
299
|
|
|
275
300
|
return header + payload_bytes
|
|
@@ -309,21 +334,21 @@ def parse_response_header(data: bytes) -> tuple[StatusCode, int, int, int]:
|
|
|
309
334
|
Raises:
|
|
310
335
|
IncompleteResponseError: If data is too short.
|
|
311
336
|
InvalidMagicError: If magic number is invalid.
|
|
312
|
-
|
|
313
|
-
|
|
337
|
+
TableMismatchError: If the server was built from another protocol
|
|
338
|
+
version or op table; its body can't be read.
|
|
314
339
|
"""
|
|
315
340
|
if len(data) < HEADER_SIZE:
|
|
316
341
|
raise IncompleteResponseError(f"Response too short: {len(data)} < {HEADER_SIZE}")
|
|
317
342
|
|
|
318
|
-
magic, data_len, request_id, crc, version, status, flags, _pad,
|
|
343
|
+
magic, data_len, request_id, crc, version, status, flags, _pad, table_hash = struct.unpack(
|
|
319
344
|
RESPONSE_HEADER_FORMAT, data[:HEADER_SIZE]
|
|
320
345
|
)
|
|
321
346
|
|
|
322
347
|
if magic != MAGIC:
|
|
323
348
|
raise InvalidMagicError(f"Invalid magic: 0x{magic:08X} != 0x{MAGIC:08X}")
|
|
324
349
|
|
|
325
|
-
if version != VERSION:
|
|
326
|
-
raise
|
|
350
|
+
if version != VERSION or table_hash != TABLE_HASH:
|
|
351
|
+
raise TableMismatchError(version, table_hash)
|
|
327
352
|
|
|
328
353
|
return StatusCode(status), data_len, request_id, crc
|
|
329
354
|
|
|
@@ -362,117 +387,12 @@ def parse_response(data: bytes) -> RawResponse:
|
|
|
362
387
|
# =============================================================================
|
|
363
388
|
|
|
364
389
|
|
|
365
|
-
def parse_scan_response(data: bytes) -> ScanResult:
|
|
366
|
-
"""Parse scan response data.
|
|
367
|
-
|
|
368
|
-
Format: [has_more:u8][cursor_len:u32][cursor:bytes][count:u32][entries...]
|
|
369
|
-
Entry format: [key_len:u16][key][value_len:u32][value]
|
|
370
|
-
"""
|
|
371
|
-
if len(data) < 9:
|
|
372
|
-
raise IncompleteResponseError("Scan response too short")
|
|
373
|
-
|
|
374
|
-
offset = 0
|
|
375
|
-
|
|
376
|
-
# has_more
|
|
377
|
-
has_more = data[offset] != 0
|
|
378
|
-
offset += 1
|
|
379
|
-
|
|
380
|
-
# cursor
|
|
381
|
-
cursor_len = struct.unpack("<I", data[offset : offset + 4])[0]
|
|
382
|
-
offset += 4
|
|
383
|
-
|
|
384
|
-
if len(data) < offset + cursor_len:
|
|
385
|
-
raise IncompleteResponseError("Scan response cursor incomplete")
|
|
386
|
-
|
|
387
|
-
cursor = data[offset : offset + cursor_len] if cursor_len > 0 else None
|
|
388
|
-
offset += cursor_len
|
|
389
|
-
|
|
390
|
-
# count
|
|
391
|
-
if len(data) < offset + 4:
|
|
392
|
-
raise IncompleteResponseError("Scan response count incomplete")
|
|
393
|
-
|
|
394
|
-
count = struct.unpack("<I", data[offset : offset + 4])[0]
|
|
395
|
-
offset += 4
|
|
396
|
-
|
|
397
|
-
# entries
|
|
398
|
-
entries: list[KVEntry] = []
|
|
399
|
-
for _ in range(count):
|
|
400
|
-
# key
|
|
401
|
-
if len(data) < offset + 2:
|
|
402
|
-
raise IncompleteResponseError("Scan response key length incomplete")
|
|
403
|
-
|
|
404
|
-
key_len = struct.unpack("<H", data[offset : offset + 2])[0]
|
|
405
|
-
offset += 2
|
|
406
|
-
|
|
407
|
-
if len(data) < offset + key_len:
|
|
408
|
-
raise IncompleteResponseError("Scan response key incomplete")
|
|
409
|
-
|
|
410
|
-
key = data[offset : offset + key_len]
|
|
411
|
-
offset += key_len
|
|
412
|
-
|
|
413
|
-
# value
|
|
414
|
-
if len(data) < offset + 4:
|
|
415
|
-
raise IncompleteResponseError("Scan response value length incomplete")
|
|
416
|
-
|
|
417
|
-
value_len = struct.unpack("<I", data[offset : offset + 4])[0]
|
|
418
|
-
offset += 4
|
|
419
|
-
|
|
420
|
-
value: bytes | None = None
|
|
421
|
-
if value_len > 0:
|
|
422
|
-
if len(data) < offset + value_len:
|
|
423
|
-
raise IncompleteResponseError("Scan response value incomplete")
|
|
424
|
-
value = data[offset : offset + value_len]
|
|
425
|
-
offset += value_len
|
|
426
|
-
|
|
427
|
-
entries.append(KVEntry(key=key, value=value))
|
|
428
|
-
|
|
429
|
-
return ScanResult(entries=entries, cursor=cursor, has_more=has_more)
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
def parse_history_response(data: bytes) -> list[VersionEntry]:
|
|
433
|
-
"""Parse history response data.
|
|
434
|
-
|
|
435
|
-
Format: [count:u32][entries...]
|
|
436
|
-
Entry format: [version:u64][timestamp:i64][value_len:u32][value]
|
|
437
|
-
"""
|
|
438
|
-
if len(data) < 4:
|
|
439
|
-
raise IncompleteResponseError("History response too short")
|
|
440
|
-
|
|
441
|
-
offset = 0
|
|
442
|
-
|
|
443
|
-
count = struct.unpack("<I", data[offset : offset + 4])[0]
|
|
444
|
-
offset += 4
|
|
445
|
-
|
|
446
|
-
entries: list[VersionEntry] = []
|
|
447
|
-
for _ in range(count):
|
|
448
|
-
if len(data) < offset + 20:
|
|
449
|
-
raise IncompleteResponseError("History response entry incomplete")
|
|
450
|
-
|
|
451
|
-
version = struct.unpack("<Q", data[offset : offset + 8])[0]
|
|
452
|
-
offset += 8
|
|
453
|
-
|
|
454
|
-
timestamp = struct.unpack("<q", data[offset : offset + 8])[0]
|
|
455
|
-
offset += 8
|
|
456
|
-
|
|
457
|
-
value_len = struct.unpack("<I", data[offset : offset + 4])[0]
|
|
458
|
-
offset += 4
|
|
459
|
-
|
|
460
|
-
if len(data) < offset + value_len:
|
|
461
|
-
raise IncompleteResponseError("History response value incomplete")
|
|
462
|
-
|
|
463
|
-
value = data[offset : offset + value_len]
|
|
464
|
-
offset += value_len
|
|
465
|
-
|
|
466
|
-
entries.append(VersionEntry(version=version, timestamp=timestamp, value=value))
|
|
467
|
-
|
|
468
|
-
return entries
|
|
469
|
-
|
|
470
|
-
|
|
471
390
|
def parse_dequeue_response(data: bytes) -> DequeueResult:
|
|
472
391
|
"""Parse dequeue response data.
|
|
473
392
|
|
|
474
393
|
Format: [count:u32][messages...]
|
|
475
394
|
Message format: [seq:u64][payload_len:u32][payload]
|
|
395
|
+
[enqueued_at_ms:i64][delivery_count:u32][priority:u8]
|
|
476
396
|
"""
|
|
477
397
|
if len(data) < 4:
|
|
478
398
|
raise IncompleteResponseError("Dequeue response too short")
|
|
@@ -499,7 +419,20 @@ def parse_dequeue_response(data: bytes) -> DequeueResult:
|
|
|
499
419
|
payload = data[offset : offset + payload_len]
|
|
500
420
|
offset += payload_len
|
|
501
421
|
|
|
502
|
-
|
|
422
|
+
if len(data) < offset + 13:
|
|
423
|
+
raise IncompleteResponseError("Dequeue response message incomplete")
|
|
424
|
+
enqueued_at_ms, delivery_count, priority = struct.unpack("<qIB", data[offset : offset + 13])
|
|
425
|
+
offset += 13
|
|
426
|
+
|
|
427
|
+
messages.append(
|
|
428
|
+
Message(
|
|
429
|
+
seq=seq,
|
|
430
|
+
payload=payload,
|
|
431
|
+
enqueued_at_ms=enqueued_at_ms,
|
|
432
|
+
delivery_count=delivery_count,
|
|
433
|
+
priority=priority,
|
|
434
|
+
)
|
|
435
|
+
)
|
|
503
436
|
|
|
504
437
|
return DequeueResult(messages=messages)
|
|
505
438
|
|
|
@@ -563,11 +496,10 @@ def parse_stream_read_response(data: bytes) -> StreamReadResult:
|
|
|
563
496
|
count = struct.unpack("<I", data[pos : pos + 4])[0]
|
|
564
497
|
pos += 4
|
|
565
498
|
|
|
499
|
+
# An answer that ends before its count of records is refused, not
|
|
500
|
+
# returned short: each field below checks it's there.
|
|
566
501
|
records: list[StreamRecord] = []
|
|
567
502
|
for _ in range(count):
|
|
568
|
-
if pos >= len(data):
|
|
569
|
-
break
|
|
570
|
-
|
|
571
503
|
# Read sequence
|
|
572
504
|
if pos + 8 > len(data):
|
|
573
505
|
raise IncompleteResponseError("Stream record: missing sequence")
|
|
@@ -583,7 +515,10 @@ def parse_stream_read_response(data: bytes) -> StreamReadResult:
|
|
|
583
515
|
# Read tier
|
|
584
516
|
if pos + 1 > len(data):
|
|
585
517
|
raise IncompleteResponseError("Stream record: missing tier")
|
|
586
|
-
|
|
518
|
+
try:
|
|
519
|
+
tier = StorageTier(data[pos])
|
|
520
|
+
except ValueError:
|
|
521
|
+
raise IncompleteResponseError(f"Stream record: unknown tier {data[pos]}") from None
|
|
587
522
|
pos += 1
|
|
588
523
|
|
|
589
524
|
# Skip partition
|
|
@@ -597,7 +532,8 @@ def parse_stream_read_response(data: bytes) -> StreamReadResult:
|
|
|
597
532
|
key_present = data[pos]
|
|
598
533
|
pos += 1
|
|
599
534
|
|
|
600
|
-
#
|
|
535
|
+
# Read key (stream name) if present
|
|
536
|
+
stream_name = ""
|
|
601
537
|
if key_present != 0:
|
|
602
538
|
if pos + 4 > len(data):
|
|
603
539
|
raise IncompleteResponseError("Stream record: missing key length")
|
|
@@ -605,6 +541,7 @@ def parse_stream_read_response(data: bytes) -> StreamReadResult:
|
|
|
605
541
|
pos += 4
|
|
606
542
|
if pos + key_len > len(data):
|
|
607
543
|
raise IncompleteResponseError("Stream record: missing key data")
|
|
544
|
+
stream_name = data[pos : pos + key_len].decode("utf-8")
|
|
608
545
|
pos += key_len
|
|
609
546
|
|
|
610
547
|
# Read payload
|
|
@@ -618,17 +555,41 @@ def parse_stream_read_response(data: bytes) -> StreamReadResult:
|
|
|
618
555
|
payload = data[pos : pos + payload_len]
|
|
619
556
|
pos += payload_len
|
|
620
557
|
|
|
621
|
-
#
|
|
558
|
+
# Read headers: [header_count:u32]([key_len:u32][key][val_len:u32][val])*
|
|
622
559
|
if pos + 4 > len(data):
|
|
623
560
|
raise IncompleteResponseError("Stream record: missing header count")
|
|
561
|
+
header_count = struct.unpack("<I", data[pos : pos + 4])[0]
|
|
624
562
|
pos += 4
|
|
625
563
|
|
|
564
|
+
headers = None
|
|
565
|
+
if header_count > 0:
|
|
566
|
+
headers = {}
|
|
567
|
+
for _ in range(header_count):
|
|
568
|
+
if pos + 4 > len(data):
|
|
569
|
+
raise IncompleteResponseError("Stream record: missing header key length")
|
|
570
|
+
hkey_len = struct.unpack("<I", data[pos : pos + 4])[0]
|
|
571
|
+
pos += 4
|
|
572
|
+
if pos + hkey_len > len(data):
|
|
573
|
+
raise IncompleteResponseError("Stream record: missing header key")
|
|
574
|
+
hkey = data[pos : pos + hkey_len].decode("utf-8")
|
|
575
|
+
pos += hkey_len
|
|
576
|
+
if pos + 4 > len(data):
|
|
577
|
+
raise IncompleteResponseError("Stream record: missing header value length")
|
|
578
|
+
hval_len = struct.unpack("<I", data[pos : pos + 4])[0]
|
|
579
|
+
pos += 4
|
|
580
|
+
if pos + hval_len > len(data):
|
|
581
|
+
raise IncompleteResponseError("Stream record: missing header value")
|
|
582
|
+
hval = data[pos : pos + hval_len].decode("utf-8")
|
|
583
|
+
pos += hval_len
|
|
584
|
+
headers[hkey] = hval
|
|
585
|
+
|
|
626
586
|
records.append(
|
|
627
587
|
StreamRecord(
|
|
628
588
|
id=StreamID(timestamp_ms=timestamp_ms, sequence=sequence),
|
|
629
589
|
tier=tier,
|
|
590
|
+
stream=stream_name,
|
|
630
591
|
payload=payload,
|
|
631
|
-
headers=
|
|
592
|
+
headers=headers,
|
|
632
593
|
)
|
|
633
594
|
)
|
|
634
595
|
|
|
@@ -661,6 +622,33 @@ def parse_stream_info_response(data: bytes) -> StreamInfo:
|
|
|
661
622
|
)
|
|
662
623
|
|
|
663
624
|
|
|
625
|
+
def build_stream_batch_value(
|
|
626
|
+
payload: bytes,
|
|
627
|
+
headers: dict[str, str] | None = None,
|
|
628
|
+
) -> bytes:
|
|
629
|
+
"""Frame a single record for stream_append (batch of one).
|
|
630
|
+
|
|
631
|
+
Wire format:
|
|
632
|
+
[record_count:u32=1][payload_len:u32][payload]
|
|
633
|
+
[header_count:u16]([key_len:u16][key][val_len:u16][val])*
|
|
634
|
+
"""
|
|
635
|
+
hdrs = headers or {}
|
|
636
|
+
parts: list[bytes] = [
|
|
637
|
+
struct.pack("<I", 1),
|
|
638
|
+
struct.pack("<I", len(payload)),
|
|
639
|
+
payload,
|
|
640
|
+
struct.pack("<H", len(hdrs)),
|
|
641
|
+
]
|
|
642
|
+
for key, val in hdrs.items():
|
|
643
|
+
key_bytes = key.encode("utf-8")
|
|
644
|
+
val_bytes = val.encode("utf-8")
|
|
645
|
+
parts.append(struct.pack("<H", len(key_bytes)))
|
|
646
|
+
parts.append(key_bytes)
|
|
647
|
+
parts.append(struct.pack("<H", len(val_bytes)))
|
|
648
|
+
parts.append(val_bytes)
|
|
649
|
+
return b"".join(parts)
|
|
650
|
+
|
|
651
|
+
|
|
664
652
|
def serialize_group_value(group: str, consumer: str) -> bytes:
|
|
665
653
|
"""Serialize group and consumer names for consumer group operations.
|
|
666
654
|
|
|
@@ -697,95 +685,140 @@ def serialize_group_ack_value(group: str, ids: list[StreamID], consumer: str = "
|
|
|
697
685
|
return bytes(result)
|
|
698
686
|
|
|
699
687
|
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
# =============================================================================
|
|
688
|
+
def serialize_group_pending_value(group: str, consumer: str = "") -> bytes:
|
|
689
|
+
"""Serialize group (and optional consumer filter) for STREAM_GROUP_PENDING.
|
|
703
690
|
|
|
691
|
+
Format: [group_len:u16][group]([consumer_len:u16][consumer])?
|
|
704
692
|
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
693
|
+
An empty ``consumer`` returns the whole group's PEL; a non-empty one filters
|
|
694
|
+
to that consumer's entries.
|
|
695
|
+
"""
|
|
696
|
+
group_bytes = group.encode("utf-8")
|
|
697
|
+
|
|
698
|
+
result = bytearray()
|
|
699
|
+
result.extend(struct.pack("<H", len(group_bytes)))
|
|
700
|
+
result.extend(group_bytes)
|
|
701
|
+
if consumer:
|
|
702
|
+
consumer_bytes = consumer.encode("utf-8")
|
|
703
|
+
result.extend(struct.pack("<H", len(consumer_bytes)))
|
|
704
|
+
result.extend(consumer_bytes)
|
|
705
|
+
return bytes(result)
|
|
706
|
+
|
|
707
|
+
|
|
708
|
+
def serialize_group_claim_value(
|
|
709
|
+
group: str,
|
|
710
|
+
consumer: str,
|
|
711
|
+
min_idle_ms: int,
|
|
712
|
+
start_id: StreamID,
|
|
713
|
+
count: int,
|
|
710
714
|
) -> bytes:
|
|
711
|
-
"""Serialize
|
|
715
|
+
"""Serialize the request value for STREAM_GROUP_CLAIM.
|
|
712
716
|
|
|
713
|
-
Format: [
|
|
714
|
-
[
|
|
715
|
-
[has_wasm_module:u8=0]...(reserved zero fields for wire compat)
|
|
717
|
+
Format: [group_len:u16][group][consumer_len:u16][consumer]
|
|
718
|
+
[min_idle_ms:u32][start_ts:u64][start_seq:u64][count:u32]
|
|
716
719
|
"""
|
|
720
|
+
group_bytes = group.encode("utf-8")
|
|
721
|
+
consumer_bytes = consumer.encode("utf-8")
|
|
722
|
+
|
|
717
723
|
result = bytearray()
|
|
724
|
+
result.extend(struct.pack("<H", len(group_bytes)))
|
|
725
|
+
result.extend(group_bytes)
|
|
726
|
+
result.extend(struct.pack("<H", len(consumer_bytes)))
|
|
727
|
+
result.extend(consumer_bytes)
|
|
728
|
+
result.extend(struct.pack("<I", min_idle_ms))
|
|
729
|
+
result.extend(struct.pack("<Q", start_id.timestamp_ms))
|
|
730
|
+
result.extend(struct.pack("<Q", start_id.sequence))
|
|
731
|
+
result.extend(struct.pack("<I", count))
|
|
732
|
+
return bytes(result)
|
|
718
733
|
|
|
719
|
-
# action_type
|
|
720
|
-
result.append(action_type & 0xFF)
|
|
721
734
|
|
|
722
|
-
|
|
723
|
-
|
|
735
|
+
def parse_pending_entries(data: bytes) -> list[PendingEntry]:
|
|
736
|
+
"""Parse the STREAM_GROUP_PENDING response.
|
|
724
737
|
|
|
725
|
-
|
|
726
|
-
|
|
738
|
+
Format: [count:u32]([ts:u64][seq:u64][delivery_count:u32]
|
|
739
|
+
[consumer_len:u16][consumer])*
|
|
740
|
+
"""
|
|
741
|
+
if len(data) < 4:
|
|
742
|
+
return []
|
|
727
743
|
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
result.append(1) # has_desc
|
|
732
|
-
result.extend(struct.pack("<H", len(desc_bytes)))
|
|
733
|
-
result.extend(desc_bytes)
|
|
734
|
-
else:
|
|
735
|
-
result.append(0)
|
|
744
|
+
pos = 0
|
|
745
|
+
count = struct.unpack("<I", data[pos : pos + 4])[0]
|
|
746
|
+
pos += 4
|
|
736
747
|
|
|
737
|
-
|
|
738
|
-
|
|
748
|
+
entries: list[PendingEntry] = []
|
|
749
|
+
for _ in range(count):
|
|
750
|
+
if pos + 8 + 8 + 4 + 2 > len(data):
|
|
751
|
+
raise IncompleteResponseError("Pending entry: truncated header")
|
|
752
|
+
ts = struct.unpack("<Q", data[pos : pos + 8])[0]
|
|
753
|
+
pos += 8
|
|
754
|
+
seq = struct.unpack("<Q", data[pos : pos + 8])[0]
|
|
755
|
+
pos += 8
|
|
756
|
+
delivery_count = struct.unpack("<I", data[pos : pos + 4])[0]
|
|
757
|
+
pos += 4
|
|
758
|
+
clen = struct.unpack("<H", data[pos : pos + 2])[0]
|
|
759
|
+
pos += 2
|
|
760
|
+
if pos + clen > len(data):
|
|
761
|
+
raise IncompleteResponseError("Pending entry: truncated consumer")
|
|
762
|
+
consumer = data[pos : pos + clen].decode("utf-8")
|
|
763
|
+
pos += clen
|
|
764
|
+
entries.append(
|
|
765
|
+
PendingEntry(
|
|
766
|
+
id=StreamID(timestamp_ms=ts, sequence=seq),
|
|
767
|
+
consumer=consumer,
|
|
768
|
+
delivery_count=delivery_count,
|
|
769
|
+
)
|
|
770
|
+
)
|
|
771
|
+
return entries
|
|
739
772
|
|
|
740
|
-
# wasm_entrypoint (reserved, always zero)
|
|
741
|
-
result.append(0)
|
|
742
773
|
|
|
743
|
-
|
|
744
|
-
|
|
774
|
+
# =============================================================================
|
|
775
|
+
# Action/Worker Value Serialization
|
|
776
|
+
# =============================================================================
|
|
745
777
|
|
|
746
|
-
# trigger_stream (optional, not used)
|
|
747
|
-
result.append(0)
|
|
748
778
|
|
|
749
|
-
|
|
750
|
-
|
|
779
|
+
def serialize_action_register_value(
|
|
780
|
+
action_type: int,
|
|
781
|
+
timeout_ms: int,
|
|
782
|
+
max_retries: int,
|
|
783
|
+
owner: str | None = None,
|
|
784
|
+
) -> bytes:
|
|
785
|
+
"""Serialize action register value.
|
|
751
786
|
|
|
752
|
-
|
|
787
|
+
Format: [action_type:u8][timeout_ms:u32][max_retries:u32][owner_len:u16][owner]
|
|
788
|
+
"""
|
|
789
|
+
for name, value in (("timeout_ms", timeout_ms), ("max_retries", max_retries)):
|
|
790
|
+
if not 0 <= value < 2**32:
|
|
791
|
+
raise ValueError(f"{name} must be in [0, 2**32), not {value}")
|
|
792
|
+
owner_bytes = (owner or "").encode("utf-8")
|
|
793
|
+
if len(owner_bytes) > 0xFFFF:
|
|
794
|
+
raise ValueError("owner is longer than 65535 bytes")
|
|
795
|
+
return (
|
|
796
|
+
struct.pack("<BIIH", action_type & 0xFF, timeout_ms, max_retries, len(owner_bytes))
|
|
797
|
+
+ owner_bytes
|
|
798
|
+
)
|
|
753
799
|
|
|
754
800
|
|
|
755
801
|
def serialize_action_invoke_value(
|
|
756
802
|
input_data: bytes,
|
|
757
|
-
|
|
758
|
-
idempotency_key: str | None = None,
|
|
803
|
+
labels: dict[str, Any] | None = None,
|
|
759
804
|
) -> bytes:
|
|
760
805
|
"""Serialize action invoke value.
|
|
761
806
|
|
|
762
|
-
Format: [
|
|
763
|
-
[has_idempotency_key:u8][key_len:u16]?[key]?[input...]
|
|
764
|
-
"""
|
|
765
|
-
result = bytearray()
|
|
766
|
-
|
|
767
|
-
# priority
|
|
768
|
-
result.append(priority & 0xFF)
|
|
807
|
+
Format: [has_labels:u8]([labels_len:u16][labels...])?[input...]
|
|
769
808
|
|
|
770
|
-
|
|
771
|
-
|
|
809
|
+
has_labels is 0 or 1. labels is a compact JSON object; only workers whose
|
|
810
|
+
registered labels contain every key/value receive the run.
|
|
772
811
|
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
key_bytes = idempotency_key.encode("utf-8")
|
|
779
|
-
result.append(1)
|
|
780
|
-
result.extend(struct.pack("<H", len(key_bytes)))
|
|
781
|
-
result.extend(key_bytes)
|
|
782
|
-
else:
|
|
783
|
-
result.append(0)
|
|
784
|
-
|
|
785
|
-
# input
|
|
786
|
-
result.extend(input_data)
|
|
812
|
+
Raises:
|
|
813
|
+
ValueError: If the encoded labels exceed 65535 bytes.
|
|
814
|
+
"""
|
|
815
|
+
if labels is None:
|
|
816
|
+
return b"\x00" + input_data
|
|
787
817
|
|
|
788
|
-
|
|
818
|
+
labels_bytes = json.dumps(labels, separators=(",", ":")).encode("utf-8")
|
|
819
|
+
if len(labels_bytes) > 0xFFFF:
|
|
820
|
+
raise ValueError(f"invoke labels are {len(labels_bytes)} bytes; the limit is 65535")
|
|
821
|
+
return b"\x01" + struct.pack("<H", len(labels_bytes)) + labels_bytes + input_data
|
|
789
822
|
|
|
790
823
|
|
|
791
824
|
def serialize_action_list_value(limit: int = 100) -> bytes:
|
|
@@ -933,19 +966,13 @@ def serialize_worker_fail_value(
|
|
|
933
966
|
return bytes(result)
|
|
934
967
|
|
|
935
968
|
|
|
936
|
-
def serialize_worker_list_value(limit: int = 100) -> bytes:
|
|
937
|
-
"""Serialize worker list value.
|
|
938
|
-
|
|
939
|
-
Format: [limit:u32]
|
|
940
|
-
"""
|
|
941
|
-
return struct.pack("<I", limit)
|
|
942
|
-
|
|
943
|
-
|
|
944
969
|
def parse_task_assignment(data: bytes) -> "types.TaskAssignment":
|
|
945
970
|
"""Parse task assignment from server response.
|
|
946
971
|
|
|
947
972
|
Format: [task_id_len:u16][task_id][task_type_len:u16][task_type]
|
|
948
|
-
[created_at:i64][attempt:u32]
|
|
973
|
+
[created_at:i64][attempt:u32]
|
|
974
|
+
[has_caller:u8]([run_id_len:u16][run_id][wf_name_len:u16][wf_name])?
|
|
975
|
+
[payload...]
|
|
949
976
|
"""
|
|
950
977
|
if len(data) < 10:
|
|
951
978
|
raise ValueError("Incomplete task assignment response")
|
|
@@ -982,6 +1009,32 @@ def parse_task_assignment(data: bytes) -> "types.TaskAssignment":
|
|
|
982
1009
|
attempt = struct.unpack_from("<I", data, pos)[0]
|
|
983
1010
|
pos += 4
|
|
984
1011
|
|
|
1012
|
+
# caller block: [has_caller:u8][run_id_len:u16][run_id][wf_name_len:u16][wf_name]
|
|
1013
|
+
if pos + 1 > len(data):
|
|
1014
|
+
raise ValueError("Incomplete task assignment: missing has_caller")
|
|
1015
|
+
has_caller = data[pos]
|
|
1016
|
+
pos += 1
|
|
1017
|
+
caller_run_id = ""
|
|
1018
|
+
caller_workflow_name = ""
|
|
1019
|
+
if has_caller == 1:
|
|
1020
|
+
if pos + 2 > len(data):
|
|
1021
|
+
raise ValueError("Incomplete task assignment: missing caller_run_id length")
|
|
1022
|
+
crid_len = struct.unpack_from("<H", data, pos)[0]
|
|
1023
|
+
pos += 2
|
|
1024
|
+
if pos + crid_len > len(data):
|
|
1025
|
+
raise ValueError("Incomplete task assignment: missing caller_run_id")
|
|
1026
|
+
caller_run_id = data[pos : pos + crid_len].decode("utf-8")
|
|
1027
|
+
pos += crid_len
|
|
1028
|
+
|
|
1029
|
+
if pos + 2 > len(data):
|
|
1030
|
+
raise ValueError("Incomplete task assignment: missing caller_workflow_name length")
|
|
1031
|
+
cwf_len = struct.unpack_from("<H", data, pos)[0]
|
|
1032
|
+
pos += 2
|
|
1033
|
+
if pos + cwf_len > len(data):
|
|
1034
|
+
raise ValueError("Incomplete task assignment: missing caller_workflow_name")
|
|
1035
|
+
caller_workflow_name = data[pos : pos + cwf_len].decode("utf-8")
|
|
1036
|
+
pos += cwf_len
|
|
1037
|
+
|
|
985
1038
|
# payload (rest of data)
|
|
986
1039
|
payload = data[pos:]
|
|
987
1040
|
|
|
@@ -991,4 +1044,6 @@ def parse_task_assignment(data: bytes) -> "types.TaskAssignment":
|
|
|
991
1044
|
payload=payload,
|
|
992
1045
|
created_at=created_at,
|
|
993
1046
|
attempt=attempt,
|
|
1047
|
+
caller_run_id=caller_run_id,
|
|
1048
|
+
caller_workflow_name=caller_workflow_name,
|
|
994
1049
|
)
|