flo-python 0.1.2__tar.gz → 0.1.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. {flo_python-0.1.2 → flo_python-0.1.3}/.github/flo-ref.env +1 -1
  2. {flo_python-0.1.2 → flo_python-0.1.3}/.github/workflows/ci.yml +14 -6
  3. {flo_python-0.1.2 → flo_python-0.1.3}/PKG-INFO +95 -18
  4. {flo_python-0.1.2 → flo_python-0.1.3}/README.md +93 -17
  5. {flo_python-0.1.2 → flo_python-0.1.3}/pyproject.toml +4 -1
  6. {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/__init__.py +15 -1
  7. {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/actions.py +55 -3
  8. {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/client.py +13 -6
  9. {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/exceptions.py +27 -2
  10. {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/kv.py +73 -34
  11. {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/kv_body.py +82 -38
  12. {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/kv_txn.py +2 -1
  13. {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/processing.py +27 -24
  14. {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/retry.py +25 -3
  15. {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/types.py +89 -2
  16. {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/wire.py +42 -30
  17. {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/worker.py +606 -226
  18. {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_blocking_deadline.py +6 -6
  19. {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_empty_poll_backoff.py +2 -2
  20. flo_python-0.1.3/tests/test_halts.py +579 -0
  21. flo_python-0.1.3/tests/test_kv_live.py +56 -0
  22. {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_kv_ops.py +101 -18
  23. {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_kv_table.py +3 -2
  24. flo_python-0.1.3/tests/test_ops_drift.py +45 -0
  25. flo_python-0.1.3/tests/test_poll_loops.py +129 -0
  26. flo_python-0.1.3/tests/test_processing_decode.py +71 -0
  27. {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_retry.py +3 -3
  28. {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_status_codes.py +6 -4
  29. {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_stream_worker_ack.py +5 -3
  30. {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_wire.py +23 -6
  31. flo_python-0.1.3/tests/test_worker_drain.py +634 -0
  32. flo_python-0.1.3/tests/test_worker_info.py +115 -0
  33. flo_python-0.1.3/tests/test_worker_info_live.py +127 -0
  34. {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_worker_list.py +11 -2
  35. {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_worker_live.py +173 -1
  36. {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_worker_liveness.py +2 -0
  37. flo_python-0.1.2/tests/test_processing_decode.py +0 -33
  38. {flo_python-0.1.2 → flo_python-0.1.3}/.github/workflows/release.yml +0 -0
  39. {flo_python-0.1.2 → flo_python-0.1.3}/.gitignore +0 -0
  40. {flo_python-0.1.2 → flo_python-0.1.3}/LICENSE +0 -0
  41. {flo_python-0.1.2 → flo_python-0.1.3}/examples/basic.py +0 -0
  42. {flo_python-0.1.2 → flo_python-0.1.3}/examples/stream_worker.py +0 -0
  43. {flo_python-0.1.2 → flo_python-0.1.3}/examples/test_decorator.py +0 -0
  44. {flo_python-0.1.2 → flo_python-0.1.3}/examples/worker.py +0 -0
  45. {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/queue.py +0 -0
  46. {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/streams.py +0 -0
  47. {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/workflows.py +0 -0
  48. {flo_python-0.1.2 → flo_python-0.1.3}/tests/__init__.py +0 -0
  49. {flo_python-0.1.2 → flo_python-0.1.3}/tests/conftest.py +0 -0
  50. {flo_python-0.1.2 → flo_python-0.1.3}/tests/order-workflow.yaml +0 -0
  51. {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_action_status.py +0 -0
  52. {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_auto_touch.py +0 -0
  53. {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_block_ms.py +0 -0
  54. {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_kv_ttl.py +0 -0
  55. {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_list_paging.py +0 -0
  56. {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_protocol_version.py +0 -0
  57. {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_result_resend.py +0 -0
  58. {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_stream_tier.py +0 -0
  59. {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_stream_trim.py +0 -0
  60. {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_workflows.py +0 -0
@@ -1,4 +1,4 @@
1
1
  # The flo commit the live tests run against: CI checks floruntime/flo out at
2
2
  # this commit, builds it and starts it. Bump it in the same commit as
3
3
  # TABLE_HASH, so the SDK and the server agree.
4
- FLO_REF=b02486074eb42eadb777133ee585e2a01cd4dfc9
4
+ FLO_REF=51735a8132c7826b175e53a9b690d120dd9cf52d
@@ -52,10 +52,16 @@ jobs:
52
52
  - name: Build flo
53
53
  working-directory: flo-src
54
54
  run: zig build --release=safe -Dcpu=x86_64_v2
55
+ # The server's status codes and refusal reasons, for the drift test.
56
+ - name: Write ops.json
57
+ working-directory: flo-src
58
+ run: zig build gen
55
59
  - uses: actions/upload-artifact@v4
56
60
  with:
57
61
  name: flo
58
- path: flo-src/zig-out/bin/flo
62
+ path: |
63
+ flo-src/zig-out/bin/flo
64
+ flo-src/zig-out/gen/ops.json
59
65
  retention-days: 1
60
66
 
61
67
  test:
@@ -81,14 +87,15 @@ jobs:
81
87
  - name: Start the pinned flo
82
88
  run: |
83
89
  cd "$RUNNER_TEMP"
84
- chmod +x flo-bin/flo
85
- echo "FLO_BINARY=$RUNNER_TEMP/flo-bin/flo" >> "$GITHUB_ENV"
90
+ chmod +x flo-bin/bin/flo
91
+ echo "FLO_BINARY=$RUNNER_TEMP/flo-bin/bin/flo" >> "$GITHUB_ENV"
92
+ echo "FLO_OPS_JSON=$RUNNER_TEMP/flo-bin/gen/ops.json" >> "$GITHUB_ENV"
86
93
  mkdir -p flo-data flo-expiry-data
87
- ./flo-bin/flo server start --port 4453 --data-dir "$RUNNER_TEMP/flo-data" > flo-server.log 2>&1 &
94
+ ./flo-bin/bin/flo server start --port 4453 --data-dir "$RUNNER_TEMP/flo-data" > flo-server.log 2>&1 &
88
95
  # A second server that removes a worker a second after its last
89
96
  # heartbeat, for the re-register test.
90
97
  printf '[workers]\nheartbeat_timeout_ms = 1000\nremove_after_ms = 1000\n' > flo-fast-expiry.toml
91
- ./flo-bin/flo server start --port 4463 --data-dir "$RUNNER_TEMP/flo-expiry-data" \
98
+ ./flo-bin/bin/flo server start --port 4463 --data-dir "$RUNNER_TEMP/flo-expiry-data" \
92
99
  --config "$RUNNER_TEMP/flo-fast-expiry.toml" > flo-expiry-server.log 2>&1 &
93
100
  for port in 4453 4463; do
94
101
  for _ in $(seq 1 50); do
@@ -99,7 +106,8 @@ jobs:
99
106
  done
100
107
  - name: Run tests
101
108
  # CI=1 turns a missing server into a failure, not a skip; FLO_BINARY
102
- # runs the version test against the pinned flo.
109
+ # runs the version test against the pinned flo, FLO_OPS_JSON the
110
+ # status and reason drift test.
103
111
  env:
104
112
  CI: "1"
105
113
  FLO_ENDPOINT: localhost:4453
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: flo-python
3
- Version: 0.1.2
3
+ Version: 0.1.3
4
4
  Summary: Python SDK for the Flo distributed systems platform
5
5
  Project-URL: Homepage, https://github.com/floruntime/flo-python
6
6
  Project-URL: Documentation, https://github.com/floruntime/flo-python#readme
@@ -20,6 +20,7 @@ Classifier: Programming Language :: Python :: 3.13
20
20
  Classifier: Topic :: Database
21
21
  Classifier: Topic :: System :: Distributed Computing
22
22
  Requires-Python: >=3.10
23
+ Requires-Dist: google-crc32c>=1.7.0
23
24
  Provides-Extra: dev
24
25
  Requires-Dist: mypy>=1.0; extra == 'dev'
25
26
  Requires-Dist: pytest-asyncio>=0.21; extra == 'dev'
@@ -131,14 +132,16 @@ await client.kv.delete("lock:resource", DeleteOptions(cas_version=held_version))
131
132
  ```python
132
133
  from flo import ScanOptions
133
134
 
134
- # Names of keys with a prefix ("" for all), up to 1024 per page by default
135
- result = await client.kv.scan("user:")
136
- for name in result.names:
137
- print(name)
138
-
139
- # Next pages: the cursor is None on the last page (limit is 1..1024)
140
- while result.cursor is not None:
141
- result = await client.kv.scan("user:", ScanOptions(cursor=result.cursor, limit=100))
135
+ # Names of keys with a prefix ("" for all), a page at a time (limit 1..1024,
136
+ # default 1024); the cursor is None on the last page
137
+ opts = ScanOptions(limit=100)
138
+ while True:
139
+ result = await client.kv.scan("user:", opts)
140
+ for name in result.names:
141
+ print(name)
142
+ if result.cursor is None or not result.names:
143
+ break
144
+ opts = ScanOptions(cursor=result.cursor, limit=100)
142
145
  ```
143
146
 
144
147
  ### MGet
@@ -161,13 +164,21 @@ await client.kv.mget(["user:1:name", "user:1:email"], KVMGetOptions(routing_key=
161
164
  ```python
162
165
  from flo import HistoryOptions
163
166
 
164
- # Versions newest first (limit 1..65, default 10), each with its write time in ms;
165
- # empty for a missing key
166
- history = await client.kv.history("user:123", HistoryOptions(limit=10))
167
- for entry in history:
168
- print(f"v{entry.version} at {entry.at_ms}: {entry.value.decode()}")
167
+ # A page of versions newest first (limit 1..65, default 10), each with its write
168
+ # time in ms and whether it was a delete; empty for a missing key. `more` says
169
+ # older kept versions follow: ask again from before the last one.
170
+ opts = HistoryOptions(limit=10)
171
+ while True:
172
+ page = await client.kv.history("user:123", opts)
173
+ for entry in page.versions:
174
+ print(f"v{entry.version} at {entry.at_ms}: {entry.value.decode()}")
175
+ if not page.more or not page.versions:
176
+ break
177
+ opts = HistoryOptions(limit=10, before_version=page.versions[-1].version)
169
178
  ```
170
179
 
180
+ A key keeps at most 64 older versions, and 64 KiB of them; the oldest go first.
181
+
171
182
  ### Counters, TTL and exists
172
183
 
173
184
  ```python
@@ -182,8 +193,9 @@ await client.kv.expire("session:abc", 60_000)
182
193
  await client.kv.expire("lease", 30_000, KVExpireOptions(cas_version=lease_version))
183
194
  await client.kv.expire("session:abc", 0)
184
195
 
185
- if await client.kv.exists("session:abc"):
186
- ...
196
+ res = await client.kv.exists("session:abc")
197
+ if res: # truthy when the key exists; res.exists says the same
198
+ print(res.version)
187
199
  ```
188
200
 
189
201
  ### JSON documents
@@ -423,6 +435,34 @@ await client.worker.register(
423
435
  )
424
436
  ```
425
437
 
438
+ ### Inspect a Worker
439
+
440
+ ```python
441
+ # Its row (as client.worker.list shows it), its metadata and its processes
442
+ info = await client.worker.info("worker-1")
443
+ print(info.worker.status, info.worker.machine_id)
444
+ print(info.metadata) # bytes as registered, or None; decode it if you know it's text
445
+ for p in info.processes: # p.kind: ProcessKind.ACTION or STREAM_CONSUMER
446
+ print(p.name, p.kind.name, p.run_count, p.fail_count, p.last_run_at_ms)
447
+ ```
448
+
449
+ List the namespace's workers a page at a time, optionally only those that run an action:
450
+
451
+ ```python
452
+ from flo import WorkerListOptions
453
+
454
+ opts = WorkerListOptions(limit=100, action="process-image")
455
+ while True:
456
+ page = await client.worker.list(opts)
457
+ for w in page.workers:
458
+ print(w.worker_id, w.status)
459
+ if page.cursor is None or not page.workers:
460
+ break
461
+ opts = WorkerListOptions(limit=100, action="process-image", cursor=page.cursor)
462
+ ```
463
+
464
+ The server allows at most 1024 processes per worker and refuses a registration past its limits; the refusal comes back as a `BadRequestError` carrying its reason (`OUT_OF_RANGE` for too many processes, `FIELD_TOO_LONG` for a process name, the processes together or a machine id too long) and message.
465
+
426
466
  ### Await and Process Tasks
427
467
 
428
468
  ```python
@@ -491,7 +531,9 @@ A run whose handler goes past the action's `timeout_ms` without touching it is t
491
531
 
492
532
  Delivery is at-least-once: a result lost to a dropped connection is resent, and one that never arrives lets the lease run out, so a handler may see the same task or record again. `worker.stop()` waits for running handlers, and their results are still sent.
493
533
 
494
- An action worker heartbeats every 30 s, reporting the handlers it is running. One the server removed after missed heartbeats registers again, with the same options, when its next heartbeat is answered not found. One the server has drained stops taking work, and `start()` returns once its running handlers finish.
534
+ Both workers register in the worker registry and heartbeat every 30 s, reporting the handlers they are running. One the server removed after missed heartbeats registers again, with the same options, when its next heartbeat is answered not found.
535
+
536
+ `worker.drain()` stops taking work and lets the running handlers finish, heartbeats going on; the server is told, so it assigns the worker nothing more, and `start()` returns once the last handler has reported. A drain started on the server does the same. Neither waits out `block_ms`: `drain()` interrupts the worker's own poll connection, so an action worker's held await and a stream worker's held read end at once. A drain started on the server answers an action worker's held await at once, refused as `WORKER_DRAINING`; it doesn't wake a stream worker's held group read, so a stream worker learns of it from its next heartbeat (every 30 s) and only then interrupts its read and leaves. `worker.stop()` ends polling at once, also mid-reconnect or mid-drain; running handlers still finish, and a stream worker hands back the records of its last batch it hasn't started. Either one before or during `start()` ends that run, and a worker can be started again afterwards.
495
537
 
496
538
  ### StreamWorker Example
497
539
 
@@ -582,11 +624,13 @@ A call that times out (`RequestTimeoutError`, also an `asyncio.TimeoutError`) or
582
624
  | `UnauthorizedError` | Authentication failed |
583
625
  | `OverloadedError` | Server is overloaded, and the client's retries ran out |
584
626
  | `UnavailableError` | No leader, or the shard isn't taking writes or is offline, and the client's retries ran out (the message says why, and whether an operator must act first) |
627
+ | `StaleNotFoundError` | `kv.get` or `kv.json_get` found no key, but a halted shard answered: absent only as of the entry it halted at |
628
+ | `OutcomePendingError` | The write committed but hasn't applied: its shard halted at or before it (reason `SHARD_HALTED`, ran `UNKNOWN`; the message says whether it applies when the shard resumes). Never retried |
585
629
  | `InternalServerError` | Server internal error; not retryable (may mean committed but not applied) |
586
630
  | `GenericServerError` | Any other error status, including one this SDK doesn't know (`status_code` carries it) |
587
631
  | `ConnectionFailedError` | Failed to connect |
588
632
  | `RequestTimeoutError` | No response within the timeout |
589
- | `InvalidChecksumError` | Response CRC32 mismatch |
633
+ | `InvalidChecksumError` | Response CRC32C mismatch |
590
634
 
591
635
  ### Retries
592
636
 
@@ -596,6 +640,27 @@ new request, after a jittered wait. Because the server never ran it, this is
596
640
  safe for writes too. Any other refusal, or one that may have run, comes
597
641
  straight back.
598
642
 
643
+ Two refusals from a halted shard are never retried, and come back after one
644
+ attempt for the caller to decide:
645
+
646
+ - `OutcomePendingError`: the write committed, so resending it would write it
647
+ twice, or halt the shard again.
648
+ - `UnavailableError` with `error.reason == Reason.SHARD_HALTED`: the shard
649
+ takes no writes until an operator repairs it. The reason tells it apart from
650
+ a leader change.
651
+
652
+ A halted shard still answers reads, from its state just before the entry it
653
+ halted at. Such a read succeeds; its result says so with `stale=True` (on
654
+ `GetResult`, `GetJsonResult`, each `MGetEntry`, `ScanResult`, `HistoryResult`,
655
+ `ExistsResult`, and the raw response). It is not an error. A key absent on
656
+ such an answer is absent only as of the halt, so `kv.get` and `kv.json_get`
657
+ raise `StaleNotFoundError` (a `NotFoundError` with `stale` True) instead of
658
+ returning `None`; `kv.history` returns an empty page with `stale=True`, and
659
+ `kv.exists` and `kv.mget` entries say `stale` too. A refusal from a halted
660
+ shard carries `error.stale` as well. Reads other than KV ones (streams,
661
+ queues, workers and the rest) don't surface the flag yet; they drop it this
662
+ release.
663
+
599
664
  | `RetryPolicy` field | Default | |
600
665
  |---|---|---|
601
666
  | `max_attempts` | 4 | Tries in all, the first included; 1 turns retrying off |
@@ -613,6 +678,18 @@ between failed polls, from 100 ms up to 5 s, starting again from 100 ms after a
613
678
  poll that succeeds; `stop()` cuts the wait short. A stream worker retries a
614
679
  group join refused this way instead of stopping.
615
680
 
681
+ Workers are the exception to the halted-shard rule: registering, polling and
682
+ joining a group are safe to repeat, so on `SHARD_HALTED` either worker's
683
+ registration at start and poll, and a stream worker's group join, keep
684
+ trying, waiting from 100 ms up to 30 s, logging each refusal at
685
+ warning with the server's message. A worker started during a halt comes up,
686
+ and one running through it resumes, by itself once the shard is repaired; no
687
+ restart is needed. They wait only on a write the shard refused without running it
688
+ (`UNAVAILABLE`, ran `NO`); an `OutcomePendingError` (an action registration or
689
+ group join that committed but hasn't applied) ends `start()` after one try,
690
+ with the server's message, since sending it again could halt the shard again.
691
+ Results and acks the shard refuses are not resent.
692
+
616
693
  ```python
617
694
  from flo import FloClient, RetryPolicy
618
695
 
@@ -101,14 +101,16 @@ await client.kv.delete("lock:resource", DeleteOptions(cas_version=held_version))
101
101
  ```python
102
102
  from flo import ScanOptions
103
103
 
104
- # Names of keys with a prefix ("" for all), up to 1024 per page by default
105
- result = await client.kv.scan("user:")
106
- for name in result.names:
107
- print(name)
108
-
109
- # Next pages: the cursor is None on the last page (limit is 1..1024)
110
- while result.cursor is not None:
111
- result = await client.kv.scan("user:", ScanOptions(cursor=result.cursor, limit=100))
104
+ # Names of keys with a prefix ("" for all), a page at a time (limit 1..1024,
105
+ # default 1024); the cursor is None on the last page
106
+ opts = ScanOptions(limit=100)
107
+ while True:
108
+ result = await client.kv.scan("user:", opts)
109
+ for name in result.names:
110
+ print(name)
111
+ if result.cursor is None or not result.names:
112
+ break
113
+ opts = ScanOptions(cursor=result.cursor, limit=100)
112
114
  ```
113
115
 
114
116
  ### MGet
@@ -131,13 +133,21 @@ await client.kv.mget(["user:1:name", "user:1:email"], KVMGetOptions(routing_key=
131
133
  ```python
132
134
  from flo import HistoryOptions
133
135
 
134
- # Versions newest first (limit 1..65, default 10), each with its write time in ms;
135
- # empty for a missing key
136
- history = await client.kv.history("user:123", HistoryOptions(limit=10))
137
- for entry in history:
138
- print(f"v{entry.version} at {entry.at_ms}: {entry.value.decode()}")
136
+ # A page of versions newest first (limit 1..65, default 10), each with its write
137
+ # time in ms and whether it was a delete; empty for a missing key. `more` says
138
+ # older kept versions follow: ask again from before the last one.
139
+ opts = HistoryOptions(limit=10)
140
+ while True:
141
+ page = await client.kv.history("user:123", opts)
142
+ for entry in page.versions:
143
+ print(f"v{entry.version} at {entry.at_ms}: {entry.value.decode()}")
144
+ if not page.more or not page.versions:
145
+ break
146
+ opts = HistoryOptions(limit=10, before_version=page.versions[-1].version)
139
147
  ```
140
148
 
149
+ A key keeps at most 64 older versions, and 64 KiB of them; the oldest go first.
150
+
141
151
  ### Counters, TTL and exists
142
152
 
143
153
  ```python
@@ -152,8 +162,9 @@ await client.kv.expire("session:abc", 60_000)
152
162
  await client.kv.expire("lease", 30_000, KVExpireOptions(cas_version=lease_version))
153
163
  await client.kv.expire("session:abc", 0)
154
164
 
155
- if await client.kv.exists("session:abc"):
156
- ...
165
+ res = await client.kv.exists("session:abc")
166
+ if res: # truthy when the key exists; res.exists says the same
167
+ print(res.version)
157
168
  ```
158
169
 
159
170
  ### JSON documents
@@ -393,6 +404,34 @@ await client.worker.register(
393
404
  )
394
405
  ```
395
406
 
407
+ ### Inspect a Worker
408
+
409
+ ```python
410
+ # Its row (as client.worker.list shows it), its metadata and its processes
411
+ info = await client.worker.info("worker-1")
412
+ print(info.worker.status, info.worker.machine_id)
413
+ print(info.metadata) # bytes as registered, or None; decode it if you know it's text
414
+ for p in info.processes: # p.kind: ProcessKind.ACTION or STREAM_CONSUMER
415
+ print(p.name, p.kind.name, p.run_count, p.fail_count, p.last_run_at_ms)
416
+ ```
417
+
418
+ List the namespace's workers a page at a time, optionally only those that run an action:
419
+
420
+ ```python
421
+ from flo import WorkerListOptions
422
+
423
+ opts = WorkerListOptions(limit=100, action="process-image")
424
+ while True:
425
+ page = await client.worker.list(opts)
426
+ for w in page.workers:
427
+ print(w.worker_id, w.status)
428
+ if page.cursor is None or not page.workers:
429
+ break
430
+ opts = WorkerListOptions(limit=100, action="process-image", cursor=page.cursor)
431
+ ```
432
+
433
+ The server allows at most 1024 processes per worker and refuses a registration past its limits; the refusal comes back as a `BadRequestError` carrying its reason (`OUT_OF_RANGE` for too many processes, `FIELD_TOO_LONG` for a process name, the processes together or a machine id too long) and message.
434
+
396
435
  ### Await and Process Tasks
397
436
 
398
437
  ```python
@@ -461,7 +500,9 @@ A run whose handler goes past the action's `timeout_ms` without touching it is t
461
500
 
462
501
  Delivery is at-least-once: a result lost to a dropped connection is resent, and one that never arrives lets the lease run out, so a handler may see the same task or record again. `worker.stop()` waits for running handlers, and their results are still sent.
463
502
 
464
- An action worker heartbeats every 30 s, reporting the handlers it is running. One the server removed after missed heartbeats registers again, with the same options, when its next heartbeat is answered not found. One the server has drained stops taking work, and `start()` returns once its running handlers finish.
503
+ Both workers register in the worker registry and heartbeat every 30 s, reporting the handlers they are running. One the server removed after missed heartbeats registers again, with the same options, when its next heartbeat is answered not found.
504
+
505
+ `worker.drain()` stops taking work and lets the running handlers finish, heartbeats going on; the server is told, so it assigns the worker nothing more, and `start()` returns once the last handler has reported. A drain started on the server does the same. Neither waits out `block_ms`: `drain()` interrupts the worker's own poll connection, so an action worker's held await and a stream worker's held read end at once. A drain started on the server answers an action worker's held await at once, refused as `WORKER_DRAINING`; it doesn't wake a stream worker's held group read, so a stream worker learns of it from its next heartbeat (every 30 s) and only then interrupts its read and leaves. `worker.stop()` ends polling at once, also mid-reconnect or mid-drain; running handlers still finish, and a stream worker hands back the records of its last batch it hasn't started. Either one before or during `start()` ends that run, and a worker can be started again afterwards.
465
506
 
466
507
  ### StreamWorker Example
467
508
 
@@ -552,11 +593,13 @@ A call that times out (`RequestTimeoutError`, also an `asyncio.TimeoutError`) or
552
593
  | `UnauthorizedError` | Authentication failed |
553
594
  | `OverloadedError` | Server is overloaded, and the client's retries ran out |
554
595
  | `UnavailableError` | No leader, or the shard isn't taking writes or is offline, and the client's retries ran out (the message says why, and whether an operator must act first) |
596
+ | `StaleNotFoundError` | `kv.get` or `kv.json_get` found no key, but a halted shard answered: absent only as of the entry it halted at |
597
+ | `OutcomePendingError` | The write committed but hasn't applied: its shard halted at or before it (reason `SHARD_HALTED`, ran `UNKNOWN`; the message says whether it applies when the shard resumes). Never retried |
555
598
  | `InternalServerError` | Server internal error; not retryable (may mean committed but not applied) |
556
599
  | `GenericServerError` | Any other error status, including one this SDK doesn't know (`status_code` carries it) |
557
600
  | `ConnectionFailedError` | Failed to connect |
558
601
  | `RequestTimeoutError` | No response within the timeout |
559
- | `InvalidChecksumError` | Response CRC32 mismatch |
602
+ | `InvalidChecksumError` | Response CRC32C mismatch |
560
603
 
561
604
  ### Retries
562
605
 
@@ -566,6 +609,27 @@ new request, after a jittered wait. Because the server never ran it, this is
566
609
  safe for writes too. Any other refusal, or one that may have run, comes
567
610
  straight back.
568
611
 
612
+ Two refusals from a halted shard are never retried, and come back after one
613
+ attempt for the caller to decide:
614
+
615
+ - `OutcomePendingError`: the write committed, so resending it would write it
616
+ twice, or halt the shard again.
617
+ - `UnavailableError` with `error.reason == Reason.SHARD_HALTED`: the shard
618
+ takes no writes until an operator repairs it. The reason tells it apart from
619
+ a leader change.
620
+
621
+ A halted shard still answers reads, from its state just before the entry it
622
+ halted at. Such a read succeeds; its result says so with `stale=True` (on
623
+ `GetResult`, `GetJsonResult`, each `MGetEntry`, `ScanResult`, `HistoryResult`,
624
+ `ExistsResult`, and the raw response). It is not an error. A key absent on
625
+ such an answer is absent only as of the halt, so `kv.get` and `kv.json_get`
626
+ raise `StaleNotFoundError` (a `NotFoundError` with `stale` True) instead of
627
+ returning `None`; `kv.history` returns an empty page with `stale=True`, and
628
+ `kv.exists` and `kv.mget` entries say `stale` too. A refusal from a halted
629
+ shard carries `error.stale` as well. Reads other than KV ones (streams,
630
+ queues, workers and the rest) don't surface the flag yet; they drop it this
631
+ release.
632
+
569
633
  | `RetryPolicy` field | Default | |
570
634
  |---|---|---|
571
635
  | `max_attempts` | 4 | Tries in all, the first included; 1 turns retrying off |
@@ -583,6 +647,18 @@ between failed polls, from 100 ms up to 5 s, starting again from 100 ms after a
583
647
  poll that succeeds; `stop()` cuts the wait short. A stream worker retries a
584
648
  group join refused this way instead of stopping.
585
649
 
650
+ Workers are the exception to the halted-shard rule: registering, polling and
651
+ joining a group are safe to repeat, so on `SHARD_HALTED` either worker's
652
+ registration at start and poll, and a stream worker's group join, keep
653
+ trying, waiting from 100 ms up to 30 s, logging each refusal at
654
+ warning with the server's message. A worker started during a halt comes up,
655
+ and one running through it resumes, by itself once the shard is repaired; no
656
+ restart is needed. They wait only on a write the shard refused without running it
657
+ (`UNAVAILABLE`, ran `NO`); an `OutcomePendingError` (an action registration or
658
+ group join that committed but hasn't applied) ends `start()` after one try,
659
+ with the server's message, since sending it again could halt the shard again.
660
+ Results and acks the shard refuses are not resent.
661
+
586
662
  ```python
587
663
  from flo import FloClient, RetryPolicy
588
664
 
@@ -4,12 +4,15 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "flo-python"
7
- version = "0.1.2"
7
+ version = "0.1.3"
8
8
  description = "Python SDK for the Flo distributed systems platform"
9
9
  readme = "README.md"
10
10
  license = "Apache-2.0"
11
11
  license-files = ["LICENSE"]
12
12
  requires-python = ">=3.10"
13
+ # Frame checksums: a C CRC32C, with its own pure-Python fallback where no
14
+ # wheel fits.
15
+ dependencies = ["google-crc32c>=1.7.0"]
13
16
  authors = [
14
17
  { name = "Flo Team" }
15
18
  ]
@@ -49,12 +49,14 @@ from .exceptions import (
49
49
  NonRetryableError,
50
50
  NotConnectedError,
51
51
  NotFoundError,
52
+ OutcomePendingError,
52
53
  OverloadedError,
53
54
  PayloadTooLargeError,
54
55
  ProtocolError,
55
56
  RateLimitedError,
56
57
  RequestTimeoutError,
57
58
  ServerError,
59
+ StaleNotFoundError,
58
60
  TableMismatchError,
59
61
  UnauthorizedError,
60
62
  UnavailableError,
@@ -85,10 +87,12 @@ from .types import (
85
87
  DlqListOptions,
86
88
  DlqRequeueOptions,
87
89
  EnqueueOptions,
90
+ ExistsResult,
88
91
  GetJsonResult,
89
92
  GetOptions,
90
93
  GetResult,
91
94
  HistoryOptions,
95
+ HistoryResult,
92
96
  IncrResult,
93
97
  KVBeginResult,
94
98
  KVCommitResult,
@@ -113,6 +117,7 @@ from .types import (
113
117
  ProcessingSubmitOptions,
114
118
  ProcessingSyncOptions,
115
119
  ProcessingSyncResult,
120
+ ProcessKind,
116
121
  PutOptions,
117
122
  PutResult,
118
123
  Ran,
@@ -143,10 +148,12 @@ from .types import (
143
148
  WorkerAwaitOptions,
144
149
  WorkerAwaitResult,
145
150
  WorkerCompleteOptions,
151
+ WorkerDetails,
146
152
  WorkerFailOptions,
147
153
  WorkerInfo,
148
154
  WorkerListOptions,
149
155
  WorkerListResult,
156
+ WorkerProcess,
150
157
  WorkerRegisterOptions,
151
158
  WorkerStatus,
152
159
  WorkerTouchOptions,
@@ -177,7 +184,7 @@ from .worker import (
177
184
  )
178
185
  from .workflows import WorkflowOperations
179
186
 
180
- __version__ = "0.1.2"
187
+ __version__ = "0.1.3"
181
188
 
182
189
  __all__ = [
183
190
  "Transaction",
@@ -191,6 +198,8 @@ __all__ = [
191
198
  "KVMGetOptions",
192
199
  "KVExpireOptions",
193
200
  "GetResult",
201
+ "HistoryResult",
202
+ "ExistsResult",
194
203
  "GetJsonResult",
195
204
  "MGetEntry",
196
205
  "PutResult",
@@ -235,6 +244,8 @@ __all__ = [
235
244
  "ConflictError",
236
245
  "UnauthorizedError",
237
246
  "OverloadedError",
247
+ "OutcomePendingError",
248
+ "StaleNotFoundError",
238
249
  "UnavailableError",
239
250
  "RateLimitedError",
240
251
  "InternalServerError",
@@ -301,7 +312,10 @@ __all__ = [
301
312
  # Worker types
302
313
  "TaskAssignment",
303
314
  "WorkerAwaitResult",
315
+ "ProcessKind",
316
+ "WorkerDetails",
304
317
  "WorkerInfo",
318
+ "WorkerProcess",
305
319
  "WorkerListResult",
306
320
  "WorkerStatus",
307
321
  "WorkerType",
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING
12
12
  from .kv_body import (
13
13
  decode_action_invoke_answer,
14
14
  decode_action_status_answer,
15
+ decode_worker_info_answer,
15
16
  decode_worker_list_answer,
16
17
  encode_worker_list_body,
17
18
  )
@@ -30,10 +31,12 @@ from .types import (
30
31
  WorkerAwaitOptions,
31
32
  WorkerAwaitResult,
32
33
  WorkerCompleteOptions,
34
+ WorkerDetails,
33
35
  WorkerFailOptions,
34
36
  WorkerListOptions,
35
37
  WorkerListResult,
36
38
  WorkerRegisterOptions,
39
+ WorkerStatus,
37
40
  WorkerTouchOptions,
38
41
  )
39
42
  from .wire import (
@@ -229,6 +232,7 @@ class WorkerOperations:
229
232
 
230
233
  value = serialize_worker_register_value(
231
234
  task_types,
235
+ worker_type=int(opts.worker_type),
232
236
  max_concurrency=opts.concurrency,
233
237
  metadata=opts.metadata,
234
238
  machine_id=opts.machine_id,
@@ -246,24 +250,57 @@ class WorkerOperations:
246
250
  worker_id: str,
247
251
  current_load: int = 0,
248
252
  namespace: str | None = None,
249
- ) -> None:
253
+ ) -> WorkerStatus:
250
254
  """Send a worker heartbeat to keep the registration alive.
251
255
 
252
256
  Args:
253
257
  worker_id: Worker identifier.
254
258
  current_load: Current number of active tasks.
255
259
  namespace: Optional namespace override.
260
+
261
+ Returns:
262
+ The worker's status on the server: DRAINING once it is drained.
263
+
264
+ Raises:
265
+ NotFoundError: The server doesn't know the worker (it never
266
+ registered, or was removed after going silent).
256
267
  """
257
268
  import struct
258
269
 
259
270
  ns = self._client.get_namespace(namespace)
260
271
  value = struct.pack("<I", current_load)
261
- await self._client._send_and_check(
272
+ response = await self._client._send_and_check(
262
273
  OpCode.WORKER_HEARTBEAT,
263
274
  ns,
264
275
  worker_id.encode("utf-8"),
265
276
  value,
266
277
  )
278
+ if len(response.data) != 1:
279
+ raise ValueError(f"heartbeat answer is {len(response.data)} bytes, want 1")
280
+ return WorkerStatus(response.data[0])
281
+
282
+ async def drain(self, worker_id: str, namespace: str | None = None) -> None:
283
+ """Mark a worker draining: the server gives it no more work, and
284
+ its heartbeats answer DRAINING until it deregisters.
285
+
286
+ Raises:
287
+ NotFoundError: The server doesn't know the worker.
288
+ """
289
+ await self._client._send_and_check(
290
+ OpCode.WORKER_DRAIN,
291
+ self._client.get_namespace(namespace),
292
+ worker_id.encode("utf-8"),
293
+ b"",
294
+ )
295
+
296
+ async def deregister(self, worker_id: str, namespace: str | None = None) -> None:
297
+ """Remove a worker from the registry."""
298
+ await self._client._send_and_check(
299
+ OpCode.WORKER_DEREGISTER,
300
+ self._client.get_namespace(namespace),
301
+ worker_id.encode("utf-8"),
302
+ b"",
303
+ )
267
304
 
268
305
  async def await_task(
269
306
  self,
@@ -412,8 +449,23 @@ class WorkerOperations:
412
449
  OpCode.WORKER_LIST,
413
450
  namespace,
414
451
  b"",
415
- encode_worker_list_body(opts.limit, opts.cursor),
452
+ encode_worker_list_body(opts.limit, opts.cursor, opts.action),
416
453
  )
417
454
 
418
455
  workers, cursor = decode_worker_list_answer(response.data)
419
456
  return WorkerListResult(workers=workers, cursor=cursor)
457
+
458
+ async def info(self, worker_id: str, namespace: str | None = None) -> WorkerDetails:
459
+ """A worker's record: its row, as :meth:`list` shows it, the metadata
460
+ it registered with, and its processes with their run and fail counts.
461
+
462
+ Raises:
463
+ NotFoundError: The server doesn't know the worker.
464
+ """
465
+ response = await self._client._send_and_check(
466
+ OpCode.WORKER_INFO,
467
+ self._client.get_namespace(namespace),
468
+ worker_id.encode("utf-8"),
469
+ b"",
470
+ )
471
+ return decode_worker_info_answer(response.data)