flo-python 0.1.2__tar.gz → 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {flo_python-0.1.2 → flo_python-0.1.3}/.github/flo-ref.env +1 -1
- {flo_python-0.1.2 → flo_python-0.1.3}/.github/workflows/ci.yml +14 -6
- {flo_python-0.1.2 → flo_python-0.1.3}/PKG-INFO +95 -18
- {flo_python-0.1.2 → flo_python-0.1.3}/README.md +93 -17
- {flo_python-0.1.2 → flo_python-0.1.3}/pyproject.toml +4 -1
- {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/__init__.py +15 -1
- {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/actions.py +55 -3
- {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/client.py +13 -6
- {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/exceptions.py +27 -2
- {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/kv.py +73 -34
- {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/kv_body.py +82 -38
- {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/kv_txn.py +2 -1
- {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/processing.py +27 -24
- {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/retry.py +25 -3
- {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/types.py +89 -2
- {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/wire.py +42 -30
- {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/worker.py +606 -226
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_blocking_deadline.py +6 -6
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_empty_poll_backoff.py +2 -2
- flo_python-0.1.3/tests/test_halts.py +579 -0
- flo_python-0.1.3/tests/test_kv_live.py +56 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_kv_ops.py +101 -18
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_kv_table.py +3 -2
- flo_python-0.1.3/tests/test_ops_drift.py +45 -0
- flo_python-0.1.3/tests/test_poll_loops.py +129 -0
- flo_python-0.1.3/tests/test_processing_decode.py +71 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_retry.py +3 -3
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_status_codes.py +6 -4
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_stream_worker_ack.py +5 -3
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_wire.py +23 -6
- flo_python-0.1.3/tests/test_worker_drain.py +634 -0
- flo_python-0.1.3/tests/test_worker_info.py +115 -0
- flo_python-0.1.3/tests/test_worker_info_live.py +127 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_worker_list.py +11 -2
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_worker_live.py +173 -1
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_worker_liveness.py +2 -0
- flo_python-0.1.2/tests/test_processing_decode.py +0 -33
- {flo_python-0.1.2 → flo_python-0.1.3}/.github/workflows/release.yml +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/.gitignore +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/LICENSE +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/examples/basic.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/examples/stream_worker.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/examples/test_decorator.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/examples/worker.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/queue.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/streams.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/src/flo/workflows.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/__init__.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/conftest.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/order-workflow.yaml +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_action_status.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_auto_touch.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_block_ms.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_kv_ttl.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_list_paging.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_protocol_version.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_result_resend.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_stream_tier.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_stream_trim.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.3}/tests/test_workflows.py +0 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
# The flo commit the live tests run against: CI checks floruntime/flo out at
|
|
2
2
|
# this commit, builds it and starts it. Bump it in the same commit as
|
|
3
3
|
# TABLE_HASH, so the SDK and the server agree.
|
|
4
|
-
FLO_REF=
|
|
4
|
+
FLO_REF=51735a8132c7826b175e53a9b690d120dd9cf52d
|
|
@@ -52,10 +52,16 @@ jobs:
|
|
|
52
52
|
- name: Build flo
|
|
53
53
|
working-directory: flo-src
|
|
54
54
|
run: zig build --release=safe -Dcpu=x86_64_v2
|
|
55
|
+
# The server's status codes and refusal reasons, for the drift test.
|
|
56
|
+
- name: Write ops.json
|
|
57
|
+
working-directory: flo-src
|
|
58
|
+
run: zig build gen
|
|
55
59
|
- uses: actions/upload-artifact@v4
|
|
56
60
|
with:
|
|
57
61
|
name: flo
|
|
58
|
-
path:
|
|
62
|
+
path: |
|
|
63
|
+
flo-src/zig-out/bin/flo
|
|
64
|
+
flo-src/zig-out/gen/ops.json
|
|
59
65
|
retention-days: 1
|
|
60
66
|
|
|
61
67
|
test:
|
|
@@ -81,14 +87,15 @@ jobs:
|
|
|
81
87
|
- name: Start the pinned flo
|
|
82
88
|
run: |
|
|
83
89
|
cd "$RUNNER_TEMP"
|
|
84
|
-
chmod +x flo-bin/flo
|
|
85
|
-
echo "FLO_BINARY=$RUNNER_TEMP/flo-bin/flo" >> "$GITHUB_ENV"
|
|
90
|
+
chmod +x flo-bin/bin/flo
|
|
91
|
+
echo "FLO_BINARY=$RUNNER_TEMP/flo-bin/bin/flo" >> "$GITHUB_ENV"
|
|
92
|
+
echo "FLO_OPS_JSON=$RUNNER_TEMP/flo-bin/gen/ops.json" >> "$GITHUB_ENV"
|
|
86
93
|
mkdir -p flo-data flo-expiry-data
|
|
87
|
-
./flo-bin/flo server start --port 4453 --data-dir "$RUNNER_TEMP/flo-data" > flo-server.log 2>&1 &
|
|
94
|
+
./flo-bin/bin/flo server start --port 4453 --data-dir "$RUNNER_TEMP/flo-data" > flo-server.log 2>&1 &
|
|
88
95
|
# A second server that removes a worker a second after its last
|
|
89
96
|
# heartbeat, for the re-register test.
|
|
90
97
|
printf '[workers]\nheartbeat_timeout_ms = 1000\nremove_after_ms = 1000\n' > flo-fast-expiry.toml
|
|
91
|
-
./flo-bin/flo server start --port 4463 --data-dir "$RUNNER_TEMP/flo-expiry-data" \
|
|
98
|
+
./flo-bin/bin/flo server start --port 4463 --data-dir "$RUNNER_TEMP/flo-expiry-data" \
|
|
92
99
|
--config "$RUNNER_TEMP/flo-fast-expiry.toml" > flo-expiry-server.log 2>&1 &
|
|
93
100
|
for port in 4453 4463; do
|
|
94
101
|
for _ in $(seq 1 50); do
|
|
@@ -99,7 +106,8 @@ jobs:
|
|
|
99
106
|
done
|
|
100
107
|
- name: Run tests
|
|
101
108
|
# CI=1 turns a missing server into a failure, not a skip; FLO_BINARY
|
|
102
|
-
# runs the version test against the pinned flo
|
|
109
|
+
# runs the version test against the pinned flo, FLO_OPS_JSON the
|
|
110
|
+
# status and reason drift test.
|
|
103
111
|
env:
|
|
104
112
|
CI: "1"
|
|
105
113
|
FLO_ENDPOINT: localhost:4453
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: flo-python
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: Python SDK for the Flo distributed systems platform
|
|
5
5
|
Project-URL: Homepage, https://github.com/floruntime/flo-python
|
|
6
6
|
Project-URL: Documentation, https://github.com/floruntime/flo-python#readme
|
|
@@ -20,6 +20,7 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
20
20
|
Classifier: Topic :: Database
|
|
21
21
|
Classifier: Topic :: System :: Distributed Computing
|
|
22
22
|
Requires-Python: >=3.10
|
|
23
|
+
Requires-Dist: google-crc32c>=1.7.0
|
|
23
24
|
Provides-Extra: dev
|
|
24
25
|
Requires-Dist: mypy>=1.0; extra == 'dev'
|
|
25
26
|
Requires-Dist: pytest-asyncio>=0.21; extra == 'dev'
|
|
@@ -131,14 +132,16 @@ await client.kv.delete("lock:resource", DeleteOptions(cas_version=held_version))
|
|
|
131
132
|
```python
|
|
132
133
|
from flo import ScanOptions
|
|
133
134
|
|
|
134
|
-
# Names of keys with a prefix ("" for all),
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
result
|
|
135
|
+
# Names of keys with a prefix ("" for all), a page at a time (limit 1..1024,
|
|
136
|
+
# default 1024); the cursor is None on the last page
|
|
137
|
+
opts = ScanOptions(limit=100)
|
|
138
|
+
while True:
|
|
139
|
+
result = await client.kv.scan("user:", opts)
|
|
140
|
+
for name in result.names:
|
|
141
|
+
print(name)
|
|
142
|
+
if result.cursor is None or not result.names:
|
|
143
|
+
break
|
|
144
|
+
opts = ScanOptions(cursor=result.cursor, limit=100)
|
|
142
145
|
```
|
|
143
146
|
|
|
144
147
|
### MGet
|
|
@@ -161,13 +164,21 @@ await client.kv.mget(["user:1:name", "user:1:email"], KVMGetOptions(routing_key=
|
|
|
161
164
|
```python
|
|
162
165
|
from flo import HistoryOptions
|
|
163
166
|
|
|
164
|
-
#
|
|
165
|
-
# empty for a missing key
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
167
|
+
# A page of versions newest first (limit 1..65, default 10), each with its write
|
|
168
|
+
# time in ms and whether it was a delete; empty for a missing key. `more` says
|
|
169
|
+
# older kept versions follow: ask again from before the last one.
|
|
170
|
+
opts = HistoryOptions(limit=10)
|
|
171
|
+
while True:
|
|
172
|
+
page = await client.kv.history("user:123", opts)
|
|
173
|
+
for entry in page.versions:
|
|
174
|
+
print(f"v{entry.version} at {entry.at_ms}: {entry.value.decode()}")
|
|
175
|
+
if not page.more or not page.versions:
|
|
176
|
+
break
|
|
177
|
+
opts = HistoryOptions(limit=10, before_version=page.versions[-1].version)
|
|
169
178
|
```
|
|
170
179
|
|
|
180
|
+
A key keeps at most 64 older versions, and 64 KiB of them; the oldest go first.
|
|
181
|
+
|
|
171
182
|
### Counters, TTL and exists
|
|
172
183
|
|
|
173
184
|
```python
|
|
@@ -182,8 +193,9 @@ await client.kv.expire("session:abc", 60_000)
|
|
|
182
193
|
await client.kv.expire("lease", 30_000, KVExpireOptions(cas_version=lease_version))
|
|
183
194
|
await client.kv.expire("session:abc", 0)
|
|
184
195
|
|
|
185
|
-
|
|
186
|
-
|
|
196
|
+
res = await client.kv.exists("session:abc")
|
|
197
|
+
if res: # truthy when the key exists; res.exists says the same
|
|
198
|
+
print(res.version)
|
|
187
199
|
```
|
|
188
200
|
|
|
189
201
|
### JSON documents
|
|
@@ -423,6 +435,34 @@ await client.worker.register(
|
|
|
423
435
|
)
|
|
424
436
|
```
|
|
425
437
|
|
|
438
|
+
### Inspect a Worker
|
|
439
|
+
|
|
440
|
+
```python
|
|
441
|
+
# Its row (as client.worker.list shows it), its metadata and its processes
|
|
442
|
+
info = await client.worker.info("worker-1")
|
|
443
|
+
print(info.worker.status, info.worker.machine_id)
|
|
444
|
+
print(info.metadata) # bytes as registered, or None; decode it if you know it's text
|
|
445
|
+
for p in info.processes: # p.kind: ProcessKind.ACTION or STREAM_CONSUMER
|
|
446
|
+
print(p.name, p.kind.name, p.run_count, p.fail_count, p.last_run_at_ms)
|
|
447
|
+
```
|
|
448
|
+
|
|
449
|
+
List the namespace's workers a page at a time, optionally only those that run an action:
|
|
450
|
+
|
|
451
|
+
```python
|
|
452
|
+
from flo import WorkerListOptions
|
|
453
|
+
|
|
454
|
+
opts = WorkerListOptions(limit=100, action="process-image")
|
|
455
|
+
while True:
|
|
456
|
+
page = await client.worker.list(opts)
|
|
457
|
+
for w in page.workers:
|
|
458
|
+
print(w.worker_id, w.status)
|
|
459
|
+
if page.cursor is None or not page.workers:
|
|
460
|
+
break
|
|
461
|
+
opts = WorkerListOptions(limit=100, action="process-image", cursor=page.cursor)
|
|
462
|
+
```
|
|
463
|
+
|
|
464
|
+
The server allows at most 1024 processes per worker and refuses a registration past its limits; the refusal comes back as a `BadRequestError` carrying its reason (`OUT_OF_RANGE` for too many processes, `FIELD_TOO_LONG` for a process name, the processes together or a machine id too long) and message.
|
|
465
|
+
|
|
426
466
|
### Await and Process Tasks
|
|
427
467
|
|
|
428
468
|
```python
|
|
@@ -491,7 +531,9 @@ A run whose handler goes past the action's `timeout_ms` without touching it is t
|
|
|
491
531
|
|
|
492
532
|
Delivery is at-least-once: a result lost to a dropped connection is resent, and one that never arrives lets the lease run out, so a handler may see the same task or record again. `worker.stop()` waits for running handlers, and their results are still sent.
|
|
493
533
|
|
|
494
|
-
|
|
534
|
+
Both workers register in the worker registry and heartbeat every 30 s, reporting the handlers they are running. One the server removed after missed heartbeats registers again, with the same options, when its next heartbeat is answered not found.
|
|
535
|
+
|
|
536
|
+
`worker.drain()` stops taking work and lets the running handlers finish, heartbeats going on; the server is told, so it assigns the worker nothing more, and `start()` returns once the last handler has reported. A drain started on the server does the same. Neither waits out `block_ms`: `drain()` interrupts the worker's own poll connection, so an action worker's held await and a stream worker's held read end at once. A drain started on the server answers an action worker's held await at once, refused as `WORKER_DRAINING`; it doesn't wake a stream worker's held group read, so a stream worker learns of it from its next heartbeat (every 30 s) and only then interrupts its read and leaves. `worker.stop()` ends polling at once, also mid-reconnect or mid-drain; running handlers still finish, and a stream worker hands back the records of its last batch it hasn't started. Either one before or during `start()` ends that run, and a worker can be started again afterwards.
|
|
495
537
|
|
|
496
538
|
### StreamWorker Example
|
|
497
539
|
|
|
@@ -582,11 +624,13 @@ A call that times out (`RequestTimeoutError`, also an `asyncio.TimeoutError`) or
|
|
|
582
624
|
| `UnauthorizedError` | Authentication failed |
|
|
583
625
|
| `OverloadedError` | Server is overloaded, and the client's retries ran out |
|
|
584
626
|
| `UnavailableError` | No leader, or the shard isn't taking writes or is offline, and the client's retries ran out (the message says why, and whether an operator must act first) |
|
|
627
|
+
| `StaleNotFoundError` | `kv.get` or `kv.json_get` found no key, but a halted shard answered: absent only as of the entry it halted at |
|
|
628
|
+
| `OutcomePendingError` | The write committed but hasn't applied: its shard halted at or before it (reason `SHARD_HALTED`, ran `UNKNOWN`; the message says whether it applies when the shard resumes). Never retried |
|
|
585
629
|
| `InternalServerError` | Server internal error; not retryable (may mean committed but not applied) |
|
|
586
630
|
| `GenericServerError` | Any other error status, including one this SDK doesn't know (`status_code` carries it) |
|
|
587
631
|
| `ConnectionFailedError` | Failed to connect |
|
|
588
632
|
| `RequestTimeoutError` | No response within the timeout |
|
|
589
|
-
| `InvalidChecksumError` | Response
|
|
633
|
+
| `InvalidChecksumError` | Response CRC32C mismatch |
|
|
590
634
|
|
|
591
635
|
### Retries
|
|
592
636
|
|
|
@@ -596,6 +640,27 @@ new request, after a jittered wait. Because the server never ran it, this is
|
|
|
596
640
|
safe for writes too. Any other refusal, or one that may have run, comes
|
|
597
641
|
straight back.
|
|
598
642
|
|
|
643
|
+
Two refusals from a halted shard are never retried, and come back after one
|
|
644
|
+
attempt for the caller to decide:
|
|
645
|
+
|
|
646
|
+
- `OutcomePendingError`: the write committed, so resending it would write it
|
|
647
|
+
twice, or halt the shard again.
|
|
648
|
+
- `UnavailableError` with `error.reason == Reason.SHARD_HALTED`: the shard
|
|
649
|
+
takes no writes until an operator repairs it. The reason tells it apart from
|
|
650
|
+
a leader change.
|
|
651
|
+
|
|
652
|
+
A halted shard still answers reads, from its state just before the entry it
|
|
653
|
+
halted at. Such a read succeeds; its result says so with `stale=True` (on
|
|
654
|
+
`GetResult`, `GetJsonResult`, each `MGetEntry`, `ScanResult`, `HistoryResult`,
|
|
655
|
+
`ExistsResult`, and the raw response). It is not an error. A key absent on
|
|
656
|
+
such an answer is absent only as of the halt, so `kv.get` and `kv.json_get`
|
|
657
|
+
raise `StaleNotFoundError` (a `NotFoundError` with `stale` True) instead of
|
|
658
|
+
returning `None`; `kv.history` returns an empty page with `stale=True`, and
|
|
659
|
+
`kv.exists` and `kv.mget` entries say `stale` too. A refusal from a halted
|
|
660
|
+
shard carries `error.stale` as well. Reads other than KV ones (streams,
|
|
661
|
+
queues, workers and the rest) don't surface the flag yet; they drop it this
|
|
662
|
+
release.
|
|
663
|
+
|
|
599
664
|
| `RetryPolicy` field | Default | |
|
|
600
665
|
|---|---|---|
|
|
601
666
|
| `max_attempts` | 4 | Tries in all, the first included; 1 turns retrying off |
|
|
@@ -613,6 +678,18 @@ between failed polls, from 100 ms up to 5 s, starting again from 100 ms after a
|
|
|
613
678
|
poll that succeeds; `stop()` cuts the wait short. A stream worker retries a
|
|
614
679
|
group join refused this way instead of stopping.
|
|
615
680
|
|
|
681
|
+
Workers are the exception to the halted-shard rule: registering, polling and
|
|
682
|
+
joining a group are safe to repeat, so on `SHARD_HALTED` either worker's
|
|
683
|
+
registration at start and poll, and a stream worker's group join, keep
|
|
684
|
+
trying, waiting from 100 ms up to 30 s, logging each refusal at
|
|
685
|
+
warning with the server's message. A worker started during a halt comes up,
|
|
686
|
+
and one running through it resumes, by itself once the shard is repaired; no
|
|
687
|
+
restart is needed. They wait only on a write the shard refused without running it
|
|
688
|
+
(`UNAVAILABLE`, ran `NO`); an `OutcomePendingError` (an action registration or
|
|
689
|
+
group join that committed but hasn't applied) ends `start()` after one try,
|
|
690
|
+
with the server's message, since sending it again could halt the shard again.
|
|
691
|
+
Results and acks the shard refuses are not resent.
|
|
692
|
+
|
|
616
693
|
```python
|
|
617
694
|
from flo import FloClient, RetryPolicy
|
|
618
695
|
|
|
@@ -101,14 +101,16 @@ await client.kv.delete("lock:resource", DeleteOptions(cas_version=held_version))
|
|
|
101
101
|
```python
|
|
102
102
|
from flo import ScanOptions
|
|
103
103
|
|
|
104
|
-
# Names of keys with a prefix ("" for all),
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
result
|
|
104
|
+
# Names of keys with a prefix ("" for all), a page at a time (limit 1..1024,
|
|
105
|
+
# default 1024); the cursor is None on the last page
|
|
106
|
+
opts = ScanOptions(limit=100)
|
|
107
|
+
while True:
|
|
108
|
+
result = await client.kv.scan("user:", opts)
|
|
109
|
+
for name in result.names:
|
|
110
|
+
print(name)
|
|
111
|
+
if result.cursor is None or not result.names:
|
|
112
|
+
break
|
|
113
|
+
opts = ScanOptions(cursor=result.cursor, limit=100)
|
|
112
114
|
```
|
|
113
115
|
|
|
114
116
|
### MGet
|
|
@@ -131,13 +133,21 @@ await client.kv.mget(["user:1:name", "user:1:email"], KVMGetOptions(routing_key=
|
|
|
131
133
|
```python
|
|
132
134
|
from flo import HistoryOptions
|
|
133
135
|
|
|
134
|
-
#
|
|
135
|
-
# empty for a missing key
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
136
|
+
# A page of versions newest first (limit 1..65, default 10), each with its write
|
|
137
|
+
# time in ms and whether it was a delete; empty for a missing key. `more` says
|
|
138
|
+
# older kept versions follow: ask again from before the last one.
|
|
139
|
+
opts = HistoryOptions(limit=10)
|
|
140
|
+
while True:
|
|
141
|
+
page = await client.kv.history("user:123", opts)
|
|
142
|
+
for entry in page.versions:
|
|
143
|
+
print(f"v{entry.version} at {entry.at_ms}: {entry.value.decode()}")
|
|
144
|
+
if not page.more or not page.versions:
|
|
145
|
+
break
|
|
146
|
+
opts = HistoryOptions(limit=10, before_version=page.versions[-1].version)
|
|
139
147
|
```
|
|
140
148
|
|
|
149
|
+
A key keeps at most 64 older versions, and 64 KiB of them; the oldest go first.
|
|
150
|
+
|
|
141
151
|
### Counters, TTL and exists
|
|
142
152
|
|
|
143
153
|
```python
|
|
@@ -152,8 +162,9 @@ await client.kv.expire("session:abc", 60_000)
|
|
|
152
162
|
await client.kv.expire("lease", 30_000, KVExpireOptions(cas_version=lease_version))
|
|
153
163
|
await client.kv.expire("session:abc", 0)
|
|
154
164
|
|
|
155
|
-
|
|
156
|
-
|
|
165
|
+
res = await client.kv.exists("session:abc")
|
|
166
|
+
if res: # truthy when the key exists; res.exists says the same
|
|
167
|
+
print(res.version)
|
|
157
168
|
```
|
|
158
169
|
|
|
159
170
|
### JSON documents
|
|
@@ -393,6 +404,34 @@ await client.worker.register(
|
|
|
393
404
|
)
|
|
394
405
|
```
|
|
395
406
|
|
|
407
|
+
### Inspect a Worker
|
|
408
|
+
|
|
409
|
+
```python
|
|
410
|
+
# Its row (as client.worker.list shows it), its metadata and its processes
|
|
411
|
+
info = await client.worker.info("worker-1")
|
|
412
|
+
print(info.worker.status, info.worker.machine_id)
|
|
413
|
+
print(info.metadata) # bytes as registered, or None; decode it if you know it's text
|
|
414
|
+
for p in info.processes: # p.kind: ProcessKind.ACTION or STREAM_CONSUMER
|
|
415
|
+
print(p.name, p.kind.name, p.run_count, p.fail_count, p.last_run_at_ms)
|
|
416
|
+
```
|
|
417
|
+
|
|
418
|
+
List the namespace's workers a page at a time, optionally only those that run an action:
|
|
419
|
+
|
|
420
|
+
```python
|
|
421
|
+
from flo import WorkerListOptions
|
|
422
|
+
|
|
423
|
+
opts = WorkerListOptions(limit=100, action="process-image")
|
|
424
|
+
while True:
|
|
425
|
+
page = await client.worker.list(opts)
|
|
426
|
+
for w in page.workers:
|
|
427
|
+
print(w.worker_id, w.status)
|
|
428
|
+
if page.cursor is None or not page.workers:
|
|
429
|
+
break
|
|
430
|
+
opts = WorkerListOptions(limit=100, action="process-image", cursor=page.cursor)
|
|
431
|
+
```
|
|
432
|
+
|
|
433
|
+
The server allows at most 1024 processes per worker and refuses a registration past its limits; the refusal comes back as a `BadRequestError` carrying its reason (`OUT_OF_RANGE` for too many processes, `FIELD_TOO_LONG` for a process name, the processes together or a machine id too long) and message.
|
|
434
|
+
|
|
396
435
|
### Await and Process Tasks
|
|
397
436
|
|
|
398
437
|
```python
|
|
@@ -461,7 +500,9 @@ A run whose handler goes past the action's `timeout_ms` without touching it is t
|
|
|
461
500
|
|
|
462
501
|
Delivery is at-least-once: a result lost to a dropped connection is resent, and one that never arrives lets the lease run out, so a handler may see the same task or record again. `worker.stop()` waits for running handlers, and their results are still sent.
|
|
463
502
|
|
|
464
|
-
|
|
503
|
+
Both workers register in the worker registry and heartbeat every 30 s, reporting the handlers they are running. One the server removed after missed heartbeats registers again, with the same options, when its next heartbeat is answered not found.
|
|
504
|
+
|
|
505
|
+
`worker.drain()` stops taking work and lets the running handlers finish, heartbeats going on; the server is told, so it assigns the worker nothing more, and `start()` returns once the last handler has reported. A drain started on the server does the same. Neither waits out `block_ms`: `drain()` interrupts the worker's own poll connection, so an action worker's held await and a stream worker's held read end at once. A drain started on the server answers an action worker's held await at once, refused as `WORKER_DRAINING`; it doesn't wake a stream worker's held group read, so a stream worker learns of it from its next heartbeat (every 30 s) and only then interrupts its read and leaves. `worker.stop()` ends polling at once, also mid-reconnect or mid-drain; running handlers still finish, and a stream worker hands back the records of its last batch it hasn't started. Either one before or during `start()` ends that run, and a worker can be started again afterwards.
|
|
465
506
|
|
|
466
507
|
### StreamWorker Example
|
|
467
508
|
|
|
@@ -552,11 +593,13 @@ A call that times out (`RequestTimeoutError`, also an `asyncio.TimeoutError`) or
|
|
|
552
593
|
| `UnauthorizedError` | Authentication failed |
|
|
553
594
|
| `OverloadedError` | Server is overloaded, and the client's retries ran out |
|
|
554
595
|
| `UnavailableError` | No leader, or the shard isn't taking writes or is offline, and the client's retries ran out (the message says why, and whether an operator must act first) |
|
|
596
|
+
| `StaleNotFoundError` | `kv.get` or `kv.json_get` found no key, but a halted shard answered: absent only as of the entry it halted at |
|
|
597
|
+
| `OutcomePendingError` | The write committed but hasn't applied: its shard halted at or before it (reason `SHARD_HALTED`, ran `UNKNOWN`; the message says whether it applies when the shard resumes). Never retried |
|
|
555
598
|
| `InternalServerError` | Server internal error; not retryable (may mean committed but not applied) |
|
|
556
599
|
| `GenericServerError` | Any other error status, including one this SDK doesn't know (`status_code` carries it) |
|
|
557
600
|
| `ConnectionFailedError` | Failed to connect |
|
|
558
601
|
| `RequestTimeoutError` | No response within the timeout |
|
|
559
|
-
| `InvalidChecksumError` | Response
|
|
602
|
+
| `InvalidChecksumError` | Response CRC32C mismatch |
|
|
560
603
|
|
|
561
604
|
### Retries
|
|
562
605
|
|
|
@@ -566,6 +609,27 @@ new request, after a jittered wait. Because the server never ran it, this is
|
|
|
566
609
|
safe for writes too. Any other refusal, or one that may have run, comes
|
|
567
610
|
straight back.
|
|
568
611
|
|
|
612
|
+
Two refusals from a halted shard are never retried, and come back after one
|
|
613
|
+
attempt for the caller to decide:
|
|
614
|
+
|
|
615
|
+
- `OutcomePendingError`: the write committed, so resending it would write it
|
|
616
|
+
twice, or halt the shard again.
|
|
617
|
+
- `UnavailableError` with `error.reason == Reason.SHARD_HALTED`: the shard
|
|
618
|
+
takes no writes until an operator repairs it. The reason tells it apart from
|
|
619
|
+
a leader change.
|
|
620
|
+
|
|
621
|
+
A halted shard still answers reads, from its state just before the entry it
|
|
622
|
+
halted at. Such a read succeeds; its result says so with `stale=True` (on
|
|
623
|
+
`GetResult`, `GetJsonResult`, each `MGetEntry`, `ScanResult`, `HistoryResult`,
|
|
624
|
+
`ExistsResult`, and the raw response). It is not an error. A key absent on
|
|
625
|
+
such an answer is absent only as of the halt, so `kv.get` and `kv.json_get`
|
|
626
|
+
raise `StaleNotFoundError` (a `NotFoundError` with `stale` True) instead of
|
|
627
|
+
returning `None`; `kv.history` returns an empty page with `stale=True`, and
|
|
628
|
+
`kv.exists` and `kv.mget` entries say `stale` too. A refusal from a halted
|
|
629
|
+
shard carries `error.stale` as well. Reads other than KV ones (streams,
|
|
630
|
+
queues, workers and the rest) don't surface the flag yet; they drop it this
|
|
631
|
+
release.
|
|
632
|
+
|
|
569
633
|
| `RetryPolicy` field | Default | |
|
|
570
634
|
|---|---|---|
|
|
571
635
|
| `max_attempts` | 4 | Tries in all, the first included; 1 turns retrying off |
|
|
@@ -583,6 +647,18 @@ between failed polls, from 100 ms up to 5 s, starting again from 100 ms after a
|
|
|
583
647
|
poll that succeeds; `stop()` cuts the wait short. A stream worker retries a
|
|
584
648
|
group join refused this way instead of stopping.
|
|
585
649
|
|
|
650
|
+
Workers are the exception to the halted-shard rule: registering, polling and
|
|
651
|
+
joining a group are safe to repeat, so on `SHARD_HALTED` either worker's
|
|
652
|
+
registration at start and poll, and a stream worker's group join, keep
|
|
653
|
+
trying, waiting from 100 ms up to 30 s, logging each refusal at
|
|
654
|
+
warning with the server's message. A worker started during a halt comes up,
|
|
655
|
+
and one running through it resumes, by itself once the shard is repaired; no
|
|
656
|
+
restart is needed. They wait only on a write the shard refused without running it
|
|
657
|
+
(`UNAVAILABLE`, ran `NO`); an `OutcomePendingError` (an action registration or
|
|
658
|
+
group join that committed but hasn't applied) ends `start()` after one try,
|
|
659
|
+
with the server's message, since sending it again could halt the shard again.
|
|
660
|
+
Results and acks the shard refuses are not resent.
|
|
661
|
+
|
|
586
662
|
```python
|
|
587
663
|
from flo import FloClient, RetryPolicy
|
|
588
664
|
|
|
@@ -4,12 +4,15 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "flo-python"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.3"
|
|
8
8
|
description = "Python SDK for the Flo distributed systems platform"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "Apache-2.0"
|
|
11
11
|
license-files = ["LICENSE"]
|
|
12
12
|
requires-python = ">=3.10"
|
|
13
|
+
# Frame checksums: a C CRC32C, with its own pure-Python fallback where no
|
|
14
|
+
# wheel fits.
|
|
15
|
+
dependencies = ["google-crc32c>=1.7.0"]
|
|
13
16
|
authors = [
|
|
14
17
|
{ name = "Flo Team" }
|
|
15
18
|
]
|
|
@@ -49,12 +49,14 @@ from .exceptions import (
|
|
|
49
49
|
NonRetryableError,
|
|
50
50
|
NotConnectedError,
|
|
51
51
|
NotFoundError,
|
|
52
|
+
OutcomePendingError,
|
|
52
53
|
OverloadedError,
|
|
53
54
|
PayloadTooLargeError,
|
|
54
55
|
ProtocolError,
|
|
55
56
|
RateLimitedError,
|
|
56
57
|
RequestTimeoutError,
|
|
57
58
|
ServerError,
|
|
59
|
+
StaleNotFoundError,
|
|
58
60
|
TableMismatchError,
|
|
59
61
|
UnauthorizedError,
|
|
60
62
|
UnavailableError,
|
|
@@ -85,10 +87,12 @@ from .types import (
|
|
|
85
87
|
DlqListOptions,
|
|
86
88
|
DlqRequeueOptions,
|
|
87
89
|
EnqueueOptions,
|
|
90
|
+
ExistsResult,
|
|
88
91
|
GetJsonResult,
|
|
89
92
|
GetOptions,
|
|
90
93
|
GetResult,
|
|
91
94
|
HistoryOptions,
|
|
95
|
+
HistoryResult,
|
|
92
96
|
IncrResult,
|
|
93
97
|
KVBeginResult,
|
|
94
98
|
KVCommitResult,
|
|
@@ -113,6 +117,7 @@ from .types import (
|
|
|
113
117
|
ProcessingSubmitOptions,
|
|
114
118
|
ProcessingSyncOptions,
|
|
115
119
|
ProcessingSyncResult,
|
|
120
|
+
ProcessKind,
|
|
116
121
|
PutOptions,
|
|
117
122
|
PutResult,
|
|
118
123
|
Ran,
|
|
@@ -143,10 +148,12 @@ from .types import (
|
|
|
143
148
|
WorkerAwaitOptions,
|
|
144
149
|
WorkerAwaitResult,
|
|
145
150
|
WorkerCompleteOptions,
|
|
151
|
+
WorkerDetails,
|
|
146
152
|
WorkerFailOptions,
|
|
147
153
|
WorkerInfo,
|
|
148
154
|
WorkerListOptions,
|
|
149
155
|
WorkerListResult,
|
|
156
|
+
WorkerProcess,
|
|
150
157
|
WorkerRegisterOptions,
|
|
151
158
|
WorkerStatus,
|
|
152
159
|
WorkerTouchOptions,
|
|
@@ -177,7 +184,7 @@ from .worker import (
|
|
|
177
184
|
)
|
|
178
185
|
from .workflows import WorkflowOperations
|
|
179
186
|
|
|
180
|
-
__version__ = "0.1.
|
|
187
|
+
__version__ = "0.1.3"
|
|
181
188
|
|
|
182
189
|
__all__ = [
|
|
183
190
|
"Transaction",
|
|
@@ -191,6 +198,8 @@ __all__ = [
|
|
|
191
198
|
"KVMGetOptions",
|
|
192
199
|
"KVExpireOptions",
|
|
193
200
|
"GetResult",
|
|
201
|
+
"HistoryResult",
|
|
202
|
+
"ExistsResult",
|
|
194
203
|
"GetJsonResult",
|
|
195
204
|
"MGetEntry",
|
|
196
205
|
"PutResult",
|
|
@@ -235,6 +244,8 @@ __all__ = [
|
|
|
235
244
|
"ConflictError",
|
|
236
245
|
"UnauthorizedError",
|
|
237
246
|
"OverloadedError",
|
|
247
|
+
"OutcomePendingError",
|
|
248
|
+
"StaleNotFoundError",
|
|
238
249
|
"UnavailableError",
|
|
239
250
|
"RateLimitedError",
|
|
240
251
|
"InternalServerError",
|
|
@@ -301,7 +312,10 @@ __all__ = [
|
|
|
301
312
|
# Worker types
|
|
302
313
|
"TaskAssignment",
|
|
303
314
|
"WorkerAwaitResult",
|
|
315
|
+
"ProcessKind",
|
|
316
|
+
"WorkerDetails",
|
|
304
317
|
"WorkerInfo",
|
|
318
|
+
"WorkerProcess",
|
|
305
319
|
"WorkerListResult",
|
|
306
320
|
"WorkerStatus",
|
|
307
321
|
"WorkerType",
|
|
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING
|
|
|
12
12
|
from .kv_body import (
|
|
13
13
|
decode_action_invoke_answer,
|
|
14
14
|
decode_action_status_answer,
|
|
15
|
+
decode_worker_info_answer,
|
|
15
16
|
decode_worker_list_answer,
|
|
16
17
|
encode_worker_list_body,
|
|
17
18
|
)
|
|
@@ -30,10 +31,12 @@ from .types import (
|
|
|
30
31
|
WorkerAwaitOptions,
|
|
31
32
|
WorkerAwaitResult,
|
|
32
33
|
WorkerCompleteOptions,
|
|
34
|
+
WorkerDetails,
|
|
33
35
|
WorkerFailOptions,
|
|
34
36
|
WorkerListOptions,
|
|
35
37
|
WorkerListResult,
|
|
36
38
|
WorkerRegisterOptions,
|
|
39
|
+
WorkerStatus,
|
|
37
40
|
WorkerTouchOptions,
|
|
38
41
|
)
|
|
39
42
|
from .wire import (
|
|
@@ -229,6 +232,7 @@ class WorkerOperations:
|
|
|
229
232
|
|
|
230
233
|
value = serialize_worker_register_value(
|
|
231
234
|
task_types,
|
|
235
|
+
worker_type=int(opts.worker_type),
|
|
232
236
|
max_concurrency=opts.concurrency,
|
|
233
237
|
metadata=opts.metadata,
|
|
234
238
|
machine_id=opts.machine_id,
|
|
@@ -246,24 +250,57 @@ class WorkerOperations:
|
|
|
246
250
|
worker_id: str,
|
|
247
251
|
current_load: int = 0,
|
|
248
252
|
namespace: str | None = None,
|
|
249
|
-
) ->
|
|
253
|
+
) -> WorkerStatus:
|
|
250
254
|
"""Send a worker heartbeat to keep the registration alive.
|
|
251
255
|
|
|
252
256
|
Args:
|
|
253
257
|
worker_id: Worker identifier.
|
|
254
258
|
current_load: Current number of active tasks.
|
|
255
259
|
namespace: Optional namespace override.
|
|
260
|
+
|
|
261
|
+
Returns:
|
|
262
|
+
The worker's status on the server: DRAINING once it is drained.
|
|
263
|
+
|
|
264
|
+
Raises:
|
|
265
|
+
NotFoundError: The server doesn't know the worker (it never
|
|
266
|
+
registered, or was removed after going silent).
|
|
256
267
|
"""
|
|
257
268
|
import struct
|
|
258
269
|
|
|
259
270
|
ns = self._client.get_namespace(namespace)
|
|
260
271
|
value = struct.pack("<I", current_load)
|
|
261
|
-
await self._client._send_and_check(
|
|
272
|
+
response = await self._client._send_and_check(
|
|
262
273
|
OpCode.WORKER_HEARTBEAT,
|
|
263
274
|
ns,
|
|
264
275
|
worker_id.encode("utf-8"),
|
|
265
276
|
value,
|
|
266
277
|
)
|
|
278
|
+
if len(response.data) != 1:
|
|
279
|
+
raise ValueError(f"heartbeat answer is {len(response.data)} bytes, want 1")
|
|
280
|
+
return WorkerStatus(response.data[0])
|
|
281
|
+
|
|
282
|
+
async def drain(self, worker_id: str, namespace: str | None = None) -> None:
|
|
283
|
+
"""Mark a worker draining: the server gives it no more work, and
|
|
284
|
+
its heartbeats answer DRAINING until it deregisters.
|
|
285
|
+
|
|
286
|
+
Raises:
|
|
287
|
+
NotFoundError: The server doesn't know the worker.
|
|
288
|
+
"""
|
|
289
|
+
await self._client._send_and_check(
|
|
290
|
+
OpCode.WORKER_DRAIN,
|
|
291
|
+
self._client.get_namespace(namespace),
|
|
292
|
+
worker_id.encode("utf-8"),
|
|
293
|
+
b"",
|
|
294
|
+
)
|
|
295
|
+
|
|
296
|
+
async def deregister(self, worker_id: str, namespace: str | None = None) -> None:
|
|
297
|
+
"""Remove a worker from the registry."""
|
|
298
|
+
await self._client._send_and_check(
|
|
299
|
+
OpCode.WORKER_DEREGISTER,
|
|
300
|
+
self._client.get_namespace(namespace),
|
|
301
|
+
worker_id.encode("utf-8"),
|
|
302
|
+
b"",
|
|
303
|
+
)
|
|
267
304
|
|
|
268
305
|
async def await_task(
|
|
269
306
|
self,
|
|
@@ -412,8 +449,23 @@ class WorkerOperations:
|
|
|
412
449
|
OpCode.WORKER_LIST,
|
|
413
450
|
namespace,
|
|
414
451
|
b"",
|
|
415
|
-
encode_worker_list_body(opts.limit, opts.cursor),
|
|
452
|
+
encode_worker_list_body(opts.limit, opts.cursor, opts.action),
|
|
416
453
|
)
|
|
417
454
|
|
|
418
455
|
workers, cursor = decode_worker_list_answer(response.data)
|
|
419
456
|
return WorkerListResult(workers=workers, cursor=cursor)
|
|
457
|
+
|
|
458
|
+
async def info(self, worker_id: str, namespace: str | None = None) -> WorkerDetails:
|
|
459
|
+
"""A worker's record: its row, as :meth:`list` shows it, the metadata
|
|
460
|
+
it registered with, and its processes with their run and fail counts.
|
|
461
|
+
|
|
462
|
+
Raises:
|
|
463
|
+
NotFoundError: The server doesn't know the worker.
|
|
464
|
+
"""
|
|
465
|
+
response = await self._client._send_and_check(
|
|
466
|
+
OpCode.WORKER_INFO,
|
|
467
|
+
self._client.get_namespace(namespace),
|
|
468
|
+
worker_id.encode("utf-8"),
|
|
469
|
+
b"",
|
|
470
|
+
)
|
|
471
|
+
return decode_worker_info_answer(response.data)
|