flo-python 0.1.2__tar.gz → 0.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {flo_python-0.1.2 → flo_python-0.1.4}/.github/flo-ref.env +1 -1
- {flo_python-0.1.2 → flo_python-0.1.4}/.github/workflows/ci.yml +14 -6
- flo_python-0.1.4/CHANGELOG.md +74 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/PKG-INFO +108 -22
- {flo_python-0.1.2 → flo_python-0.1.4}/README.md +106 -21
- {flo_python-0.1.2 → flo_python-0.1.4}/pyproject.toml +4 -1
- {flo_python-0.1.2 → flo_python-0.1.4}/src/flo/__init__.py +29 -3
- {flo_python-0.1.2 → flo_python-0.1.4}/src/flo/actions.py +65 -12
- {flo_python-0.1.2 → flo_python-0.1.4}/src/flo/client.py +13 -6
- {flo_python-0.1.2 → flo_python-0.1.4}/src/flo/exceptions.py +37 -2
- {flo_python-0.1.2 → flo_python-0.1.4}/src/flo/kv.py +74 -34
- {flo_python-0.1.2 → flo_python-0.1.4}/src/flo/kv_body.py +116 -44
- {flo_python-0.1.2 → flo_python-0.1.4}/src/flo/kv_txn.py +2 -1
- {flo_python-0.1.2 → flo_python-0.1.4}/src/flo/processing.py +91 -60
- {flo_python-0.1.2 → flo_python-0.1.4}/src/flo/retry.py +25 -3
- {flo_python-0.1.2 → flo_python-0.1.4}/src/flo/streams.py +33 -8
- {flo_python-0.1.2 → flo_python-0.1.4}/src/flo/types.py +194 -26
- {flo_python-0.1.2 → flo_python-0.1.4}/src/flo/wire.py +46 -33
- {flo_python-0.1.2 → flo_python-0.1.4}/src/flo/worker.py +606 -226
- {flo_python-0.1.2 → flo_python-0.1.4}/src/flo/workflows.py +173 -24
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/test_blocking_deadline.py +6 -6
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/test_empty_poll_backoff.py +2 -2
- flo_python-0.1.4/tests/test_halts.py +579 -0
- flo_python-0.1.4/tests/test_kv_live.py +56 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/test_kv_ops.py +139 -26
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/test_kv_table.py +4 -2
- flo_python-0.1.4/tests/test_list_paging.py +227 -0
- flo_python-0.1.4/tests/test_list_paging_live.py +157 -0
- flo_python-0.1.4/tests/test_missing_answers_live.py +22 -0
- flo_python-0.1.4/tests/test_not_found_answers.py +39 -0
- flo_python-0.1.4/tests/test_op_codes.py +28 -0
- flo_python-0.1.4/tests/test_ops_drift.py +45 -0
- flo_python-0.1.4/tests/test_poll_loops.py +129 -0
- flo_python-0.1.4/tests/test_processing_decode.py +112 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/test_retry.py +3 -3
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/test_status_codes.py +18 -4
- flo_python-0.1.4/tests/test_stream_delete.py +60 -0
- flo_python-0.1.4/tests/test_stream_delete_live.py +24 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/test_stream_worker_ack.py +5 -3
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/test_wire.py +23 -6
- flo_python-0.1.4/tests/test_worker_drain.py +634 -0
- flo_python-0.1.4/tests/test_worker_info.py +115 -0
- flo_python-0.1.4/tests/test_worker_info_live.py +127 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/test_worker_list.py +11 -2
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/test_worker_live.py +173 -1
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/test_worker_liveness.py +2 -0
- flo_python-0.1.4/tests/test_workflow_list_runs.py +110 -0
- flo_python-0.1.4/tests/test_workflow_runs_live.py +129 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/test_workflows.py +20 -110
- flo_python-0.1.2/tests/test_list_paging.py +0 -66
- flo_python-0.1.2/tests/test_processing_decode.py +0 -33
- {flo_python-0.1.2 → flo_python-0.1.4}/.github/workflows/release.yml +0 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/.gitignore +0 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/LICENSE +0 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/examples/basic.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/examples/stream_worker.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/examples/test_decorator.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/examples/worker.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/src/flo/queue.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/__init__.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/conftest.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/order-workflow.yaml +0 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/test_action_status.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/test_auto_touch.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/test_block_ms.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/test_kv_ttl.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/test_protocol_version.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/test_result_resend.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/test_stream_tier.py +0 -0
- {flo_python-0.1.2 → flo_python-0.1.4}/tests/test_stream_trim.py +0 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
# The flo commit the live tests run against: CI checks floruntime/flo out at
|
|
2
2
|
# this commit, builds it and starts it. Bump it in the same commit as
|
|
3
3
|
# TABLE_HASH, so the SDK and the server agree.
|
|
4
|
-
FLO_REF=
|
|
4
|
+
FLO_REF=3614d952e56553b3a32d86e644f85c8dbc48f825
|
|
@@ -52,10 +52,16 @@ jobs:
|
|
|
52
52
|
- name: Build flo
|
|
53
53
|
working-directory: flo-src
|
|
54
54
|
run: zig build --release=safe -Dcpu=x86_64_v2
|
|
55
|
+
# The server's status codes and refusal reasons, for the drift test.
|
|
56
|
+
- name: Write ops.json
|
|
57
|
+
working-directory: flo-src
|
|
58
|
+
run: zig build gen
|
|
55
59
|
- uses: actions/upload-artifact@v4
|
|
56
60
|
with:
|
|
57
61
|
name: flo
|
|
58
|
-
path:
|
|
62
|
+
path: |
|
|
63
|
+
flo-src/zig-out/bin/flo
|
|
64
|
+
flo-src/zig-out/gen/ops.json
|
|
59
65
|
retention-days: 1
|
|
60
66
|
|
|
61
67
|
test:
|
|
@@ -81,14 +87,15 @@ jobs:
|
|
|
81
87
|
- name: Start the pinned flo
|
|
82
88
|
run: |
|
|
83
89
|
cd "$RUNNER_TEMP"
|
|
84
|
-
chmod +x flo-bin/flo
|
|
85
|
-
echo "FLO_BINARY=$RUNNER_TEMP/flo-bin/flo" >> "$GITHUB_ENV"
|
|
90
|
+
chmod +x flo-bin/bin/flo
|
|
91
|
+
echo "FLO_BINARY=$RUNNER_TEMP/flo-bin/bin/flo" >> "$GITHUB_ENV"
|
|
92
|
+
echo "FLO_OPS_JSON=$RUNNER_TEMP/flo-bin/gen/ops.json" >> "$GITHUB_ENV"
|
|
86
93
|
mkdir -p flo-data flo-expiry-data
|
|
87
|
-
./flo-bin/flo server start --port 4453 --data-dir "$RUNNER_TEMP/flo-data" > flo-server.log 2>&1 &
|
|
94
|
+
./flo-bin/bin/flo server start --port 4453 --data-dir "$RUNNER_TEMP/flo-data" --shards 12 > flo-server.log 2>&1 &
|
|
88
95
|
# A second server that removes a worker a second after its last
|
|
89
96
|
# heartbeat, for the re-register test.
|
|
90
97
|
printf '[workers]\nheartbeat_timeout_ms = 1000\nremove_after_ms = 1000\n' > flo-fast-expiry.toml
|
|
91
|
-
./flo-bin/flo server start --port 4463 --data-dir "$RUNNER_TEMP/flo-expiry-data" \
|
|
98
|
+
./flo-bin/bin/flo server start --port 4463 --data-dir "$RUNNER_TEMP/flo-expiry-data" \
|
|
92
99
|
--config "$RUNNER_TEMP/flo-fast-expiry.toml" > flo-expiry-server.log 2>&1 &
|
|
93
100
|
for port in 4453 4463; do
|
|
94
101
|
for _ in $(seq 1 50); do
|
|
@@ -99,7 +106,8 @@ jobs:
|
|
|
99
106
|
done
|
|
100
107
|
- name: Run tests
|
|
101
108
|
# CI=1 turns a missing server into a failure, not a skip; FLO_BINARY
|
|
102
|
-
# runs the version test against the pinned flo
|
|
109
|
+
# runs the version test against the pinned flo, FLO_OPS_JSON the
|
|
110
|
+
# status and reason drift test.
|
|
103
111
|
env:
|
|
104
112
|
CI: "1"
|
|
105
113
|
FLO_ENDPOINT: localhost:4453
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to the Flo Python SDK will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
The op table hash moves with the server in this release: this SDK talks only
|
|
11
|
+
to the flo it pins, and the pin lands with the release.
|
|
12
|
+
The server's op table hash is now `0xc751512225a42d04` (`TABLE_HASH`).
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
|
|
16
|
+
- `NoNamespaceError` for status `NO_NAMESPACE` (14), its own error rather
|
|
17
|
+
than a `NotFoundError` (a missing key)
|
|
18
|
+
- `WorkflowListRunsOptions.search`: only runs whose id, workflow, step,
|
|
19
|
+
input or tags hold it, ignoring case
|
|
20
|
+
- `WorkflowListRunsResult` carries a run list's page (`runs` of
|
|
21
|
+
`WorkflowRunEntry`: `run_id`, `workflow`, `status`, `created_at`) with
|
|
22
|
+
`cursor` and `has_more`
|
|
23
|
+
- `Reason.ADMIN_ONLY` (30): the op is served only on a node's admin socket
|
|
24
|
+
- `ActionListOptions.cursor` and `ActionListResult.has_more`: `action.list`
|
|
25
|
+
pages
|
|
26
|
+
- `stream.delete` (`STREAM_DELETE`, 0x140) deletes a stream's records and
|
|
27
|
+
metadata; a missing stream succeeds, and one that still has records raises
|
|
28
|
+
`ConflictError` unless `StreamDeleteOptions(force=True)`
|
|
29
|
+
|
|
30
|
+
### Changed
|
|
31
|
+
|
|
32
|
+
- `action.list` returns the page's action names (`ActionListResult.names`)
|
|
33
|
+
and the next page's cursor; it returned an empty list whatever the server
|
|
34
|
+
answered
|
|
35
|
+
- A `kv.scan` cursor holds the shard and the last key returned, so it may be
|
|
36
|
+
up to 4098 bytes (was 64); its length prefix is a u16 in the request and
|
|
37
|
+
the answer
|
|
38
|
+
- `NAMESPACE_ENSURE` (0x016) is a known op code; shards send it to each
|
|
39
|
+
other, and no client API sends it
|
|
40
|
+
|
|
41
|
+
### Breaking
|
|
42
|
+
|
|
43
|
+
- Listing workers in a namespace never created raises `NoNamespaceError`, so
|
|
44
|
+
a list right after starting a worker there can raise it until the worker
|
|
45
|
+
has registered (the write that creates the namespace).
|
|
46
|
+
- `workflow.history` of a missing run, and a
|
|
47
|
+
`NOT_FOUND` answer to `stream.append`, `read`, `info`, `group_join`,
|
|
48
|
+
`group_leave`, `group_read`, `group_pending` or `group_claim`, raise
|
|
49
|
+
`NotFoundError`; the refusal's body was parsed as the answer
|
|
50
|
+
(`workflow.history` of a missing run returned it as the history)
|
|
51
|
+
- The cluster op codes are the server's: `CLUSTER_PROMOTE` (0x032),
|
|
52
|
+
`CLUSTER_REMOVE` (0x033) and `CLUSTER_REPAIR_SKIP` (0x034, admin socket
|
|
53
|
+
only) replace `CLUSTER_JOIN`, `CLUSTER_LEAVE`, `CLUSTER_TRANSFER_LEADER`,
|
|
54
|
+
`CLUSTER_ADD_NODE` and `CLUSTER_REMOVE_NODE`, which the server never
|
|
55
|
+
served. No client API sends them
|
|
56
|
+
- `NAMESPACE_CONFIG_SET` and `NAMESPACE_CONFIG_GET` (0x014, 0x015) are gone:
|
|
57
|
+
the server retired them
|
|
58
|
+
- `ActionListResult.actions` is now `ActionListResult.names`, and
|
|
59
|
+
`ActionInfo` is gone: the action list carries names only
|
|
60
|
+
- `processing.list` returns a `ProcessingListResult` (`jobs`, `cursor`,
|
|
61
|
+
`has_more`) in place of a bare list, so a caller can page past the first
|
|
62
|
+
page with `ProcessingListOptions.cursor`; it dropped the server's cursor
|
|
63
|
+
- `workflow.list_definitions` returns a `WorkflowListDefinitionsResult`
|
|
64
|
+
(`definitions` of `WorkflowDefinitionEntry`, `cursor`, `has_more`) in place of
|
|
65
|
+
the raw answer bytes
|
|
66
|
+
- `workflow.list_runs` returns a `WorkflowListRunsResult` in place of the raw
|
|
67
|
+
answer bytes, newest first, and refuses a malformed answer.
|
|
68
|
+
`WorkflowListRunsOptions.status_filter` is now `status`. A page can come
|
|
69
|
+
back short with more after it: page until `has_more` is False
|
|
70
|
+
- `stream.info` of a stream that doesn't exist raises `NotFoundError`; the
|
|
71
|
+
server answered it with zeros
|
|
72
|
+
- A read, list or scan in a namespace that was never created raises
|
|
73
|
+
`NoNamespaceError`; the server answered it as a missing key or an empty
|
|
74
|
+
page. A write still creates the namespace
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: flo-python
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.4
|
|
4
4
|
Summary: Python SDK for the Flo distributed systems platform
|
|
5
5
|
Project-URL: Homepage, https://github.com/floruntime/flo-python
|
|
6
6
|
Project-URL: Documentation, https://github.com/floruntime/flo-python#readme
|
|
@@ -20,6 +20,7 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
20
20
|
Classifier: Topic :: Database
|
|
21
21
|
Classifier: Topic :: System :: Distributed Computing
|
|
22
22
|
Requires-Python: >=3.10
|
|
23
|
+
Requires-Dist: google-crc32c>=1.7.0
|
|
23
24
|
Provides-Extra: dev
|
|
24
25
|
Requires-Dist: mypy>=1.0; extra == 'dev'
|
|
25
26
|
Requires-Dist: pytest-asyncio>=0.21; extra == 'dev'
|
|
@@ -131,14 +132,16 @@ await client.kv.delete("lock:resource", DeleteOptions(cas_version=held_version))
|
|
|
131
132
|
```python
|
|
132
133
|
from flo import ScanOptions
|
|
133
134
|
|
|
134
|
-
# Names of keys with a prefix ("" for all),
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
135
|
+
# Names of keys with a prefix ("" for all), a page at a time (limit 1..1024,
|
|
136
|
+
# default 1024); the cursor is None on the last page
|
|
137
|
+
opts = ScanOptions(limit=100)
|
|
138
|
+
while True:
|
|
139
|
+
result = await client.kv.scan("user:", opts)
|
|
140
|
+
for name in result.names:
|
|
141
|
+
print(name)
|
|
142
|
+
if result.cursor is None:
|
|
143
|
+
break
|
|
144
|
+
opts = ScanOptions(cursor=result.cursor, limit=100)
|
|
142
145
|
```
|
|
143
146
|
|
|
144
147
|
### MGet
|
|
@@ -161,13 +164,21 @@ await client.kv.mget(["user:1:name", "user:1:email"], KVMGetOptions(routing_key=
|
|
|
161
164
|
```python
|
|
162
165
|
from flo import HistoryOptions
|
|
163
166
|
|
|
164
|
-
#
|
|
165
|
-
# empty for a missing key
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
167
|
+
# A page of versions newest first (limit 1..65, default 10), each with its write
|
|
168
|
+
# time in ms and whether it was a delete; empty for a missing key. `more` says
|
|
169
|
+
# older kept versions follow: ask again from before the last one.
|
|
170
|
+
opts = HistoryOptions(limit=10)
|
|
171
|
+
while True:
|
|
172
|
+
page = await client.kv.history("user:123", opts)
|
|
173
|
+
for entry in page.versions:
|
|
174
|
+
print(f"v{entry.version} at {entry.at_ms}: {entry.value.decode()}")
|
|
175
|
+
if not page.more or not page.versions:
|
|
176
|
+
break
|
|
177
|
+
opts = HistoryOptions(limit=10, before_version=page.versions[-1].version)
|
|
169
178
|
```
|
|
170
179
|
|
|
180
|
+
A key keeps at most 64 older versions, and 64 KiB of them; the oldest go first.
|
|
181
|
+
|
|
171
182
|
### Counters, TTL and exists
|
|
172
183
|
|
|
173
184
|
```python
|
|
@@ -182,8 +193,9 @@ await client.kv.expire("session:abc", 60_000)
|
|
|
182
193
|
await client.kv.expire("lease", 30_000, KVExpireOptions(cas_version=lease_version))
|
|
183
194
|
await client.kv.expire("session:abc", 0)
|
|
184
195
|
|
|
185
|
-
|
|
186
|
-
|
|
196
|
+
res = await client.kv.exists("session:abc")
|
|
197
|
+
if res: # truthy when the key exists; res.exists says the same
|
|
198
|
+
print(res.version)
|
|
187
199
|
```
|
|
188
200
|
|
|
189
201
|
### JSON documents
|
|
@@ -399,11 +411,19 @@ if status.error_message is not None:
|
|
|
399
411
|
```python
|
|
400
412
|
from flo import ActionListOptions
|
|
401
413
|
|
|
402
|
-
# List
|
|
403
|
-
|
|
414
|
+
# List the namespace's actions a page at a time
|
|
415
|
+
opts = ActionListOptions(limit=100)
|
|
416
|
+
while True:
|
|
417
|
+
page = await client.action.list(opts)
|
|
418
|
+
for name in page.names:
|
|
419
|
+
print(name)
|
|
420
|
+
if page.cursor is None:
|
|
421
|
+
break
|
|
422
|
+
opts = ActionListOptions(limit=100, cursor=page.cursor)
|
|
404
423
|
|
|
405
|
-
#
|
|
406
|
-
|
|
424
|
+
# Only actions whose names start with a prefix (needs the next server
|
|
425
|
+
# release; the current one lists every action)
|
|
426
|
+
page = await client.action.list(ActionListOptions(prefix="process-"))
|
|
407
427
|
|
|
408
428
|
# Delete an action
|
|
409
429
|
await client.action.delete("process-image")
|
|
@@ -423,6 +443,34 @@ await client.worker.register(
|
|
|
423
443
|
)
|
|
424
444
|
```
|
|
425
445
|
|
|
446
|
+
### Inspect a Worker
|
|
447
|
+
|
|
448
|
+
```python
|
|
449
|
+
# Its row (as client.worker.list shows it), its metadata and its processes
|
|
450
|
+
info = await client.worker.info("worker-1")
|
|
451
|
+
print(info.worker.status, info.worker.machine_id)
|
|
452
|
+
print(info.metadata) # bytes as registered, or None; decode it if you know it's text
|
|
453
|
+
for p in info.processes: # p.kind: ProcessKind.ACTION or STREAM_CONSUMER
|
|
454
|
+
print(p.name, p.kind.name, p.run_count, p.fail_count, p.last_run_at_ms)
|
|
455
|
+
```
|
|
456
|
+
|
|
457
|
+
List the namespace's workers a page at a time, optionally only those that run an action:
|
|
458
|
+
|
|
459
|
+
```python
|
|
460
|
+
from flo import WorkerListOptions
|
|
461
|
+
|
|
462
|
+
opts = WorkerListOptions(limit=100, action="process-image")
|
|
463
|
+
while True:
|
|
464
|
+
page = await client.worker.list(opts)
|
|
465
|
+
for w in page.workers:
|
|
466
|
+
print(w.worker_id, w.status)
|
|
467
|
+
if page.cursor is None:
|
|
468
|
+
break
|
|
469
|
+
opts = WorkerListOptions(limit=100, action="process-image", cursor=page.cursor)
|
|
470
|
+
```
|
|
471
|
+
|
|
472
|
+
The server allows at most 1024 processes per worker and refuses a registration past its limits; the refusal comes back as a `BadRequestError` carrying its reason (`OUT_OF_RANGE` for too many processes, `FIELD_TOO_LONG` for a process name, the processes together or a machine id too long) and message.
|
|
473
|
+
|
|
426
474
|
### Await and Process Tasks
|
|
427
475
|
|
|
428
476
|
```python
|
|
@@ -491,7 +539,9 @@ A run whose handler goes past the action's `timeout_ms` without touching it is t
|
|
|
491
539
|
|
|
492
540
|
Delivery is at-least-once: a result lost to a dropped connection is resent, and one that never arrives lets the lease run out, so a handler may see the same task or record again. `worker.stop()` waits for running handlers, and their results are still sent.
|
|
493
541
|
|
|
494
|
-
|
|
542
|
+
Both workers register in the worker registry and heartbeat every 30 s, reporting the handlers they are running. One the server removed after missed heartbeats registers again, with the same options, when its next heartbeat is answered not found.
|
|
543
|
+
|
|
544
|
+
`worker.drain()` stops taking work and lets the running handlers finish, heartbeats going on; the server is told, so it assigns the worker nothing more, and `start()` returns once the last handler has reported. A drain started on the server does the same. Neither waits out `block_ms`: `drain()` interrupts the worker's own poll connection, so an action worker's held await and a stream worker's held read end at once. A drain started on the server answers an action worker's held await at once, refused as `WORKER_DRAINING`; it doesn't wake a stream worker's held group read, so a stream worker learns of it from its next heartbeat (every 30 s) and only then interrupts its read and leaves. `worker.stop()` ends polling at once, also mid-reconnect or mid-drain; running handlers still finish, and a stream worker hands back the records of its last batch it hasn't started. Either one before or during `start()` ends that run, and a worker can be started again afterwards.
|
|
495
545
|
|
|
496
546
|
### StreamWorker Example
|
|
497
547
|
|
|
@@ -577,16 +627,19 @@ A call that times out (`RequestTimeoutError`, also an `asyncio.TimeoutError`) or
|
|
|
577
627
|
| Error | Description |
|
|
578
628
|
|-------|-------------|
|
|
579
629
|
| `NotFoundError` | Key or resource not found |
|
|
630
|
+
| `NoNamespaceError` | Status `NO_NAMESPACE` (14), not a `NotFoundError`: a read, list or scan in a namespace that was never created (a write creates it) |
|
|
580
631
|
| `BadRequestError` | Invalid request parameters |
|
|
581
632
|
| `ConflictError` | CAS version mismatch |
|
|
582
633
|
| `UnauthorizedError` | Authentication failed |
|
|
583
634
|
| `OverloadedError` | Server is overloaded, and the client's retries ran out |
|
|
584
635
|
| `UnavailableError` | No leader, or the shard isn't taking writes or is offline, and the client's retries ran out (the message says why, and whether an operator must act first) |
|
|
636
|
+
| `StaleNotFoundError` | `kv.get` or `kv.json_get` found no key, but a halted shard answered: absent only as of the entry it halted at |
|
|
637
|
+
| `OutcomePendingError` | The write committed but hasn't applied: its shard halted at or before it (reason `SHARD_HALTED`, ran `UNKNOWN`; the message says whether it applies when the shard resumes). Never retried |
|
|
585
638
|
| `InternalServerError` | Server internal error; not retryable (may mean committed but not applied) |
|
|
586
639
|
| `GenericServerError` | Any other error status, including one this SDK doesn't know (`status_code` carries it) |
|
|
587
640
|
| `ConnectionFailedError` | Failed to connect |
|
|
588
641
|
| `RequestTimeoutError` | No response within the timeout |
|
|
589
|
-
| `InvalidChecksumError` | Response
|
|
642
|
+
| `InvalidChecksumError` | Response CRC32C mismatch |
|
|
590
643
|
|
|
591
644
|
### Retries
|
|
592
645
|
|
|
@@ -596,6 +649,27 @@ new request, after a jittered wait. Because the server never ran it, this is
|
|
|
596
649
|
safe for writes too. Any other refusal, or one that may have run, comes
|
|
597
650
|
straight back.
|
|
598
651
|
|
|
652
|
+
Two refusals from a halted shard are never retried, and come back after one
|
|
653
|
+
attempt for the caller to decide:
|
|
654
|
+
|
|
655
|
+
- `OutcomePendingError`: the write committed, so resending it would write it
|
|
656
|
+
twice, or halt the shard again.
|
|
657
|
+
- `UnavailableError` with `error.reason == Reason.SHARD_HALTED`: the shard
|
|
658
|
+
takes no writes until an operator repairs it. The reason tells it apart from
|
|
659
|
+
a leader change.
|
|
660
|
+
|
|
661
|
+
A halted shard still answers reads, from its state just before the entry it
|
|
662
|
+
halted at. Such a read succeeds; its result says so with `stale=True` (on
|
|
663
|
+
`GetResult`, `GetJsonResult`, each `MGetEntry`, `ScanResult`, `HistoryResult`,
|
|
664
|
+
`ExistsResult`, and the raw response). It is not an error. A key absent on
|
|
665
|
+
such an answer is absent only as of the halt, so `kv.get` and `kv.json_get`
|
|
666
|
+
raise `StaleNotFoundError` (a `NotFoundError` with `stale` True) instead of
|
|
667
|
+
returning `None`; `kv.history` returns an empty page with `stale=True`, and
|
|
668
|
+
`kv.exists` and `kv.mget` entries say `stale` too. A refusal from a halted
|
|
669
|
+
shard carries `error.stale` as well. Reads other than KV ones (streams,
|
|
670
|
+
queues, workers and the rest) don't surface the flag yet; they drop it this
|
|
671
|
+
release.
|
|
672
|
+
|
|
599
673
|
| `RetryPolicy` field | Default | |
|
|
600
674
|
|---|---|---|
|
|
601
675
|
| `max_attempts` | 4 | Tries in all, the first included; 1 turns retrying off |
|
|
@@ -613,6 +687,18 @@ between failed polls, from 100 ms up to 5 s, starting again from 100 ms after a
|
|
|
613
687
|
poll that succeeds; `stop()` cuts the wait short. A stream worker retries a
|
|
614
688
|
group join refused this way instead of stopping.
|
|
615
689
|
|
|
690
|
+
Workers are the exception to the halted-shard rule: registering, polling and
|
|
691
|
+
joining a group are safe to repeat, so on `SHARD_HALTED` either worker's
|
|
692
|
+
registration at start and poll, and a stream worker's group join, keep
|
|
693
|
+
trying, waiting from 100 ms up to 30 s, logging each refusal at
|
|
694
|
+
warning with the server's message. A worker started during a halt comes up,
|
|
695
|
+
and one running through it resumes, by itself once the shard is repaired; no
|
|
696
|
+
restart is needed. They wait only on a write the shard refused without running it
|
|
697
|
+
(`UNAVAILABLE`, ran `NO`); an `OutcomePendingError` (an action registration or
|
|
698
|
+
group join that committed but hasn't applied) ends `start()` after one try,
|
|
699
|
+
with the server's message, since sending it again could halt the shard again.
|
|
700
|
+
Results and acks the shard refuses are not resent.
|
|
701
|
+
|
|
616
702
|
```python
|
|
617
703
|
from flo import FloClient, RetryPolicy
|
|
618
704
|
|
|
@@ -101,14 +101,16 @@ await client.kv.delete("lock:resource", DeleteOptions(cas_version=held_version))
|
|
|
101
101
|
```python
|
|
102
102
|
from flo import ScanOptions
|
|
103
103
|
|
|
104
|
-
# Names of keys with a prefix ("" for all),
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
104
|
+
# Names of keys with a prefix ("" for all), a page at a time (limit 1..1024,
|
|
105
|
+
# default 1024); the cursor is None on the last page
|
|
106
|
+
opts = ScanOptions(limit=100)
|
|
107
|
+
while True:
|
|
108
|
+
result = await client.kv.scan("user:", opts)
|
|
109
|
+
for name in result.names:
|
|
110
|
+
print(name)
|
|
111
|
+
if result.cursor is None:
|
|
112
|
+
break
|
|
113
|
+
opts = ScanOptions(cursor=result.cursor, limit=100)
|
|
112
114
|
```
|
|
113
115
|
|
|
114
116
|
### MGet
|
|
@@ -131,13 +133,21 @@ await client.kv.mget(["user:1:name", "user:1:email"], KVMGetOptions(routing_key=
|
|
|
131
133
|
```python
|
|
132
134
|
from flo import HistoryOptions
|
|
133
135
|
|
|
134
|
-
#
|
|
135
|
-
# empty for a missing key
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
136
|
+
# A page of versions newest first (limit 1..65, default 10), each with its write
|
|
137
|
+
# time in ms and whether it was a delete; empty for a missing key. `more` says
|
|
138
|
+
# older kept versions follow: ask again from before the last one.
|
|
139
|
+
opts = HistoryOptions(limit=10)
|
|
140
|
+
while True:
|
|
141
|
+
page = await client.kv.history("user:123", opts)
|
|
142
|
+
for entry in page.versions:
|
|
143
|
+
print(f"v{entry.version} at {entry.at_ms}: {entry.value.decode()}")
|
|
144
|
+
if not page.more or not page.versions:
|
|
145
|
+
break
|
|
146
|
+
opts = HistoryOptions(limit=10, before_version=page.versions[-1].version)
|
|
139
147
|
```
|
|
140
148
|
|
|
149
|
+
A key keeps at most 64 older versions, and 64 KiB of them; the oldest go first.
|
|
150
|
+
|
|
141
151
|
### Counters, TTL and exists
|
|
142
152
|
|
|
143
153
|
```python
|
|
@@ -152,8 +162,9 @@ await client.kv.expire("session:abc", 60_000)
|
|
|
152
162
|
await client.kv.expire("lease", 30_000, KVExpireOptions(cas_version=lease_version))
|
|
153
163
|
await client.kv.expire("session:abc", 0)
|
|
154
164
|
|
|
155
|
-
|
|
156
|
-
|
|
165
|
+
res = await client.kv.exists("session:abc")
|
|
166
|
+
if res: # truthy when the key exists; res.exists says the same
|
|
167
|
+
print(res.version)
|
|
157
168
|
```
|
|
158
169
|
|
|
159
170
|
### JSON documents
|
|
@@ -369,11 +380,19 @@ if status.error_message is not None:
|
|
|
369
380
|
```python
|
|
370
381
|
from flo import ActionListOptions
|
|
371
382
|
|
|
372
|
-
# List
|
|
373
|
-
|
|
383
|
+
# List the namespace's actions a page at a time
|
|
384
|
+
opts = ActionListOptions(limit=100)
|
|
385
|
+
while True:
|
|
386
|
+
page = await client.action.list(opts)
|
|
387
|
+
for name in page.names:
|
|
388
|
+
print(name)
|
|
389
|
+
if page.cursor is None:
|
|
390
|
+
break
|
|
391
|
+
opts = ActionListOptions(limit=100, cursor=page.cursor)
|
|
374
392
|
|
|
375
|
-
#
|
|
376
|
-
|
|
393
|
+
# Only actions whose names start with a prefix (needs the next server
|
|
394
|
+
# release; the current one lists every action)
|
|
395
|
+
page = await client.action.list(ActionListOptions(prefix="process-"))
|
|
377
396
|
|
|
378
397
|
# Delete an action
|
|
379
398
|
await client.action.delete("process-image")
|
|
@@ -393,6 +412,34 @@ await client.worker.register(
|
|
|
393
412
|
)
|
|
394
413
|
```
|
|
395
414
|
|
|
415
|
+
### Inspect a Worker
|
|
416
|
+
|
|
417
|
+
```python
|
|
418
|
+
# Its row (as client.worker.list shows it), its metadata and its processes
|
|
419
|
+
info = await client.worker.info("worker-1")
|
|
420
|
+
print(info.worker.status, info.worker.machine_id)
|
|
421
|
+
print(info.metadata) # bytes as registered, or None; decode it if you know it's text
|
|
422
|
+
for p in info.processes: # p.kind: ProcessKind.ACTION or STREAM_CONSUMER
|
|
423
|
+
print(p.name, p.kind.name, p.run_count, p.fail_count, p.last_run_at_ms)
|
|
424
|
+
```
|
|
425
|
+
|
|
426
|
+
List the namespace's workers a page at a time, optionally only those that run an action:
|
|
427
|
+
|
|
428
|
+
```python
|
|
429
|
+
from flo import WorkerListOptions
|
|
430
|
+
|
|
431
|
+
opts = WorkerListOptions(limit=100, action="process-image")
|
|
432
|
+
while True:
|
|
433
|
+
page = await client.worker.list(opts)
|
|
434
|
+
for w in page.workers:
|
|
435
|
+
print(w.worker_id, w.status)
|
|
436
|
+
if page.cursor is None:
|
|
437
|
+
break
|
|
438
|
+
opts = WorkerListOptions(limit=100, action="process-image", cursor=page.cursor)
|
|
439
|
+
```
|
|
440
|
+
|
|
441
|
+
The server allows at most 1024 processes per worker and refuses a registration past its limits; the refusal comes back as a `BadRequestError` carrying its reason (`OUT_OF_RANGE` for too many processes, `FIELD_TOO_LONG` for a process name, the processes together or a machine id too long) and message.
|
|
442
|
+
|
|
396
443
|
### Await and Process Tasks
|
|
397
444
|
|
|
398
445
|
```python
|
|
@@ -461,7 +508,9 @@ A run whose handler goes past the action's `timeout_ms` without touching it is t
|
|
|
461
508
|
|
|
462
509
|
Delivery is at-least-once: a result lost to a dropped connection is resent, and one that never arrives lets the lease run out, so a handler may see the same task or record again. `worker.stop()` waits for running handlers, and their results are still sent.
|
|
463
510
|
|
|
464
|
-
|
|
511
|
+
Both workers register in the worker registry and heartbeat every 30 s, reporting the handlers they are running. One the server removed after missed heartbeats registers again, with the same options, when its next heartbeat is answered not found.
|
|
512
|
+
|
|
513
|
+
`worker.drain()` stops taking work and lets the running handlers finish, heartbeats going on; the server is told, so it assigns the worker nothing more, and `start()` returns once the last handler has reported. A drain started on the server does the same. Neither waits out `block_ms`: `drain()` interrupts the worker's own poll connection, so an action worker's held await and a stream worker's held read end at once. A drain started on the server answers an action worker's held await at once, refused as `WORKER_DRAINING`; it doesn't wake a stream worker's held group read, so a stream worker learns of it from its next heartbeat (every 30 s) and only then interrupts its read and leaves. `worker.stop()` ends polling at once, also mid-reconnect or mid-drain; running handlers still finish, and a stream worker hands back the records of its last batch it hasn't started. Either one before or during `start()` ends that run, and a worker can be started again afterwards.
|
|
465
514
|
|
|
466
515
|
### StreamWorker Example
|
|
467
516
|
|
|
@@ -547,16 +596,19 @@ A call that times out (`RequestTimeoutError`, also an `asyncio.TimeoutError`) or
|
|
|
547
596
|
| Error | Description |
|
|
548
597
|
|-------|-------------|
|
|
549
598
|
| `NotFoundError` | Key or resource not found |
|
|
599
|
+
| `NoNamespaceError` | Status `NO_NAMESPACE` (14), not a `NotFoundError`: a read, list or scan in a namespace that was never created (a write creates it) |
|
|
550
600
|
| `BadRequestError` | Invalid request parameters |
|
|
551
601
|
| `ConflictError` | CAS version mismatch |
|
|
552
602
|
| `UnauthorizedError` | Authentication failed |
|
|
553
603
|
| `OverloadedError` | Server is overloaded, and the client's retries ran out |
|
|
554
604
|
| `UnavailableError` | No leader, or the shard isn't taking writes or is offline, and the client's retries ran out (the message says why, and whether an operator must act first) |
|
|
605
|
+
| `StaleNotFoundError` | `kv.get` or `kv.json_get` found no key, but a halted shard answered: absent only as of the entry it halted at |
|
|
606
|
+
| `OutcomePendingError` | The write committed but hasn't applied: its shard halted at or before it (reason `SHARD_HALTED`, ran `UNKNOWN`; the message says whether it applies when the shard resumes). Never retried |
|
|
555
607
|
| `InternalServerError` | Server internal error; not retryable (may mean committed but not applied) |
|
|
556
608
|
| `GenericServerError` | Any other error status, including one this SDK doesn't know (`status_code` carries it) |
|
|
557
609
|
| `ConnectionFailedError` | Failed to connect |
|
|
558
610
|
| `RequestTimeoutError` | No response within the timeout |
|
|
559
|
-
| `InvalidChecksumError` | Response
|
|
611
|
+
| `InvalidChecksumError` | Response CRC32C mismatch |
|
|
560
612
|
|
|
561
613
|
### Retries
|
|
562
614
|
|
|
@@ -566,6 +618,27 @@ new request, after a jittered wait. Because the server never ran it, this is
|
|
|
566
618
|
safe for writes too. Any other refusal, or one that may have run, comes
|
|
567
619
|
straight back.
|
|
568
620
|
|
|
621
|
+
Two refusals from a halted shard are never retried, and come back after one
|
|
622
|
+
attempt for the caller to decide:
|
|
623
|
+
|
|
624
|
+
- `OutcomePendingError`: the write committed, so resending it would write it
|
|
625
|
+
twice, or halt the shard again.
|
|
626
|
+
- `UnavailableError` with `error.reason == Reason.SHARD_HALTED`: the shard
|
|
627
|
+
takes no writes until an operator repairs it. The reason tells it apart from
|
|
628
|
+
a leader change.
|
|
629
|
+
|
|
630
|
+
A halted shard still answers reads, from its state just before the entry it
|
|
631
|
+
halted at. Such a read succeeds; its result says so with `stale=True` (on
|
|
632
|
+
`GetResult`, `GetJsonResult`, each `MGetEntry`, `ScanResult`, `HistoryResult`,
|
|
633
|
+
`ExistsResult`, and the raw response). It is not an error. A key absent on
|
|
634
|
+
such an answer is absent only as of the halt, so `kv.get` and `kv.json_get`
|
|
635
|
+
raise `StaleNotFoundError` (a `NotFoundError` with `stale` True) instead of
|
|
636
|
+
returning `None`; `kv.history` returns an empty page with `stale=True`, and
|
|
637
|
+
`kv.exists` and `kv.mget` entries say `stale` too. A refusal from a halted
|
|
638
|
+
shard carries `error.stale` as well. Reads other than KV ones (streams,
|
|
639
|
+
queues, workers and the rest) don't surface the flag yet; they drop it this
|
|
640
|
+
release.
|
|
641
|
+
|
|
569
642
|
| `RetryPolicy` field | Default | |
|
|
570
643
|
|---|---|---|
|
|
571
644
|
| `max_attempts` | 4 | Tries in all, the first included; 1 turns retrying off |
|
|
@@ -583,6 +656,18 @@ between failed polls, from 100 ms up to 5 s, starting again from 100 ms after a
|
|
|
583
656
|
poll that succeeds; `stop()` cuts the wait short. A stream worker retries a
|
|
584
657
|
group join refused this way instead of stopping.
|
|
585
658
|
|
|
659
|
+
Workers are the exception to the halted-shard rule: registering, polling and
|
|
660
|
+
joining a group are safe to repeat, so on `SHARD_HALTED` either worker's
|
|
661
|
+
registration at start and poll, and a stream worker's group join, keep
|
|
662
|
+
trying, waiting from 100 ms up to 30 s, logging each refusal at
|
|
663
|
+
warning with the server's message. A worker started during a halt comes up,
|
|
664
|
+
and one running through it resumes, by itself once the shard is repaired; no
|
|
665
|
+
restart is needed. They wait only on a write the shard refused without running it
|
|
666
|
+
(`UNAVAILABLE`, ran `NO`); an `OutcomePendingError` (an action registration or
|
|
667
|
+
group join that committed but hasn't applied) ends `start()` after one try,
|
|
668
|
+
with the server's message, since sending it again could halt the shard again.
|
|
669
|
+
Results and acks the shard refuses are not resent.
|
|
670
|
+
|
|
586
671
|
```python
|
|
587
672
|
from flo import FloClient, RetryPolicy
|
|
588
673
|
|
|
@@ -4,12 +4,15 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "flo-python"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.4"
|
|
8
8
|
description = "Python SDK for the Flo distributed systems platform"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "Apache-2.0"
|
|
11
11
|
license-files = ["LICENSE"]
|
|
12
12
|
requires-python = ">=3.10"
|
|
13
|
+
# Frame checksums: a C CRC32C, with its own pure-Python fallback where no
|
|
14
|
+
# wheel fits.
|
|
15
|
+
dependencies = ["google-crc32c>=1.7.0"]
|
|
13
16
|
authors = [
|
|
14
17
|
{ name = "Flo Team" }
|
|
15
18
|
]
|