flo-python 0.1.0__tar.gz → 0.1.0.dev2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flo_python-0.1.0.dev2/.gitignore +142 -0
- flo_python-0.1.0.dev2/LICENSE +21 -0
- {flo_python-0.1.0 → flo_python-0.1.0.dev2}/PKG-INFO +14 -32
- {flo_python-0.1.0 → flo_python-0.1.0.dev2}/README.md +12 -31
- {flo_python-0.1.0 → flo_python-0.1.0.dev2}/examples/worker.py +11 -5
- {flo_python-0.1.0 → flo_python-0.1.0.dev2}/pyproject.toml +1 -1
- {flo_python-0.1.0 → flo_python-0.1.0.dev2}/src/flo/__init__.py +19 -9
- {flo_python-0.1.0 → flo_python-0.1.0.dev2}/src/flo/client.py +42 -0
- {flo_python-0.1.0 → flo_python-0.1.0.dev2}/src/flo/streams.py +50 -3
- {flo_python-0.1.0 → flo_python-0.1.0.dev2}/src/flo/types.py +15 -4
- {flo_python-0.1.0 → flo_python-0.1.0.dev2}/src/flo/wire.py +17 -7
- {flo_python-0.1.0 → flo_python-0.1.0.dev2}/src/flo/worker.py +30 -40
- {flo_python-0.1.0 → flo_python-0.1.0.dev2}/.github/workflows/ci.yml +0 -0
- {flo_python-0.1.0 → flo_python-0.1.0.dev2}/.github/workflows/release.yml +0 -0
- {flo_python-0.1.0 → flo_python-0.1.0.dev2}/examples/basic.py +0 -0
- {flo_python-0.1.0 → flo_python-0.1.0.dev2}/src/flo/actions.py +0 -0
- {flo_python-0.1.0 → flo_python-0.1.0.dev2}/src/flo/exceptions.py +0 -0
- {flo_python-0.1.0 → flo_python-0.1.0.dev2}/src/flo/kv.py +0 -0
- {flo_python-0.1.0 → flo_python-0.1.0.dev2}/src/flo/queue.py +0 -0
- {flo_python-0.1.0 → flo_python-0.1.0.dev2}/tests/__init__.py +0 -0
- {flo_python-0.1.0 → flo_python-0.1.0.dev2}/tests/test_wire.py +0 -0
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
share/python-wheels/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.installed.cfg
|
|
26
|
+
*.egg
|
|
27
|
+
MANIFEST
|
|
28
|
+
|
|
29
|
+
# PyInstaller
|
|
30
|
+
*.manifest
|
|
31
|
+
*.spec
|
|
32
|
+
|
|
33
|
+
# Unit test / coverage reports
|
|
34
|
+
htmlcov/
|
|
35
|
+
.tox/
|
|
36
|
+
.nox/
|
|
37
|
+
.coverage
|
|
38
|
+
.coverage.*
|
|
39
|
+
.cache
|
|
40
|
+
nosetests.xml
|
|
41
|
+
coverage.xml
|
|
42
|
+
*.cover
|
|
43
|
+
*.log
|
|
44
|
+
.hypothesis/
|
|
45
|
+
.pytest_cache/
|
|
46
|
+
cover/
|
|
47
|
+
|
|
48
|
+
# Translations
|
|
49
|
+
*.mo
|
|
50
|
+
*.pot
|
|
51
|
+
|
|
52
|
+
# Django stuff:
|
|
53
|
+
local_settings.py
|
|
54
|
+
db.sqlite3
|
|
55
|
+
db.sqlite3-journal
|
|
56
|
+
|
|
57
|
+
# Flask stuff:
|
|
58
|
+
instance/
|
|
59
|
+
.webassets-cache
|
|
60
|
+
|
|
61
|
+
# Scrapy stuff:
|
|
62
|
+
.scrapy
|
|
63
|
+
|
|
64
|
+
# Sphinx documentation
|
|
65
|
+
docs/_build/
|
|
66
|
+
|
|
67
|
+
# PyBuilder
|
|
68
|
+
.pybuilder/
|
|
69
|
+
target/
|
|
70
|
+
|
|
71
|
+
# Jupyter Notebook
|
|
72
|
+
.ipynb_checkpoints
|
|
73
|
+
|
|
74
|
+
# IPython
|
|
75
|
+
profile_default/
|
|
76
|
+
ipython_config.py
|
|
77
|
+
|
|
78
|
+
# pyenv
|
|
79
|
+
.python-version
|
|
80
|
+
|
|
81
|
+
# pipenv
|
|
82
|
+
Pipfile.lock
|
|
83
|
+
|
|
84
|
+
# poetry
|
|
85
|
+
poetry.lock
|
|
86
|
+
|
|
87
|
+
# pdm
|
|
88
|
+
.pdm.toml
|
|
89
|
+
|
|
90
|
+
# PEP 582
|
|
91
|
+
__pypackages__/
|
|
92
|
+
|
|
93
|
+
# Celery stuff
|
|
94
|
+
celerybeat-schedule
|
|
95
|
+
celerybeat.pid
|
|
96
|
+
|
|
97
|
+
# SageMath parsed files
|
|
98
|
+
*.sage.py
|
|
99
|
+
|
|
100
|
+
# Environments
|
|
101
|
+
.env
|
|
102
|
+
.venv
|
|
103
|
+
env/
|
|
104
|
+
venv/
|
|
105
|
+
ENV/
|
|
106
|
+
env.bak/
|
|
107
|
+
venv.bak/
|
|
108
|
+
|
|
109
|
+
# Spyder project settings
|
|
110
|
+
.spyderproject
|
|
111
|
+
.spyproject
|
|
112
|
+
|
|
113
|
+
# Rope project settings
|
|
114
|
+
.ropeproject
|
|
115
|
+
|
|
116
|
+
# mkdocs documentation
|
|
117
|
+
/site
|
|
118
|
+
|
|
119
|
+
# mypy
|
|
120
|
+
.mypy_cache/
|
|
121
|
+
.dmypy.json
|
|
122
|
+
dmypy.json
|
|
123
|
+
|
|
124
|
+
# Pyre type checker
|
|
125
|
+
.pyre/
|
|
126
|
+
|
|
127
|
+
# pytype static type analyzer
|
|
128
|
+
.pytype/
|
|
129
|
+
|
|
130
|
+
# Cython debug symbols
|
|
131
|
+
cython_debug/
|
|
132
|
+
|
|
133
|
+
# ruff
|
|
134
|
+
.ruff_cache/
|
|
135
|
+
|
|
136
|
+
# IDE
|
|
137
|
+
.vscode/
|
|
138
|
+
.idea/
|
|
139
|
+
*.swp
|
|
140
|
+
*.swo
|
|
141
|
+
*~
|
|
142
|
+
.DS_Store
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Flo Team
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: flo-python
|
|
3
|
-
Version: 0.1.0
|
|
3
|
+
Version: 0.1.0.dev2
|
|
4
4
|
Summary: Python SDK for the Flo distributed systems platform
|
|
5
5
|
Project-URL: Homepage, https://github.com/floruntime/flo-python
|
|
6
6
|
Project-URL: Documentation, https://github.com/floruntime/flo-python#readme
|
|
7
7
|
Project-URL: Repository, https://github.com/floruntime/flo-python
|
|
8
8
|
Author: Flo Team
|
|
9
9
|
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
10
11
|
Keywords: async,distributed-systems,flo,kv-store,queue
|
|
11
12
|
Classifier: Development Status :: 3 - Alpha
|
|
12
13
|
Classifier: Framework :: AsyncIO
|
|
@@ -452,39 +453,20 @@ await client.worker.touch(
|
|
|
452
453
|
|
|
453
454
|
```python
|
|
454
455
|
import asyncio
|
|
455
|
-
from flo import FloClient
|
|
456
|
+
from flo import FloClient
|
|
456
457
|
|
|
457
458
|
async def run_worker():
|
|
458
|
-
async with FloClient("localhost:9000") as client:
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
task_types,
|
|
470
|
-
WorkerAwaitOptions(block_ms=30000)
|
|
471
|
-
)
|
|
472
|
-
|
|
473
|
-
if not result.task:
|
|
474
|
-
continue
|
|
475
|
-
|
|
476
|
-
task = result.task
|
|
477
|
-
try:
|
|
478
|
-
# Process task
|
|
479
|
-
output = await process_image(task.input)
|
|
480
|
-
await client.worker.complete(worker_id, task.task_id, output)
|
|
481
|
-
except Exception as e:
|
|
482
|
-
await client.worker.fail(
|
|
483
|
-
worker_id,
|
|
484
|
-
task.task_id,
|
|
485
|
-
str(e),
|
|
486
|
-
WorkerFailOptions(retry=True)
|
|
487
|
-
)
|
|
459
|
+
async with FloClient("localhost:9000", namespace="myapp") as client:
|
|
460
|
+
# Create a worker from the client
|
|
461
|
+
worker = client.new_worker(concurrency=5)
|
|
462
|
+
|
|
463
|
+
@worker.action("process-image")
|
|
464
|
+
async def process_image(ctx):
|
|
465
|
+
data = ctx.json()
|
|
466
|
+
result = await do_processing(data)
|
|
467
|
+
return ctx.to_bytes({"status": "done"})
|
|
468
|
+
|
|
469
|
+
await worker.start()
|
|
488
470
|
|
|
489
471
|
asyncio.run(run_worker())
|
|
490
472
|
```
|
|
@@ -423,39 +423,20 @@ await client.worker.touch(
|
|
|
423
423
|
|
|
424
424
|
```python
|
|
425
425
|
import asyncio
|
|
426
|
-
from flo import FloClient
|
|
426
|
+
from flo import FloClient
|
|
427
427
|
|
|
428
428
|
async def run_worker():
|
|
429
|
-
async with FloClient("localhost:9000") as client:
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
task_types,
|
|
441
|
-
WorkerAwaitOptions(block_ms=30000)
|
|
442
|
-
)
|
|
443
|
-
|
|
444
|
-
if not result.task:
|
|
445
|
-
continue
|
|
446
|
-
|
|
447
|
-
task = result.task
|
|
448
|
-
try:
|
|
449
|
-
# Process task
|
|
450
|
-
output = await process_image(task.input)
|
|
451
|
-
await client.worker.complete(worker_id, task.task_id, output)
|
|
452
|
-
except Exception as e:
|
|
453
|
-
await client.worker.fail(
|
|
454
|
-
worker_id,
|
|
455
|
-
task.task_id,
|
|
456
|
-
str(e),
|
|
457
|
-
WorkerFailOptions(retry=True)
|
|
458
|
-
)
|
|
429
|
+
async with FloClient("localhost:9000", namespace="myapp") as client:
|
|
430
|
+
# Create a worker from the client
|
|
431
|
+
worker = client.new_worker(concurrency=5)
|
|
432
|
+
|
|
433
|
+
@worker.action("process-image")
|
|
434
|
+
async def process_image(ctx):
|
|
435
|
+
data = ctx.json()
|
|
436
|
+
result = await do_processing(data)
|
|
437
|
+
return ctx.to_bytes({"status": "done"})
|
|
438
|
+
|
|
439
|
+
await worker.start()
|
|
459
440
|
|
|
460
441
|
asyncio.run(run_worker())
|
|
461
442
|
```
|
|
@@ -9,7 +9,7 @@ import os
|
|
|
9
9
|
import signal
|
|
10
10
|
from dataclasses import dataclass
|
|
11
11
|
|
|
12
|
-
from flo import ActionContext,
|
|
12
|
+
from flo import ActionContext, FloClient
|
|
13
13
|
|
|
14
14
|
# Configure logging
|
|
15
15
|
logging.basicConfig(
|
|
@@ -158,13 +158,18 @@ async def generate_report(ctx: ActionContext) -> bytes:
|
|
|
158
158
|
|
|
159
159
|
|
|
160
160
|
async def main():
|
|
161
|
-
# Create
|
|
162
|
-
|
|
163
|
-
|
|
161
|
+
# Create and connect client (shared configuration)
|
|
162
|
+
client = FloClient(
|
|
163
|
+
os.getenv("FLO_ENDPOINT", "localhost:3000"),
|
|
164
164
|
namespace=os.getenv("FLO_NAMESPACE", "myapp"),
|
|
165
|
+
debug=os.getenv("FLO_DEBUG", "").lower() in ("1", "true"),
|
|
166
|
+
)
|
|
167
|
+
await client.connect()
|
|
168
|
+
|
|
169
|
+
# Create a worker from the client
|
|
170
|
+
worker = client.new_worker(
|
|
165
171
|
concurrency=5,
|
|
166
172
|
action_timeout=300, # 5 minutes
|
|
167
|
-
debug=os.getenv("FLO_DEBUG", "").lower() in ("1", "true"),
|
|
168
173
|
)
|
|
169
174
|
|
|
170
175
|
# Register action handlers using register_action()
|
|
@@ -202,6 +207,7 @@ async def main():
|
|
|
202
207
|
logger.info("Interrupted")
|
|
203
208
|
finally:
|
|
204
209
|
await worker.close()
|
|
210
|
+
await client.close()
|
|
205
211
|
logger.info("Worker shutdown complete")
|
|
206
212
|
|
|
207
213
|
|
|
@@ -2,23 +2,31 @@
|
|
|
2
2
|
|
|
3
3
|
A Python client for the Flo distributed systems platform.
|
|
4
4
|
|
|
5
|
+
All primitives are accessed as attributes on a connected FloClient:
|
|
6
|
+
|
|
5
7
|
Example:
|
|
6
8
|
import asyncio
|
|
7
9
|
from flo import FloClient
|
|
8
10
|
|
|
9
11
|
async def main():
|
|
10
|
-
async with FloClient("localhost:9000") as client:
|
|
12
|
+
async with FloClient("localhost:9000", namespace="myapp") as client:
|
|
11
13
|
# KV operations
|
|
12
14
|
await client.kv.put("key", b"value")
|
|
13
15
|
value = await client.kv.get("key")
|
|
14
|
-
print(f"Got: {value}")
|
|
15
16
|
|
|
16
17
|
# Queue operations
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
18
|
+
await client.queue.enqueue("tasks", b'{"task": "process"}')
|
|
19
|
+
|
|
20
|
+
# Stream operations
|
|
21
|
+
await client.stream.append("events", b'{"event": "click"}')
|
|
22
|
+
|
|
23
|
+
# Action operations
|
|
24
|
+
await client.action.invoke("process", b'{}')
|
|
25
|
+
|
|
26
|
+
# Worker (created from client)
|
|
27
|
+
worker = client.new_worker(concurrency=5)
|
|
28
|
+
worker.register_action("my-action", handler)
|
|
29
|
+
await worker.start()
|
|
22
30
|
|
|
23
31
|
asyncio.run(main())
|
|
24
32
|
"""
|
|
@@ -88,6 +96,7 @@ from .types import (
|
|
|
88
96
|
StreamAppendResult,
|
|
89
97
|
StreamGroupAckOptions,
|
|
90
98
|
StreamGroupJoinOptions,
|
|
99
|
+
StreamGroupNackOptions,
|
|
91
100
|
StreamGroupReadOptions,
|
|
92
101
|
StreamID,
|
|
93
102
|
StreamInfo,
|
|
@@ -111,7 +120,7 @@ from .types import (
|
|
|
111
120
|
WorkerTask,
|
|
112
121
|
WorkerTouchOptions,
|
|
113
122
|
)
|
|
114
|
-
from .worker import ActionContext, Worker,
|
|
123
|
+
from .worker import ActionContext, Worker, WorkerOptions
|
|
115
124
|
|
|
116
125
|
__version__ = "0.1.0"
|
|
117
126
|
|
|
@@ -120,7 +129,7 @@ __all__ = [
|
|
|
120
129
|
"FloClient",
|
|
121
130
|
# High-level Worker API
|
|
122
131
|
"Worker",
|
|
123
|
-
"
|
|
132
|
+
"WorkerOptions",
|
|
124
133
|
"ActionContext",
|
|
125
134
|
# Exceptions
|
|
126
135
|
"FloError",
|
|
@@ -186,6 +195,7 @@ __all__ = [
|
|
|
186
195
|
"StreamGroupJoinOptions",
|
|
187
196
|
"StreamGroupReadOptions",
|
|
188
197
|
"StreamGroupAckOptions",
|
|
198
|
+
"StreamGroupNackOptions",
|
|
189
199
|
# Action types
|
|
190
200
|
"ActionType",
|
|
191
201
|
"ActionInfo",
|
|
@@ -6,6 +6,7 @@ Async client for connecting to Flo servers.
|
|
|
6
6
|
import asyncio
|
|
7
7
|
import contextlib
|
|
8
8
|
import logging
|
|
9
|
+
from typing import TYPE_CHECKING
|
|
9
10
|
|
|
10
11
|
from .exceptions import (
|
|
11
12
|
ConnectionFailedError,
|
|
@@ -17,6 +18,9 @@ from .exceptions import (
|
|
|
17
18
|
from .types import HEADER_SIZE, OpCode, StatusCode
|
|
18
19
|
from .wire import RawResponse, parse_response_header, serialize_request
|
|
19
20
|
|
|
21
|
+
if TYPE_CHECKING:
|
|
22
|
+
from .worker import Worker
|
|
23
|
+
|
|
20
24
|
logger = logging.getLogger("flo")
|
|
21
25
|
|
|
22
26
|
|
|
@@ -282,3 +286,41 @@ class FloClient:
|
|
|
282
286
|
def get_namespace(self, override: str | None) -> str:
|
|
283
287
|
"""Get effective namespace, using override if provided."""
|
|
284
288
|
return override if override is not None else self._namespace
|
|
289
|
+
|
|
290
|
+
def new_worker(
|
|
291
|
+
self,
|
|
292
|
+
*,
|
|
293
|
+
worker_id: str | None = None,
|
|
294
|
+
concurrency: int = 10,
|
|
295
|
+
action_timeout: float = 300.0,
|
|
296
|
+
block_ms: int = 30000,
|
|
297
|
+
) -> "Worker":
|
|
298
|
+
"""Create a new Worker from this client.
|
|
299
|
+
|
|
300
|
+
The worker inherits the client's endpoint and namespace, and creates
|
|
301
|
+
a dedicated connection for polling tasks.
|
|
302
|
+
|
|
303
|
+
Args:
|
|
304
|
+
worker_id: Unique worker identifier (auto-generated if not provided).
|
|
305
|
+
concurrency: Maximum number of concurrent actions.
|
|
306
|
+
action_timeout: Timeout for action handlers in seconds.
|
|
307
|
+
block_ms: Timeout for blocking dequeue in milliseconds.
|
|
308
|
+
|
|
309
|
+
Returns:
|
|
310
|
+
A new Worker instance ready to register actions and start.
|
|
311
|
+
|
|
312
|
+
Example:
|
|
313
|
+
async with FloClient("localhost:3000", namespace="myapp") as client:
|
|
314
|
+
worker = client.new_worker(concurrency=5)
|
|
315
|
+
worker.register_action("process-order", process_order)
|
|
316
|
+
await worker.start()
|
|
317
|
+
"""
|
|
318
|
+
from .worker import Worker
|
|
319
|
+
|
|
320
|
+
return Worker(
|
|
321
|
+
self,
|
|
322
|
+
worker_id=worker_id,
|
|
323
|
+
concurrency=concurrency,
|
|
324
|
+
action_timeout=action_timeout,
|
|
325
|
+
block_ms=block_ms,
|
|
326
|
+
)
|
|
@@ -13,6 +13,7 @@ from .types import (
|
|
|
13
13
|
StreamAppendResult,
|
|
14
14
|
StreamGroupAckOptions,
|
|
15
15
|
StreamGroupJoinOptions,
|
|
16
|
+
StreamGroupNackOptions,
|
|
16
17
|
StreamGroupReadOptions,
|
|
17
18
|
StreamInfo,
|
|
18
19
|
StreamInfoOptions,
|
|
@@ -351,14 +352,15 @@ class StreamOperations:
|
|
|
351
352
|
stream: Stream name.
|
|
352
353
|
group: Consumer group name.
|
|
353
354
|
seqs: Sequence numbers of records to acknowledge.
|
|
354
|
-
options: Optional ack options.
|
|
355
|
+
options: Optional ack options (including consumer name).
|
|
355
356
|
|
|
356
357
|
Example:
|
|
357
358
|
result = await client.stream.group_read("events", "processors", "worker-1")
|
|
358
359
|
for record in result.records:
|
|
359
360
|
try:
|
|
360
361
|
process(record.payload)
|
|
361
|
-
await client.stream.group_ack("events", "processors", [record.seq]
|
|
362
|
+
await client.stream.group_ack("events", "processors", [record.seq],
|
|
363
|
+
StreamGroupAckOptions(consumer="worker-1"))
|
|
362
364
|
except Exception:
|
|
363
365
|
pass # Record will be redelivered
|
|
364
366
|
"""
|
|
@@ -368,7 +370,7 @@ class StreamOperations:
|
|
|
368
370
|
opts = options or StreamGroupAckOptions()
|
|
369
371
|
namespace = self._client.get_namespace(opts.namespace)
|
|
370
372
|
|
|
371
|
-
value = serialize_group_ack_value(group, seqs)
|
|
373
|
+
value = serialize_group_ack_value(group, seqs, consumer=opts.consumer)
|
|
372
374
|
|
|
373
375
|
await self._client._send_and_check(
|
|
374
376
|
OpCode.STREAM_GROUP_ACK,
|
|
@@ -377,3 +379,48 @@ class StreamOperations:
|
|
|
377
379
|
value,
|
|
378
380
|
allow_not_found=True,
|
|
379
381
|
)
|
|
382
|
+
|
|
383
|
+
async def group_nack(
|
|
384
|
+
self,
|
|
385
|
+
stream: str,
|
|
386
|
+
group: str,
|
|
387
|
+
seqs: list[int],
|
|
388
|
+
options: StreamGroupNackOptions | None = None,
|
|
389
|
+
) -> None:
|
|
390
|
+
"""Negatively acknowledge records in a consumer group for redelivery.
|
|
391
|
+
|
|
392
|
+
Args:
|
|
393
|
+
stream: Stream name.
|
|
394
|
+
group: Consumer group name.
|
|
395
|
+
seqs: Sequence numbers of records to negatively acknowledge.
|
|
396
|
+
options: Optional nack options (consumer name, redelivery delay).
|
|
397
|
+
|
|
398
|
+
Example:
|
|
399
|
+
result = await client.stream.group_read("events", "processors", "worker-1")
|
|
400
|
+
for record in result.records:
|
|
401
|
+
try:
|
|
402
|
+
process(record.payload)
|
|
403
|
+
except Exception:
|
|
404
|
+
await client.stream.group_nack("events", "processors", [record.seq],
|
|
405
|
+
StreamGroupNackOptions(consumer="worker-1", redelivery_delay_ms=5000))
|
|
406
|
+
"""
|
|
407
|
+
if not seqs:
|
|
408
|
+
return
|
|
409
|
+
|
|
410
|
+
opts = options or StreamGroupNackOptions()
|
|
411
|
+
namespace = self._client.get_namespace(opts.namespace)
|
|
412
|
+
|
|
413
|
+
value = serialize_group_ack_value(group, seqs, consumer=opts.consumer)
|
|
414
|
+
|
|
415
|
+
builder = OptionsBuilder()
|
|
416
|
+
if opts.redelivery_delay_ms is not None:
|
|
417
|
+
builder.add_u32(OptionTag.REDELIVERY_DELAY_MS, opts.redelivery_delay_ms)
|
|
418
|
+
|
|
419
|
+
await self._client._send_and_check(
|
|
420
|
+
OpCode.STREAM_GROUP_NACK,
|
|
421
|
+
namespace,
|
|
422
|
+
stream.encode("utf-8"),
|
|
423
|
+
value,
|
|
424
|
+
builder.build(),
|
|
425
|
+
allow_not_found=True,
|
|
426
|
+
)
|
|
@@ -368,15 +368,15 @@ class StreamID:
|
|
|
368
368
|
sequence: int = 0
|
|
369
369
|
|
|
370
370
|
def to_bytes(self) -> bytes:
|
|
371
|
-
"""Serialize the StreamID to 16 bytes (
|
|
372
|
-
return struct.pack("
|
|
371
|
+
"""Serialize the StreamID to 16 bytes (big-endian for lexicographic sorting)."""
|
|
372
|
+
return struct.pack(">QQ", self.timestamp_ms, self.sequence)
|
|
373
373
|
|
|
374
374
|
@classmethod
|
|
375
375
|
def from_bytes(cls, data: bytes) -> "StreamID":
|
|
376
|
-
"""Parse a StreamID from 16 bytes (
|
|
376
|
+
"""Parse a StreamID from 16 bytes (big-endian)."""
|
|
377
377
|
if len(data) < 16:
|
|
378
378
|
raise ValueError(f"Invalid StreamID: expected 16 bytes, got {len(data)}")
|
|
379
|
-
ts, seq = struct.unpack("
|
|
379
|
+
ts, seq = struct.unpack(">QQ", data[:16])
|
|
380
380
|
return cls(timestamp_ms=ts, sequence=seq)
|
|
381
381
|
|
|
382
382
|
@classmethod
|
|
@@ -431,6 +431,7 @@ class StreamInfo:
|
|
|
431
431
|
last_seq: int
|
|
432
432
|
count: int
|
|
433
433
|
bytes_size: int
|
|
434
|
+
partition_count: int = 1
|
|
434
435
|
|
|
435
436
|
|
|
436
437
|
# =============================================================================
|
|
@@ -613,6 +614,16 @@ class StreamGroupAckOptions:
|
|
|
613
614
|
"""Options for acknowledging records in a consumer group."""
|
|
614
615
|
|
|
615
616
|
namespace: str | None = None
|
|
617
|
+
consumer: str = "" # Consumer ID (required for correct ack matching)
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
@dataclass
|
|
621
|
+
class StreamGroupNackOptions:
|
|
622
|
+
"""Options for negatively acknowledging records in a consumer group."""
|
|
623
|
+
|
|
624
|
+
namespace: str | None = None
|
|
625
|
+
consumer: str = "" # Consumer ID (required for correct nack matching)
|
|
626
|
+
redelivery_delay_ms: int | None = None # Delay before message becomes visible again
|
|
616
627
|
|
|
617
628
|
|
|
618
629
|
# =============================================================================
|
|
@@ -633,17 +633,24 @@ def parse_stream_read_response(data: bytes) -> StreamReadResult:
|
|
|
633
633
|
def parse_stream_info_response(data: bytes) -> StreamInfo:
|
|
634
634
|
"""Parse stream info response data.
|
|
635
635
|
|
|
636
|
-
Format: [first_seq:u64][last_seq:u64][count:u64][bytes:u64]
|
|
636
|
+
Format: [first_seq:u64][last_seq:u64][count:u64][bytes:u64][partition_count:u32]
|
|
637
637
|
"""
|
|
638
|
-
if len(data) <
|
|
638
|
+
if len(data) < 36:
|
|
639
639
|
raise IncompleteResponseError("Stream info response too short")
|
|
640
640
|
|
|
641
641
|
first_seq = struct.unpack("<Q", data[0:8])[0]
|
|
642
642
|
last_seq = struct.unpack("<Q", data[8:16])[0]
|
|
643
643
|
count = struct.unpack("<Q", data[16:24])[0]
|
|
644
644
|
bytes_size = struct.unpack("<Q", data[24:32])[0]
|
|
645
|
-
|
|
646
|
-
|
|
645
|
+
partition_count = struct.unpack("<I", data[32:36])[0]
|
|
646
|
+
|
|
647
|
+
return StreamInfo(
|
|
648
|
+
first_seq=first_seq,
|
|
649
|
+
last_seq=last_seq,
|
|
650
|
+
count=count,
|
|
651
|
+
bytes_size=bytes_size,
|
|
652
|
+
partition_count=partition_count,
|
|
653
|
+
)
|
|
647
654
|
|
|
648
655
|
|
|
649
656
|
def serialize_group_value(group: str, consumer: str) -> bytes:
|
|
@@ -662,16 +669,19 @@ def serialize_group_value(group: str, consumer: str) -> bytes:
|
|
|
662
669
|
return bytes(result)
|
|
663
670
|
|
|
664
671
|
|
|
665
|
-
def serialize_group_ack_value(group: str, seqs: list[int]) -> bytes:
|
|
666
|
-
"""Serialize group name and sequence numbers for group ack.
|
|
672
|
+
def serialize_group_ack_value(group: str, seqs: list[int], consumer: str = "") -> bytes:
|
|
673
|
+
"""Serialize group name, consumer, and sequence numbers for group ack/nack.
|
|
667
674
|
|
|
668
|
-
Format: [group_len:u16][group][count:u32][seq:u64]*
|
|
675
|
+
Format: [group_len:u16][group][consumer_len:u16][consumer][count:u32][seq:u64]*
|
|
669
676
|
"""
|
|
670
677
|
group_bytes = group.encode("utf-8")
|
|
678
|
+
consumer_bytes = consumer.encode("utf-8")
|
|
671
679
|
|
|
672
680
|
result = bytearray()
|
|
673
681
|
result.extend(struct.pack("<H", len(group_bytes)))
|
|
674
682
|
result.extend(group_bytes)
|
|
683
|
+
result.extend(struct.pack("<H", len(consumer_bytes)))
|
|
684
|
+
result.extend(consumer_bytes)
|
|
675
685
|
result.extend(struct.pack("<I", len(seqs)))
|
|
676
686
|
for seq in seqs:
|
|
677
687
|
result.extend(struct.pack("<Q", seq))
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
Provides an easy-to-use Worker class for executing actions.
|
|
4
4
|
|
|
5
5
|
Example:
|
|
6
|
-
from flo import
|
|
6
|
+
from flo import FloClient, ActionContext
|
|
7
7
|
|
|
8
8
|
async def process_order(ctx: ActionContext) -> bytes:
|
|
9
9
|
order = ctx.json()
|
|
@@ -11,13 +11,10 @@ Example:
|
|
|
11
11
|
return ctx.to_bytes({"status": "completed"})
|
|
12
12
|
|
|
13
13
|
async def main():
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
worker.action("process-order")(process_order)
|
|
19
|
-
|
|
20
|
-
await worker.start()
|
|
14
|
+
async with FloClient("localhost:3000", namespace="myapp") as client:
|
|
15
|
+
worker = client.new_worker(concurrency=5)
|
|
16
|
+
worker.action("process-order")(process_order)
|
|
17
|
+
await worker.start()
|
|
21
18
|
"""
|
|
22
19
|
|
|
23
20
|
import asyncio
|
|
@@ -40,16 +37,16 @@ ActionHandler = Callable[["ActionContext"], Awaitable[bytes]]
|
|
|
40
37
|
|
|
41
38
|
|
|
42
39
|
@dataclass
|
|
43
|
-
class
|
|
44
|
-
"""Configuration for a Flo worker.
|
|
40
|
+
class WorkerOptions:
|
|
41
|
+
"""Configuration for a Flo worker.
|
|
42
|
+
|
|
43
|
+
Endpoint and namespace are inherited from the parent FloClient.
|
|
44
|
+
"""
|
|
45
45
|
|
|
46
|
-
endpoint: str
|
|
47
|
-
namespace: str = "default"
|
|
48
46
|
worker_id: str = ""
|
|
49
47
|
concurrency: int = 10
|
|
50
48
|
action_timeout: float = 300.0 # 5 minutes
|
|
51
49
|
block_ms: int = 30000
|
|
52
|
-
debug: bool = False
|
|
53
50
|
|
|
54
51
|
|
|
55
52
|
@dataclass
|
|
@@ -122,48 +119,44 @@ class ActionContext:
|
|
|
122
119
|
class Worker:
|
|
123
120
|
"""High-level Flo worker for executing actions.
|
|
124
121
|
|
|
122
|
+
Created from a FloClient via ``client.new_worker()``.
|
|
123
|
+
|
|
125
124
|
Example:
|
|
126
|
-
|
|
125
|
+
async with FloClient("localhost:3000", namespace="myapp") as client:
|
|
126
|
+
worker = client.new_worker(concurrency=5)
|
|
127
127
|
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
return ctx.to_bytes({"status": "completed"})
|
|
128
|
+
@worker.action("process-order")
|
|
129
|
+
async def process_order(ctx: ActionContext) -> bytes:
|
|
130
|
+
order = ctx.json()
|
|
131
|
+
return ctx.to_bytes({"status": "completed"})
|
|
133
132
|
|
|
134
|
-
|
|
133
|
+
await worker.start()
|
|
135
134
|
"""
|
|
136
135
|
|
|
137
136
|
def __init__(
|
|
138
137
|
self,
|
|
139
|
-
|
|
138
|
+
parent_client: "FloClient",
|
|
140
139
|
*,
|
|
141
|
-
namespace: str = "default",
|
|
142
140
|
worker_id: str | None = None,
|
|
143
141
|
concurrency: int = 10,
|
|
144
142
|
action_timeout: float = 300.0,
|
|
145
143
|
block_ms: int = 30000,
|
|
146
|
-
debug: bool = False,
|
|
147
144
|
):
|
|
148
|
-
"""Initialize a Flo worker.
|
|
145
|
+
"""Initialize a Flo worker from a connected client.
|
|
149
146
|
|
|
150
147
|
Args:
|
|
151
|
-
|
|
152
|
-
namespace: Namespace for operations.
|
|
148
|
+
parent_client: The parent FloClient whose endpoint and namespace are used.
|
|
153
149
|
worker_id: Unique worker identifier (auto-generated if not provided).
|
|
154
150
|
concurrency: Maximum number of concurrent actions.
|
|
155
151
|
action_timeout: Timeout for action handlers in seconds.
|
|
156
152
|
block_ms: Timeout for blocking dequeue in milliseconds.
|
|
157
|
-
debug: Enable debug logging.
|
|
158
153
|
"""
|
|
159
|
-
self.
|
|
160
|
-
|
|
161
|
-
namespace=namespace,
|
|
154
|
+
self._parent_client = parent_client
|
|
155
|
+
self.config = WorkerOptions(
|
|
162
156
|
worker_id=worker_id or self._generate_worker_id(),
|
|
163
157
|
concurrency=concurrency,
|
|
164
158
|
action_timeout=action_timeout,
|
|
165
159
|
block_ms=block_ms,
|
|
166
|
-
debug=debug,
|
|
167
160
|
)
|
|
168
161
|
|
|
169
162
|
self._client: FloClient | None = None
|
|
@@ -173,9 +166,6 @@ class Worker:
|
|
|
173
166
|
self._tasks: set[asyncio.Task[None]] = set()
|
|
174
167
|
self._semaphore: asyncio.Semaphore | None = None
|
|
175
168
|
|
|
176
|
-
if debug:
|
|
177
|
-
logging.basicConfig(level=logging.DEBUG)
|
|
178
|
-
|
|
179
169
|
@staticmethod
|
|
180
170
|
def _generate_worker_id() -> str:
|
|
181
171
|
"""Generate a unique worker ID."""
|
|
@@ -235,14 +225,14 @@ class Worker:
|
|
|
235
225
|
|
|
236
226
|
logger.info(
|
|
237
227
|
f"Starting Flo worker (id={self.config.worker_id}, "
|
|
238
|
-
f"namespace={self.
|
|
228
|
+
f"namespace={self._parent_client.namespace}, concurrency={self.config.concurrency})"
|
|
239
229
|
)
|
|
240
230
|
|
|
241
|
-
#
|
|
231
|
+
# Create a dedicated connection using the parent client's endpoint and namespace
|
|
242
232
|
self._client = FloClient(
|
|
243
|
-
self.
|
|
244
|
-
namespace=self.
|
|
245
|
-
debug=self.
|
|
233
|
+
self._parent_client._endpoint,
|
|
234
|
+
namespace=self._parent_client.namespace,
|
|
235
|
+
debug=self._parent_client._debug,
|
|
246
236
|
)
|
|
247
237
|
await self._client.connect()
|
|
248
238
|
|
|
@@ -344,7 +334,7 @@ class Worker:
|
|
|
344
334
|
payload=task.payload,
|
|
345
335
|
attempt=task.attempt,
|
|
346
336
|
created_at=task.created_at,
|
|
347
|
-
namespace=self.
|
|
337
|
+
namespace=self._parent_client.namespace,
|
|
348
338
|
_worker=self,
|
|
349
339
|
)
|
|
350
340
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|