flo-python 0.1.4__py3-none-any.whl → 0.1.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flo/__init__.py +63 -13
- flo/actions.py +78 -95
- flo/client.py +56 -36
- flo/exceptions.py +59 -10
- flo/kv.py +31 -24
- flo/kv_body.py +520 -81
- flo/kv_txn.py +34 -34
- flo/processing.py +57 -6
- flo/queue.py +204 -167
- flo/retry.py +37 -6
- flo/streams.py +222 -226
- flo/types.py +520 -124
- flo/wire.py +198 -439
- flo/worker.py +596 -315
- flo/workflows.py +11 -3
- {flo_python-0.1.4.dist-info → flo_python-0.1.5.dist-info}/METADATA +262 -110
- flo_python-0.1.5.dist-info/RECORD +19 -0
- flo_python-0.1.4.dist-info/RECORD +0 -19
- {flo_python-0.1.4.dist-info → flo_python-0.1.5.dist-info}/WHEEL +0 -0
- {flo_python-0.1.4.dist-info → flo_python-0.1.5.dist-info}/licenses/LICENSE +0 -0
flo/__init__.py
CHANGED
|
@@ -39,12 +39,14 @@ from .exceptions import (
|
|
|
39
39
|
ConnectionFailedError,
|
|
40
40
|
FloError,
|
|
41
41
|
GenericServerError,
|
|
42
|
+
GroupHeldFullError,
|
|
42
43
|
IncompleteResponseError,
|
|
43
44
|
InternalServerError,
|
|
44
45
|
InvalidChecksumError,
|
|
45
46
|
InvalidEndpointError,
|
|
46
47
|
InvalidMagicError,
|
|
47
48
|
KeyTooLargeError,
|
|
49
|
+
LimitError,
|
|
48
50
|
NamespaceTooLargeError,
|
|
49
51
|
NoNamespaceError,
|
|
50
52
|
NonRetryableError,
|
|
@@ -54,10 +56,12 @@ from .exceptions import (
|
|
|
54
56
|
OverloadedError,
|
|
55
57
|
PayloadTooLargeError,
|
|
56
58
|
ProtocolError,
|
|
59
|
+
QueueFullError,
|
|
57
60
|
RateLimitedError,
|
|
58
61
|
RequestTimeoutError,
|
|
59
62
|
ServerError,
|
|
60
63
|
StaleNotFoundError,
|
|
64
|
+
StaleReportError,
|
|
61
65
|
TableMismatchError,
|
|
62
66
|
UnauthorizedError,
|
|
63
67
|
UnavailableError,
|
|
@@ -70,10 +74,11 @@ from .kv_txn import Transaction, TxnFinishedError, TxnUnsupportedOpError
|
|
|
70
74
|
from .processing import ProcessingOperations
|
|
71
75
|
from .retry import DEFAULT_RETRY_POLICY, RetryPolicy
|
|
72
76
|
from .types import (
|
|
73
|
-
AckOptions,
|
|
74
77
|
ActionDeleteOptions,
|
|
78
|
+
ActionInfo,
|
|
75
79
|
ActionInvokeOptions,
|
|
76
80
|
ActionInvokeResult,
|
|
81
|
+
ActionListEntry,
|
|
77
82
|
ActionListOptions,
|
|
78
83
|
ActionListResult,
|
|
79
84
|
ActionRegisterOptions,
|
|
@@ -81,16 +86,23 @@ from .types import (
|
|
|
81
86
|
ActionStatusOptions,
|
|
82
87
|
ActionType,
|
|
83
88
|
Condition,
|
|
89
|
+
DeadLetter,
|
|
84
90
|
DeleteOptions,
|
|
85
91
|
DequeueOptions,
|
|
86
92
|
DequeueResult,
|
|
87
93
|
DlqListOptions,
|
|
88
|
-
|
|
94
|
+
DlqPage,
|
|
89
95
|
EnqueueOptions,
|
|
90
96
|
ExistsResult,
|
|
91
97
|
GetJsonResult,
|
|
92
98
|
GetOptions,
|
|
93
99
|
GetResult,
|
|
100
|
+
GroupMemberState,
|
|
101
|
+
GroupReadResult,
|
|
102
|
+
GroupRecord,
|
|
103
|
+
GroupRefusal,
|
|
104
|
+
GroupReportResult,
|
|
105
|
+
GroupTouchResult,
|
|
94
106
|
HistoryOptions,
|
|
95
107
|
HistoryResult,
|
|
96
108
|
IncrResult,
|
|
@@ -103,15 +115,18 @@ from .types import (
|
|
|
103
115
|
KVMGetOptions,
|
|
104
116
|
Message,
|
|
105
117
|
MGetEntry,
|
|
106
|
-
NackOptions,
|
|
107
118
|
OpCode,
|
|
108
119
|
OptionTag,
|
|
120
|
+
PeekedMessage,
|
|
109
121
|
PeekOptions,
|
|
122
|
+
PeekResult,
|
|
110
123
|
PendingEntry,
|
|
111
124
|
ProcessingCancelOptions,
|
|
112
125
|
ProcessingListEntry,
|
|
113
126
|
ProcessingListOptions,
|
|
114
127
|
ProcessingListResult,
|
|
128
|
+
ProcessingRemoveOptions,
|
|
129
|
+
ProcessingRuntime,
|
|
115
130
|
ProcessingStatusOptions,
|
|
116
131
|
ProcessingStatusResult,
|
|
117
132
|
ProcessingStopOptions,
|
|
@@ -119,8 +134,15 @@ from .types import (
|
|
|
119
134
|
ProcessingSyncOptions,
|
|
120
135
|
ProcessingSyncResult,
|
|
121
136
|
ProcessKind,
|
|
137
|
+
PurgeOptions,
|
|
122
138
|
PutOptions,
|
|
123
139
|
PutResult,
|
|
140
|
+
QueueFailOptions,
|
|
141
|
+
QueueRefusal,
|
|
142
|
+
QueueRefusalReason,
|
|
143
|
+
QueueReportOptions,
|
|
144
|
+
QueueReportResult,
|
|
145
|
+
QueueTouchResult,
|
|
124
146
|
Ran,
|
|
125
147
|
Reason,
|
|
126
148
|
RunStatus,
|
|
@@ -130,10 +152,14 @@ from .types import (
|
|
|
130
152
|
StorageTier,
|
|
131
153
|
StreamAppendOptions,
|
|
132
154
|
StreamAppendResult,
|
|
133
|
-
StreamClaimResult,
|
|
134
155
|
StreamDeleteOptions,
|
|
135
156
|
StreamGroupAckOptions,
|
|
136
|
-
|
|
157
|
+
StreamGroupCreateOptions,
|
|
158
|
+
StreamGroupEntry,
|
|
159
|
+
StreamGroupInfo,
|
|
160
|
+
StreamGroupListResult,
|
|
161
|
+
StreamGroupMember,
|
|
162
|
+
StreamGroupMembersResult,
|
|
137
163
|
StreamGroupNackOptions,
|
|
138
164
|
StreamGroupReadOptions,
|
|
139
165
|
StreamID,
|
|
@@ -158,7 +184,6 @@ from .types import (
|
|
|
158
184
|
WorkerProcess,
|
|
159
185
|
WorkerRegisterOptions,
|
|
160
186
|
WorkerStatus,
|
|
161
|
-
WorkerTouchOptions,
|
|
162
187
|
WorkerType,
|
|
163
188
|
WorkflowCancelOptions,
|
|
164
189
|
WorkflowCreateOptions,
|
|
@@ -190,7 +215,7 @@ from .worker import (
|
|
|
190
215
|
)
|
|
191
216
|
from .workflows import WorkflowOperations
|
|
192
217
|
|
|
193
|
-
__version__ = "0.1.
|
|
218
|
+
__version__ = "0.1.5"
|
|
194
219
|
|
|
195
220
|
__all__ = [
|
|
196
221
|
"Transaction",
|
|
@@ -250,12 +275,16 @@ __all__ = [
|
|
|
250
275
|
"BadRequestError",
|
|
251
276
|
"ConflictError",
|
|
252
277
|
"UnauthorizedError",
|
|
278
|
+
"GroupHeldFullError",
|
|
253
279
|
"OverloadedError",
|
|
280
|
+
"QueueFullError",
|
|
254
281
|
"OutcomePendingError",
|
|
255
282
|
"StaleNotFoundError",
|
|
283
|
+
"StaleReportError",
|
|
256
284
|
"UnavailableError",
|
|
257
285
|
"RateLimitedError",
|
|
258
286
|
"InternalServerError",
|
|
287
|
+
"LimitError",
|
|
259
288
|
"GenericServerError",
|
|
260
289
|
"is_connection_error",
|
|
261
290
|
# Types
|
|
@@ -271,6 +300,7 @@ __all__ = [
|
|
|
271
300
|
"DequeueResult",
|
|
272
301
|
# Options
|
|
273
302
|
"Condition",
|
|
303
|
+
"DeadLetter",
|
|
274
304
|
"GetOptions",
|
|
275
305
|
"WaitFor",
|
|
276
306
|
"PutOptions",
|
|
@@ -279,18 +309,29 @@ __all__ = [
|
|
|
279
309
|
"HistoryOptions",
|
|
280
310
|
"EnqueueOptions",
|
|
281
311
|
"DequeueOptions",
|
|
282
|
-
"AckOptions",
|
|
283
|
-
"NackOptions",
|
|
284
312
|
"DlqListOptions",
|
|
285
|
-
"
|
|
313
|
+
"DlqPage",
|
|
286
314
|
"PeekOptions",
|
|
315
|
+
"PurgeOptions",
|
|
316
|
+
"QueueFailOptions",
|
|
317
|
+
"QueueRefusal",
|
|
318
|
+
"QueueRefusalReason",
|
|
319
|
+
"QueueReportOptions",
|
|
320
|
+
"QueueReportResult",
|
|
321
|
+
"QueueTouchResult",
|
|
322
|
+
"PeekedMessage",
|
|
323
|
+
"PeekResult",
|
|
287
324
|
# Stream types
|
|
288
325
|
"StreamID",
|
|
289
326
|
"StorageTier",
|
|
290
327
|
"StreamRecord",
|
|
291
328
|
"StreamAppendResult",
|
|
292
329
|
"StreamReadResult",
|
|
293
|
-
"
|
|
330
|
+
"GroupRecord",
|
|
331
|
+
"GroupReadResult",
|
|
332
|
+
"GroupRefusal",
|
|
333
|
+
"GroupReportResult",
|
|
334
|
+
"GroupTouchResult",
|
|
294
335
|
"PendingEntry",
|
|
295
336
|
"StreamInfo",
|
|
296
337
|
# Stream options
|
|
@@ -300,9 +341,15 @@ __all__ = [
|
|
|
300
341
|
"StreamTrimResult",
|
|
301
342
|
"StreamInfoOptions",
|
|
302
343
|
"StreamDeleteOptions",
|
|
303
|
-
"
|
|
344
|
+
"StreamGroupCreateOptions",
|
|
345
|
+
"StreamGroupInfo",
|
|
304
346
|
"StreamGroupReadOptions",
|
|
305
347
|
"StreamGroupAckOptions",
|
|
348
|
+
"StreamGroupEntry",
|
|
349
|
+
"StreamGroupListResult",
|
|
350
|
+
"StreamGroupMember",
|
|
351
|
+
"StreamGroupMembersResult",
|
|
352
|
+
"GroupMemberState",
|
|
306
353
|
"StreamGroupNackOptions",
|
|
307
354
|
# Action types
|
|
308
355
|
"ActionType",
|
|
@@ -315,6 +362,8 @@ __all__ = [
|
|
|
315
362
|
"ActionInvokeOptions",
|
|
316
363
|
"ActionStatusOptions",
|
|
317
364
|
"ActionListOptions",
|
|
365
|
+
"ActionListEntry",
|
|
366
|
+
"ActionInfo",
|
|
318
367
|
"ActionDeleteOptions",
|
|
319
368
|
# Worker types
|
|
320
369
|
"TaskAssignment",
|
|
@@ -329,7 +378,6 @@ __all__ = [
|
|
|
329
378
|
# Worker options
|
|
330
379
|
"WorkerRegisterOptions",
|
|
331
380
|
"WorkerAwaitOptions",
|
|
332
|
-
"WorkerTouchOptions",
|
|
333
381
|
"WorkerCompleteOptions",
|
|
334
382
|
"WorkerFailOptions",
|
|
335
383
|
"WorkerListOptions",
|
|
@@ -363,5 +411,7 @@ __all__ = [
|
|
|
363
411
|
"ProcessingStatusResult",
|
|
364
412
|
"ProcessingListEntry",
|
|
365
413
|
"ProcessingListResult",
|
|
414
|
+
"ProcessingRemoveOptions",
|
|
415
|
+
"ProcessingRuntime",
|
|
366
416
|
"ProcessingSyncResult",
|
|
367
417
|
]
|
flo/actions.py
CHANGED
|
@@ -10,15 +10,24 @@ For a higher-level API, see `Worker` in worker.py.
|
|
|
10
10
|
from typing import TYPE_CHECKING
|
|
11
11
|
|
|
12
12
|
from .kv_body import (
|
|
13
|
+
decode_action_info_answer,
|
|
13
14
|
decode_action_invoke_answer,
|
|
14
15
|
decode_action_list_answer,
|
|
15
16
|
decode_action_status_answer,
|
|
17
|
+
decode_await_answer,
|
|
18
|
+
decode_empty_answer,
|
|
19
|
+
decode_touch_answer,
|
|
16
20
|
decode_worker_info_answer,
|
|
17
21
|
decode_worker_list_answer,
|
|
22
|
+
encode_await_body,
|
|
23
|
+
encode_complete_body,
|
|
24
|
+
encode_fail_body,
|
|
25
|
+
encode_touch_body,
|
|
18
26
|
encode_worker_list_body,
|
|
19
27
|
)
|
|
20
28
|
from .types import (
|
|
21
29
|
ActionDeleteOptions,
|
|
30
|
+
ActionInfo,
|
|
22
31
|
ActionInvokeOptions,
|
|
23
32
|
ActionInvokeResult,
|
|
24
33
|
ActionListOptions,
|
|
@@ -28,7 +37,7 @@ from .types import (
|
|
|
28
37
|
ActionStatusOptions,
|
|
29
38
|
ActionType,
|
|
30
39
|
OpCode,
|
|
31
|
-
|
|
40
|
+
TaskAssignment,
|
|
32
41
|
WorkerAwaitOptions,
|
|
33
42
|
WorkerAwaitResult,
|
|
34
43
|
WorkerCompleteOptions,
|
|
@@ -38,19 +47,12 @@ from .types import (
|
|
|
38
47
|
WorkerListResult,
|
|
39
48
|
WorkerRegisterOptions,
|
|
40
49
|
WorkerStatus,
|
|
41
|
-
WorkerTouchOptions,
|
|
42
50
|
)
|
|
43
51
|
from .wire import (
|
|
44
|
-
OptionsBuilder,
|
|
45
|
-
parse_task_assignment,
|
|
46
52
|
serialize_action_invoke_value,
|
|
47
53
|
serialize_action_list_value,
|
|
48
54
|
serialize_action_register_value,
|
|
49
|
-
serialize_worker_await_value,
|
|
50
|
-
serialize_worker_complete_value,
|
|
51
|
-
serialize_worker_fail_value,
|
|
52
55
|
serialize_worker_register_value,
|
|
53
|
-
serialize_worker_touch_value,
|
|
54
56
|
)
|
|
55
57
|
|
|
56
58
|
if TYPE_CHECKING:
|
|
@@ -86,7 +88,7 @@ class ActionOperations:
|
|
|
86
88
|
value = serialize_action_register_value(
|
|
87
89
|
action_type=action_type,
|
|
88
90
|
timeout_ms=opts.timeout_ms,
|
|
89
|
-
|
|
91
|
+
max_attempts=opts.max_attempts,
|
|
90
92
|
owner=opts.owner,
|
|
91
93
|
)
|
|
92
94
|
|
|
@@ -180,8 +182,23 @@ class ActionOperations:
|
|
|
180
182
|
serialize_action_list_value(limit=opts.limit, cursor=opts.cursor),
|
|
181
183
|
)
|
|
182
184
|
|
|
183
|
-
|
|
184
|
-
return ActionListResult(
|
|
185
|
+
actions, cursor = decode_action_list_answer(response.data)
|
|
186
|
+
return ActionListResult(actions=actions, cursor=cursor)
|
|
187
|
+
|
|
188
|
+
async def info(self, name: str, namespace: str | None = None) -> ActionInfo:
|
|
189
|
+
"""An action's definition, its runs by status and how long its
|
|
190
|
+
completed runs took.
|
|
191
|
+
|
|
192
|
+
Raises:
|
|
193
|
+
NotFoundError: No action has that name.
|
|
194
|
+
"""
|
|
195
|
+
response = await self._client._send_and_check(
|
|
196
|
+
OpCode.ACTION_INFO,
|
|
197
|
+
self._client.get_namespace(namespace),
|
|
198
|
+
name.encode("utf-8"),
|
|
199
|
+
b"",
|
|
200
|
+
)
|
|
201
|
+
return decode_action_info_answer(response.data)
|
|
185
202
|
|
|
186
203
|
async def delete(
|
|
187
204
|
self,
|
|
@@ -306,128 +323,94 @@ class WorkerOperations:
|
|
|
306
323
|
async def await_task(
|
|
307
324
|
self,
|
|
308
325
|
worker_id: str,
|
|
309
|
-
|
|
326
|
+
actions: list[str],
|
|
310
327
|
options: WorkerAwaitOptions | None = None,
|
|
311
328
|
) -> WorkerAwaitResult:
|
|
312
|
-
"""Wait for a
|
|
313
|
-
|
|
314
|
-
Args:
|
|
315
|
-
worker_id: Worker identifier.
|
|
316
|
-
task_types: Task types to listen for.
|
|
317
|
-
options: Optional await options (block_ms).
|
|
329
|
+
"""Wait up to ``block_ms`` for a run of one of ``actions`` (the first
|
|
330
|
+
tried first), and take it.
|
|
318
331
|
|
|
319
332
|
Returns:
|
|
320
|
-
WorkerAwaitResult
|
|
333
|
+
WorkerAwaitResult whose task is None if no run was ready in time.
|
|
321
334
|
"""
|
|
322
335
|
opts = options or WorkerAwaitOptions()
|
|
323
|
-
namespace = self._client.get_namespace(opts.namespace)
|
|
324
|
-
|
|
325
|
-
value = serialize_worker_await_value(task_types)
|
|
326
|
-
|
|
327
|
-
options_builder = OptionsBuilder()
|
|
328
|
-
if opts.block_ms is not None:
|
|
329
|
-
options_builder.add_u32(OptionTag.BLOCK_MS, opts.block_ms)
|
|
330
|
-
|
|
331
336
|
response = await self._client._send_and_check(
|
|
332
337
|
OpCode.ACTION_AWAIT,
|
|
333
|
-
namespace,
|
|
338
|
+
self._client.get_namespace(opts.namespace),
|
|
334
339
|
worker_id.encode("utf-8"),
|
|
335
|
-
|
|
336
|
-
options_builder.build(),
|
|
340
|
+
encode_await_body(actions, opts.block_ms, opts.request_key),
|
|
337
341
|
)
|
|
342
|
+
return WorkerAwaitResult(task=decode_await_answer(response.data))
|
|
338
343
|
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
344
|
+
async def touch(self, worker_id: str, task: TaskAssignment) -> int:
|
|
345
|
+
"""Keep a task. With half its lease or less left, its lease runs a
|
|
346
|
+
full lease from now (up to 12 h from its delivery); with more left,
|
|
347
|
+
nothing changes. Touch at least every half lease.
|
|
342
348
|
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
return WorkerAwaitResult(task=task)
|
|
346
|
-
|
|
347
|
-
async def touch(
|
|
348
|
-
self,
|
|
349
|
-
worker_id: str,
|
|
350
|
-
action_name: str,
|
|
351
|
-
task_id: str,
|
|
352
|
-
options: WorkerTouchOptions | None = None,
|
|
353
|
-
) -> None:
|
|
354
|
-
"""Extend task lease (heartbeat).
|
|
349
|
+
Returns:
|
|
350
|
+
When the lease runs out without another touch, in Unix ms.
|
|
355
351
|
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
action_name: Action name for routing.
|
|
359
|
-
task_id: Task identifier to extend.
|
|
360
|
-
options: Optional touch options (extend_ms).
|
|
352
|
+
Raises:
|
|
353
|
+
StaleReportError: The run moved on from this delivery.
|
|
361
354
|
"""
|
|
362
|
-
|
|
363
|
-
namespace = self._client.get_namespace(opts.namespace)
|
|
364
|
-
|
|
365
|
-
value = serialize_worker_touch_value(action_name, task_id, opts.extend_ms)
|
|
366
|
-
|
|
367
|
-
await self._client._send_and_check(
|
|
355
|
+
response = await self._client._send_and_check(
|
|
368
356
|
OpCode.ACTION_TOUCH,
|
|
369
|
-
namespace,
|
|
370
|
-
|
|
371
|
-
|
|
357
|
+
task.namespace,
|
|
358
|
+
task.run.encode("utf-8"),
|
|
359
|
+
encode_touch_body(worker_id, task),
|
|
372
360
|
)
|
|
361
|
+
return decode_touch_answer(response.data)
|
|
373
362
|
|
|
374
363
|
async def complete(
|
|
375
364
|
self,
|
|
376
365
|
worker_id: str,
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
result: bytes,
|
|
366
|
+
task: TaskAssignment,
|
|
367
|
+
result: bytes = b"",
|
|
380
368
|
options: WorkerCompleteOptions | None = None,
|
|
381
369
|
) -> None:
|
|
382
|
-
"""Complete a task
|
|
370
|
+
"""Complete a task: its run is done, with this result.
|
|
383
371
|
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
372
|
+
Sending it again after it applied answers ok, so a report whose
|
|
373
|
+
answer was lost can be resent unchanged.
|
|
374
|
+
|
|
375
|
+
Raises:
|
|
376
|
+
StaleReportError: The run moved on from this delivery, or was
|
|
377
|
+
completed with another result.
|
|
390
378
|
"""
|
|
391
379
|
opts = options or WorkerCompleteOptions()
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
value = serialize_worker_complete_value(action_name, task_id, result, opts.outcome)
|
|
395
|
-
|
|
396
|
-
await self._client._send_and_check(
|
|
380
|
+
response = await self._client._send_and_check(
|
|
397
381
|
OpCode.ACTION_COMPLETE,
|
|
398
|
-
namespace,
|
|
399
|
-
|
|
400
|
-
|
|
382
|
+
task.namespace,
|
|
383
|
+
task.run.encode("utf-8"),
|
|
384
|
+
encode_complete_body(worker_id, task, opts.outcome, result),
|
|
401
385
|
)
|
|
386
|
+
decode_empty_answer("action.complete", response.data)
|
|
402
387
|
|
|
403
388
|
async def fail(
|
|
404
389
|
self,
|
|
405
390
|
worker_id: str,
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
error_message: str,
|
|
391
|
+
task: TaskAssignment,
|
|
392
|
+
error: str = "",
|
|
409
393
|
options: WorkerFailOptions | None = None,
|
|
410
394
|
) -> None:
|
|
411
|
-
"""Fail a task
|
|
395
|
+
"""Fail a task: its run is ready again after ``delay_ms``, unless its
|
|
396
|
+
attempts are spent or ``retry`` is False. With ``abandoned`` it is
|
|
397
|
+
handed back untried, and no failure is counted.
|
|
412
398
|
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
error_message: Error message describing the failure.
|
|
418
|
-
options: Optional fail options (retry flag).
|
|
399
|
+
Sending it again after it applied answers ok.
|
|
400
|
+
|
|
401
|
+
Raises:
|
|
402
|
+
StaleReportError: The run moved on from this delivery.
|
|
419
403
|
"""
|
|
420
404
|
opts = options or WorkerFailOptions()
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
value = serialize_worker_fail_value(action_name, task_id, error_message, opts.retry)
|
|
424
|
-
|
|
425
|
-
await self._client._send_and_check(
|
|
405
|
+
response = await self._client._send_and_check(
|
|
426
406
|
OpCode.ACTION_FAIL,
|
|
427
|
-
namespace,
|
|
428
|
-
|
|
429
|
-
|
|
407
|
+
task.namespace,
|
|
408
|
+
task.run.encode("utf-8"),
|
|
409
|
+
encode_fail_body(
|
|
410
|
+
worker_id, task, error.encode("utf-8"), opts.retry, opts.abandoned, opts.delay_ms
|
|
411
|
+
),
|
|
430
412
|
)
|
|
413
|
+
decode_empty_answer("action.fail", response.data)
|
|
431
414
|
|
|
432
415
|
async def list(
|
|
433
416
|
self,
|
flo/client.py
CHANGED
|
@@ -19,6 +19,7 @@ from .exceptions import (
|
|
|
19
19
|
UnexpectedEofError,
|
|
20
20
|
raise_for_status,
|
|
21
21
|
)
|
|
22
|
+
from .kv_body import await_block_ms, dequeue_block_ms
|
|
22
23
|
from .retry import DEFAULT_RETRY_POLICY, RetryPolicy, _policy_backoff, _refused_unrun
|
|
23
24
|
from .types import _DEFAULT_WORKER_BLOCK_MS, HEADER_SIZE, OpCode, OptionTag, StatusCode
|
|
24
25
|
from .wire import (
|
|
@@ -34,7 +35,30 @@ if TYPE_CHECKING:
|
|
|
34
35
|
|
|
35
36
|
logger = logging.getLogger("flo")
|
|
36
37
|
|
|
37
|
-
|
|
38
|
+
|
|
39
|
+
async def _open_within(
|
|
40
|
+
host: str, port: int, timeout: float
|
|
41
|
+
) -> tuple[asyncio.StreamReader, asyncio.StreamWriter]:
|
|
42
|
+
"""open_connection, given up after timeout seconds (asyncio.TimeoutError).
|
|
43
|
+
|
|
44
|
+
Not wait_for: before Python 3.12, a cancel that lands as the connect
|
|
45
|
+
finishes, as a refused one does at once, is dropped and the connect's
|
|
46
|
+
result or error returned instead. A reconnect loop would then retry on
|
|
47
|
+
past the deadline of whoever cancelled it.
|
|
48
|
+
"""
|
|
49
|
+
connect = asyncio.ensure_future(asyncio.open_connection(host, port))
|
|
50
|
+
try:
|
|
51
|
+
done, _ = await asyncio.wait({connect}, timeout=timeout)
|
|
52
|
+
except asyncio.CancelledError:
|
|
53
|
+
connect.cancel()
|
|
54
|
+
# Connected just as the cancel landed: don't leak the connection.
|
|
55
|
+
if connect.done() and not connect.cancelled() and connect.exception() is None:
|
|
56
|
+
connect.result()[1].close()
|
|
57
|
+
raise
|
|
58
|
+
if not done:
|
|
59
|
+
connect.cancel()
|
|
60
|
+
raise asyncio.TimeoutError
|
|
61
|
+
return connect.result()
|
|
38
62
|
|
|
39
63
|
|
|
40
64
|
class FloClient:
|
|
@@ -165,10 +189,7 @@ class FloClient:
|
|
|
165
189
|
ConnectionFailedError: If connection fails.
|
|
166
190
|
"""
|
|
167
191
|
try:
|
|
168
|
-
self._reader, self._writer = await
|
|
169
|
-
asyncio.open_connection(self._host, self._port),
|
|
170
|
-
timeout=self._timeout,
|
|
171
|
-
)
|
|
192
|
+
self._reader, self._writer = await _open_within(self._host, self._port, self._timeout)
|
|
172
193
|
if self._debug:
|
|
173
194
|
logger.debug(f"[flo] Connected to {self._endpoint}")
|
|
174
195
|
return self
|
|
@@ -228,10 +249,7 @@ class FloClient:
|
|
|
228
249
|
while True:
|
|
229
250
|
attempt += 1
|
|
230
251
|
try:
|
|
231
|
-
reader, writer = await
|
|
232
|
-
asyncio.open_connection(self._host, self._port),
|
|
233
|
-
timeout=self._timeout,
|
|
234
|
-
)
|
|
252
|
+
reader, writer = await _open_within(self._host, self._port, self._timeout)
|
|
235
253
|
# A concurrent reconnect may have installed its connection
|
|
236
254
|
# first; close it rather than leak it.
|
|
237
255
|
replaced = self._writer
|
|
@@ -283,15 +301,14 @@ class FloClient:
|
|
|
283
301
|
if len(value) >= 9 and value[0] == 1:
|
|
284
302
|
return int.from_bytes(value[1:9], "little")
|
|
285
303
|
return 0
|
|
286
|
-
waits = {
|
|
287
|
-
opt.tag: opt.as_u32() or 0
|
|
288
|
-
for opt in OptionsIterator(options)
|
|
289
|
-
if opt.tag == OptionTag.BLOCK_MS
|
|
290
|
-
}
|
|
291
304
|
if op_code == OpCode.ACTION_AWAIT:
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
305
|
+
return await_block_ms(value)
|
|
306
|
+
if op_code == OpCode.QUEUE_DEQUEUE:
|
|
307
|
+
return dequeue_block_ms(value)
|
|
308
|
+
waits = [
|
|
309
|
+
opt.as_u32() or 0 for opt in OptionsIterator(options) if opt.tag == OptionTag.BLOCK_MS
|
|
310
|
+
]
|
|
311
|
+
return max(waits, default=0)
|
|
295
312
|
|
|
296
313
|
async def _send_request(
|
|
297
314
|
self,
|
|
@@ -505,9 +522,11 @@ class FloClient:
|
|
|
505
522
|
action_timeout: Timeout for action handlers in seconds.
|
|
506
523
|
block_ms: Long-poll wait per await in milliseconds, at most 300000.
|
|
507
524
|
0 means the 30000 default.
|
|
508
|
-
auto_touch: Touch each
|
|
509
|
-
|
|
510
|
-
past the
|
|
525
|
+
auto_touch: Touch each task every sixth of its lease while its
|
|
526
|
+
handler runs, counted from its arrival. Off by default: a
|
|
527
|
+
handler that runs past the lease without touching is taken
|
|
528
|
+
back and run again. A hold ends 12 h after its delivery
|
|
529
|
+
whatever the touches.
|
|
511
530
|
|
|
512
531
|
Returns:
|
|
513
532
|
A new ActionWorker instance ready to register actions and start.
|
|
@@ -543,14 +562,16 @@ class FloClient:
|
|
|
543
562
|
batch_size: int = 10,
|
|
544
563
|
block_ms: int = _DEFAULT_WORKER_BLOCK_MS,
|
|
545
564
|
message_timeout: float = 300.0,
|
|
546
|
-
|
|
547
|
-
|
|
565
|
+
ack_timeout_ms: int | None = None,
|
|
566
|
+
max_deliver: int | None = None,
|
|
567
|
+
auto_touch: bool = False,
|
|
548
568
|
) -> "StreamWorker":
|
|
549
569
|
"""Create a new StreamWorker from this client.
|
|
550
570
|
|
|
551
|
-
The worker
|
|
552
|
-
|
|
553
|
-
|
|
571
|
+
The worker reads a consumer group on the specified stream and
|
|
572
|
+
dispatches its records to the handler. Each record is acked on
|
|
573
|
+
success and nacked on error; one not acked within the group's lease
|
|
574
|
+
is delivered again.
|
|
554
575
|
|
|
555
576
|
Args:
|
|
556
577
|
stream: Stream name to consume from.
|
|
@@ -559,18 +580,16 @@ class FloClient:
|
|
|
559
580
|
consumer: Consumer ID (auto-generated if not provided).
|
|
560
581
|
worker_id: Unique worker identifier (auto-generated if not provided).
|
|
561
582
|
concurrency: Maximum number of concurrent record handlers.
|
|
562
|
-
batch_size:
|
|
583
|
+
batch_size: Records to read per poll, 1..100.
|
|
563
584
|
block_ms: Long-poll wait per read in milliseconds, at most 300000.
|
|
564
585
|
0 means the 30000 default.
|
|
565
586
|
message_timeout: Timeout for record handlers in seconds.
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
claims an entry. 0 drains everything this consumer owns; > 0
|
|
573
|
-
also steals entries abandoned by dead consumers.
|
|
587
|
+
ack_timeout_ms: The group's lease, 5000..43200000. With it or
|
|
588
|
+
``max_deliver``, the worker creates the group at start, and
|
|
589
|
+
fails if it exists with other settings.
|
|
590
|
+
max_deliver: Nacks or expiries before a record is dead, 1..255.
|
|
591
|
+
auto_touch: Touch each record every sixth of the group's lease
|
|
592
|
+
while its handler runs, so a long handler keeps its record.
|
|
574
593
|
|
|
575
594
|
Returns:
|
|
576
595
|
A new StreamWorker instance ready to start.
|
|
@@ -598,6 +617,7 @@ class FloClient:
|
|
|
598
617
|
batch_size=batch_size,
|
|
599
618
|
block_ms=block_ms,
|
|
600
619
|
message_timeout=message_timeout,
|
|
601
|
-
|
|
602
|
-
|
|
620
|
+
ack_timeout_ms=ack_timeout_ms,
|
|
621
|
+
max_deliver=max_deliver,
|
|
622
|
+
auto_touch=auto_touch,
|
|
603
623
|
)
|