flo-python 0.1.5__py3-none-any.whl → 0.1.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flo/__init__.py +33 -1
- flo/client.py +2 -0
- flo/exceptions.py +26 -7
- flo/kv.py +5 -14
- flo/kv_body.py +2 -1
- flo/processing.py +16 -3
- flo/streams.py +14 -1
- flo/ts.py +465 -0
- flo/types.py +189 -6
- flo/worker.py +68 -16
- {flo_python-0.1.5.dist-info → flo_python-0.1.6.dist-info}/METADATA +141 -3
- flo_python-0.1.6.dist-info/RECORD +20 -0
- flo_python-0.1.5.dist-info/RECORD +0 -19
- {flo_python-0.1.5.dist-info → flo_python-0.1.6.dist-info}/WHEEL +0 -0
- {flo_python-0.1.5.dist-info → flo_python-0.1.6.dist-info}/licenses/LICENSE +0 -0
flo/__init__.py
CHANGED
|
@@ -73,6 +73,7 @@ from .exceptions import (
|
|
|
73
73
|
from .kv_txn import Transaction, TxnFinishedError, TxnUnsupportedOpError
|
|
74
74
|
from .processing import ProcessingOperations
|
|
75
75
|
from .retry import DEFAULT_RETRY_POLICY, RetryPolicy
|
|
76
|
+
from .ts import TSOperations
|
|
76
77
|
from .types import (
|
|
77
78
|
ActionDeleteOptions,
|
|
78
79
|
ActionInfo,
|
|
@@ -94,6 +95,8 @@ from .types import (
|
|
|
94
95
|
DlqPage,
|
|
95
96
|
EnqueueOptions,
|
|
96
97
|
ExistsResult,
|
|
98
|
+
FloQLPoint,
|
|
99
|
+
FloQLSeries,
|
|
97
100
|
GetJsonResult,
|
|
98
101
|
GetOptions,
|
|
99
102
|
GetResult,
|
|
@@ -171,6 +174,16 @@ from .types import (
|
|
|
171
174
|
StreamTrimOptions,
|
|
172
175
|
StreamTrimResult,
|
|
173
176
|
TaskAssignment,
|
|
177
|
+
TSAggregation,
|
|
178
|
+
TSBucket,
|
|
179
|
+
TSInfo,
|
|
180
|
+
TSOrder,
|
|
181
|
+
TSPoint,
|
|
182
|
+
TSQueryOptions,
|
|
183
|
+
TSReadOptions,
|
|
184
|
+
TSReadResult,
|
|
185
|
+
TSWriteOptions,
|
|
186
|
+
TSWriteResult,
|
|
174
187
|
VersionEntry,
|
|
175
188
|
WaitFor,
|
|
176
189
|
WorkerAwaitOptions,
|
|
@@ -202,6 +215,9 @@ from .types import (
|
|
|
202
215
|
WorkflowStatusOptions,
|
|
203
216
|
WorkflowSyncOptions,
|
|
204
217
|
WorkflowSyncResult,
|
|
218
|
+
datetime_from_ns,
|
|
219
|
+
ns_from_datetime,
|
|
220
|
+
ns_from_timedelta,
|
|
205
221
|
)
|
|
206
222
|
from .worker import (
|
|
207
223
|
ActionContext,
|
|
@@ -215,7 +231,7 @@ from .worker import (
|
|
|
215
231
|
)
|
|
216
232
|
from .workflows import WorkflowOperations
|
|
217
233
|
|
|
218
|
-
__version__ = "0.1.
|
|
234
|
+
__version__ = "0.1.6"
|
|
219
235
|
|
|
220
236
|
__all__ = [
|
|
221
237
|
"Transaction",
|
|
@@ -413,5 +429,21 @@ __all__ = [
|
|
|
413
429
|
"ProcessingListResult",
|
|
414
430
|
"ProcessingRemoveOptions",
|
|
415
431
|
"ProcessingRuntime",
|
|
432
|
+
"FloQLPoint",
|
|
433
|
+
"FloQLSeries",
|
|
434
|
+
"TSOrder",
|
|
435
|
+
"TSAggregation",
|
|
436
|
+
"TSBucket",
|
|
437
|
+
"TSInfo",
|
|
438
|
+
"TSQueryOptions",
|
|
439
|
+
"ns_from_timedelta",
|
|
440
|
+
"TSPoint",
|
|
441
|
+
"TSReadOptions",
|
|
442
|
+
"TSReadResult",
|
|
443
|
+
"TSWriteOptions",
|
|
444
|
+
"TSWriteResult",
|
|
445
|
+
"datetime_from_ns",
|
|
446
|
+
"ns_from_datetime",
|
|
447
|
+
"TSOperations",
|
|
416
448
|
"ProcessingSyncResult",
|
|
417
449
|
]
|
flo/client.py
CHANGED
|
@@ -114,6 +114,7 @@ class FloClient:
|
|
|
114
114
|
from .processing import ProcessingOperations
|
|
115
115
|
from .queue import QueueOperations
|
|
116
116
|
from .streams import StreamOperations
|
|
117
|
+
from .ts import TSOperations
|
|
117
118
|
from .workflows import WorkflowOperations
|
|
118
119
|
|
|
119
120
|
self.kv = KVOperations(self)
|
|
@@ -123,6 +124,7 @@ class FloClient:
|
|
|
123
124
|
self.worker = WorkerOperations(self)
|
|
124
125
|
self.workflow = WorkflowOperations(self)
|
|
125
126
|
self.processing = ProcessingOperations(self)
|
|
127
|
+
self.ts = TSOperations(self)
|
|
126
128
|
|
|
127
129
|
if debug:
|
|
128
130
|
logging.basicConfig(level=logging.DEBUG)
|
flo/exceptions.py
CHANGED
|
@@ -183,10 +183,12 @@ class NotFoundError(ServerError):
|
|
|
183
183
|
|
|
184
184
|
|
|
185
185
|
class StaleNotFoundError(NotFoundError):
|
|
186
|
-
"""A
|
|
187
|
-
|
|
188
|
-
that
|
|
189
|
-
|
|
186
|
+
"""A node answered not found from state it knows may be behind (``stale``
|
|
187
|
+
is True): a halted or read-blocked shard, as of the entry it couldn't
|
|
188
|
+
apply, or a follower that couldn't reach the leader. It may exist: read
|
|
189
|
+
again rather than treat it as gone. kv.get and kv.json_get raise it
|
|
190
|
+
rather than report the key absent; stream.read, ts.read and
|
|
191
|
+
ts.iter_points raise it for the stream or series."""
|
|
190
192
|
|
|
191
193
|
stale = True
|
|
192
194
|
|
|
@@ -250,9 +252,10 @@ class GroupHeldFullError(OverloadedError):
|
|
|
250
252
|
names which cap. At the group's cap (records ready, leased and dead
|
|
251
253
|
together), acks and nacks make room, but dead records stay until the
|
|
252
254
|
group is deleted or a trim of the stream cuts them. At the shard's cap
|
|
253
|
-
(its queues and groups together),
|
|
254
|
-
|
|
255
|
-
it; a stream worker logs it and
|
|
255
|
+
(its queues and groups together), this group's own acks free room, as do
|
|
256
|
+
other groups and queues on the shard draining. Reading again won't help
|
|
257
|
+
until then, so the client never retries it; a stream worker logs it and
|
|
258
|
+
backs off."""
|
|
256
259
|
|
|
257
260
|
|
|
258
261
|
class RateLimitedError(ServerError):
|
|
@@ -329,6 +332,22 @@ def is_connection_error(exc: BaseException) -> bool:
|
|
|
329
332
|
return isinstance(exc, (UnexpectedEofError, NotConnectedError, RequestTimeoutError, OSError))
|
|
330
333
|
|
|
331
334
|
|
|
335
|
+
def not_found_error(status: StatusCode, data: bytes, stale: bool) -> ServerError:
|
|
336
|
+
"""The error a not-found answer raises. On a stale answer, a thing not
|
|
337
|
+
found (reason NOT_FOUND) is a StaleNotFoundError: the node hadn't seen it
|
|
338
|
+
yet, so it may exist."""
|
|
339
|
+
try:
|
|
340
|
+
raise_for_status(status, data)
|
|
341
|
+
except ServerError as e:
|
|
342
|
+
e.stale = stale
|
|
343
|
+
if stale and e.reason == Reason.NOT_FOUND:
|
|
344
|
+
found = StaleNotFoundError(str(e))
|
|
345
|
+
found.reason, found.ran = e.reason, e.ran
|
|
346
|
+
return found
|
|
347
|
+
return e
|
|
348
|
+
raise AssertionError("a not-found answer always raises") # pragma: no cover
|
|
349
|
+
|
|
350
|
+
|
|
332
351
|
def raise_for_status(status: StatusCode, data: bytes = b"") -> None:
|
|
333
352
|
"""Raise appropriate exception for non-OK status codes.
|
|
334
353
|
|
flo/kv.py
CHANGED
|
@@ -8,9 +8,8 @@ from typing import TYPE_CHECKING, Any
|
|
|
8
8
|
|
|
9
9
|
from .exceptions import (
|
|
10
10
|
IncompleteResponseError,
|
|
11
|
-
ServerError,
|
|
12
11
|
StaleNotFoundError,
|
|
13
|
-
|
|
12
|
+
not_found_error,
|
|
14
13
|
)
|
|
15
14
|
from .kv_body import (
|
|
16
15
|
decode_delete_answer,
|
|
@@ -79,18 +78,10 @@ def _absent(response: RawResponse, *, stale_raises: bool = False) -> bool:
|
|
|
79
78
|
absent."""
|
|
80
79
|
if not response.is_not_found():
|
|
81
80
|
return False
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
if e.reason != Reason.NOT_FOUND:
|
|
87
|
-
raise
|
|
88
|
-
if stale_raises and response.stale:
|
|
89
|
-
stale = StaleNotFoundError(str(e))
|
|
90
|
-
stale.reason, stale.ran = e.reason, e.ran
|
|
91
|
-
raise stale from None
|
|
92
|
-
return True
|
|
93
|
-
return False # pragma: no cover - raise_for_status always raises here
|
|
81
|
+
e = not_found_error(response.status, response.data, response.stale)
|
|
82
|
+
if e.reason != Reason.NOT_FOUND or (stale_raises and isinstance(e, StaleNotFoundError)):
|
|
83
|
+
raise e
|
|
84
|
+
return True
|
|
94
85
|
|
|
95
86
|
|
|
96
87
|
async def kv_get(
|
flo/kv_body.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""The ops on the server's op table: KV, the action reports and reads,
|
|
2
2
|
worker.list and worker.info, stream group reads, the queue's dequeue,
|
|
3
|
-
reports and peek, and processing.remove.
|
|
3
|
+
reports and peek, ts.read and ts.info, and processing.remove.
|
|
4
4
|
|
|
5
5
|
The request is ``[namespace][key][body]``, the body coded by the op's
|
|
6
6
|
schema, field by field in order. An optional field is a presence byte then
|
|
@@ -123,6 +123,7 @@ _TABLE_OPS = frozenset(
|
|
|
123
123
|
OpCode.QUEUE_COMPLETE,
|
|
124
124
|
OpCode.QUEUE_FAIL,
|
|
125
125
|
OpCode.QUEUE_TOUCH,
|
|
126
|
+
OpCode.TS_READ,
|
|
126
127
|
OpCode.TS_INFO,
|
|
127
128
|
OpCode.PROCESSING_REMOVE,
|
|
128
129
|
}
|
flo/processing.py
CHANGED
|
@@ -34,6 +34,7 @@ if TYPE_CHECKING:
|
|
|
34
34
|
_PROCESSING_STATUS_NAMES = ["running", "stopped", "cancelled", "failed", "completed"]
|
|
35
35
|
# The most jobs one list page holds.
|
|
36
36
|
_MAX_PROCESSING_LIST = 1024
|
|
37
|
+
_NO_LATE_EDGE = (1 << 64) - 1
|
|
37
38
|
|
|
38
39
|
|
|
39
40
|
class ProcessingOperations:
|
|
@@ -211,6 +212,7 @@ def _parse_processing_status(data: bytes) -> ProcessingStatusResult:
|
|
|
211
212
|
[reason_len:u16][reason]
|
|
212
213
|
[running:u8][records_in:u64][records_out:u64][watermark_ms:i64]
|
|
213
214
|
[tick_avg_us:u64][tick_max_us:u64][uptime_ms:u64]
|
|
215
|
+
[late_edge_ms:u64][late_points:u64]
|
|
214
216
|
[yaml_len:u32][yaml]
|
|
215
217
|
The reason is a failed job's, why a running one is held here, or empty.
|
|
216
218
|
"""
|
|
@@ -246,9 +248,18 @@ def _parse_processing_status(data: bytes) -> ProcessingStatusResult:
|
|
|
246
248
|
running = take(1)[0]
|
|
247
249
|
if running > 1:
|
|
248
250
|
raise IncompleteResponseError(f"processing.status answer has a running of {running}")
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
251
|
+
(
|
|
252
|
+
records_in,
|
|
253
|
+
records_out,
|
|
254
|
+
watermark_ms,
|
|
255
|
+
tick_avg_us,
|
|
256
|
+
tick_max_us,
|
|
257
|
+
uptime_ms,
|
|
258
|
+
late_edge_ms,
|
|
259
|
+
late_points,
|
|
260
|
+
) = struct.unpack("<QQqQQQQQ", take(64))
|
|
261
|
+
# maxInt(u64) for the edge: no TS source here, so no count.
|
|
262
|
+
counted = late_edge_ms != _NO_LATE_EDGE
|
|
252
263
|
(yaml_len,) = struct.unpack("<I", take(4))
|
|
253
264
|
yaml = take(yaml_len)
|
|
254
265
|
if pos != len(data):
|
|
@@ -272,6 +283,8 @@ def _parse_processing_status(data: bytes) -> ProcessingStatusResult:
|
|
|
272
283
|
tick_avg_us=tick_avg_us,
|
|
273
284
|
tick_max_us=tick_max_us,
|
|
274
285
|
uptime_ms=uptime_ms,
|
|
286
|
+
late_edge_ms=late_edge_ms if counted else None,
|
|
287
|
+
late_points=late_points if counted else None,
|
|
275
288
|
),
|
|
276
289
|
yaml=yaml,
|
|
277
290
|
)
|
flo/streams.py
CHANGED
|
@@ -7,7 +7,7 @@ Uses StreamID-native positioning (timestamp_ms + sequence).
|
|
|
7
7
|
import struct
|
|
8
8
|
from typing import TYPE_CHECKING
|
|
9
9
|
|
|
10
|
-
from .exceptions import IncompleteResponseError, ValidationError
|
|
10
|
+
from .exceptions import IncompleteResponseError, ValidationError, not_found_error
|
|
11
11
|
from .kv_body import (
|
|
12
12
|
decode_group_list_answer,
|
|
13
13
|
decode_group_members_answer,
|
|
@@ -117,6 +117,16 @@ class StreamOperations:
|
|
|
117
117
|
Returns:
|
|
118
118
|
StreamReadResult with list of records.
|
|
119
119
|
|
|
120
|
+
Raises:
|
|
121
|
+
NotFoundError: The stream doesn't exist (never created or
|
|
122
|
+
appended to, or deleted); one that exists with nothing to
|
|
123
|
+
read is empty. A blocking read (block_ms > 0) of a stream not
|
|
124
|
+
created yet waits for its first append instead. A consumer
|
|
125
|
+
that may start before its stream exists catches it and starts
|
|
126
|
+
again.
|
|
127
|
+
StaleNotFoundError: A not found from a node that may be behind:
|
|
128
|
+
the stream may exist. Read again rather than treat it as gone.
|
|
129
|
+
|
|
120
130
|
Example:
|
|
121
131
|
# Read from beginning
|
|
122
132
|
result = await client.stream.read("events")
|
|
@@ -169,7 +179,10 @@ class StreamOperations:
|
|
|
169
179
|
stream.encode("utf-8"),
|
|
170
180
|
b"",
|
|
171
181
|
builder.build(),
|
|
182
|
+
allow_not_found=True,
|
|
172
183
|
)
|
|
184
|
+
if response.is_not_found():
|
|
185
|
+
raise not_found_error(response.status, response.data, response.stale)
|
|
173
186
|
|
|
174
187
|
return parse_stream_read_response(response.data)
|
|
175
188
|
|