kernable-sdk 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kernable/__init__.py +3923 -0
- kernable/__main__.py +7 -0
- kernable/cli.py +106 -0
- kernable/messages.py +199 -0
- kernable/processes.py +349 -0
- kernable/signing.py +221 -0
- kernable/store.py +149 -0
- kernable/v1/kernable_pb2.py +222 -0
- kernable/v1/kernable_pb2_grpc.py +1627 -0
- kernable_sdk-0.1.0.dist-info/METADATA +82 -0
- kernable_sdk-0.1.0.dist-info/RECORD +15 -0
- kernable_sdk-0.1.0.dist-info/WHEEL +5 -0
- kernable_sdk-0.1.0.dist-info/entry_points.txt +2 -0
- kernable_sdk-0.1.0.dist-info/licenses/LICENSE +21 -0
- kernable_sdk-0.1.0.dist-info/top_level.txt +1 -0
kernable/__init__.py
ADDED
|
@@ -0,0 +1,3923 @@
|
|
|
1
|
+
"""kernable SDK for Python: write event-sourced business logic in Python; the kernel does the rest.
|
|
2
|
+
|
|
3
|
+
The API mirrors the TypeScript SDK:
|
|
4
|
+
|
|
5
|
+
account_aggregate = domain("identity").aggregate("account", {"status": "none"})
|
|
6
|
+
|
|
7
|
+
@account_aggregate.apply("AccountRegistered")
|
|
8
|
+
def _(state, data):
|
|
9
|
+
return {**state, "status": "active", "email": data["email"]}
|
|
10
|
+
|
|
11
|
+
@account_aggregate.command("RegisterAccount")
|
|
12
|
+
def _(state, data):
|
|
13
|
+
if state["status"] != "none":
|
|
14
|
+
raise Rejected("Account already exists")
|
|
15
|
+
return [{"type": "AccountRegistered", "data": {"email": data["email"]}}]
|
|
16
|
+
|
|
17
|
+
k = Kernable(url, key_path)
|
|
18
|
+
k.worker(account_aggregate).run()
|
|
19
|
+
|
|
20
|
+
Listeners are defined on a domain too and run through the connection:
|
|
21
|
+
|
|
22
|
+
accounts_readmodel = domain("identity").read_model("accounts")
|
|
23
|
+
k.listener(accounts_readmodel).run()
|
|
24
|
+
|
|
25
|
+
Stubs: generate them first (`make py-stubs`), see sdks/python/README.md.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
import concurrent.futures
|
|
29
|
+
import contextvars
|
|
30
|
+
import copy
|
|
31
|
+
import dataclasses
|
|
32
|
+
import hashlib
|
|
33
|
+
import inspect
|
|
34
|
+
import json
|
|
35
|
+
import os
|
|
36
|
+
import queue
|
|
37
|
+
import random
|
|
38
|
+
import re
|
|
39
|
+
import sqlite3
|
|
40
|
+
import sys
|
|
41
|
+
import threading
|
|
42
|
+
import time
|
|
43
|
+
import traceback
|
|
44
|
+
import uuid
|
|
45
|
+
from dataclasses import dataclass, field
|
|
46
|
+
from datetime import datetime, timedelta, timezone
|
|
47
|
+
from typing import ClassVar, Any, Callable, Iterable
|
|
48
|
+
|
|
49
|
+
import grpc
|
|
50
|
+
|
|
51
|
+
from kernable.signing import (Key, cache_message, cancel_message, challenge_message, command_message, data_message, derived_id,
|
|
52
|
+
index_message, membership_message, process_act_message, process_message, query_message, vault_ref, rejection_message,
|
|
53
|
+
result_message, snapshot_message, subscribe_message)
|
|
54
|
+
from kernable.store import Store
|
|
55
|
+
from kernable.v1 import kernable_pb2 as pb
|
|
56
|
+
from kernable.v1 import kernable_pb2_grpc as rpc
|
|
57
|
+
|
|
58
|
+
from .messages import ( # noqa: E402
|
|
59
|
+
Command, Event, Query, bind as _bind, binding as _binding, declare_command as _declare_command,
|
|
60
|
+
declare_event as _declare_event, from_wire as _from_wire, is_message_class as _is_message_class,
|
|
61
|
+
query_target as _query_target, to_wire as _to_wire,
|
|
62
|
+
)
|
|
63
|
+
from .processes import Process, Value, value # noqa: E402
|
|
64
|
+
|
|
65
|
+
__all__ = ["PROTOCOL", "waits_for_switch", "Event", "Command", "Query", "StoredEvent", "Aggregate", "AggregateType", "MiddlewareContext", "App", "ApplyContext", "CacheClient", "CancelOutcome", "Check", "ClientClosed", "CommandContext", "CommandMeta",
|
|
66
|
+
"CommandSchema", "QuerySchema", "Target",
|
|
67
|
+
"Data", "DataClient", "Domain", "Listed", "Listing",
|
|
68
|
+
"EventMeta", "EventSchema", "HeldKey", "Index", "Kernable", "Key", "Listener", "ListenerContext", "ListenerHandle", "Manifest", "NotDue", "NotSettled", "Outcome",
|
|
69
|
+
"Process", "QueryContext", "QueryMeta", "QueryOutcome", "RegisteredProcess", "Rejected", "RequestContext", "Requirement", "Store", "StoredEventMeta", "Subscription", "Tls", "Worker",
|
|
70
|
+
"Value", "domain", "reindex_type", "value"]
|
|
71
|
+
|
|
72
|
+
#: The key of a channel in `Kernable._seen`: the builtin `id`, which a parameter named `id` (a
|
|
73
|
+
#: command ID) would hide inside the method that keeps it.
|
|
74
|
+
_channel_id = id
|
|
75
|
+
|
|
76
|
+
#: Time zero (UTC): the default of times not known.
|
|
77
|
+
_EPOCH = datetime.fromtimestamp(0, timezone.utc)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class Rejected(Exception):
|
|
81
|
+
"""A business reason to refuse a command or query: raise it from a command handler, a query
|
|
82
|
+
handler or a middleware, e.g. `raise Rejected("Cart is checked out", "checked_out")`.
|
|
83
|
+
|
|
84
|
+
`code` (default `rejected`) is what callers act on, the message (`str(err)`) what they show.
|
|
85
|
+
`Kernable.query` raises it when the read model or a middleware refuses the query; a rejected
|
|
86
|
+
command does not raise: `Kernable.submit` returns an `Outcome` with `committed=False`. Any
|
|
87
|
+
other exception from a command or query handler rejects too, with the code `handler_error`
|
|
88
|
+
(`middleware_error` from a middleware)."""
|
|
89
|
+
|
|
90
|
+
def __init__(self, message: str, code: str = "rejected"):
|
|
91
|
+
super().__init__(message)
|
|
92
|
+
self.code = code
|
|
93
|
+
#: For a rejected follow-up command (`ListenerContext.submit`): the key the kernel named
|
|
94
|
+
#: (`unique_taken`).
|
|
95
|
+
self.key = ""
|
|
96
|
+
#: For a rejected follow-up command: the requirement the kernel named (`requirement_missing`).
|
|
97
|
+
self.requirement: "Requirement | None" = None
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass
|
|
101
|
+
class RequestContext:
|
|
102
|
+
"""Who sends a request and in which context: signed with the command and stored with every
|
|
103
|
+
event it causes. `attributes` are business data only, never security decisions.
|
|
104
|
+
|
|
105
|
+
Set a default for a client with `Kernable(context=...)`, or one per request with `context=`
|
|
106
|
+
of `submit`, `enqueue` and `query`. Inside a listener's handler, submit follow-up commands
|
|
107
|
+
with `ctx.follow()`. Example: `RequestContext(sender_identity="customer-7",
|
|
108
|
+
expected_version=3)`."""
|
|
109
|
+
|
|
110
|
+
#: The tenant the sender belongs to.
|
|
111
|
+
sender_tenant: str = ""
|
|
112
|
+
#: Who sends the request, e.g. a user or service ID; middlewares check permissions on it.
|
|
113
|
+
sender_identity: str = ""
|
|
114
|
+
sender_account: str = ""
|
|
115
|
+
sender_session: str = ""
|
|
116
|
+
#: Authorize as this identity of the target tenant; the sender stays the real principal.
|
|
117
|
+
on_behalf_of: str = ""
|
|
118
|
+
#: Reject the command (`version_mismatch`) unless the aggregate is at this version.
|
|
119
|
+
expected_version: int | None = None
|
|
120
|
+
#: Shared by everything one original request causes; defaults to the command ID.
|
|
121
|
+
correlation_id: str = ""
|
|
122
|
+
#: The command that caused this one.
|
|
123
|
+
causation_id: str = ""
|
|
124
|
+
trace_id: str = ""
|
|
125
|
+
#: Privileged: only keys granted `privileged` for it may set it.
|
|
126
|
+
bypass_authorization: bool = False
|
|
127
|
+
attributes: dict[str, str] = field(default_factory=dict)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@dataclass
|
|
131
|
+
class CommandMeta:
|
|
132
|
+
"""What a command handler gets about the command besides state and data (as an optional third
|
|
133
|
+
parameter)."""
|
|
134
|
+
|
|
135
|
+
#: The command's ID: given as `id` to `submit`, else a new UUID.
|
|
136
|
+
command_id: str
|
|
137
|
+
#: The command type within the domain.
|
|
138
|
+
type: str
|
|
139
|
+
domain: str = ""
|
|
140
|
+
aggregate_type: str = ""
|
|
141
|
+
aggregate_id: str = ""
|
|
142
|
+
tenant: str = ""
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@dataclass
|
|
146
|
+
class CommandContext:
|
|
147
|
+
"""What a command handler gets besides state, data and the command (as an optional fourth
|
|
148
|
+
parameter)."""
|
|
149
|
+
|
|
150
|
+
#: Who sends the command: check permissions here.
|
|
151
|
+
request: RequestContext
|
|
152
|
+
#: When the kernel hands the command to another worker if this one has not answered.
|
|
153
|
+
deadline: datetime
|
|
154
|
+
#: When the kernel received the command (UTC); `now()` returns it.
|
|
155
|
+
received_at: datetime
|
|
156
|
+
#: The command whose IDs `id()` derives.
|
|
157
|
+
_command_id: str = field(default="", repr=False)
|
|
158
|
+
_ids: int = field(default=0, repr=False)
|
|
159
|
+
|
|
160
|
+
@classmethod
|
|
161
|
+
def new(cls, command_id: str, request: RequestContext | None = None,
|
|
162
|
+
received_at: datetime = _EPOCH) -> "CommandContext":
|
|
163
|
+
"""A context for calling a command handler directly, e.g. in a unit test with
|
|
164
|
+
`Aggregate.handle`: `id()` derives from `command_id` as in a real command, `now()` returns
|
|
165
|
+
`received_at` (default: the epoch), and its deadline is 30 s after that."""
|
|
166
|
+
return cls(request or RequestContext(), received_at + timedelta(seconds=30), received_at, command_id)
|
|
167
|
+
|
|
168
|
+
def now(self) -> datetime:
|
|
169
|
+
"""When the kernel received the command: the same every time the command runs.
|
|
170
|
+
Use it instead of `datetime.now()`."""
|
|
171
|
+
return self.received_at
|
|
172
|
+
|
|
173
|
+
def id(self) -> str:
|
|
174
|
+
"""A new ID that is the same every time the command runs (1st call, 2nd call, ...).
|
|
175
|
+
Use it instead of random IDs."""
|
|
176
|
+
n, self._ids = self._ids, self._ids + 1
|
|
177
|
+
return derived_id(self._command_id, n)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
@dataclass
|
|
181
|
+
class EventMeta:
|
|
182
|
+
"""What `apply` gets besides state and data (as an optional third parameter): the event
|
|
183
|
+
without its payload. Only fields known before the kernel stores the event, so a snapshot and
|
|
184
|
+
a replay compute the same state; put a time into the data (`ctx.now()`) if the state needs one."""
|
|
185
|
+
|
|
186
|
+
type: str
|
|
187
|
+
#: The aggregate's version after this event: 1 for its first event.
|
|
188
|
+
version: int = 0
|
|
189
|
+
tenant: str = ""
|
|
190
|
+
domain: str = ""
|
|
191
|
+
aggregate_type: str = ""
|
|
192
|
+
aggregate_id: str = ""
|
|
193
|
+
command_id: str = ""
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
@dataclass
|
|
197
|
+
class ApplyContext:
|
|
198
|
+
"""What `apply` gets besides state, data and the event (as an optional fourth parameter).
|
|
199
|
+
No clock and no IDs: `apply` must compute the same state on every replay."""
|
|
200
|
+
|
|
201
|
+
#: The request context of the command that caused the event.
|
|
202
|
+
request: RequestContext = field(default_factory=RequestContext)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
@dataclass
|
|
206
|
+
class QueryMeta:
|
|
207
|
+
"""What a query handler gets about the query besides its data (as an optional second
|
|
208
|
+
parameter)."""
|
|
209
|
+
|
|
210
|
+
query_id: str
|
|
211
|
+
#: The query type within the domain.
|
|
212
|
+
type: str
|
|
213
|
+
domain: str = ""
|
|
214
|
+
tenant: str = ""
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
@dataclass
|
|
218
|
+
class QueryContext:
|
|
219
|
+
"""What a query handler gets besides the query's data and meta (as an optional third
|
|
220
|
+
parameter)."""
|
|
221
|
+
|
|
222
|
+
#: Who asks: check permissions here.
|
|
223
|
+
request: RequestContext
|
|
224
|
+
#: The listener's store, if it has one (`store=` of `Domain.read_model` and `Domain.reactor`).
|
|
225
|
+
store: "Store | None" = None
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
@dataclass
|
|
229
|
+
class MiddlewareContext:
|
|
230
|
+
"""What a middleware handler gets besides the check (an optional third parameter)."""
|
|
231
|
+
|
|
232
|
+
#: The listener's store, if it has one.
|
|
233
|
+
store: "Store | None" = None
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
@dataclass(frozen=True)
|
|
237
|
+
class Target:
|
|
238
|
+
"""The aggregate a request is about, as `Check.target` holds it."""
|
|
239
|
+
|
|
240
|
+
#: Qualified: `shop.order`.
|
|
241
|
+
type: str
|
|
242
|
+
id: str
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
@dataclass
|
|
246
|
+
class Check:
|
|
247
|
+
"""A command, query, file or cache request a middleware checks, before the kernel accepts,
|
|
248
|
+
routes or runs it (or, for commands and queries, after): the first parameter of a
|
|
249
|
+
`Listener.middleware` handler."""
|
|
250
|
+
|
|
251
|
+
#: `"command"`, `"query"`, `"data"` (a file request of `k.data`), `"cache"` (a cache
|
|
252
|
+
#: request of `k.cache`) or `"cancel"` (`Kernable.cancel` of a scheduled command: `id`,
|
|
253
|
+
#: `domain`, `type`, the aggregate, `target` and `data` are the scheduled command's,
|
|
254
|
+
#: `request` the cancel's sender; decide it as the command).
|
|
255
|
+
kind: str
|
|
256
|
+
#: The command or query ID; empty for a file or cache request.
|
|
257
|
+
id: str
|
|
258
|
+
tenant: str
|
|
259
|
+
domain: str
|
|
260
|
+
#: The type within the domain: the command or query type, or for a file request `PutData`,
|
|
261
|
+
#: `GetData`, `DeleteData` or `ListData`, for a cache request `SetCache`, `GetCache` or
|
|
262
|
+
#: `DeleteCache`.
|
|
263
|
+
type: str
|
|
264
|
+
#: The request's payload; vault fields are `None` unless the key is granted `vault_read`.
|
|
265
|
+
#: Always `None` for a file or cache request: the middleware decides on who asks for which
|
|
266
|
+
#: key, never sees the file or value.
|
|
267
|
+
data: Any
|
|
268
|
+
#: Who sends it: check permissions here.
|
|
269
|
+
request: RequestContext
|
|
270
|
+
#: Commands only; empty otherwise.
|
|
271
|
+
aggregate_type: str = ""
|
|
272
|
+
#: Commands only; empty otherwise.
|
|
273
|
+
aggregate_id: str = ""
|
|
274
|
+
#: File and cache requests only: the file or cache key; empty otherwise.
|
|
275
|
+
key: str = ""
|
|
276
|
+
#: The aggregate the request is about: a command's aggregate (`Target("shop.order", "o-17")`),
|
|
277
|
+
#: or the one a query declares (`Query.target`, `QuerySchema.target`). None for queries that
|
|
278
|
+
#: declare none, and for file and cache requests.
|
|
279
|
+
target: "Target | None" = None
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
@dataclass
|
|
283
|
+
class StoredEvent:
|
|
284
|
+
"""An event as stored by the kernel, as `Outcome.events` holds it."""
|
|
285
|
+
|
|
286
|
+
#: Global position of the event in the kernel's log.
|
|
287
|
+
position: int
|
|
288
|
+
tenant: str
|
|
289
|
+
#: Always the domain of the command that caused the event.
|
|
290
|
+
domain: str
|
|
291
|
+
aggregate_type: str
|
|
292
|
+
aggregate_id: str
|
|
293
|
+
#: The aggregate's version after this event: 1 for its first event.
|
|
294
|
+
version: int
|
|
295
|
+
#: The event type within the domain, e.g. `ItemAdded`.
|
|
296
|
+
type: str
|
|
297
|
+
#: The payload.
|
|
298
|
+
data: Any
|
|
299
|
+
#: The command that caused the event.
|
|
300
|
+
command_id: str
|
|
301
|
+
#: The request context of the command that caused the event.
|
|
302
|
+
request: RequestContext = field(default_factory=RequestContext)
|
|
303
|
+
#: Version of the payload format, as stored (see `Domain.event`).
|
|
304
|
+
schema: int = 1
|
|
305
|
+
#: Fields kept in the vault whose value is not here (None in `data`): erased, or not readable
|
|
306
|
+
#: with this key (granted `vault_read`).
|
|
307
|
+
redacted: list[str] = field(default_factory=list)
|
|
308
|
+
#: When the kernel committed the event (UTC).
|
|
309
|
+
created_at: datetime = _EPOCH
|
|
310
|
+
#: The code version (deployment) of the worker that wrote it; empty if none.
|
|
311
|
+
deployment: str = ""
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
@dataclass
|
|
315
|
+
class StoredEventMeta(EventMeta):
|
|
316
|
+
"""What a listener handler gets about the event besides its data (as an optional second
|
|
317
|
+
parameter): the stored event without its payload and request context. It has every field
|
|
318
|
+
of `EventMeta` (`type`, `version`, `tenant`, `domain`, `aggregate_type`, `aggregate_id`,
|
|
319
|
+
`command_id`), and these. Unlike `EventMeta`, it says when the event was committed: a read
|
|
320
|
+
model may show that time, `apply` may not depend on it."""
|
|
321
|
+
|
|
322
|
+
#: Global position of the event in the kernel's log.
|
|
323
|
+
position: int = 0
|
|
324
|
+
#: Version of the payload format: the version the listener knows (see `Domain.event`), or the
|
|
325
|
+
#: stored one for a `raw` handler.
|
|
326
|
+
schema: int = 1
|
|
327
|
+
#: Fields kept in the vault whose value is not in the data (None there): erased, or not
|
|
328
|
+
#: readable with this key (granted `vault_read`).
|
|
329
|
+
redacted: list[str] = field(default_factory=list)
|
|
330
|
+
#: When the kernel committed the event (UTC).
|
|
331
|
+
created_at: datetime = _EPOCH
|
|
332
|
+
#: The code version (deployment) of the worker that wrote it; empty if none.
|
|
333
|
+
deployment: str = ""
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
@dataclass
|
|
337
|
+
class ListenerContext:
|
|
338
|
+
"""What a listener handler gets besides the event's data and meta (as an optional third
|
|
339
|
+
parameter)."""
|
|
340
|
+
|
|
341
|
+
#: The event was committed before this connection began: a read model rebuilding from the
|
|
342
|
+
#: start, and equally the backlog a reactor catches up on after a restart or a lost
|
|
343
|
+
#: connection. A reactor must not skip side effects because of it (those events are new to
|
|
344
|
+
#: it); submitting with fixed command IDs makes them safe to repeat instead. Use it to skip
|
|
345
|
+
#: work that only matters live, e.g. a notification about something long past, in a listener
|
|
346
|
+
#: that starts at 0.
|
|
347
|
+
replaying: bool
|
|
348
|
+
#: The request context of the command that caused the event.
|
|
349
|
+
request: RequestContext = field(default_factory=RequestContext)
|
|
350
|
+
#: The listener's store, if it has one (`store=` of `Domain.read_model` and `Domain.reactor`).
|
|
351
|
+
store: "Store | None" = None
|
|
352
|
+
#: The command that caused the event, for `follow()`.
|
|
353
|
+
_command_id: str = field(default="", repr=False)
|
|
354
|
+
#: The event's tenant: the default for follow-up commands and queries.
|
|
355
|
+
_tenant: str = field(default="", repr=False)
|
|
356
|
+
#: The listener's domain: the default for follow-up commands and queries.
|
|
357
|
+
_domain: str = field(default="", repr=False)
|
|
358
|
+
#: The listener's client, for follow-up commands and queries.
|
|
359
|
+
_kernable: "Kernable | None" = field(default=None, repr=False)
|
|
360
|
+
|
|
361
|
+
def submit(self, aggregate_type, aggregate_id: str | None = None, type: str | None = None, data: Any = None, *, id: str,
|
|
362
|
+
domain: str | None = None, tenant: str | None = None, context: "RequestContext | None" = None,
|
|
363
|
+
wait: str = "settled", timeout_s: float | None = None) -> "list":
|
|
364
|
+
"""Submits a command this event causes, on the listener's client, and returns its events.
|
|
365
|
+
It needs a fixed `id` (`f"confirm-{evt.aggregate_id}"`), so a redelivered event runs it
|
|
366
|
+
once. `domain` defaults to the listener's, `context` to `follow()`, `tenant` to the
|
|
367
|
+
event's; the rest as for `Kernable.submit`.
|
|
368
|
+
|
|
369
|
+
Raises `Rejected` (with the kernel's `key` or `requirement`, if any) if the command is
|
|
370
|
+
rejected: a reactor then parks the event at once (see `park_after` of
|
|
371
|
+
`Domain.reactor`); catch it to accept a rejection. Raises like `Kernable.submit` otherwise.
|
|
372
|
+
Example: `ctx.submit("order", evt.aggregate_id, "Confirm", id=f"confirm-{evt.aggregate_id}")`."""
|
|
373
|
+
if not id:
|
|
374
|
+
raise ValueError(f"{type or aggregate_type}: a follow-up command needs a fixed id, so a redelivered event runs it once")
|
|
375
|
+
assert self._kernable is not None, "only a running listener's handlers submit follow-ups"
|
|
376
|
+
if isinstance(aggregate_type, Command):
|
|
377
|
+
domain, aggregate_type, type, data = self._kernable._command_of(aggregate_type)
|
|
378
|
+
outcome = self._kernable.submit(domain or self._domain, aggregate_type, aggregate_id, type, data,
|
|
379
|
+
tenant=tenant or self._tenant, id=id, wait=wait,
|
|
380
|
+
context=context or self.follow(), timeout_s=timeout_s)
|
|
381
|
+
if outcome.committed:
|
|
382
|
+
return outcome.events
|
|
383
|
+
rejected = Rejected(outcome.message, outcome.code)
|
|
384
|
+
rejected.key = outcome.key
|
|
385
|
+
rejected.requirement = outcome.requirement
|
|
386
|
+
raise rejected
|
|
387
|
+
|
|
388
|
+
def query(self, type, data: Any = None, *, tenant: str | None = None,
|
|
389
|
+
context: "RequestContext | None" = None, min_position: int = 0,
|
|
390
|
+
timeout_s: float | None = None, all_partitions: bool = False) -> Any:
|
|
391
|
+
"""Asks a query on the listener's client, like `Kernable.query`: `type` is of the
|
|
392
|
+
listener's domain (`GetOrder`) or qualified (`crm.GetCustomer`). `tenant` defaults to the
|
|
393
|
+
event's. It is the listener's own read: it keeps the client's context, not the event's
|
|
394
|
+
sender."""
|
|
395
|
+
assert self._kernable is not None, "only a running listener's handlers ask queries"
|
|
396
|
+
if isinstance(type, Query):
|
|
397
|
+
return self._kernable.query(type, tenant=tenant or self._tenant, context=context, min_position=min_position,
|
|
398
|
+
timeout_s=timeout_s, all_partitions=all_partitions)
|
|
399
|
+
domain, _, name = type.rpartition(".")
|
|
400
|
+
return self._kernable.query(domain or self._domain, name, data, tenant=tenant or self._tenant,
|
|
401
|
+
context=context, min_position=min_position,
|
|
402
|
+
timeout_s=timeout_s)
|
|
403
|
+
|
|
404
|
+
def process_act(self, name: str, context: RequestContext) -> bool:
|
|
405
|
+
"""Sets as whom the process `name` acts, with the listener's client; see
|
|
406
|
+
`Kernable.process_act`. Returns whether it changed."""
|
|
407
|
+
assert self._kernable is not None, "only a running listener's handlers set as whom a process acts"
|
|
408
|
+
return self._kernable.process_act(name, context)
|
|
409
|
+
|
|
410
|
+
def follow(self) -> RequestContext:
|
|
411
|
+
"""A request context for commands caused by this event: same sender and correlation."""
|
|
412
|
+
r = self.request
|
|
413
|
+
return RequestContext(r.sender_tenant, r.sender_identity, r.sender_account, r.sender_session,
|
|
414
|
+
r.on_behalf_of, None, r.correlation_id or self._command_id,
|
|
415
|
+
self._command_id, r.trace_id, False, dict(r.attributes))
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
@dataclass
|
|
419
|
+
class Outcome:
|
|
420
|
+
"""What became of a command, returned by `Kernable.submit`: committed with its events, or
|
|
421
|
+
rejected with a code and a message. Example: `Outcome(committed=False, events=[],
|
|
422
|
+
code='rejected', message='Cart is checked out')`."""
|
|
423
|
+
|
|
424
|
+
#: Whether the command's events were stored; `False` if it was rejected.
|
|
425
|
+
committed: bool
|
|
426
|
+
#: The committed events, in order; empty if the command was rejected. Vault fields in them
|
|
427
|
+
#: are `None` (see `StoredEvent.redacted`).
|
|
428
|
+
events: list[StoredEvent] = field(default_factory=list)
|
|
429
|
+
#: The rejection code (see `Rejected`), if rejected: a handler's, or the kernel's, e.g.
|
|
430
|
+
#: `unknown_history` when the aggregate holds an event the deployment of the workers that run
|
|
431
|
+
#: the command does not know (a rollback after a newer deployment wrote it).
|
|
432
|
+
code: str = ""
|
|
433
|
+
#: The rejection message, if rejected.
|
|
434
|
+
message: str = ""
|
|
435
|
+
#: For `unique_taken`: the name of the key another aggregate holds (never its value). Set by
|
|
436
|
+
#: the kernel only, and stored with the rejection.
|
|
437
|
+
key: str = ""
|
|
438
|
+
#: For `requirement_missing`: the first requirement that does not hold. Set by the kernel only.
|
|
439
|
+
requirement: "Requirement | None" = None
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
@dataclass
|
|
443
|
+
class Requirement:
|
|
444
|
+
"""A fact another aggregate of the command's tenant must hold (or, `absent`, must not hold)
|
|
445
|
+
when the command commits, as `requires=` of `Aggregate.command` returns them:
|
|
446
|
+
`Requirement("funds", "vehicle", data["vehicleId"], "open")`. The kernel checks it atomically
|
|
447
|
+
with the commit; if it does not hold, the command is rejected with `requirement_missing`.
|
|
448
|
+
|
|
449
|
+
The aggregate is named by its ID, or by a unique key of its type (`Requirement.by_key`, not a
|
|
450
|
+
personal key), which the kernel resolves at commit: no aggregate holds the key, none holds
|
|
451
|
+
the fact; an aggregate without events holds none either. An empty `domain` is the command's
|
|
452
|
+
own (`Requirement.of`)."""
|
|
453
|
+
|
|
454
|
+
#: Empty: the domain of the command's aggregate.
|
|
455
|
+
domain: str
|
|
456
|
+
#: Unqualified, e.g. `vehicle`.
|
|
457
|
+
aggregate_type: str
|
|
458
|
+
#: Empty when the aggregate is named by `key`.
|
|
459
|
+
aggregate_id: str
|
|
460
|
+
#: A fact the aggregate declares with `Aggregate.fact`.
|
|
461
|
+
fact: str
|
|
462
|
+
#: A unique key of the aggregate type that names the aggregate instead of its ID.
|
|
463
|
+
key: str = ""
|
|
464
|
+
#: The key's value, normalised as the aggregate type's key function does: the kernel gets only
|
|
465
|
+
#: its SHA-256. Empty in a rejection, which never carries the value.
|
|
466
|
+
value: str = ""
|
|
467
|
+
#: The aggregate must not hold the fact.
|
|
468
|
+
absent: bool = False
|
|
469
|
+
|
|
470
|
+
@classmethod
|
|
471
|
+
def of(cls, aggregate_type: str, aggregate_id: str, fact: str, *, absent: bool = False) -> "Requirement":
|
|
472
|
+
"""A requirement of an aggregate type of the command's own domain:
|
|
473
|
+
`Requirement.of("vehicle", data["vehicleId"], "woundUp", absent=True)`."""
|
|
474
|
+
return cls("", aggregate_type, aggregate_id, fact, absent=absent)
|
|
475
|
+
|
|
476
|
+
@classmethod
|
|
477
|
+
def by_key(cls, aggregate_type: str, key: str, value: str, fact: str, *, domain: str = "",
|
|
478
|
+
absent: bool = False) -> "Requirement":
|
|
479
|
+
"""The fact of the aggregate that holds the unique key `key` with `value`, normalised as
|
|
480
|
+
its key function does: `Requirement.by_key("vehicle", "isin", data["vehicle"], "open")`."""
|
|
481
|
+
return cls(domain, aggregate_type, "", fact, key=key, value=value, absent=absent)
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
@dataclass
|
|
485
|
+
class HeldKey:
|
|
486
|
+
"""A unique key an aggregate holds for a state, as `Aggregate.index` returns it: the value in
|
|
487
|
+
clear (the worker sends only its SHA-256 hash)."""
|
|
488
|
+
|
|
489
|
+
#: The name declared with `Aggregate.unique`.
|
|
490
|
+
name: str
|
|
491
|
+
value: str
|
|
492
|
+
#: `"tenant"` or `"global"`; see `Aggregate.unique`.
|
|
493
|
+
scope: str = "tenant"
|
|
494
|
+
#: The person the value belongs to, for a personal key; empty otherwise.
|
|
495
|
+
subject: str = ""
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
@dataclass
|
|
499
|
+
class Index:
|
|
500
|
+
"""The unique keys and facts an aggregate holds for a state, returned by `Aggregate.index`."""
|
|
501
|
+
|
|
502
|
+
keys: list[HeldKey] = field(default_factory=list)
|
|
503
|
+
#: The names of the facts that are true, in the order they were declared.
|
|
504
|
+
facts: list[str] = field(default_factory=list)
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
@dataclass
|
|
508
|
+
class RegisteredProcess:
|
|
509
|
+
"""What became of `Kernable.register_process`: the process's version."""
|
|
510
|
+
|
|
511
|
+
#: Qualified: `shop.fulfilment`.
|
|
512
|
+
name: str
|
|
513
|
+
#: From 1; the newest version, whether registered now or before.
|
|
514
|
+
version: int
|
|
515
|
+
#: False if the model equals the newest version: nothing was registered.
|
|
516
|
+
new: bool
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
@dataclass
|
|
520
|
+
class CancelOutcome:
|
|
521
|
+
"""What became of `Kernable.cancel`: cancelled, or rejected with a code and a message
|
|
522
|
+
(`too_late`, `not_found`, or a middleware's code)."""
|
|
523
|
+
|
|
524
|
+
#: Whether the scheduled command was cancelled (or had been already): it completes as
|
|
525
|
+
#: rejected `cancelled`.
|
|
526
|
+
cancelled: bool
|
|
527
|
+
#: The rejection code, if not cancelled.
|
|
528
|
+
code: str = ""
|
|
529
|
+
#: The rejection message, if not cancelled.
|
|
530
|
+
message: str = ""
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
@dataclass
|
|
534
|
+
class QueryOutcome:
|
|
535
|
+
"""What came of a query, as a middleware that runs after requests (`when="after"`) gets it:
|
|
536
|
+
its response data or why it was rejected."""
|
|
537
|
+
|
|
538
|
+
#: Whether the read model answered; `False` if the query was rejected.
|
|
539
|
+
answered: bool
|
|
540
|
+
#: The answer, if answered.
|
|
541
|
+
data: Any = None
|
|
542
|
+
#: The rejection code, if rejected.
|
|
543
|
+
code: str = ""
|
|
544
|
+
#: The rejection message, if rejected.
|
|
545
|
+
message: str = ""
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
@dataclass
|
|
549
|
+
class Tls:
|
|
550
|
+
"""Paths of PEM files for mutual TLS with a kernel at `host:port`: the CA that signed the
|
|
551
|
+
kernel's certificate, and this process's own certificate and key. Pass it as
|
|
552
|
+
`Kernable(tls=Tls(ca="ca.pem", cert="app.pem", key="app-key.pem"))`, or set
|
|
553
|
+
`KERNABLE_TLS_CA`, `KERNABLE_TLS_CERT` and `KERNABLE_TLS_KEY`."""
|
|
554
|
+
|
|
555
|
+
ca: str
|
|
556
|
+
cert: str
|
|
557
|
+
key: str
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
@dataclass
|
|
561
|
+
class Data:
|
|
562
|
+
"""A file as stored by the kernel, returned by `k.data.get`."""
|
|
563
|
+
|
|
564
|
+
data: bytes
|
|
565
|
+
#: The media type given when the file was put, e.g. `image/png`.
|
|
566
|
+
content_type: str
|
|
567
|
+
#: The SHA-256 of `data`, hex, as the kernel computed it.
|
|
568
|
+
sha256: str = ""
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
@dataclass
|
|
572
|
+
class Listed:
|
|
573
|
+
"""A file as `k.data.list` names it."""
|
|
574
|
+
|
|
575
|
+
key: str
|
|
576
|
+
content_type: str
|
|
577
|
+
#: The file's length in bytes.
|
|
578
|
+
size: int
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
@dataclass
|
|
582
|
+
class Listing:
|
|
583
|
+
"""One page of `k.data.list`: up to 1,000 files, and `next` (to pass as `after=`) if there
|
|
584
|
+
may be more."""
|
|
585
|
+
|
|
586
|
+
files: list[Listed]
|
|
587
|
+
next: str | None
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
class ClientClosed(RuntimeError):
|
|
591
|
+
"""Raised by a request on a client after `Kernable.close()`. Workers and listeners of a
|
|
592
|
+
closed client end their `run()`."""
|
|
593
|
+
|
|
594
|
+
def __init__(self):
|
|
595
|
+
super().__init__("the kernable client is closed")
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
class NotSettled(Exception):
|
|
599
|
+
"""Raised by a settling `Kernable.submit` (the default `wait="settled"`): the command was
|
|
600
|
+
committed, but not every listener that counts processed its events within the kernel's
|
|
601
|
+
`--settle-timeout-ms`.
|
|
602
|
+
|
|
603
|
+
Read models may not show the change yet; `outcome` (an `Outcome`) has the committed events,
|
|
604
|
+
e.g. `except NotSettled as err: events = err.outcome.events`. Submitting again with the
|
|
605
|
+
same `id` returns the same outcome, without running the command again.
|
|
606
|
+
"""
|
|
607
|
+
|
|
608
|
+
def __init__(self, outcome: Outcome):
|
|
609
|
+
super().__init__("command committed, but not all listeners have processed its events yet")
|
|
610
|
+
self.outcome = outcome
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
class NotDue(Exception):
|
|
614
|
+
"""Raised by `Kernable.submit` for the `id` of a scheduled command (`enqueue` with `at`) that
|
|
615
|
+
is not due yet: it has no outcome to wait for. Submit its `id` again after its time;
|
|
616
|
+
`command_id` is the scheduled command's ID."""
|
|
617
|
+
|
|
618
|
+
def __init__(self, command_id: str):
|
|
619
|
+
super().__init__(f"command {command_id} is scheduled and not due yet: ask again after its time")
|
|
620
|
+
self.command_id = command_id
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
#: The protocol version this SDK speaks; see "Protocol versions" in `kernable.proto`.
|
|
624
|
+
PROTOCOL = 1
|
|
625
|
+
|
|
626
|
+
#: Attempts before a reactor parks an event its handler keeps failing on.
|
|
627
|
+
_PARK_AFTER = 5
|
|
628
|
+
# The event types and schemas a listener of this process skipped already: each is logged once.
|
|
629
|
+
_SKIP_WARNED: set[tuple[str, int]] = set()
|
|
630
|
+
_SKIP_LOCK = threading.Lock()
|
|
631
|
+
|
|
632
|
+
#: gRPC codes of a kernel that is away or busy.
|
|
633
|
+
_BUSY = frozenset({grpc.StatusCode.CANCELLED, grpc.StatusCode.DEADLINE_EXCEEDED, grpc.StatusCode.RESOURCE_EXHAUSTED,
|
|
634
|
+
grpc.StatusCode.ABORTED, grpc.StatusCode.INTERNAL, grpc.StatusCode.UNAVAILABLE})
|
|
635
|
+
|
|
636
|
+
#: The name of the listener whose handler is running, for requests sent from it.
|
|
637
|
+
_HANDLING: contextvars.ContextVar[str] = contextvars.ContextVar("kernable_listener", default="")
|
|
638
|
+
|
|
639
|
+
_INITIAL = object()
|
|
640
|
+
#: Room for the largest file a kernel takes (`--max-data-bytes`, at most 128 MiB) and the rest of
|
|
641
|
+
#: its message; grpcio receives 4 MiB by default.
|
|
642
|
+
_MAX_MESSAGE = 129 * 1024 * 1024
|
|
643
|
+
#: Pings on every connection, also idle ones: a connection the network dropped without a word (a
|
|
644
|
+
#: dead host, a NAT that forgot it) ends within 40 s, and the workers and listeners on it reconnect.
|
|
645
|
+
_CHANNEL_OPTIONS = [("grpc.max_send_message_length", _MAX_MESSAGE),
|
|
646
|
+
("grpc.max_receive_message_length", _MAX_MESSAGE),
|
|
647
|
+
("grpc.keepalive_time_ms", 30_000),
|
|
648
|
+
("grpc.keepalive_timeout_ms", 10_000),
|
|
649
|
+
# grpcio times a keepalive ping out by this, not by the keepalive timeout
|
|
650
|
+
# (default 60 s).
|
|
651
|
+
("grpc.http2.ping_timeout_ms", 10_000),
|
|
652
|
+
("grpc.keepalive_permit_without_calls", 1)]
|
|
653
|
+
#: A client of several kernels (a pair) pings every 2 s and waits 2 s for the answer: a connection
|
|
654
|
+
#: the network cut off (a partition) ends within 4 s, and the client turns to the other kernel.
|
|
655
|
+
_PAIR_CHANNEL_OPTIONS = [o for o in _CHANNEL_OPTIONS
|
|
656
|
+
if o[0] not in ("grpc.keepalive_time_ms", "grpc.keepalive_timeout_ms",
|
|
657
|
+
"grpc.http2.ping_timeout_ms")] + [
|
|
658
|
+
("grpc.keepalive_time_ms", 2_000),
|
|
659
|
+
("grpc.keepalive_timeout_ms", 2_000),
|
|
660
|
+
("grpc.http2.ping_timeout_ms", 2_000),
|
|
661
|
+
]
|
|
662
|
+
#: gRPC codes that a retry does not change.
|
|
663
|
+
_REFUSALS = {grpc.StatusCode.INVALID_ARGUMENT, grpc.StatusCode.PERMISSION_DENIED, grpc.StatusCode.FAILED_PRECONDITION,
|
|
664
|
+
grpc.StatusCode.UNIMPLEMENTED, grpc.StatusCode.UNAUTHENTICATED}
|
|
665
|
+
#: How long a request waits, for the connection and the answer, unless told otherwise: above the
|
|
666
|
+
#: kernel's own submit and settle timeouts.
|
|
667
|
+
DEFAULT_TIMEOUT_S = 60.0
|
|
668
|
+
_DOMAIN = re.compile(r"^[A-Za-z][A-Za-z0-9_-]*$")
|
|
669
|
+
_UNQUALIFIED = re.compile(r"^[^.*]+$")
|
|
670
|
+
_LISTENED = re.compile(r"^[A-Za-z][A-Za-z0-9_-]*\.([^.]+|\*)$")
|
|
671
|
+
_QUERY = re.compile(r"^[A-Za-z][A-Za-z0-9_-]*\.[^.*]+$")
|
|
672
|
+
_CHECKED = re.compile(r"^(\*|[A-Za-z][A-Za-z0-9_-]*\.([^.]+|\*))$")
|
|
673
|
+
_DEPLOYMENT = re.compile(r"^[A-Za-z0-9_.-]{1,128}$")
|
|
674
|
+
|
|
675
|
+
|
|
676
|
+
def _deployment(value: str) -> str:
|
|
677
|
+
"""`value` if it names a deployment (or is empty); raises `ValueError` otherwise."""
|
|
678
|
+
if value and not _DEPLOYMENT.match(value):
|
|
679
|
+
raise ValueError(f"a deployment is 1 to 128 letters, digits, `-`, `_` and `.`, not {value!r}")
|
|
680
|
+
return value
|
|
681
|
+
def reindex_type(aggregate_type: str) -> str:
|
|
682
|
+
"""The type of the built-in command an aggregate with unique keys or facts runs without
|
|
683
|
+
events, `_reindex_<aggregate_type>` (see `Aggregate.unique` and `Kernable.reindex`): one per
|
|
684
|
+
aggregate, as command types are unique within a domain. `reindex_type("fund")` is
|
|
685
|
+
`"_reindex_fund"`."""
|
|
686
|
+
return f"_reindex_{aggregate_type}"
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
class EventSchema:
|
|
690
|
+
"""The payload format of one event type: its current version and the steps that bring older
|
|
691
|
+
payloads up to it. Events are stored as written and never changed; readers upcast them.
|
|
692
|
+
Made with `Domain.event`; its methods return it, so they chain:
|
|
693
|
+
`shop_domain.event("CustomerRegistered", 1).personal(["email"], lambda d, id: id)`."""
|
|
694
|
+
|
|
695
|
+
def __init__(self, type: str, schema: int):
|
|
696
|
+
"""Use `Domain.event`. Raises `ValueError` if `type` is qualified or `schema` is below 1."""
|
|
697
|
+
if not re.match(r"^[^.*]+$", type):
|
|
698
|
+
raise ValueError(f"event types are unqualified within their domain, not {type!r}")
|
|
699
|
+
if not isinstance(schema, int) or schema < 1:
|
|
700
|
+
raise ValueError(f"a schema is a version from 1, not {schema!r}")
|
|
701
|
+
self.type = type
|
|
702
|
+
self.schema = schema
|
|
703
|
+
self.steps: dict[int, Callable] = {}
|
|
704
|
+
#: The fields kept in the vault; see `personal`.
|
|
705
|
+
self.vault = _VaultFields()
|
|
706
|
+
#: The subject this event erases, if it is an erasing event; see `erases`.
|
|
707
|
+
self.eraser: Callable | None = None
|
|
708
|
+
|
|
709
|
+
def personal(self, fields: list[str], subject: Callable) -> "EventSchema":
|
|
710
|
+
"""Keeps these top-level fields in the vault, as personal data of the subject `subject`
|
|
711
|
+
returns from the payload and, optionally, the aggregate's ID (e.g.
|
|
712
|
+
`lambda d: d["customerId"]`, or `lambda d, id: id` for the aggregate itself). Events
|
|
713
|
+
then hold only references; the kernel hands the values to workers and to listeners
|
|
714
|
+
allowed to read them, until an event erases the subject."""
|
|
715
|
+
self.vault.personal.append((fields, subject))
|
|
716
|
+
return self
|
|
717
|
+
|
|
718
|
+
def erases(self, subject: Callable) -> "EventSchema":
|
|
719
|
+
"""Makes this an erasing event: it deletes every vault value of the subject `subject`
|
|
720
|
+
returns, in every stream, and the snapshots that may hold them. Read models should drop
|
|
721
|
+
the subject's data when they see it; rebuilt, they read None."""
|
|
722
|
+
self.eraser = subject
|
|
723
|
+
return self
|
|
724
|
+
|
|
725
|
+
def upcast(self, from_version: int, fn: Callable | None = None):
|
|
726
|
+
"""Turns a payload of version `from_version` into one of the next version: `fn(data)`
|
|
727
|
+
returns the new payload. Works as a decorator too: `@item_added.upcast(1)` over
|
|
728
|
+
`def _(data): return {**data, "qty": data.get("qty", 1)}`. Declare one step per older
|
|
729
|
+
version. Raises `ValueError` if `from_version` is not between 1 and the current version
|
|
730
|
+
minus 1."""
|
|
731
|
+
if not isinstance(from_version, int) or not 1 <= from_version < self.schema:
|
|
732
|
+
raise ValueError(f"upcast {self.type} from a version between 1 and {self.schema - 1}, not {from_version!r}")
|
|
733
|
+
return _register(self, self.steps, from_version, fn)
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
class _VaultFields:
|
|
737
|
+
"""Payload fields kept in the vault instead of the signed log."""
|
|
738
|
+
|
|
739
|
+
def __init__(self):
|
|
740
|
+
self.personal: list[tuple[list[str], Callable]] = []
|
|
741
|
+
self.secret: list[str] = []
|
|
742
|
+
|
|
743
|
+
|
|
744
|
+
class CommandSchema:
|
|
745
|
+
"""How a command's payload is kept: which fields go to the vault. Made with
|
|
746
|
+
`Domain.command`; its methods return it, so they chain:
|
|
747
|
+
`shop_domain.command("Register").personal(["email"], lambda d, id: id).secret(["password"])`.
|
|
748
|
+
Clients that submit these commands need the domain: `k.use(shop_domain)`."""
|
|
749
|
+
|
|
750
|
+
def __init__(self, type: str):
|
|
751
|
+
"""Use `Domain.command`. Raises `ValueError` if `type` is qualified."""
|
|
752
|
+
if not re.match(r"^[^.*]+$", type):
|
|
753
|
+
raise ValueError(f"command types are unqualified within their domain, not {type!r}")
|
|
754
|
+
self.type = type
|
|
755
|
+
self.vault = _VaultFields()
|
|
756
|
+
|
|
757
|
+
def personal(self, fields: list[str], subject: Callable) -> "CommandSchema":
|
|
758
|
+
"""Keeps these top-level fields in the vault as personal data of `subject`; see
|
|
759
|
+
`EventSchema.personal`. Declare them on the events that carry them too: a worker rejects
|
|
760
|
+
the command (`personal_in_clear`) if one of its events holds such a field, under the same
|
|
761
|
+
name, without declaring it."""
|
|
762
|
+
self.vault.personal.append((fields, subject))
|
|
763
|
+
return self
|
|
764
|
+
|
|
765
|
+
def secret(self, fields: list[str]) -> "CommandSchema":
|
|
766
|
+
"""Keeps these top-level fields (passwords, one-time tokens) in the vault only until the
|
|
767
|
+
command completed. The handler gets them in clear; the stored command keeps a reference."""
|
|
768
|
+
self.vault.secret.extend(fields)
|
|
769
|
+
return self
|
|
770
|
+
|
|
771
|
+
|
|
772
|
+
class QuerySchema:
|
|
773
|
+
"""What a domain declares about a query type given by name: the aggregate it is about. Made
|
|
774
|
+
with `Domain.query`; query classes declare it as `target` instead (see `Query`):
|
|
775
|
+
`shop_domain.query("GetOrder").target("order", "id")` (`GetOrder {"id": "o-17"}` is about
|
|
776
|
+
order o-17)."""
|
|
777
|
+
|
|
778
|
+
def __init__(self, type: str):
|
|
779
|
+
"""Use `Domain.query`. Raises `ValueError` if `type` is qualified."""
|
|
780
|
+
if not re.match(r"^[^.*]+$", type):
|
|
781
|
+
raise ValueError(f"query types are unqualified within their domain, not {type!r}")
|
|
782
|
+
self.type = type
|
|
783
|
+
#: The aggregate the query is about, `(aggregate_type, field)`; see `target`.
|
|
784
|
+
self.target_of: tuple[str, str] | None = None
|
|
785
|
+
|
|
786
|
+
def target(self, aggregate_type: str, field: str) -> "QuerySchema":
|
|
787
|
+
"""The query is about one aggregate of `aggregate_type` (`order`, or qualified
|
|
788
|
+
`shop.order`), whose ID is in the data's `field` (as sent: `orderId`). The SDK sends it
|
|
789
|
+
with the query, signed, and a middleware decides on it as on a command's aggregate.
|
|
790
|
+
Clients that ask it need the domain: `kernable.use(domain)`."""
|
|
791
|
+
self.target_of = (aggregate_type, field)
|
|
792
|
+
return self
|
|
793
|
+
|
|
794
|
+
|
|
795
|
+
def _query_target_of(declared: tuple[str, str] | None, domain: str, data: Any) -> "pb.Target | None":
|
|
796
|
+
"""The wire target of a query: the declared aggregate type, qualified, and the ID from `data`."""
|
|
797
|
+
if declared is None:
|
|
798
|
+
return None
|
|
799
|
+
aggregate_type, field_ = declared
|
|
800
|
+
id = data.get(field_) if isinstance(data, dict) else None
|
|
801
|
+
if not isinstance(id, str) or not id:
|
|
802
|
+
return None # no ID: no target, so a rule that needs one refuses
|
|
803
|
+
return pb.Target(aggregate_type=aggregate_type if "." in aggregate_type else f"{domain}.{aggregate_type}",
|
|
804
|
+
aggregate_id=id)
|
|
805
|
+
|
|
806
|
+
|
|
807
|
+
_issued_lock = threading.Lock()
|
|
808
|
+
_issued_last = 0
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
def _issued_now() -> int:
|
|
812
|
+
"""A request's issued_at: now, but later than every one this process issued before. Two calls in the
|
|
813
|
+
same millisecond would otherwise carry the same signature, and the kernel refuses the second as a replay."""
|
|
814
|
+
global _issued_last
|
|
815
|
+
with _issued_lock:
|
|
816
|
+
_issued_last = max(int(time.time() * 1000), _issued_last + 1)
|
|
817
|
+
return _issued_last
|
|
818
|
+
|
|
819
|
+
|
|
820
|
+
def _seal(data: Any, vault: _VaultFields | None, aggregate_id: str) -> tuple[bytes, list[str], list]:
|
|
821
|
+
"""Moves the vault fields of `data` into vault values; the payload keeps `{"$vault": ref}`.
|
|
822
|
+
Absent and None fields stay as they are."""
|
|
823
|
+
if vault is None or not (vault.personal or vault.secret) or not isinstance(data, dict):
|
|
824
|
+
return _encode(data), [], []
|
|
825
|
+
sealed = dict(data)
|
|
826
|
+
values = []
|
|
827
|
+
|
|
828
|
+
def keep(f: str, subject: str, secret: bool):
|
|
829
|
+
if sealed.get(f) is None:
|
|
830
|
+
return
|
|
831
|
+
salt = os.urandom(16)
|
|
832
|
+
value = json.dumps(sealed[f], separators=(",", ":")).encode()
|
|
833
|
+
ref = vault_ref(salt, subject, secret, value)
|
|
834
|
+
values.append(pb.VaultValue(ref=ref, subject=subject, secret=secret, salt=salt, value=value))
|
|
835
|
+
sealed[f] = {"$vault": ref}
|
|
836
|
+
|
|
837
|
+
for fields, subject in vault.personal:
|
|
838
|
+
who = str(_call(subject, data, aggregate_id) or "")
|
|
839
|
+
if not who:
|
|
840
|
+
raise ValueError(f"no subject for the personal fields {', '.join(fields)}")
|
|
841
|
+
for f in fields:
|
|
842
|
+
keep(f, who, False)
|
|
843
|
+
for f in vault.secret:
|
|
844
|
+
keep(f, "", True)
|
|
845
|
+
return _encode(sealed), [v.ref for v in values], values
|
|
846
|
+
|
|
847
|
+
|
|
848
|
+
def _unseal(data: Any, vault) -> tuple[Any, list[str]]:
|
|
849
|
+
"""Puts vault values back into `data`; references without a value become None."""
|
|
850
|
+
values = {v.ref: v.value for v in vault or []}
|
|
851
|
+
redacted: list[str] = []
|
|
852
|
+
|
|
853
|
+
def walk(v, path: str):
|
|
854
|
+
if isinstance(v, list):
|
|
855
|
+
return [walk(x, f"{path}[{i}]") for i, x in enumerate(v)]
|
|
856
|
+
if not isinstance(v, dict):
|
|
857
|
+
return v
|
|
858
|
+
if len(v) == 1 and isinstance(v.get("$vault"), str):
|
|
859
|
+
if v["$vault"] not in values:
|
|
860
|
+
redacted.append(path)
|
|
861
|
+
return None
|
|
862
|
+
return json.loads(values[v["$vault"]])
|
|
863
|
+
return {k: walk(x, f"{path}.{k}" if path else k) for k, x in v.items()}
|
|
864
|
+
|
|
865
|
+
return walk(data, ""), redacted
|
|
866
|
+
|
|
867
|
+
|
|
868
|
+
class SchemaError(Exception):
|
|
869
|
+
"""An event a process cannot read: its schema is newer than known, or a step is missing."""
|
|
870
|
+
|
|
871
|
+
#: The event's qualified type (`shop.ItemAdded`).
|
|
872
|
+
type: str = ""
|
|
873
|
+
#: The event's schema, as stored.
|
|
874
|
+
schema: int = 0
|
|
875
|
+
#: The newest schema this process reads; `newer` is set when the event's is above it (a
|
|
876
|
+
#: newer deployment wrote it), unset when an upcast step is missing.
|
|
877
|
+
known: int = 0
|
|
878
|
+
newer: bool = False
|
|
879
|
+
|
|
880
|
+
|
|
881
|
+
def _upcast(events: dict | None, domain: str, type: str, schema: int, data: Any) -> tuple[int, Any]:
|
|
882
|
+
"""`data` of a `type` event stored at `schema`, upcast to the current version in `events`."""
|
|
883
|
+
spec = (events or {}).get(type)
|
|
884
|
+
current = spec.schema if spec else 1
|
|
885
|
+
if schema > current:
|
|
886
|
+
err = SchemaError(f"{domain}.{type} has schema {schema}, but this process knows it only up to {current}: "
|
|
887
|
+
"update it")
|
|
888
|
+
err.type, err.schema, err.known, err.newer = f"{domain}.{type}", schema, current, True
|
|
889
|
+
raise err
|
|
890
|
+
for v in range(schema, current):
|
|
891
|
+
if v not in spec.steps:
|
|
892
|
+
raise SchemaError(f"no upcast for {domain}.{type} from schema {v}")
|
|
893
|
+
data = spec.steps[v](data)
|
|
894
|
+
return current, data
|
|
895
|
+
|
|
896
|
+
|
|
897
|
+
class Domain:
|
|
898
|
+
"""A domain: a name that scopes its aggregates, commands and events, lowercase by convention
|
|
899
|
+
(`orders`) like aggregates, while message types are capitalized (`orders.PlaceOrder`). Both
|
|
900
|
+
cases are valid; names are case-sensitive. Types are unique within a domain; elsewhere they
|
|
901
|
+
are named `domain.Type`.
|
|
902
|
+
|
|
903
|
+
Make it with `domain("shop")`. It remembers the aggregates, read models and reactors defined
|
|
904
|
+
on it, so `Kernable.app` runs all of them."""
|
|
905
|
+
|
|
906
|
+
def __init__(self, name: str):
|
|
907
|
+
"""Use `domain(name)`. Raises `ValueError` unless `name` starts with a letter and holds
|
|
908
|
+
only letters, digits, `-` and `_`."""
|
|
909
|
+
if not _DOMAIN.match(name):
|
|
910
|
+
raise ValueError(f"invalid domain name {name!r}")
|
|
911
|
+
self.name = name
|
|
912
|
+
#: Payload versions and upcasts per event type; types not listed are at version 1.
|
|
913
|
+
self.events: dict[str, EventSchema] = {}
|
|
914
|
+
#: Vault fields per command type; see `command`.
|
|
915
|
+
self.commands: dict[str, CommandSchema] = {}
|
|
916
|
+
#: What query types given by name are about; see `query`.
|
|
917
|
+
self.queries: dict[str, QuerySchema] = {}
|
|
918
|
+
#: The aggregates defined on it, which `Kernable.app` runs.
|
|
919
|
+
self.aggregates: list[AggregateType] = []
|
|
920
|
+
#: The read models and reactors defined on it, which `Kernable.app` runs.
|
|
921
|
+
self.listeners: list[Listener] = []
|
|
922
|
+
#: The business processes defined on it, which `Kernable.app` registers.
|
|
923
|
+
self.processes: list[Process] = []
|
|
924
|
+
|
|
925
|
+
def command(self, type: str) -> CommandSchema:
|
|
926
|
+
"""Declares which fields of `type` commands the vault keeps (`personal`, `secret`); see
|
|
927
|
+
`CommandSchema`. Clients that submit them need the domain: `kernable.use(domain)`."""
|
|
928
|
+
spec = CommandSchema(type)
|
|
929
|
+
self.commands[type] = spec
|
|
930
|
+
return spec
|
|
931
|
+
|
|
932
|
+
def process(self, name: str) -> Process:
|
|
933
|
+
"""Defines a business process `name` of this domain (`shop.fulfilment` for `fulfilment`):
|
|
934
|
+
returns the `Process` builder to chain its steps on (`start`, `command`, `wait`, `timer`,
|
|
935
|
+
`decide`, `query`, `end`). `Kernable.app` registers the domain's processes; see
|
|
936
|
+
`kernable.processes` for an example. Raises `ValueError` if the domain has a process of
|
|
937
|
+
that name already."""
|
|
938
|
+
if any(p.name == name for p in self.processes):
|
|
939
|
+
raise ValueError(f"{self.name} has a process {name!r} already")
|
|
940
|
+
p = Process(self.name, name)
|
|
941
|
+
self.processes.append(p)
|
|
942
|
+
return p
|
|
943
|
+
|
|
944
|
+
def query(self, type: str) -> QuerySchema:
|
|
945
|
+
"""Declares what `type` queries (unqualified) are about: returns the `QuerySchema` to chain
|
|
946
|
+
`target` on. Query classes declare it as `target` instead. Clients that ask such queries
|
|
947
|
+
by name need the domain: `kernable.use(domain)`."""
|
|
948
|
+
spec = QuerySchema(type)
|
|
949
|
+
self.queries[type] = spec
|
|
950
|
+
return spec
|
|
951
|
+
|
|
952
|
+
def event(self, type, schema: int | None = None, *more) -> EventSchema | None:
|
|
953
|
+
"""Declares that this domain writes `type` events at version `schema` (from 1), with an
|
|
954
|
+
`upcast` per older version, and which of their fields the vault keeps; see
|
|
955
|
+
`EventSchema`. The domain's aggregates and listeners use it; listeners of other domains
|
|
956
|
+
get it with `Listener.use(domain)`. Types not declared are at version 1. Example:
|
|
957
|
+
`shop_domain.event("ItemAdded", 2).upcast(1, lambda d: {**d, "qty": 1})`.
|
|
958
|
+
|
|
959
|
+
With event classes instead (`shop_domain.event(ItemAdded, CartCleared)`), declares them
|
|
960
|
+
from their class attributes, e.g. in a process that only reads them; an aggregate's
|
|
961
|
+
`apply` declares them too."""
|
|
962
|
+
if _is_message_class(type):
|
|
963
|
+
schemas = self
|
|
964
|
+
for cls in (type, schema, *more):
|
|
965
|
+
if cls is None:
|
|
966
|
+
continue
|
|
967
|
+
_declare_event(cls, self.events, EventSchema)
|
|
968
|
+
_bind(cls, self.name, schemas)
|
|
969
|
+
return None
|
|
970
|
+
spec = EventSchema(type, schema)
|
|
971
|
+
self.events[type] = spec
|
|
972
|
+
return spec
|
|
973
|
+
|
|
974
|
+
def aggregate(self, name: str, initial: Any, snapshot_every: int = 100, schema: str = "1") -> "AggregateType":
|
|
975
|
+
"""An aggregate of this domain named `name` (lowercase by convention, like `cart`), whose
|
|
976
|
+
state starts as `initial` (a JSON-like value, usually a dict); see `Aggregate` for
|
|
977
|
+
`snapshot_every` and `schema`. Run it with `kernable.worker(aggregate)` or in an app.
|
|
978
|
+
Example: `cart_aggregate = shop_domain.aggregate("cart", {"items": 0})`. Raises
|
|
979
|
+
`ValueError` if the domain has an aggregate of that name already."""
|
|
980
|
+
if any(a.name == name for a in self.aggregates):
|
|
981
|
+
raise ValueError(f"{self.name} has an aggregate {name} already")
|
|
982
|
+
aggregate = AggregateType(self.name, name, initial, snapshot_every, schema, self.events, self.commands)
|
|
983
|
+
self.aggregates.append(aggregate)
|
|
984
|
+
return aggregate
|
|
985
|
+
|
|
986
|
+
def read_model(self, name: str, *, order: int = 0, start: int | None = 0, reconnect_s: float = 1.0,
|
|
987
|
+
store: str | None = None, schema: str = "1", live: bool = False,
|
|
988
|
+
park_after: int = 0, instance: str | None = None, single: bool = False) -> "Listener":
|
|
989
|
+
"""A read model of this domain named `name`: builds a view from events and answers
|
|
990
|
+
queries from it (`on`, `query`; see `Listener`). Kept in memory and rebuilt from the
|
|
991
|
+
first event on every start (default). Run it with `kernable.listener(read_model)` or in
|
|
992
|
+
an app. Example: `carts_readmodel = shop_domain.read_model("carts")`.
|
|
993
|
+
|
|
994
|
+
`order` (default 0): listeners with a lower order process each event first. `start`:
|
|
995
|
+
the position to start after on the first connect (default 0, from the beginning; `None`:
|
|
996
|
+
the kernel's stored position, or the newest event for a new listener). `reconnect_s`:
|
|
997
|
+
seconds to wait before reconnecting after the stream drops (default 1), twice as long
|
|
998
|
+
after each further failed attempt, at most 30.
|
|
999
|
+
|
|
1000
|
+
With `store` (`sqlite:<path>` or `memory:`), its state lives in the running listener's
|
|
1001
|
+
`store`: data and position are written together per batch, and a restart continues after
|
|
1002
|
+
the stored position. Change `schema` when the handlers store data differently: the store is
|
|
1003
|
+
cleared and the read model rebuilds from the first event. `live` and `park_after` (default
|
|
1004
|
+
0: never; a read model that parks misses the event's change): see `reactor`.
|
|
1005
|
+
|
|
1006
|
+
`instance`: runs this process's read model as one instance of a group under its name,
|
|
1007
|
+
e.g. one per node of a pair (1 to 128 letters, digits, `-`, `_` and `.`; default
|
|
1008
|
+
`KERNABLE_INSTANCE`, which `kernable node` sets; without either, the name has one
|
|
1009
|
+
connection). Every instance gets every event, from its own position, and answers
|
|
1010
|
+
queries; the group is through an event once one instance has handled it. `single`: see
|
|
1011
|
+
`reactor`. Raises `ValueError` if the domain has a listener (read model or reactor) of
|
|
1012
|
+
that name already."""
|
|
1013
|
+
return self._listener(Listener(self, name, order, start, reconnect_s, store, schema, live=live,
|
|
1014
|
+
park_after=park_after, instance=instance, single=single))
|
|
1015
|
+
|
|
1016
|
+
def reactor(self, name: str, *, order: int = 10, start: int | None = None, reconnect_s: float = 1.0,
|
|
1017
|
+
store: str | None = None, schema: str = "1", live: bool = False,
|
|
1018
|
+
park_after: int = _PARK_AFTER, instance: str | None = None, single: bool = True) -> "Listener":
|
|
1019
|
+
"""A reactor of this domain named `name`: triggers side effects and follow-up commands
|
|
1020
|
+
(`on`; see `Listener`). It starts at the newest event when it is new, then resumes after
|
|
1021
|
+
the last acknowledged event (`start=None`, default), so it never reacts to what happened
|
|
1022
|
+
before it existed. Run it with `kernable.listener(reactor)` or in an app.
|
|
1023
|
+
|
|
1024
|
+
A settling submit from its handler waits for the listeners before it (the read models),
|
|
1025
|
+
not for the reactor itself, which handles one event at a time. Submit follow-ups with
|
|
1026
|
+
`ctx.submit` and a fixed `id`, so a redelivered event does not run the command twice.
|
|
1027
|
+
`order` (default 10), `start`, `reconnect_s`, `store` and `schema`: see `read_model`.
|
|
1028
|
+
|
|
1029
|
+
`park_after` (default 5): how often a handler may fail on one event before the reactor
|
|
1030
|
+
parks it and moves on (0: never, it keeps getting the event and everything behind it
|
|
1031
|
+
waits). A `Rejected` (also from `ctx.submit`) parks at once; the kernel being unreachable
|
|
1032
|
+
or busy (`UNAVAILABLE`, `DEADLINE_EXCEEDED`, `RESOURCE_EXHAUSTED`), `NotSettled` or the
|
|
1033
|
+
store failing never does: the reactor waits and tries again. The kernel keeps parked
|
|
1034
|
+
events with the rejection's code and message (for other errors `handler_error` and the
|
|
1035
|
+
exception's type; the log has the whole error) until an operator retries or dismisses
|
|
1036
|
+
them (`kernable listeners retry`). With a store, a parked event's writes are dropped, the
|
|
1037
|
+
rest of its batch kept. Live listeners never park, and no listener parks an event that
|
|
1038
|
+
erases a subject because its handler failed: the handler must get past it, so no personal
|
|
1039
|
+
data stays behind. An event newer than the code (`unknown_event`) is parked, erasing or
|
|
1040
|
+
not: no handler ran on it.
|
|
1041
|
+
|
|
1042
|
+
`live`: not registered with the kernel: no stored position, always starts at the newest
|
|
1043
|
+
event, nobody waits for it, and it is forgotten when it disconnects. For reactions that
|
|
1044
|
+
only matter while the process runs, e.g. pushing to connected browsers. Cannot be a
|
|
1045
|
+
middleware.
|
|
1046
|
+
|
|
1047
|
+
`instance`: see `read_model`. `single` (default `True` for reactors): the instances take
|
|
1048
|
+
turns: one gets events, the others wait and the first of them takes over when it leaves,
|
|
1049
|
+
so a side effect happens once. `False`: every instance gets every event; such a listener
|
|
1050
|
+
cannot park events. Live listeners are never single. Raises `ValueError` if the domain
|
|
1051
|
+
has a listener of that name already."""
|
|
1052
|
+
return self._listener(Listener(self, name, order, start, reconnect_s, store, schema, live=live,
|
|
1053
|
+
park_after=park_after, instance=instance, single=single,
|
|
1054
|
+
reactor=True))
|
|
1055
|
+
|
|
1056
|
+
def _listener(self, listener: "Listener") -> "Listener":
|
|
1057
|
+
if any(l.name == listener.name for l in self.listeners):
|
|
1058
|
+
raise ValueError(f"{self.name} has a listener {listener.name} already")
|
|
1059
|
+
self.listeners.append(listener)
|
|
1060
|
+
return listener
|
|
1061
|
+
|
|
1062
|
+
|
|
1063
|
+
def domain(name: str) -> Domain:
|
|
1064
|
+
"""A domain named `name`, on which you define its aggregates, read models, reactors and
|
|
1065
|
+
event and command schemas: `shop_domain = domain("shop")`. Raises `ValueError` unless `name`
|
|
1066
|
+
starts with a letter and holds only letters, digits, `-` and `_`."""
|
|
1067
|
+
return Domain(name)
|
|
1068
|
+
|
|
1069
|
+
|
|
1070
|
+
class AggregateType:
|
|
1071
|
+
"""How events change an aggregate's state (`apply`) and which events a command produces
|
|
1072
|
+
(`command`). Handlers must be deterministic and free of side effects; they may run again.
|
|
1073
|
+
|
|
1074
|
+
Make it with `Domain.aggregate`. Both work as decorators (`@agg.apply("Type")`) or directly
|
|
1075
|
+
(`agg.apply("Type", fn)`). A command handler returns a list of `{"type": ..., "data": ...}`
|
|
1076
|
+
and refuses with `raise Rejected(...)`. Handlers take only the parameters they need:
|
|
1077
|
+
`def _(state, data)` is enough, `def _(state, data, cmd, ctx)` gets the rest.
|
|
1078
|
+
|
|
1079
|
+
Every `snapshot_every` events (0 turns it off) the worker sends a snapshot of the state, so
|
|
1080
|
+
later commands fold only the events after it. The state must survive a JSON round trip.
|
|
1081
|
+
Change `schema` when `apply` produces differently shaped state: older snapshots are then
|
|
1082
|
+
ignored and the state is rebuilt from the events.
|
|
1083
|
+
|
|
1084
|
+
Uniqueness and conditions across aggregates: `unique` declares keys no other aggregate may
|
|
1085
|
+
hold at the same time (an email, an order number), `fact` what other aggregates' commands may
|
|
1086
|
+
require of this one (`requires=` of `command`). The kernel checks both atomically with the
|
|
1087
|
+
commit.
|
|
1088
|
+
"""
|
|
1089
|
+
|
|
1090
|
+
def __init__(self, domain: str, name: str, initial: Any, snapshot_every: int = 100, schema: str = "1",
|
|
1091
|
+
events: dict[str, EventSchema] | None = None, commands: dict[str, CommandSchema] | None = None):
|
|
1092
|
+
"""Use `Domain.aggregate`, which passes the domain's event and command schemas."""
|
|
1093
|
+
self.domain = domain
|
|
1094
|
+
#: The domain's event schemas.
|
|
1095
|
+
self.events = {} if events is None else events
|
|
1096
|
+
#: The domain's command schemas: which command fields are personal.
|
|
1097
|
+
self.commands = {} if commands is None else commands
|
|
1098
|
+
self.name = name
|
|
1099
|
+
self.initial = initial
|
|
1100
|
+
self.snapshot_every = snapshot_every
|
|
1101
|
+
self.schema = schema
|
|
1102
|
+
self.appliers: dict[str, Callable] = {}
|
|
1103
|
+
self.handlers: dict[str, Callable] = {}
|
|
1104
|
+
#: Per command type, the facts of other aggregates it requires; see `command`.
|
|
1105
|
+
self.requirements: dict[str, Callable] = {}
|
|
1106
|
+
#: The unique keys, by name: `(fn, scope, personal)`; see `unique`.
|
|
1107
|
+
self.keys: dict[str, tuple[Callable, str, Callable | None]] = {}
|
|
1108
|
+
#: The facts, by name; see `fact`.
|
|
1109
|
+
self.facts: dict[str, Callable] = {}
|
|
1110
|
+
#: How a state becomes the JSON of a snapshot, and back (an `Aggregate` class: its fields).
|
|
1111
|
+
self._to_json: Callable[[Any], Any] = lambda state: state
|
|
1112
|
+
self._from_json: Callable[[Any], Any] = lambda data: data
|
|
1113
|
+
|
|
1114
|
+
def apply(self, event_type: str, fn: Callable | None = None):
|
|
1115
|
+
"""How an event of `event_type` changes the state: `fn(state, data, evt, ctx)` changes
|
|
1116
|
+
`state` in place and returns nothing, or returns a new state that replaces it. `state` is a
|
|
1117
|
+
copy, so changing it never touches the state it came from; `data` is the payload, upcast
|
|
1118
|
+
to the current version; `evt` (an `EventMeta`) and `ctx` (an `ApplyContext`) are
|
|
1119
|
+
optional. Works as a decorator too: `@cart_aggregate.apply("ItemAdded")` over
|
|
1120
|
+
`def _(state, data): state["items"] += data["qty"]`. Returns the decorated function, or
|
|
1121
|
+
the aggregate when `fn` is given.
|
|
1122
|
+
|
|
1123
|
+
With an event class (`@cart_aggregate.apply(ItemAdded)`), `data` is an instance of it."""
|
|
1124
|
+
if _is_message_class(event_type, Event):
|
|
1125
|
+
cls = event_type
|
|
1126
|
+
name = _declare_event(cls, self.events, EventSchema)
|
|
1127
|
+
_bind(cls, self.domain, self._schemas())
|
|
1128
|
+
|
|
1129
|
+
def wrap(f: Callable) -> Callable:
|
|
1130
|
+
return lambda state, data, evt=None, ctx=None: _call(f, state, cls.from_wire(data), evt, ctx)
|
|
1131
|
+
|
|
1132
|
+
return _register_wrapped(self, self.appliers, name, fn, wrap)
|
|
1133
|
+
return _register(self, self.appliers, event_type, fn)
|
|
1134
|
+
|
|
1135
|
+
def _schemas(self):
|
|
1136
|
+
"""This aggregate's domain's schemas, as a class binding carries them."""
|
|
1137
|
+
from types import SimpleNamespace
|
|
1138
|
+
return SimpleNamespace(name=self.domain, events=self.events, commands=self.commands)
|
|
1139
|
+
|
|
1140
|
+
def _plain(self, produced: Any) -> Any:
|
|
1141
|
+
"""Event class instances a handler returned, as `{"type": ..., "data": ...}`."""
|
|
1142
|
+
if not isinstance(produced, list):
|
|
1143
|
+
return produced
|
|
1144
|
+
out = []
|
|
1145
|
+
for e in produced:
|
|
1146
|
+
if isinstance(e, Event):
|
|
1147
|
+
name = _declare_event(type(e), self.events, EventSchema)
|
|
1148
|
+
_bind(type(e), self.domain, self._schemas())
|
|
1149
|
+
out.append({"type": name, "data": e.to_wire()})
|
|
1150
|
+
else:
|
|
1151
|
+
out.append(e)
|
|
1152
|
+
return out
|
|
1153
|
+
|
|
1154
|
+
def command(self, command_type: str, fn: Callable | None = None, *, requires: Callable | None = None):
|
|
1155
|
+
"""Which events a `command_type` command produces: `fn(state, data, cmd, ctx)` returns
|
|
1156
|
+
them as a list of `{"type": ..., "data": ...}`, or raises
|
|
1157
|
+
`Rejected`; `cmd` (a `CommandMeta`) and `ctx` (a `CommandContext`) are optional. `state`
|
|
1158
|
+
is the state after every earlier event; do not change it. Works as a decorator too:
|
|
1159
|
+
`@cart_aggregate.command("AddItem")` over `def _(state, data): return [{"type":
|
|
1160
|
+
"ItemAdded", "data": data}]`. Returns the decorated function, or the aggregate when `fn`
|
|
1161
|
+
is given.
|
|
1162
|
+
|
|
1163
|
+
`requires(data, state)` returns the facts other aggregates of the command's tenant must
|
|
1164
|
+
hold when it commits, as a list of `Requirement` (`state` is the state before the
|
|
1165
|
+
command): `requires=lambda data, state: [Requirement("funds", "vehicle", data["vehicleId"],
|
|
1166
|
+
"open")]`. The kernel checks them atomically with the commit and rejects the command with
|
|
1167
|
+
`requirement_missing` if one is not held, so no other command can change them in between.
|
|
1168
|
+
Raises `ValueError` for a type that starts with `_reindex_`: aggregates with keys or
|
|
1169
|
+
facts run `_reindex_<aggregate_type>` themselves (see `unique`), and one of another aggregate
|
|
1170
|
+
of the domain could clash."""
|
|
1171
|
+
if _is_message_class(command_type, Command):
|
|
1172
|
+
cls = command_type
|
|
1173
|
+
name = _declare_command(cls, self.commands, CommandSchema)
|
|
1174
|
+
_bind(cls, self.domain, self._schemas(), self.name)
|
|
1175
|
+
if requires is not None:
|
|
1176
|
+
given = requires
|
|
1177
|
+
requires = lambda data, state: given(cls.from_wire(data), state) # noqa: E731
|
|
1178
|
+
|
|
1179
|
+
def wrap(f: Callable) -> Callable:
|
|
1180
|
+
return lambda state, data, meta=None, ctx=None: self._plain(_call(f, state, cls.from_wire(data), meta, ctx))
|
|
1181
|
+
|
|
1182
|
+
if requires is not None:
|
|
1183
|
+
self.requirements[name] = requires
|
|
1184
|
+
return _register_wrapped(self, self.handlers, name, fn, wrap)
|
|
1185
|
+
if command_type.startswith("_reindex_"):
|
|
1186
|
+
raise ValueError(f"{command_type}: command types starting with _reindex_ are the built-in commands "
|
|
1187
|
+
"of aggregates with keys or facts; name yours differently")
|
|
1188
|
+
if requires is not None:
|
|
1189
|
+
self.requirements[command_type] = requires
|
|
1190
|
+
return _register(self, self.handlers, command_type, fn)
|
|
1191
|
+
|
|
1192
|
+
def unique(self, name: str, fn: Callable | None = None, *, scope: str = "tenant",
|
|
1193
|
+
personal: Callable | None = None):
|
|
1194
|
+
"""A unique key: `fn(state)` returns its value (a `str`), several values (a list of `str`), or
|
|
1195
|
+
`None` for none. After every command the aggregate holds exactly the values of its new
|
|
1196
|
+
state, and no two aggregates of this type hold the same value of `name` at the same time:
|
|
1197
|
+
a command that would take a value another one holds is rejected with `unique_taken`
|
|
1198
|
+
(the message names the key, never the value). Normalise the value in `fn` (e.g.
|
|
1199
|
+
`.lower()` for an email); the kernel compares its SHA-256 hash.
|
|
1200
|
+
|
|
1201
|
+
`scope`: `"tenant"` (default) for unique within the command's tenant, `"global"` for
|
|
1202
|
+
across tenants. `personal(state)`: the person the value belongs to, for personal data
|
|
1203
|
+
such as an email; the kernel keeps the key in the vault, and erasing that subject frees
|
|
1204
|
+
it. A personal key whose subject is empty or `None` is not held.
|
|
1205
|
+
|
|
1206
|
+
Example: `account_aggregate.unique("email", lambda s: s.get("email"), scope="global")`.
|
|
1207
|
+
Works as a decorator too (`@account_aggregate.unique("email")` over `def _(state):`) and
|
|
1208
|
+
returns the decorated function, or the aggregate when `fn` is given, so it chains.
|
|
1209
|
+
|
|
1210
|
+
Keys come from the state, so aggregates that existed before a key was declared hold it
|
|
1211
|
+
only after their next command: `Kernable.reindex` runs the built-in command
|
|
1212
|
+
`_reindex_<aggregate_type>` (`reindex_type`, e.g. `_reindex_account`; no events) on them.
|
|
1213
|
+
Workers of an aggregate with keys or facts run it too, so their key needs `run` for
|
|
1214
|
+
`<domain>._reindex_<aggregate_type>` (`<domain>.*` covers it). An aggregate whose value another
|
|
1215
|
+
one already holds (a duplicate from before) is rejected with `unique_taken` until its
|
|
1216
|
+
value changes or becomes `None`. Raises `ValueError` if the name is declared twice, is
|
|
1217
|
+
empty, holds a dot or is longer than 128 bytes, or `scope` is neither of the two."""
|
|
1218
|
+
_check_index_name("key", name, self.keys, self)
|
|
1219
|
+
if scope not in ("tenant", "global"):
|
|
1220
|
+
raise ValueError(f'the scope of a key is "tenant" or "global", not {scope!r}')
|
|
1221
|
+
|
|
1222
|
+
def define(f: Callable) -> None:
|
|
1223
|
+
self.keys[name] = (f, scope, personal)
|
|
1224
|
+
self.handlers.setdefault(reindex_type(self.name), _no_events)
|
|
1225
|
+
|
|
1226
|
+
return _define(self, define, fn)
|
|
1227
|
+
|
|
1228
|
+
def fact(self, name: str, fn: Callable | None = None):
|
|
1229
|
+
"""A fact: `fn(state)` returns whether the aggregate holds it. After every command the
|
|
1230
|
+
aggregate holds the facts that are true for its new state, and commands of other
|
|
1231
|
+
aggregates may require them (`requires=` of `command`), checked atomically with their
|
|
1232
|
+
commit (`requirement_missing` if not held). Example: `vehicle_aggregate.fact("open",
|
|
1233
|
+
lambda s: s["status"] == "open")`. Works as a decorator too and chains like `unique`;
|
|
1234
|
+
aggregates that existed before a fact was declared hold it after `Kernable.reindex`.
|
|
1235
|
+
Raises `ValueError` if the name is declared twice, is empty, holds a dot or is longer
|
|
1236
|
+
than 128 bytes."""
|
|
1237
|
+
_check_index_name("fact", name, self.facts, self)
|
|
1238
|
+
|
|
1239
|
+
def define(f: Callable) -> None:
|
|
1240
|
+
self.facts[name] = f
|
|
1241
|
+
self.handlers.setdefault(reindex_type(self.name), _no_events)
|
|
1242
|
+
|
|
1243
|
+
return _define(self, define, fn)
|
|
1244
|
+
|
|
1245
|
+
def index(self, state: Any) -> Index:
|
|
1246
|
+
"""The unique keys and facts this aggregate holds for `state`, as the worker computes them
|
|
1247
|
+
after a command; for unit tests next to `handle`: `account_aggregate.index(state).keys`.
|
|
1248
|
+
Raises the exception of a key's or fact's function, `TypeError` if a key's value is not a
|
|
1249
|
+
string."""
|
|
1250
|
+
keys: list[HeldKey] = []
|
|
1251
|
+
for name, (fn, scope, personal) in self.keys.items():
|
|
1252
|
+
values = fn(state)
|
|
1253
|
+
if values is None:
|
|
1254
|
+
continue
|
|
1255
|
+
for value in values if isinstance(values, (list, tuple)) else [values]:
|
|
1256
|
+
if not isinstance(value, str):
|
|
1257
|
+
raise TypeError(f"the key {name} of {self.domain}.{self.name} is a string, not {type(value).__name__}")
|
|
1258
|
+
subject = ""
|
|
1259
|
+
if personal is not None:
|
|
1260
|
+
subject = personal(state) or ""
|
|
1261
|
+
if not isinstance(subject, str):
|
|
1262
|
+
raise TypeError(f"the subject of the key {name} is a string, not {type(subject).__name__}")
|
|
1263
|
+
if not subject:
|
|
1264
|
+
continue
|
|
1265
|
+
held = HeldKey(name, value, scope, subject)
|
|
1266
|
+
if held not in keys:
|
|
1267
|
+
keys.append(held)
|
|
1268
|
+
return Index(keys, [name for name, fn in self.facts.items() if fn(state)])
|
|
1269
|
+
|
|
1270
|
+
def schema_of(self, event_type: str) -> int:
|
|
1271
|
+
"""The payload version this aggregate writes `event_type` events at."""
|
|
1272
|
+
spec = self.events.get(event_type)
|
|
1273
|
+
return spec.schema if spec else 1
|
|
1274
|
+
|
|
1275
|
+
def fold(self, history: list[tuple], state: Any = _INITIAL) -> Any:
|
|
1276
|
+
"""State after `history`, a list of tuples `(type, schema, data)`, optionally followed by
|
|
1277
|
+
an `EventMeta` and an `ApplyContext`, starting from `state` (default: `initial`). Events
|
|
1278
|
+
are upcast first; events without an `apply` are skipped. For unit tests without a kernel:
|
|
1279
|
+
`cart_aggregate.fold([("ItemAdded", 1, {"qty": 2})])`. Raises the handler's exception,
|
|
1280
|
+
or an error if an event's schema is newer than this process knows."""
|
|
1281
|
+
if state is _INITIAL:
|
|
1282
|
+
state = self.initial
|
|
1283
|
+
for event_type, schema, data, *rest in history:
|
|
1284
|
+
fn = self.appliers.get(event_type)
|
|
1285
|
+
if fn:
|
|
1286
|
+
meta = rest[0] if rest else EventMeta(event_type, domain=self.domain, aggregate_type=self.name)
|
|
1287
|
+
ctx = rest[1] if len(rest) > 1 else ApplyContext()
|
|
1288
|
+
draft = copy.deepcopy(state)
|
|
1289
|
+
result = _call(fn, draft, _upcast(self.events, self.domain, event_type, schema, data)[1], meta, ctx)
|
|
1290
|
+
state = draft if result is None else result
|
|
1291
|
+
return state
|
|
1292
|
+
|
|
1293
|
+
def handle(self, state: Any, command_type: str, data: Any, ctx: CommandContext, *, aggregate_id: str = "", tenant: str = "") -> list[dict]:
|
|
1294
|
+
"""Runs the `command_type` handler on `state`, e.g. in a unit test, and returns its events.
|
|
1295
|
+
`data` reaches it as from a client (after a JSON round trip; `None` as `{}`), `state` as a
|
|
1296
|
+
copy. The handler's `CommandMeta` has the command ID of `ctx` (see `CommandContext.new`),
|
|
1297
|
+
this aggregate's domain and name, and an empty aggregate ID and tenant: `cart_aggregate.handle(
|
|
1298
|
+
state, "AddItem", {"qty": 2}, CommandContext.new("add-1"))`. Raises the handler's `Rejected`
|
|
1299
|
+
(or other exception), `ValueError` if there is no handler for `command_type`."""
|
|
1300
|
+
fn = self.handlers.get(command_type)
|
|
1301
|
+
if fn is None:
|
|
1302
|
+
raise ValueError(f"no handler for {command_type} on {self.domain}.{self.name}")
|
|
1303
|
+
meta = CommandMeta(ctx._command_id, command_type, self.domain, self.name, aggregate_id, tenant)
|
|
1304
|
+
return _call(fn, copy.deepcopy(state), _decode(_encode(data)), meta, ctx)
|
|
1305
|
+
|
|
1306
|
+
|
|
1307
|
+
def _no_events(state, data) -> list[dict]:
|
|
1308
|
+
"""The built-in `_reindex_<aggregate_type>` command: no events, so the worker sends the keys and facts of the
|
|
1309
|
+
current state."""
|
|
1310
|
+
return []
|
|
1311
|
+
|
|
1312
|
+
|
|
1313
|
+
def _check_index_name(kind: str, name: str, declared: dict, agg: AggregateType) -> None:
|
|
1314
|
+
if not isinstance(name, str) or not name or "." in name or len(name.encode()) > 128:
|
|
1315
|
+
raise ValueError(f"a {kind} name is not empty, has no dots and at most 128 bytes, not {name!r}")
|
|
1316
|
+
if name in declared:
|
|
1317
|
+
raise ValueError(f"{agg.domain}.{agg.name} has a {kind} {name} already")
|
|
1318
|
+
|
|
1319
|
+
|
|
1320
|
+
def _define(owner, define: Callable, fn: Callable | None):
|
|
1321
|
+
"""Defines `fn` with `define` and returns `owner`, or, without `fn`, a decorator that does."""
|
|
1322
|
+
if fn is not None:
|
|
1323
|
+
define(fn)
|
|
1324
|
+
return owner
|
|
1325
|
+
|
|
1326
|
+
def decorator(f: Callable) -> Callable:
|
|
1327
|
+
define(f)
|
|
1328
|
+
return f
|
|
1329
|
+
|
|
1330
|
+
return decorator
|
|
1331
|
+
|
|
1332
|
+
|
|
1333
|
+
def _personal(vault: _VaultFields) -> set[str]:
|
|
1334
|
+
return {f for fields, _ in vault.personal for f in fields}
|
|
1335
|
+
|
|
1336
|
+
|
|
1337
|
+
def _check_personal(agg: AggregateType, cmd, produced: list[dict]) -> None:
|
|
1338
|
+
"""Rejects (`personal_in_clear`) an event that holds a personal field of its command, under the
|
|
1339
|
+
same name, but does not declare it personal: the value would go into the log in clear, for good."""
|
|
1340
|
+
spec = agg.commands.get(cmd.command_type)
|
|
1341
|
+
personal = _personal(spec.vault) if spec else set()
|
|
1342
|
+
if not personal:
|
|
1343
|
+
return
|
|
1344
|
+
for e in produced:
|
|
1345
|
+
data, event = e.get("data"), agg.events.get(e["type"])
|
|
1346
|
+
if not isinstance(data, dict):
|
|
1347
|
+
continue
|
|
1348
|
+
declared = _personal(event.vault) if event else set()
|
|
1349
|
+
for f in sorted(personal - declared):
|
|
1350
|
+
if data.get(f) is not None:
|
|
1351
|
+
raise Rejected(f"{cmd.command_type} declares {f} personal, but {e['type']} does not: it would be "
|
|
1352
|
+
f"stored in clear (declare it with Domain.event({e['type']!r}, …).personal)",
|
|
1353
|
+
"personal_in_clear")
|
|
1354
|
+
|
|
1355
|
+
|
|
1356
|
+
class Aggregate:
|
|
1357
|
+
"""An aggregate defined as a class: its fields are the state, and the class is registered as an
|
|
1358
|
+
aggregate of `domain` (its name: the class name in lowercase, or `name`). The state is a
|
|
1359
|
+
dataclass (no `@dataclass`); mutable defaults (`[]`, `{}`) are copied for every aggregate, and
|
|
1360
|
+
snapshots hold its fields as JSON. Handlers get and change an instance: `s.status = "active"`.
|
|
1361
|
+
|
|
1362
|
+
class Cart(Aggregate, domain=shop_domain):
|
|
1363
|
+
items: int = 0
|
|
1364
|
+
|
|
1365
|
+
@Cart.apply(ItemAdded)
|
|
1366
|
+
def _(s: Cart, d: ItemAdded):
|
|
1367
|
+
s.items += d.count
|
|
1368
|
+
|
|
1369
|
+
`Cart.command`, `Cart.unique`, `Cart.fact`, `Cart.fold`, `Cart.handle` and `Cart.index` work
|
|
1370
|
+
as on `AggregateType` (what `domain.aggregate` returns, which keeps working with a dict)."""
|
|
1371
|
+
|
|
1372
|
+
__aggregate__: "ClassVar[AggregateType]"
|
|
1373
|
+
|
|
1374
|
+
def __init_subclass__(cls, *, domain: "Domain | None" = None, name: str | None = None, snapshot_every: int = 100,
|
|
1375
|
+
schema: str = "1", **kwargs: Any) -> None:
|
|
1376
|
+
super().__init_subclass__(**kwargs)
|
|
1377
|
+
if domain is None:
|
|
1378
|
+
return
|
|
1379
|
+
for field_name in list(cls.__dict__.get("__annotations__", {})):
|
|
1380
|
+
default = cls.__dict__.get(field_name)
|
|
1381
|
+
if isinstance(default, (list, dict, set)):
|
|
1382
|
+
setattr(cls, field_name, dataclasses.field(default_factory=lambda v=default: copy.deepcopy(v)))
|
|
1383
|
+
dataclasses.dataclass(cls)
|
|
1384
|
+
definition = domain.aggregate(name or cls.__name__.lower(), cls(), snapshot_every, schema)
|
|
1385
|
+
definition._to_json = lambda state: dataclasses.asdict(state) if isinstance(state, cls) else state
|
|
1386
|
+
names = {f.name for f in dataclasses.fields(cls)}
|
|
1387
|
+
definition._from_json = lambda data: cls(**{k: v for k, v in data.items() if k in names}) if isinstance(data, dict) else data
|
|
1388
|
+
cls.__aggregate__ = definition
|
|
1389
|
+
|
|
1390
|
+
def update(self, **values: Any) -> None:
|
|
1391
|
+
"""Sets several fields at once: `s.update(status="active", slug=d.slug)`."""
|
|
1392
|
+
for key, value in values.items():
|
|
1393
|
+
setattr(self, key, value)
|
|
1394
|
+
|
|
1395
|
+
@classmethod
|
|
1396
|
+
def apply(cls, event_type, fn: Callable | None = None):
|
|
1397
|
+
return cls.__aggregate__.apply(event_type, fn)
|
|
1398
|
+
|
|
1399
|
+
@classmethod
|
|
1400
|
+
def command(cls, command_type, fn: Callable | None = None, *, requires: Callable | None = None):
|
|
1401
|
+
return cls.__aggregate__.command(command_type, fn, requires=requires)
|
|
1402
|
+
|
|
1403
|
+
@classmethod
|
|
1404
|
+
def unique(cls, name: str, fn: Callable | None = None, *, scope: str = "tenant", personal: Callable | None = None):
|
|
1405
|
+
return cls.__aggregate__.unique(name, fn, scope=scope, personal=personal)
|
|
1406
|
+
|
|
1407
|
+
@classmethod
|
|
1408
|
+
def fact(cls, name: str, fn: Callable | None = None):
|
|
1409
|
+
return cls.__aggregate__.fact(name, fn)
|
|
1410
|
+
|
|
1411
|
+
@classmethod
|
|
1412
|
+
def fold(cls, history: list[tuple], state: Any = _INITIAL) -> Any:
|
|
1413
|
+
return cls.__aggregate__.fold(history, state)
|
|
1414
|
+
|
|
1415
|
+
@classmethod
|
|
1416
|
+
def handle(cls, state: Any, command_type: str, data: Any, ctx: "CommandContext", *, aggregate_id: str = "", tenant: str = "") -> list[dict]:
|
|
1417
|
+
return cls.__aggregate__.handle(state, command_type, data, ctx, aggregate_id=aggregate_id, tenant=tenant)
|
|
1418
|
+
|
|
1419
|
+
@classmethod
|
|
1420
|
+
def index(cls, state: Any) -> "Index":
|
|
1421
|
+
return cls.__aggregate__.index(state)
|
|
1422
|
+
|
|
1423
|
+
|
|
1424
|
+
def _register_wrapped(owner, table: dict, name: str, fn: Callable | None, wrap: Callable[[Callable], Callable]):
|
|
1425
|
+
"""`_register` for a message class: the handler is stored wrapped, the decorator returns it unwrapped."""
|
|
1426
|
+
if fn is not None:
|
|
1427
|
+
table[name] = wrap(fn)
|
|
1428
|
+
return owner
|
|
1429
|
+
|
|
1430
|
+
def decorator(f: Callable) -> Callable:
|
|
1431
|
+
table[name] = wrap(f)
|
|
1432
|
+
return f
|
|
1433
|
+
|
|
1434
|
+
return decorator
|
|
1435
|
+
|
|
1436
|
+
|
|
1437
|
+
def _register(owner, table: dict, name: str, fn: Callable | None):
|
|
1438
|
+
if fn is not None:
|
|
1439
|
+
table[name] = fn
|
|
1440
|
+
return owner
|
|
1441
|
+
|
|
1442
|
+
def decorator(f: Callable) -> Callable:
|
|
1443
|
+
table[name] = f
|
|
1444
|
+
return f
|
|
1445
|
+
|
|
1446
|
+
return decorator
|
|
1447
|
+
|
|
1448
|
+
|
|
1449
|
+
class _WithGrant(grpc.UnaryUnaryClientInterceptor, grpc.UnaryStreamClientInterceptor,
|
|
1450
|
+
grpc.StreamUnaryClientInterceptor, grpc.StreamStreamClientInterceptor):
|
|
1451
|
+
"""Sends the key's grant with every call, in the `kernable-grant` metadata entry."""
|
|
1452
|
+
|
|
1453
|
+
def __init__(self, grant: str):
|
|
1454
|
+
self._grant = grant
|
|
1455
|
+
|
|
1456
|
+
def _details(self, details):
|
|
1457
|
+
metadata = [*(details.metadata or []), ("kernable-grant", self._grant)]
|
|
1458
|
+
return details._replace(metadata=metadata)
|
|
1459
|
+
|
|
1460
|
+
def intercept_unary_unary(self, continuation, details, request):
|
|
1461
|
+
return continuation(self._details(details), request)
|
|
1462
|
+
|
|
1463
|
+
def intercept_unary_stream(self, continuation, details, request):
|
|
1464
|
+
return continuation(self._details(details), request)
|
|
1465
|
+
|
|
1466
|
+
def intercept_stream_unary(self, continuation, details, requests):
|
|
1467
|
+
return continuation(self._details(details), requests)
|
|
1468
|
+
|
|
1469
|
+
def intercept_stream_stream(self, continuation, details, requests):
|
|
1470
|
+
return continuation(self._details(details), requests)
|
|
1471
|
+
|
|
1472
|
+
|
|
1473
|
+
class _Open(grpc.UnaryUnaryClientInterceptor, grpc.UnaryStreamClientInterceptor,
|
|
1474
|
+
grpc.StreamUnaryClientInterceptor, grpc.StreamStreamClientInterceptor):
|
|
1475
|
+
"""Refuses every call once the client is closed: grpcio crashes the process on a call over a
|
|
1476
|
+
closed channel."""
|
|
1477
|
+
|
|
1478
|
+
def __init__(self):
|
|
1479
|
+
self.closed = False
|
|
1480
|
+
|
|
1481
|
+
def _check(self):
|
|
1482
|
+
if self.closed:
|
|
1483
|
+
raise ClientClosed()
|
|
1484
|
+
|
|
1485
|
+
def intercept_unary_unary(self, continuation, details, request):
|
|
1486
|
+
self._check()
|
|
1487
|
+
return continuation(details, request)
|
|
1488
|
+
|
|
1489
|
+
def intercept_unary_stream(self, continuation, details, request):
|
|
1490
|
+
self._check()
|
|
1491
|
+
return continuation(details, request)
|
|
1492
|
+
|
|
1493
|
+
def intercept_stream_unary(self, continuation, details, requests):
|
|
1494
|
+
self._check()
|
|
1495
|
+
return continuation(details, requests)
|
|
1496
|
+
|
|
1497
|
+
def intercept_stream_stream(self, continuation, details, requests):
|
|
1498
|
+
self._check()
|
|
1499
|
+
return continuation(details, requests)
|
|
1500
|
+
|
|
1501
|
+
|
|
1502
|
+
class _Deadline(grpc.UnaryUnaryClientInterceptor):
|
|
1503
|
+
"""Lets every request wait for the connection (e.g. while the kernel restarts) and gives it a
|
|
1504
|
+
deadline, unless the call has its own."""
|
|
1505
|
+
|
|
1506
|
+
def __init__(self, timeout_s: float):
|
|
1507
|
+
self._timeout_s = timeout_s
|
|
1508
|
+
|
|
1509
|
+
def intercept_unary_unary(self, continuation, details, request):
|
|
1510
|
+
timeout = self._timeout_s if details.timeout is None else details.timeout
|
|
1511
|
+
return continuation(details._replace(timeout=timeout, wait_for_ready=True), request)
|
|
1512
|
+
|
|
1513
|
+
|
|
1514
|
+
class _Failover(grpc.Channel):
|
|
1515
|
+
"""The channels to the kernels of a pair (`url` with several addresses): every call goes to
|
|
1516
|
+
the current one; on `UNAVAILABLE` (a standby answers `not_primary`, a kernel is gone or
|
|
1517
|
+
shutting down) the next one becomes current. A request tries the next address within its
|
|
1518
|
+
timeout, waiting at most a second for each to connect; a stream that ends with
|
|
1519
|
+
`UNAVAILABLE` reconnects to the next."""
|
|
1520
|
+
|
|
1521
|
+
_PER_ADDRESS_S = 1.0
|
|
1522
|
+
|
|
1523
|
+
def __init__(self, channels: list, raw: list, timeout_s: float):
|
|
1524
|
+
self._channels, self._raw, self._timeout_s = channels, raw, timeout_s
|
|
1525
|
+
self._current = 0
|
|
1526
|
+
self._lock = threading.Lock()
|
|
1527
|
+
|
|
1528
|
+
def current(self) -> int:
|
|
1529
|
+
return self._current
|
|
1530
|
+
|
|
1531
|
+
def next(self, from_: int) -> None:
|
|
1532
|
+
with self._lock:
|
|
1533
|
+
if self._current == from_:
|
|
1534
|
+
self._current = (from_ + 1) % len(self._channels)
|
|
1535
|
+
|
|
1536
|
+
def _unary(self, kind: str, method: str, *args, **kwargs):
|
|
1537
|
+
calls = [getattr(c, kind)(method, *args, **kwargs) for c in self._channels]
|
|
1538
|
+
failover = self
|
|
1539
|
+
|
|
1540
|
+
def attempt(how: str, request, timeout, kw):
|
|
1541
|
+
deadline = time.monotonic() + (failover._timeout_s if timeout is None else timeout)
|
|
1542
|
+
while True:
|
|
1543
|
+
at = failover._current
|
|
1544
|
+
left = deadline - time.monotonic()
|
|
1545
|
+
try:
|
|
1546
|
+
grpc.channel_ready_future(failover._raw[at]).result(
|
|
1547
|
+
timeout=max(0.0, min(left, failover._PER_ADDRESS_S)))
|
|
1548
|
+
except grpc.FutureTimeoutError:
|
|
1549
|
+
if time.monotonic() < deadline:
|
|
1550
|
+
failover.next(at)
|
|
1551
|
+
continue
|
|
1552
|
+
try:
|
|
1553
|
+
return getattr(calls[at], how)(request, timeout=max(0.001, deadline - time.monotonic()), **kw)
|
|
1554
|
+
except grpc.RpcError as e:
|
|
1555
|
+
if e.code() != grpc.StatusCode.UNAVAILABLE or time.monotonic() >= deadline:
|
|
1556
|
+
raise
|
|
1557
|
+
failover.next(at)
|
|
1558
|
+
|
|
1559
|
+
class _Call:
|
|
1560
|
+
def __call__(self, request, timeout=None, **kw):
|
|
1561
|
+
return attempt("__call__", request, timeout, kw)
|
|
1562
|
+
|
|
1563
|
+
def with_call(self, request, timeout=None, **kw):
|
|
1564
|
+
return attempt("with_call", request, timeout, kw)
|
|
1565
|
+
|
|
1566
|
+
def future(self, request, timeout=None, **kw):
|
|
1567
|
+
return calls[failover._current].future(request, timeout=timeout, **kw)
|
|
1568
|
+
|
|
1569
|
+
return _Call()
|
|
1570
|
+
|
|
1571
|
+
def _stream(self, kind: str, method: str, *args, **kwargs):
|
|
1572
|
+
calls = [getattr(c, kind)(method, *args, **kwargs) for c in self._channels]
|
|
1573
|
+
failover = self
|
|
1574
|
+
|
|
1575
|
+
class _Call:
|
|
1576
|
+
def __init__(self, at, inner):
|
|
1577
|
+
self._at, self._inner = at, inner
|
|
1578
|
+
|
|
1579
|
+
def __iter__(self):
|
|
1580
|
+
return self
|
|
1581
|
+
|
|
1582
|
+
def __next__(self):
|
|
1583
|
+
try:
|
|
1584
|
+
return next(self._inner)
|
|
1585
|
+
except grpc.RpcError as e:
|
|
1586
|
+
if e.code() == grpc.StatusCode.UNAVAILABLE:
|
|
1587
|
+
failover.next(self._at)
|
|
1588
|
+
raise
|
|
1589
|
+
|
|
1590
|
+
def __getattr__(self, name):
|
|
1591
|
+
return getattr(self._inner, name)
|
|
1592
|
+
|
|
1593
|
+
def call(request, *a, **kw):
|
|
1594
|
+
at = failover._current
|
|
1595
|
+
return _Call(at, calls[at](request, *a, **kw))
|
|
1596
|
+
|
|
1597
|
+
return call
|
|
1598
|
+
|
|
1599
|
+
def unary_unary(self, method, *args, **kwargs):
|
|
1600
|
+
return self._unary("unary_unary", method, *args, **kwargs)
|
|
1601
|
+
|
|
1602
|
+
def stream_unary(self, method, *args, **kwargs):
|
|
1603
|
+
return self._unary("stream_unary", method, *args, **kwargs)
|
|
1604
|
+
|
|
1605
|
+
def unary_stream(self, method, *args, **kwargs):
|
|
1606
|
+
return self._stream("unary_stream", method, *args, **kwargs)
|
|
1607
|
+
|
|
1608
|
+
def stream_stream(self, method, *args, **kwargs):
|
|
1609
|
+
return self._stream("stream_stream", method, *args, **kwargs)
|
|
1610
|
+
|
|
1611
|
+
def subscribe(self, callback, try_to_connect=False):
|
|
1612
|
+
self._raw[self._current].subscribe(callback, try_to_connect)
|
|
1613
|
+
|
|
1614
|
+
def unsubscribe(self, callback):
|
|
1615
|
+
for c in self._raw:
|
|
1616
|
+
c.unsubscribe(callback)
|
|
1617
|
+
|
|
1618
|
+
def close(self):
|
|
1619
|
+
for c in self._channels:
|
|
1620
|
+
c.close()
|
|
1621
|
+
|
|
1622
|
+
|
|
1623
|
+
def _local(url: str) -> bool:
|
|
1624
|
+
"""Whether `url` (every address of a comma-separated list) is a Unix socket or a loopback
|
|
1625
|
+
address: plain text never leaves the machine."""
|
|
1626
|
+
if "," in url:
|
|
1627
|
+
return all(_local(u.strip()) for u in url.split(","))
|
|
1628
|
+
if url.startswith("unix:"):
|
|
1629
|
+
return True
|
|
1630
|
+
host = re.sub(r":\d+/?$", "", re.sub(r"^https?://", "", url)).strip("[]")
|
|
1631
|
+
return host in ("localhost", "::1") or re.fullmatch(r"127\.\d+\.\d+\.\d+", host) is not None
|
|
1632
|
+
|
|
1633
|
+
|
|
1634
|
+
def _moving(err: grpc.RpcError) -> bool:
|
|
1635
|
+
"""Whether a kernel refused a request because its tenant is moving to another partition."""
|
|
1636
|
+
return err.code() == grpc.StatusCode.UNAVAILABLE and (err.details() or "").startswith("moving")
|
|
1637
|
+
|
|
1638
|
+
|
|
1639
|
+
# How long a request waits for its tenant's move to end.
|
|
1640
|
+
_MOVE_WAIT_S = 30.0
|
|
1641
|
+
|
|
1642
|
+
|
|
1643
|
+
def _switch(switch: str) -> str:
|
|
1644
|
+
"""`switch` if it is one of `caught-up`, `now` and `never`."""
|
|
1645
|
+
if switch not in ("caught-up", "now", "never"):
|
|
1646
|
+
raise ValueError(f"switch {switch!r}: caught-up, now or never")
|
|
1647
|
+
return switch
|
|
1648
|
+
|
|
1649
|
+
|
|
1650
|
+
def waits_for_switch(err: BaseException) -> bool:
|
|
1651
|
+
"""Whether the kernel refused a membership because it would switch its service to a
|
|
1652
|
+
deployment that has not caught up yet (`not_caught_up`), back to one that served before with
|
|
1653
|
+
`switch="caught-up"` (`served_before`), or at all with `switch="never"` (`not_serving`):
|
|
1654
|
+
applying it again may succeed later, once the deployment caught up or an operator switched
|
|
1655
|
+
to it."""
|
|
1656
|
+
return (isinstance(err, grpc.RpcError) and err.code() == grpc.StatusCode.FAILED_PRECONDITION
|
|
1657
|
+
and (err.details() or "").startswith(("not_caught_up", "not_serving", "served_before")))
|
|
1658
|
+
|
|
1659
|
+
# The first pause before an idempotent call asks a kernel that answered UNAVAILABLE again; it
|
|
1660
|
+
# doubles up to the second.
|
|
1661
|
+
_RETRY_PAUSE_S = 0.05
|
|
1662
|
+
_MAX_RETRY_PAUSE_S = 0.2
|
|
1663
|
+
# How long an idempotent request whose connection dropped under way is tried again: long enough
|
|
1664
|
+
# for a kernel handing its role over, short enough that a crash reaches the caller.
|
|
1665
|
+
_DROPPED_RETRY_S = 5.0
|
|
1666
|
+
_DROPPED = re.compile(r"connection dropped|socket closed|ECONNRESET|EPIPE|RST_STREAM|GOAWAY|connection reset",
|
|
1667
|
+
re.IGNORECASE)
|
|
1668
|
+
|
|
1669
|
+
|
|
1670
|
+
def _connection_dropped(err: grpc.RpcError) -> bool:
|
|
1671
|
+
"""Whether `err` is the client's own UNAVAILABLE for a connection that went away under the
|
|
1672
|
+
request, not a kernel's answer."""
|
|
1673
|
+
return bool(_DROPPED.search(err.details() or ""))
|
|
1674
|
+
|
|
1675
|
+
|
|
1676
|
+
def _idempotent(timeout_s: float, call: Callable[[float], Any]) -> Any:
|
|
1677
|
+
"""`call(left)` (a request with `left` seconds to go) for a call that is safe to repeat: a
|
|
1678
|
+
submit, by its command ID (the kernel answers a resubmission with the stored result); a
|
|
1679
|
+
cancel; a query; a data or cache read. A kernel that answers `UNAVAILABLE` (shutting down,
|
|
1680
|
+
handing over, a read model or middleware not connected yet, a connection that dropped) is
|
|
1681
|
+
asked again with backoff until `timeout_s` passed, at a single address too; then the last error is raised. A tenant
|
|
1682
|
+
that moves is waited for by `_Partitioned.route`."""
|
|
1683
|
+
deadline = time.monotonic() + timeout_s
|
|
1684
|
+
pause = _RETRY_PAUSE_S
|
|
1685
|
+
# Since when the request may have reached a kernel that then went away: from then on it is
|
|
1686
|
+
# tried again for a short while only (a handover), not until the deadline (a crash the
|
|
1687
|
+
# caller should hear of).
|
|
1688
|
+
dropped: "tuple[float, grpc.RpcError] | None" = None
|
|
1689
|
+
while True:
|
|
1690
|
+
until = deadline if dropped is None else min(deadline, dropped[0] + _DROPPED_RETRY_S)
|
|
1691
|
+
try:
|
|
1692
|
+
return call(max(0.001, until - time.monotonic()))
|
|
1693
|
+
except grpc.RpcError as e:
|
|
1694
|
+
if dropped is not None and until < deadline and time.monotonic() >= until:
|
|
1695
|
+
raise dropped[1] from e
|
|
1696
|
+
if (e.code() != grpc.StatusCode.UNAVAILABLE or _moving(e)
|
|
1697
|
+
or time.monotonic() + pause >= deadline):
|
|
1698
|
+
raise
|
|
1699
|
+
if _connection_dropped(e):
|
|
1700
|
+
if dropped is None:
|
|
1701
|
+
dropped = (time.monotonic(), e)
|
|
1702
|
+
elif time.monotonic() - dropped[0] >= _DROPPED_RETRY_S:
|
|
1703
|
+
raise
|
|
1704
|
+
time.sleep(pause)
|
|
1705
|
+
pause = min(pause * 2, _MAX_RETRY_PAUSE_S)
|
|
1706
|
+
|
|
1707
|
+
|
|
1708
|
+
def _partition_of(m, tenant: str) -> str:
|
|
1709
|
+
"""The partition map `m` assigns `tenant` to."""
|
|
1710
|
+
return next((t.partition for t in m.tenants if t.tenant == tenant), m.default)
|
|
1711
|
+
|
|
1712
|
+
|
|
1713
|
+
def _wrong_partition(err: grpc.RpcError) -> bool:
|
|
1714
|
+
"""Whether a kernel refused a request as one for a tenant of another partition."""
|
|
1715
|
+
return err.code() == grpc.StatusCode.FAILED_PRECONDITION and (err.details() or "").startswith("wrong_partition")
|
|
1716
|
+
|
|
1717
|
+
|
|
1718
|
+
class _Partitioned:
|
|
1719
|
+
"""The channels of every partition, found by the map the first kernel holds: a request
|
|
1720
|
+
goes to its tenant's partition; refused there as `wrong_partition` (the map changed), the
|
|
1721
|
+
map is read again and the request sent anew. A kernel without partitions holds no map:
|
|
1722
|
+
everything goes to the first kernel."""
|
|
1723
|
+
|
|
1724
|
+
_UNREAD = object()
|
|
1725
|
+
|
|
1726
|
+
def __init__(self, entry: grpc.Channel, channel_for: Callable[[str], grpc.Channel], timeout_s: float):
|
|
1727
|
+
self.entry = entry
|
|
1728
|
+
self._timeout_s = timeout_s
|
|
1729
|
+
self._channel_for = channel_for
|
|
1730
|
+
self._map: Any = self._UNREAD
|
|
1731
|
+
self._parts: dict[str, grpc.Channel] = {}
|
|
1732
|
+
self._lock = threading.Lock()
|
|
1733
|
+
|
|
1734
|
+
def current(self, refresh: bool = False, timeout_s: float | None = None):
|
|
1735
|
+
"""The map (a `PartitionMap`), read once and again after `refresh`; `None` without
|
|
1736
|
+
partitions. `timeout_s`: how long to wait for the kernel (default: the client's)."""
|
|
1737
|
+
with self._lock:
|
|
1738
|
+
if refresh:
|
|
1739
|
+
self._map = self._UNREAD
|
|
1740
|
+
if self._map is self._UNREAD:
|
|
1741
|
+
try:
|
|
1742
|
+
self._map = _idempotent(timeout_s or self._timeout_s, lambda left: rpc.PartitionServiceStub(
|
|
1743
|
+
self.entry).Map(pb.MapRequest(protocol=PROTOCOL), timeout=left))
|
|
1744
|
+
except grpc.RpcError as e:
|
|
1745
|
+
if e.code() not in (grpc.StatusCode.NOT_FOUND, grpc.StatusCode.UNIMPLEMENTED):
|
|
1746
|
+
raise
|
|
1747
|
+
self._map = None
|
|
1748
|
+
return self._map
|
|
1749
|
+
|
|
1750
|
+
def _channel(self, m, name: str) -> grpc.Channel:
|
|
1751
|
+
partition = next((p for p in m.partitions if p.name == name), None)
|
|
1752
|
+
if partition is None:
|
|
1753
|
+
raise ValueError(f"the partition map names no partition {name}")
|
|
1754
|
+
# `relay:<relay>/<copy>` is how partitions at other sites reach this one: not a client's way.
|
|
1755
|
+
own = [a for a in partition.addresses if not a.startswith("relay:")]
|
|
1756
|
+
if not own:
|
|
1757
|
+
raise ValueError(f"partition {name} is reached only through a relay: a client needs one of its own addresses")
|
|
1758
|
+
addresses = ",".join(own)
|
|
1759
|
+
with self._lock:
|
|
1760
|
+
if addresses not in self._parts:
|
|
1761
|
+
self._parts[addresses] = self._channel_for(addresses)
|
|
1762
|
+
return self._parts[addresses]
|
|
1763
|
+
|
|
1764
|
+
def for_tenant(self, tenant: str) -> grpc.Channel:
|
|
1765
|
+
m = self.current()
|
|
1766
|
+
if m is None:
|
|
1767
|
+
return self.entry
|
|
1768
|
+
return self._channel(m, _partition_of(m, tenant))
|
|
1769
|
+
|
|
1770
|
+
def control(self) -> grpc.Channel:
|
|
1771
|
+
"""The control partition's channel, which takes memberships."""
|
|
1772
|
+
m = self.current()
|
|
1773
|
+
return self.entry if m is None else self._channel(m, m.control)
|
|
1774
|
+
|
|
1775
|
+
def all(self, timeout_s: float | None = None) -> "list[tuple[str, grpc.Channel]]":
|
|
1776
|
+
"""Every partition with its channel; one, named `""`, without partitions."""
|
|
1777
|
+
m = self.current(timeout_s=timeout_s)
|
|
1778
|
+
if m is None:
|
|
1779
|
+
return [("", self.entry)]
|
|
1780
|
+
return [(p.name, self._channel(m, p.name)) for p in m.partitions]
|
|
1781
|
+
|
|
1782
|
+
def route(self, tenant: str, call: Callable[[grpc.Channel], Any]) -> Any:
|
|
1783
|
+
"""`call` on `tenant`'s partition, sent anew after reading the map again on
|
|
1784
|
+
`wrong_partition`, and while the tenant moves (`moving`, refused before it took effect)
|
|
1785
|
+
for up to 30 s."""
|
|
1786
|
+
deadline = time.monotonic() + _MOVE_WAIT_S
|
|
1787
|
+
pause = 0.05
|
|
1788
|
+
attempt = 0
|
|
1789
|
+
while True:
|
|
1790
|
+
try:
|
|
1791
|
+
return call(self.for_tenant(tenant))
|
|
1792
|
+
except grpc.RpcError as e:
|
|
1793
|
+
if _moving(e) and time.monotonic() + pause < deadline:
|
|
1794
|
+
time.sleep(pause)
|
|
1795
|
+
pause = min(pause * 2, 1.0)
|
|
1796
|
+
continue
|
|
1797
|
+
if attempt == 3 or not _wrong_partition(e):
|
|
1798
|
+
raise
|
|
1799
|
+
attempt += 1
|
|
1800
|
+
self.current(refresh=True)
|
|
1801
|
+
|
|
1802
|
+
def close(self) -> None:
|
|
1803
|
+
self.entry.close()
|
|
1804
|
+
for c in self._parts.values():
|
|
1805
|
+
c.close()
|
|
1806
|
+
|
|
1807
|
+
|
|
1808
|
+
class Kernable:
|
|
1809
|
+
"""A connection to the kernel: submits commands (`submit`, `enqueue`), asks queries
|
|
1810
|
+
(`query`), runs workers, listeners and apps, and applies memberships. All calls block the
|
|
1811
|
+
calling thread; one client may be shared by threads.
|
|
1812
|
+
|
|
1813
|
+
`k.data` (a `DataClient`) holds files of a domain, `k.cache` (a `CacheClient`) its shared
|
|
1814
|
+
cache. `Kernable()` without arguments takes everything from the environment or the
|
|
1815
|
+
defaults, which fit a kernel started with `kernable start --dev` in the current directory."""
|
|
1816
|
+
|
|
1817
|
+
def __init__(self, url: str | None = None, key: str | None = None, tenant: str = "default",
|
|
1818
|
+
tls: Tls | None = None, context: RequestContext | None = None,
|
|
1819
|
+
grant: str | None = None, service: str | None = None, insecure: bool | None = None,
|
|
1820
|
+
timeout_s: float = DEFAULT_TIMEOUT_S, deployment: str | None = None):
|
|
1821
|
+
"""A client for the kernel at `url`, signing with the key in the file `key`. It
|
|
1822
|
+
connects on first use and again after the connection drops.
|
|
1823
|
+
|
|
1824
|
+
`url`: `http://host:port` (`host:port` works too) or, on Unix, `unix:/path/to.sock`;
|
|
1825
|
+
for a pair of kernels with a standby both, comma-separated (`http://a:7233,http://b:7233`):
|
|
1826
|
+
the client turns to the other one when a kernel answers `UNAVAILABLE` (a standby says
|
|
1827
|
+
`not_primary`) or cannot be reached. Default `KERNABLE_URL`, else
|
|
1828
|
+
`http://127.0.0.1:7233`, where `kernable start` listens.
|
|
1829
|
+
`key`: path to this process's `.key` file; default `KERNABLE_KEY`, else `app.key` in the
|
|
1830
|
+
current directory. `tenant`: default tenant of submits, queries, files and the cache
|
|
1831
|
+
(default `default`). `tls`: mutual TLS for `host:port`; default `KERNABLE_TLS_CA`,
|
|
1832
|
+
`KERNABLE_TLS_CERT` and `KERNABLE_TLS_KEY`, if set (all three or none). `context`:
|
|
1833
|
+
default request context for submits, queries, files and the cache, e.g. this service's
|
|
1834
|
+
identity, replaced by a request's own. `grant`: path to the key's grant (`kernable grant`), sent with every
|
|
1835
|
+
call; default `KERNABLE_GRANT`, else the `.grant` file next to the key file, if there is
|
|
1836
|
+
one. `service`: the service this process's workers and listeners belong to. The
|
|
1837
|
+
service's membership (see `apply`) decides what the kernel waits for: what it lists
|
|
1838
|
+
counts, the service's other workers and listeners are dynamic, and only listeners it lists
|
|
1839
|
+
as middlewares may check requests. Default `KERNABLE_SERVICE`, else none (everything
|
|
1840
|
+
registered counts). `insecure`: allows plain text to a kernel that is not on this machine;
|
|
1841
|
+
without TLS, grants, signed requests and vault values would cross the network readable,
|
|
1842
|
+
so such a URL raises `ValueError` unless this is set; default `KERNABLE_INSECURE=1`, else
|
|
1843
|
+
False. `timeout_s`: how long a request (submit, query, files, cache, apply) waits in all,
|
|
1844
|
+
for the connection (e.g. while the kernel restarts) and its answer; default 60 s. A
|
|
1845
|
+
connection the network dropped silently is noticed within 40 s by keepalive pings (within
|
|
1846
|
+
4 s with several kernels, a pair, whose client then turns to the other). A
|
|
1847
|
+
submit (by its command ID), a cancel or a query that a kernel answers `UNAVAILABLE`
|
|
1848
|
+
(shutting down, handing over, a middleware or read model not connected) is asked again
|
|
1849
|
+
with backoff within this time; any other request may have run, and fails at once.
|
|
1850
|
+
`deployment`: the code version this process runs (1 to 128 letters, digits, `-`, `_` and
|
|
1851
|
+
`.`, e.g. `2026-11-11.1`); default `KERNABLE_DEPLOYMENT`, else none. Its workers and
|
|
1852
|
+
listeners declare it, the events its workers write carry it, its queries prefer read
|
|
1853
|
+
models of it, and `apply` names it in the membership: the deployment a membership names
|
|
1854
|
+
runs the service's commands and answers its queries, while the others stay connected
|
|
1855
|
+
for a rollback.
|
|
1856
|
+
|
|
1857
|
+
Workers and listeners print connection lines to stderr up to the level `KERNABLE_LOG`
|
|
1858
|
+
names (`off`, `error`, `warn`, `info` (default), `debug`). Raises `OSError` if the key
|
|
1859
|
+
or grant file cannot be read, `ValueError` if only some of the TLS variables are set or
|
|
1860
|
+
`deployment` is invalid."""
|
|
1861
|
+
self._deployment = _deployment(deployment or os.environ.get("KERNABLE_DEPLOYMENT") or "")
|
|
1862
|
+
key = key or os.environ.get("KERNABLE_KEY") or "app.key"
|
|
1863
|
+
tls = tls or _tls_from_env()
|
|
1864
|
+
self._context = context or RequestContext()
|
|
1865
|
+
url = url or os.environ.get("KERNABLE_URL") or "http://127.0.0.1:7233"
|
|
1866
|
+
given_url = url
|
|
1867
|
+
url = re.sub(r"^https?://", "", url).rstrip("/")
|
|
1868
|
+
if insecure is None:
|
|
1869
|
+
insecure = os.environ.get("KERNABLE_INSECURE") == "1"
|
|
1870
|
+
if not tls and not insecure and not _local(url):
|
|
1871
|
+
raise ValueError(f"{given_url} is not on this machine: connect with TLS (tls=...), or pass "
|
|
1872
|
+
"insecure=True to send in plain text")
|
|
1873
|
+
def open_channel(url: str, several: bool = False):
|
|
1874
|
+
options = _PAIR_CHANNEL_OPTIONS if several else _CHANNEL_OPTIONS
|
|
1875
|
+
if tls:
|
|
1876
|
+
def read(path: str) -> bytes:
|
|
1877
|
+
with open(path, "rb") as f:
|
|
1878
|
+
return f.read()
|
|
1879
|
+
|
|
1880
|
+
creds = grpc.ssl_channel_credentials(read(tls.ca), read(tls.key), read(tls.cert))
|
|
1881
|
+
return grpc.secure_channel(url, creds, options=options)
|
|
1882
|
+
# grpcio sends the socket path as :authority for unix: URLs, which the kernel's
|
|
1883
|
+
# HTTP/2 server rejects; any valid authority works.
|
|
1884
|
+
return grpc.insecure_channel(url, options=[("grpc.default_authority", "localhost"),
|
|
1885
|
+
*options])
|
|
1886
|
+
|
|
1887
|
+
grant = grant or os.environ.get("KERNABLE_GRANT")
|
|
1888
|
+
if grant is None and os.path.exists(default := key.removesuffix(".key") + ".grant"):
|
|
1889
|
+
grant = default
|
|
1890
|
+
granted = None
|
|
1891
|
+
if grant is not None:
|
|
1892
|
+
with open(grant) as f:
|
|
1893
|
+
granted = f.read().strip()
|
|
1894
|
+
# The outermost interceptor of every channel: checked before anything touches it.
|
|
1895
|
+
self._open = _Open()
|
|
1896
|
+
|
|
1897
|
+
def channel_for(addresses: str) -> grpc.Channel:
|
|
1898
|
+
"""A channel to the kernels at `addresses`, comma-separated (a pair: see `_Failover`)."""
|
|
1899
|
+
urls = [re.sub(r"^https?://", "", u.strip()).rstrip("/") for u in addresses.split(",") if u.strip()]
|
|
1900
|
+
raw = [open_channel(u, len(urls) > 1) for u in urls]
|
|
1901
|
+
channels = []
|
|
1902
|
+
for c in raw:
|
|
1903
|
+
if granted is not None:
|
|
1904
|
+
c = grpc.intercept_channel(c, _WithGrant(granted))
|
|
1905
|
+
channels.append(grpc.intercept_channel(c, _Deadline(timeout_s)))
|
|
1906
|
+
channel = channels[0] if len(channels) == 1 else _Failover(channels, raw, timeout_s)
|
|
1907
|
+
return grpc.intercept_channel(channel, self._open)
|
|
1908
|
+
|
|
1909
|
+
self._timeout_s = timeout_s
|
|
1910
|
+
self._channel = channel_for(url)
|
|
1911
|
+
# With partitions, each tenant's requests go to its partition.
|
|
1912
|
+
self._parts = _Partitioned(self._channel, channel_for, timeout_s)
|
|
1913
|
+
# The newest position this client's own commands committed at, per partition's channel:
|
|
1914
|
+
# its queries wait for it, on whichever copy (writer or reader) answers.
|
|
1915
|
+
self._seen: dict[int, int] = {}
|
|
1916
|
+
self._key = Key.from_file(key)
|
|
1917
|
+
self._log = _Log(given_url, key, self._key.id)
|
|
1918
|
+
# Whoever can read the key file can sign as this process.
|
|
1919
|
+
mode = os.stat(key).st_mode & 0o777
|
|
1920
|
+
if os.name == "posix" and mode & 0o077:
|
|
1921
|
+
self._log.print("warn", "key", f"{key} can be read by other users (mode {mode:o}): chmod 600 {key}")
|
|
1922
|
+
self._tenant = tenant
|
|
1923
|
+
self._service = service or os.environ.get("KERNABLE_SERVICE") or ""
|
|
1924
|
+
self._domains: dict[str, Domain] = {}
|
|
1925
|
+
#: Files of a domain: `k.data.put(domain, key, data)`, `get`, `delete`.
|
|
1926
|
+
self.data = DataClient(self)
|
|
1927
|
+
#: A domain's shared cache: `k.cache.set(domain, key, value, ttl_ms)`, `get`, `delete`.
|
|
1928
|
+
self.cache = CacheClient(self)
|
|
1929
|
+
|
|
1930
|
+
def use(self, *domains: Domain) -> "Kernable":
|
|
1931
|
+
"""Seals the vault fields of these domains' commands when submitting them (see
|
|
1932
|
+
`Domain.command`), and sends the targets of their queries given by name (see
|
|
1933
|
+
`Domain.query`). Returns this client, so it chains: `k = Kernable().use(shop_domain)`."""
|
|
1934
|
+
for d in domains:
|
|
1935
|
+
self._domains[d.name] = d
|
|
1936
|
+
return self
|
|
1937
|
+
|
|
1938
|
+
def worker(self, *aggregates: "AggregateType | type[Aggregate]") -> "Worker":
|
|
1939
|
+
"""A worker that runs these aggregates' commands, in this client's service; see
|
|
1940
|
+
`Worker`. `run()` on it blocks until `stop()`: `k.worker(cart_aggregate).run()`."""
|
|
1941
|
+
aggregates = tuple(getattr(a, "__aggregate__", a) for a in aggregates)
|
|
1942
|
+
return Worker._new(self._parts, self._key, aggregates, self._log, self._service, self._deployment)
|
|
1943
|
+
|
|
1944
|
+
def app(self, *, domains: Iterable[Domain], service: str | None = None, version: str | None = None,
|
|
1945
|
+
description: str | None = None, metadata: dict[str, str] | None = None,
|
|
1946
|
+
switch: str | None = None) -> "App":
|
|
1947
|
+
"""An app made of whole domains: every aggregate, read model and reactor defined on them.
|
|
1948
|
+
`run()` on it applies the service's membership (the domains' aggregates as workers, their
|
|
1949
|
+
read models and reactors as listeners, those with a middleware as middlewares, their
|
|
1950
|
+
processes, and this client's deployment; live listeners are not listed), then runs all of them, and registers the
|
|
1951
|
+
domains' processes once its worker is connected (see `register_process`). `service` defaults to this client's; `version`
|
|
1952
|
+
and `description` go into the membership's metadata, followed by `metadata`.
|
|
1953
|
+
|
|
1954
|
+
Example: `k.app(service="shop", version="1.0.0", domains=[shop_domain]).run()`. Every
|
|
1955
|
+
start applies the membership, so the key needs the `manage` grant for the service. One
|
|
1956
|
+
service is one app: two processes with the same service would replace each other's
|
|
1957
|
+
membership. `switch` says when the app's deployment takes the service over from another
|
|
1958
|
+
one that serves it (see `apply`): by default (`"caught-up"`) once its read models have
|
|
1959
|
+
caught up, if the deployment never served before; meanwhile it runs beside the serving
|
|
1960
|
+
deployment, its workers idle and its read models building. One that served before (an
|
|
1961
|
+
older deployment's app restarting) runs beside until an operator rolls back. Default: `KERNABLE_SWITCH`, else `"caught-up"`. Raises `ValueError` if
|
|
1962
|
+
neither `service` nor the client names a service, or for an unknown `switch`."""
|
|
1963
|
+
switch = _switch(switch if switch is not None else os.environ.get("KERNABLE_SWITCH") or "caught-up")
|
|
1964
|
+
service = service or self._service
|
|
1965
|
+
if not service:
|
|
1966
|
+
raise ValueError("an app needs a service: pass `service` or set it on the client (KERNABLE_SERVICE)")
|
|
1967
|
+
domains = list(domains)
|
|
1968
|
+
aggregates = [a for d in domains for a in d.aggregates]
|
|
1969
|
+
listeners = [l for d in domains for l in d.listeners]
|
|
1970
|
+
durable = [l for l in listeners if not l.live]
|
|
1971
|
+
meta = {k: v for k, v in (("version", version), ("description", description)) if v is not None}
|
|
1972
|
+
manifest = Manifest(service=service, workers=aggregates,
|
|
1973
|
+
listeners=[l for l in durable if not l._middlewares],
|
|
1974
|
+
middlewares=[l for l in durable if l._middlewares],
|
|
1975
|
+
metadata={**meta, **(metadata or {})},
|
|
1976
|
+
processes=[p for d in domains for p in d.processes],
|
|
1977
|
+
deployment=self._deployment, switch=switch)
|
|
1978
|
+
worker = Worker._new(self._parts, self._key, aggregates, self._log, service, self._deployment) if aggregates else None
|
|
1979
|
+
handles = [ListenerHandle._new(self._parts, self._key, l, self._log, service, self) for l in listeners]
|
|
1980
|
+
return App(self, manifest, worker, handles, [d.name for d in domains])
|
|
1981
|
+
|
|
1982
|
+
def apply(self, *, service: str | None = None, workers: Iterable[AggregateType] = (),
|
|
1983
|
+
listeners: Iterable["Listener"] = (), middlewares: Iterable["Listener"] = (),
|
|
1984
|
+
metadata: dict[str, str] | None = None, processes: Iterable["Process | str"] = (),
|
|
1985
|
+
deployment: str | None = None, switch: str = "caught-up") -> None:
|
|
1986
|
+
"""Replaces the membership of a service (default: this client's `service`): the workers,
|
|
1987
|
+
listeners and middlewares the kernel waits for, plus free metadata such as
|
|
1988
|
+
`{"version": "1.0.0"}`. `workers` are the aggregates the service runs (the kernel waits
|
|
1989
|
+
for a worker of each), `listeners` the listeners it waits for (the service's others count
|
|
1990
|
+
for nobody), `middlewares` the service's listeners that may check requests (they count like
|
|
1991
|
+
listeners), `processes` its business processes (`Process`es or qualified names): one it no
|
|
1992
|
+
longer lists starts no new cases, its running cases finish. `deployment` (default: this
|
|
1993
|
+
client's) names the code version that serves the service: its workers run the commands,
|
|
1994
|
+
its listeners count and its read models answer queries first; the service's workers and
|
|
1995
|
+
listeners of other deployments stay connected, idle or catching up. A rollback applies a
|
|
1996
|
+
membership naming the earlier deployment. Empty: every deployment serves. `switch` says
|
|
1997
|
+
when a membership that names another deployment than the serving one switches to it:
|
|
1998
|
+
`"caught-up"` (default) once that deployment has caught up (its listed workers connected,
|
|
1999
|
+
its read models within the kernel's `--deployment-switch-lag` of the newest event),
|
|
2000
|
+
refused before as `not_caught_up`, and never back to a deployment that served before (it
|
|
2001
|
+
wrote events), refused as `served_before`: switching back is a rollback, which an
|
|
2002
|
+
operator does (`kernable apply --deployment`); `"now"` at once;
|
|
2003
|
+
`"never"`: applied only while its deployment serves already (or the service has no
|
|
2004
|
+
membership yet), refused otherwise as `not_serving`. See `waits_for_switch`.
|
|
2005
|
+
|
|
2006
|
+
Apply it once per deploy, e.g. from a deploy script, not from every process: the newest
|
|
2007
|
+
apply wins, and an older one that arrives late is refused. The key needs the `manage` grant for the service. A middleware the new
|
|
2008
|
+
membership no longer lists stops checking. Example: `k.apply(service="shop",
|
|
2009
|
+
workers=[cart_aggregate], listeners=[carts_readmodel], metadata={"version": "1.0.0"})`.
|
|
2010
|
+
Raises `ValueError` without a service or for an invalid `deployment`, `grpc.RpcError` if
|
|
2011
|
+
the kernel refuses it."""
|
|
2012
|
+
service = service or self._service
|
|
2013
|
+
deployment = self._deployment if deployment is None else _deployment(deployment)
|
|
2014
|
+
if not service:
|
|
2015
|
+
raise ValueError("apply needs a service: pass `service` or set it on the client (KERNABLE_SERVICE)")
|
|
2016
|
+
membership = pb.Membership(service=service, workers=[f"{a.domain}.{a.name}" for a in workers],
|
|
2017
|
+
listeners=[l.name for l in listeners],
|
|
2018
|
+
middlewares=[l.name for l in middlewares],
|
|
2019
|
+
metadata=[pb.Attribute(key=k, value=v) for k, v in (metadata or {}).items()],
|
|
2020
|
+
issued_at=_issued_now(),
|
|
2021
|
+
processes=[p.qualified if isinstance(p, Process) else p for p in processes],
|
|
2022
|
+
deployment=deployment)
|
|
2023
|
+
switch = _switch(switch)
|
|
2024
|
+
request = pb.ApplyRequest(membership=membership, sig=self._key.sign(membership_message(membership)),
|
|
2025
|
+
protocol=PROTOCOL, force=switch == "now", no_switch=switch == "never",
|
|
2026
|
+
no_rollback=switch == "caught-up")
|
|
2027
|
+
# The same membership again is fine (the kernel takes a retried deploy): a dropped
|
|
2028
|
+
# connection, or a kernel starting, is asked again.
|
|
2029
|
+
_idempotent(self._timeout_s, lambda left: rpc.MembershipServiceStub(self._parts.control()).Apply(
|
|
2030
|
+
request, timeout=left))
|
|
2031
|
+
|
|
2032
|
+
def process_act(self, name: str, context: RequestContext) -> bool:
|
|
2033
|
+
"""Sets as whom the process `name` (qualified: `shop.fulfilment`) acts: the request context
|
|
2034
|
+
its commands and queries carry (sender tenant, identity, session), signed with this
|
|
2035
|
+
client's key, which needs the grant `process` for it. Without it, a process acts as
|
|
2036
|
+
`process:<name>`. Returns whether it changed (False: it acted so already). The kit's
|
|
2037
|
+
identity reactor calls it for every process it gives an identity. Raises `grpc.RpcError`
|
|
2038
|
+
if the kernel refuses it (`NOT_FOUND` for a process never registered)."""
|
|
2039
|
+
req = pb.ProcessActing(name=name, context=_to_wire_context(context, ""), issued_at=_issued_now(), protocol=PROTOCOL)
|
|
2040
|
+
req.sig.CopyFrom(self._key.sign(process_act_message(req.name, req.context, req.issued_at)))
|
|
2041
|
+
return rpc.ProcessServiceStub(self._parts.control()).Act(req).changed
|
|
2042
|
+
|
|
2043
|
+
def register_process(self, process: "Process | dict") -> RegisteredProcess:
|
|
2044
|
+
"""Registers a business process (a `Process`, or its model as a dict), signed with this
|
|
2045
|
+
client's key: a model equal to the newest version registers nothing (`new` is False),
|
|
2046
|
+
a different one becomes the next version; running cases stay on theirs. `Kernable.app`
|
|
2047
|
+
does this for its domains' processes. The key needs the grant `process` for the
|
|
2048
|
+
process, and `submit`, `query` and `listen` for the commands, queries and events the
|
|
2049
|
+
model names. Raises `ValueError` if the builder finds the model invalid (see
|
|
2050
|
+
`Process.model`), `grpc.RpcError` if the kernel refuses it (`INVALID_ARGUMENT` names the
|
|
2051
|
+
step and why)."""
|
|
2052
|
+
model = process.model() if isinstance(process, Process) else process
|
|
2053
|
+
raw = json.dumps(model, separators=(",", ":"), ensure_ascii=False).encode()
|
|
2054
|
+
req = pb.ProcessRegistration(model=raw, issued_at=_issued_now(), protocol=PROTOCOL)
|
|
2055
|
+
req.sig.CopyFrom(self._key.sign(process_message(req.model, req.issued_at)))
|
|
2056
|
+
res = rpc.ProcessServiceStub(self._parts.control()).Register(req)
|
|
2057
|
+
return RegisteredProcess(name=res.name, version=res.version, new=res.new)
|
|
2058
|
+
|
|
2059
|
+
def _command_of(self, command: Command) -> tuple:
|
|
2060
|
+
"""Domain, aggregate type, type and payload of a command class instance; its domain's vault fields known."""
|
|
2061
|
+
b = _binding(type(command), "handled by an aggregate")
|
|
2062
|
+
self._domains.setdefault(b.domain, b.schemas)
|
|
2063
|
+
return b.domain, b.aggregate_type, type(command).type_name(), command.to_wire()
|
|
2064
|
+
|
|
2065
|
+
def submit(self, domain, aggregate_type: str | None = None, aggregate_id: str | None = None, type: str | None = None, data: Any = None, *,
|
|
2066
|
+
tenant: str | None = None, id: str | None = None, wait: str = "settled",
|
|
2067
|
+
context: RequestContext | None = None, timeout_s: float | None = None, at: Any = None) -> Outcome:
|
|
2068
|
+
"""Runs a `type` command of `domain` on the aggregate `aggregate_type` with ID
|
|
2069
|
+
`aggregate_id` and returns its `Outcome`. `data` is the command's payload (JSON-like; `None` sends `{}`).
|
|
2070
|
+
|
|
2071
|
+
`wait="settled"` (default) responds once every listener that counts has processed the
|
|
2072
|
+
events, so read models already show them, and raises `NotSettled` on timeout; from a
|
|
2073
|
+
listener's handler, only the listeners with a lower order count. `wait="committed"`
|
|
2074
|
+
responds once the events are stored; then query with `min_position`. `id` is an
|
|
2075
|
+
idempotency key (default: a new UUID): submitting the same `id` again returns the
|
|
2076
|
+
original outcome. `tenant` replaces the client's tenant. `context` replaces the client's
|
|
2077
|
+
default context; inside a listener, pass `ctx.follow()`. `timeout_s` replaces the
|
|
2078
|
+
client's deadline for this call.
|
|
2079
|
+
|
|
2080
|
+
A rejected command returns `Outcome(committed=False, code=..., message=...)`; it does not
|
|
2081
|
+
raise. Raises `grpc.RpcError` if the kernel refuses the call (no grant, unknown aggregate
|
|
2082
|
+
type, timeout). Example: `k.submit("shop", "cart", "c1", "AddItem", {"sku": "apple",
|
|
2083
|
+
"qty": 2})`.
|
|
2084
|
+
|
|
2085
|
+
With a command class instance: `k.submit(AddItem(sku="apple", qty=2), aggregate_id="c1")`;
|
|
2086
|
+
its class says the domain, aggregate type, type and vault fields (no `use` needed).
|
|
2087
|
+
|
|
2088
|
+
A scheduled command has no outcome yet: schedule it with `enqueue(..., at=...)` (`at`
|
|
2089
|
+
here raises `ValueError`). Submitted again by its `id`, it waits for the scheduled
|
|
2090
|
+
command's outcome, as long as the kernel holds the call; raises `NotDue` if the kernel
|
|
2091
|
+
answers that it is not due yet.
|
|
2092
|
+
"""
|
|
2093
|
+
if at is not None:
|
|
2094
|
+
raise ValueError("a scheduled command (`at`) has no outcome yet: use `enqueue`")
|
|
2095
|
+
if isinstance(domain, Command):
|
|
2096
|
+
domain, aggregate_type, type, data = self._command_of(domain)
|
|
2097
|
+
mode = pb.WAIT_COMMITTED if wait == "committed" else pb.WAIT_SETTLED
|
|
2098
|
+
res = self._send(domain, aggregate_type, aggregate_id, type, data, tenant, id, mode, context, timeout_s)
|
|
2099
|
+
if res.WhichOneof("outcome") == "rejected":
|
|
2100
|
+
return _rejected_outcome(res.rejected)
|
|
2101
|
+
if res.WhichOneof("outcome") != "committed":
|
|
2102
|
+
raise NotDue(res.command_id)
|
|
2103
|
+
outcome = Outcome(committed=True, events=[_to_event(e) for e in res.committed.events])
|
|
2104
|
+
if mode == pb.WAIT_SETTLED and not res.committed.settled:
|
|
2105
|
+
raise NotSettled(outcome)
|
|
2106
|
+
return outcome
|
|
2107
|
+
|
|
2108
|
+
def enqueue(self, domain, aggregate_type: str | None = None, aggregate_id: str | None = None, type: str | None = None, data: Any = None, *,
|
|
2109
|
+
tenant: str | None = None, id: str | None = None, context: RequestContext | None = None,
|
|
2110
|
+
timeout_s: float | None = None, at: "datetime | int | None" = None,
|
|
2111
|
+
until: "datetime | int | None" = None) -> str:
|
|
2112
|
+
"""Stores the command in the kernel's queue and returns its ID without waiting for it to
|
|
2113
|
+
run (`wait="accepted"`). Submit it again with the same `id` to get the outcome. The
|
|
2114
|
+
parameters are those of `submit`.
|
|
2115
|
+
|
|
2116
|
+
With `at` (a `datetime`, or ms since epoch), a scheduled command: it runs at that time and
|
|
2117
|
+
not before, e.g. a reminder in 24 hours, `k.enqueue(Remind(), aggregate_id="o1",
|
|
2118
|
+
at=datetime.now(timezone.utc) + timedelta(days=1))`. The kernel stores it at once; the
|
|
2119
|
+
aggregate's other commands pass meanwhile, and middlewares check it now and again when it
|
|
2120
|
+
is due. `until`: if it cannot run by then (after an outage), it is rejected `expired`.
|
|
2121
|
+
Both are signed with the command. Cancel it with `cancel`."""
|
|
2122
|
+
if until is not None and at is None:
|
|
2123
|
+
raise ValueError("`until` needs `at`")
|
|
2124
|
+
if isinstance(domain, Command):
|
|
2125
|
+
domain, aggregate_type, type, data = self._command_of(domain)
|
|
2126
|
+
return self._send(domain, aggregate_type, aggregate_id, type, data, tenant, id, pb.WAIT_ACCEPTED,
|
|
2127
|
+
context, timeout_s, _ms(at), _ms(until)).command_id
|
|
2128
|
+
|
|
2129
|
+
def cancel(self, command_id: str, *, tenant: str | None = None, context: RequestContext | None = None,
|
|
2130
|
+
timeout_s: float | None = None) -> CancelOutcome:
|
|
2131
|
+
"""Cancels a scheduled command (`enqueue` with `at`) before it is due: it completes as
|
|
2132
|
+
rejected `cancelled`, and cancelling it again answers the same. Needs `submit` for the
|
|
2133
|
+
command's type; middlewares check it as a cancel of that type (`Check.kind == "cancel"`).
|
|
2134
|
+
Returns a `CancelOutcome`: not cancelled with `too_late` (due, running or done already),
|
|
2135
|
+
`not_found`, or a middleware's code. `tenant` and `context` replace the client's;
|
|
2136
|
+
`timeout_s` its deadline for this call. Example: `k.cancel(reminder_id)`."""
|
|
2137
|
+
req = pb.CancelRequest(command_id=command_id, tenant_id=tenant or self._tenant,
|
|
2138
|
+
context=_to_wire_context(context or self._context, command_id),
|
|
2139
|
+
issued_at=_issued_now(), protocol=PROTOCOL)
|
|
2140
|
+
req.client_sig.CopyFrom(self._key.sign(cancel_message(req)))
|
|
2141
|
+
res = self._parts.route(req.tenant_id, lambda ch: _idempotent(
|
|
2142
|
+
timeout_s or self._timeout_s, lambda left: rpc.CommandServiceStub(ch).Cancel(req, timeout=left)))
|
|
2143
|
+
if res.WhichOneof("outcome") == "cancelled":
|
|
2144
|
+
return CancelOutcome(cancelled=True)
|
|
2145
|
+
return CancelOutcome(cancelled=False, code=res.rejected.code, message=res.rejected.message)
|
|
2146
|
+
|
|
2147
|
+
def _caught_up_everywhere(self, q: "pb.QueryRequest", timeout_s: float | None) -> None:
|
|
2148
|
+
"""Returns once a read model that answers `q` has processed every partition's head, as
|
|
2149
|
+
of the call: a wait-only query to each partition, at the same time."""
|
|
2150
|
+
wait = pb.QueryRequest()
|
|
2151
|
+
wait.CopyFrom(q)
|
|
2152
|
+
wait.payload, wait.min_position, wait.wait_only, wait.listener = b"", 0, True, ""
|
|
2153
|
+
wait.client_sig.CopyFrom(self._key.sign(query_message(wait)))
|
|
2154
|
+
wait.listener = q.listener
|
|
2155
|
+
parts = self._parts.all(timeout_s)
|
|
2156
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, len(parts))) as pool:
|
|
2157
|
+
waits = [pool.submit(lambda ch=ch: _idempotent(
|
|
2158
|
+
timeout_s or self._timeout_s, lambda left: rpc.QueryServiceStub(ch).Query(wait, timeout=left)))
|
|
2159
|
+
for _, ch in parts]
|
|
2160
|
+
for w in waits:
|
|
2161
|
+
w.result()
|
|
2162
|
+
|
|
2163
|
+
def query(self, domain, type: str | None = None, data: Any = None, *, tenant: str | None = None,
|
|
2164
|
+
context: RequestContext | None = None, min_position: int = 0, timeout_s: float | None = None,
|
|
2165
|
+
all_partitions: bool = False) -> Any:
|
|
2166
|
+
"""Asks the read model that answers `domain.type` with `data` and returns its answer,
|
|
2167
|
+
decoded from JSON. Queries always go through the kernel, also within one process.
|
|
2168
|
+
`min_position`: answer only after the read model has processed this global position
|
|
2169
|
+
(e.g. of an event from a `wait="committed"` submit). `tenant` and `context` replace the
|
|
2170
|
+
client's; `timeout_s` its deadline for this call. `all_partitions`, for a read model
|
|
2171
|
+
across tenants with partitions: answer only once the read model has processed every
|
|
2172
|
+
partition's events up to the query, not only those of the tenant's partition (a
|
|
2173
|
+
wait-only query to each partition first).
|
|
2174
|
+
|
|
2175
|
+
Raises `Rejected` if the read model or a middleware rejects it, `grpc.RpcError` if the
|
|
2176
|
+
kernel refuses the call (e.g. `UNAVAILABLE` when no read model answers it). Example:
|
|
2177
|
+
`cart = k.query("shop", "GetCart", {"id": "c1"})`."""
|
|
2178
|
+
if isinstance(domain, Query):
|
|
2179
|
+
asked = domain
|
|
2180
|
+
kind = asked.__class__
|
|
2181
|
+
b = _binding(kind, "answered by a read model")
|
|
2182
|
+
return _from_wire(self._ask(b.domain, kind.type_name(), asked.to_wire(), _query_target(kind), tenant,
|
|
2183
|
+
context, min_position, timeout_s, all_partitions))
|
|
2184
|
+
spec = self._domains.get(domain)
|
|
2185
|
+
# A domain known only from a command class it submitted has no queries: none declared.
|
|
2186
|
+
declared = getattr(spec, "queries", {}).get(type) if spec else None
|
|
2187
|
+
return self._ask(domain, type, data, declared.target_of if declared else None, tenant, context,
|
|
2188
|
+
min_position, timeout_s, all_partitions)
|
|
2189
|
+
|
|
2190
|
+
def _ask(self, domain: str, type: str, data: Any, declared: tuple[str, str] | None, tenant: str | None,
|
|
2191
|
+
context: RequestContext | None, min_position: int, timeout_s: float | None, all_partitions: bool) -> Any:
|
|
2192
|
+
"""`query`, with the target the query type declares: `(aggregate_type, field)`."""
|
|
2193
|
+
query_id = str(uuid.uuid4())
|
|
2194
|
+
q = pb.QueryRequest(query_id=query_id, tenant_id=tenant or self._tenant, domain=domain, query_type=type,
|
|
2195
|
+
payload=_encode(data), context=_to_wire_context(context or self._context, query_id),
|
|
2196
|
+
min_position=min_position, issued_at=_issued_now(), protocol=PROTOCOL,
|
|
2197
|
+
listener=_HANDLING.get(), target=_query_target_of(declared, domain, data),
|
|
2198
|
+
deployment=self._deployment)
|
|
2199
|
+
if all_partitions:
|
|
2200
|
+
self._caught_up_everywhere(q, timeout_s)
|
|
2201
|
+
q.issued_at = _issued_now()
|
|
2202
|
+
def ask(ch):
|
|
2203
|
+
# Read your writes: wait for this client's own newest command, on whichever copy
|
|
2204
|
+
# answers. Not from a handler, which must not wait behind its own listener.
|
|
2205
|
+
if not q.listener:
|
|
2206
|
+
q.min_position = max(min_position, self._seen.get(_channel_id(ch), 0))
|
|
2207
|
+
q.client_sig.CopyFrom(self._key.sign(query_message(q)))
|
|
2208
|
+
return _idempotent(timeout_s or self._timeout_s, lambda left: rpc.QueryServiceStub(ch).Query(q, timeout=left))
|
|
2209
|
+
res = self._parts.route(q.tenant_id, ask)
|
|
2210
|
+
if res.WhichOneof("outcome") == "rejected":
|
|
2211
|
+
raise Rejected(res.rejected.message, res.rejected.code)
|
|
2212
|
+
return _decode(res.response)
|
|
2213
|
+
|
|
2214
|
+
def reindex(self, domain: str, aggregate_type: str, aggregate_ids: Iterable[str], *,
|
|
2215
|
+
tenant: str | None = None, context: RequestContext | None = None) -> dict[str, Outcome]:
|
|
2216
|
+
"""Runs the built-in command `_reindex_<aggregate_type>` on these aggregates, one after
|
|
2217
|
+
the other, so they hold the unique keys and facts of their current state, e.g. after
|
|
2218
|
+
declaring a key (`Aggregate.unique`) for aggregates that existed before. It produces no events. Returns
|
|
2219
|
+
the `Outcome` per ID: committed, or rejected, e.g. with `unique_taken` for an aggregate
|
|
2220
|
+
whose value another one holds already. An ID without events holds nothing. `tenant` and
|
|
2221
|
+
`context` replace the client's. Raises `grpc.RpcError` if the kernel refuses a call
|
|
2222
|
+
(e.g. the aggregate's worker declares no keys or facts). Example:
|
|
2223
|
+
`k.reindex("identity", "account", ["a1", "a2"])`."""
|
|
2224
|
+
command_type = reindex_type(aggregate_type)
|
|
2225
|
+
return {id: self.submit(domain, aggregate_type, id, command_type, None, tenant=tenant, wait="committed",
|
|
2226
|
+
context=context) for id in aggregate_ids}
|
|
2227
|
+
|
|
2228
|
+
def listener(self, listener: "Listener") -> "ListenerHandle":
|
|
2229
|
+
"""A handle that runs a listener defined with `Domain.read_model` or `Domain.reactor`, in
|
|
2230
|
+
this client's service; see `ListenerHandle`. `run()` on it blocks until `stop()`:
|
|
2231
|
+
`k.listener(carts_readmodel).run()`. Each call gives a new handle."""
|
|
2232
|
+
return ListenerHandle._new(self._parts, self._key, listener, self._log, self._service, self)
|
|
2233
|
+
|
|
2234
|
+
def subscribe(self, handler: Callable[["StoredEvent"], None], *, after: int = 0,
|
|
2235
|
+
types: Iterable[str] = (), reconnect_s: float = 1.0) -> "Subscription":
|
|
2236
|
+
"""Streams committed events after `after` (backlog first, then live) to `handler`, one at
|
|
2237
|
+
a time, in a thread of its own from the call on, e.g. to export them:
|
|
2238
|
+
`k.subscribe(export, types=["shop.*"])`. Returns a `Subscription`; `stop()` ends it.
|
|
2239
|
+
|
|
2240
|
+
`after`: global position of the last event already handled (0: from the beginning).
|
|
2241
|
+
`types`: qualified event types (`orders.OrderPlaced`) or domains (`orders.*`); empty: all
|
|
2242
|
+
the key may read. Reconnects and resumes after the last handled event, waiting
|
|
2243
|
+
`reconnect_s` after the stream ends, twice as long after each further failed attempt, at
|
|
2244
|
+
most 30 s; if the handler raises, that event is delivered again. Persist `position`
|
|
2245
|
+
yourself to resume across restarts. Unlike a listener, a subscription is not registered:
|
|
2246
|
+
the kernel keeps no position, nobody waits for it, and events arrive as stored (not
|
|
2247
|
+
upcast, vault fields `None` and listed in `redacted`). The key needs `subscribe` for the
|
|
2248
|
+
types. With partitions, a stream per partition, each from the beginning (`after` names a
|
|
2249
|
+
position of one kernel); events are in order within a partition, not across them."""
|
|
2250
|
+
sub = Subscription._new(self._parts, self._key, handler, after, list(types), reconnect_s, self._log)
|
|
2251
|
+
threading.Thread(target=sub.run, name="kernable subscription", daemon=True).start()
|
|
2252
|
+
return sub
|
|
2253
|
+
|
|
2254
|
+
def close(self) -> None:
|
|
2255
|
+
"""Closes the connection now; only needed in a long-running process that discards the
|
|
2256
|
+
client. Requests afterwards raise `ClientClosed`, and its workers and listeners end."""
|
|
2257
|
+
self._open.closed = True
|
|
2258
|
+
self._parts.close()
|
|
2259
|
+
|
|
2260
|
+
def _send(self, domain, aggregate_type, aggregate_id, command_type, data, tenant, id, wait, context, timeout_s,
|
|
2261
|
+
deliver_at: int = 0, deliver_until: int = 0):
|
|
2262
|
+
command_id = id or str(uuid.uuid4())
|
|
2263
|
+
context = context or self._context
|
|
2264
|
+
spec = self._domains[domain].commands.get(command_type) if domain in self._domains else None
|
|
2265
|
+
payload, refs, values = _seal(data, spec.vault if spec else None, aggregate_id)
|
|
2266
|
+
cmd = pb.Command(command_id=command_id, tenant_id=tenant or self._tenant, domain=domain,
|
|
2267
|
+
aggregate_type=aggregate_type, aggregate_id=aggregate_id,
|
|
2268
|
+
command_type=command_type, payload=payload,
|
|
2269
|
+
context=_to_wire_context(context, command_id), vault_refs=refs,
|
|
2270
|
+
deliver_at=deliver_at, deliver_until=deliver_until)
|
|
2271
|
+
cmd.client_sig.CopyFrom(self._key.sign(command_message(cmd)))
|
|
2272
|
+
request = pb.SubmitRequest(command=cmd, wait=wait, protocol=PROTOCOL, listener=_HANDLING.get(),
|
|
2273
|
+
vault=values)
|
|
2274
|
+
def send(ch):
|
|
2275
|
+
res = _idempotent(timeout_s or self._timeout_s,
|
|
2276
|
+
lambda left: rpc.CommandServiceStub(ch).Submit(request, timeout=left))
|
|
2277
|
+
if res.WhichOneof("outcome") == "committed":
|
|
2278
|
+
events = res.committed.events
|
|
2279
|
+
position = max(res.committed.position, events[-1].global_position if events else 0)
|
|
2280
|
+
self._seen[_channel_id(ch)] = max(self._seen.get(_channel_id(ch), 0), position)
|
|
2281
|
+
return res
|
|
2282
|
+
return self._parts.route(cmd.tenant_id, send)
|
|
2283
|
+
|
|
2284
|
+
|
|
2285
|
+
class DataClient:
|
|
2286
|
+
"""Files of a domain, kept by the kernel under (tenant, domain, key): invoices, uploads,
|
|
2287
|
+
exports. They are not part of the signed event log and not erased with the vault; keep
|
|
2288
|
+
personal data in vault fields instead. Needs a grant of `data_read` (get, list) or `data_write`
|
|
2289
|
+
(put, delete) for the domain; a refused or invalid request raises `grpc.RpcError`. Each
|
|
2290
|
+
request runs the middleware chain first, as `<domain>.PutData`, `.GetData`, `.DeleteData` or `.ListData`
|
|
2291
|
+
(the middleware sees the key and the request context, never the file); a middleware's
|
|
2292
|
+
refusal raises `grpc.RpcError` with code `PERMISSION_DENIED` and details `"<code>:
|
|
2293
|
+
<message>"`. Keys are 1 to 512 bytes, files at most 16 MiB (the kernel's `--max-data-bytes`).
|
|
2294
|
+
Use it as `k.data`:
|
|
2295
|
+
`k.data.put("shop", "invoices/c1.pdf", pdf_bytes, "application/pdf")`. `tenant=` replaces the
|
|
2296
|
+
client's tenant, `context=` the client's default request context."""
|
|
2297
|
+
|
|
2298
|
+
def __init__(self, k: Kernable):
|
|
2299
|
+
"""Use `k.data` of a `Kernable`."""
|
|
2300
|
+
self._k = k
|
|
2301
|
+
self._stub = rpc.DataServiceStub
|
|
2302
|
+
|
|
2303
|
+
def put(self, domain: str, key: str, data: bytes | str, content_type: str | None = None, *,
|
|
2304
|
+
tenant: str | None = None, context: RequestContext | None = None) -> str:
|
|
2305
|
+
"""Stores `data` under `key`, replacing a file stored there. A `str` is stored as UTF-8;
|
|
2306
|
+
`content_type` defaults to `text/plain; charset=utf-8` for a `str` and
|
|
2307
|
+
`application/octet-stream` for `bytes`. Returns the SHA-256 of the data (hex) as the
|
|
2308
|
+
kernel computed it: for an event to name."""
|
|
2309
|
+
if isinstance(data, str):
|
|
2310
|
+
data, content_type = data.encode(), content_type or "text/plain; charset=utf-8"
|
|
2311
|
+
return self._call("put", "PutData", tenant, context, domain, key, data=data,
|
|
2312
|
+
content_type=content_type or "application/octet-stream").sha256.hex()
|
|
2313
|
+
|
|
2314
|
+
def get(self, domain: str, key: str, *, tenant: str | None = None,
|
|
2315
|
+
context: RequestContext | None = None) -> Data | None:
|
|
2316
|
+
"""The file stored under `key`, or `None` if there is none."""
|
|
2317
|
+
res = self._call("get", "GetData", tenant, context, domain, key)
|
|
2318
|
+
return Data(res.data, res.content_type, res.sha256.hex()) if res.found else None
|
|
2319
|
+
|
|
2320
|
+
def list(self, domain: str, prefix: str = "", *, after: str = "", tenant: str | None = None,
|
|
2321
|
+
context: RequestContext | None = None) -> Listing:
|
|
2322
|
+
"""The files whose keys start with `prefix` (granted `data_read`), in the order of their
|
|
2323
|
+
keys' bytes, up to 1,000 after the key `after`; `next` of the result, if set, is the
|
|
2324
|
+
`after` of the next page. To find files no event names, or what a tenant stores."""
|
|
2325
|
+
res = self._call("list", "ListData", tenant, context, domain, prefix, after=after)
|
|
2326
|
+
return Listing([Listed(f.key, f.content_type, f.size) for f in res.files], res.next or None)
|
|
2327
|
+
|
|
2328
|
+
def delete(self, domain: str, key: str, *, tenant: str | None = None,
|
|
2329
|
+
context: RequestContext | None = None) -> bool:
|
|
2330
|
+
"""Deletes the file stored under `key`; returns whether there was one."""
|
|
2331
|
+
return self._call("delete", "DeleteData", tenant, context, domain, key).found
|
|
2332
|
+
|
|
2333
|
+
def _call(self, op, method, tenant, context, domain, key, **fields):
|
|
2334
|
+
r = pb.DataRequest(tenant_id=tenant or self._k._tenant, domain=domain, key=key,
|
|
2335
|
+
issued_at=_issued_now(), protocol=PROTOCOL,
|
|
2336
|
+
context=_to_wire_context(context or self._k._context, ""), **fields)
|
|
2337
|
+
r.client_sig.CopyFrom(self._k._key.sign(data_message(op, r)))
|
|
2338
|
+
if op in ("get", "list"): # safe to repeat; a write is one-shot (the kernel's replay guard)
|
|
2339
|
+
return self._k._parts.route(r.tenant_id, lambda ch: _idempotent(
|
|
2340
|
+
self._k._timeout_s, lambda left: getattr(self._stub(ch), method)(r, timeout=left)))
|
|
2341
|
+
return self._k._parts.route(r.tenant_id, lambda ch: getattr(self._stub(ch), method)(r))
|
|
2342
|
+
|
|
2343
|
+
|
|
2344
|
+
class CacheClient:
|
|
2345
|
+
"""A domain's shared cache, kept in the kernel's memory under (tenant, domain, key): rate
|
|
2346
|
+
limits, short-lived tokens, computed values. Not durable: a kernel restart empties it. Values
|
|
2347
|
+
are not part of the signed event log and not erased with the vault. Needs a grant of `cache`
|
|
2348
|
+
for the domain; a refused or invalid request raises `grpc.RpcError`. Each request runs the
|
|
2349
|
+
middleware chain first, as `<domain>.SetCache`, `.GetCache` or `.DeleteCache` (the middleware
|
|
2350
|
+
sees the key and the request context, never the value); a middleware's refusal raises
|
|
2351
|
+
`grpc.RpcError` with code `PERMISSION_DENIED` and details `"<code>: <message>"`. Keys are 1 to
|
|
2352
|
+
512 bytes, values at most 1 MiB. Use it as `k.cache`: `k.cache.set("shop", "rate:c1", "3",
|
|
2353
|
+
ttl_ms=60_000)`, then `k.cache.get("shop", "rate:c1")` returns `b"3"`. `tenant=` replaces the
|
|
2354
|
+
client's tenant, `context=` the client's default request context."""
|
|
2355
|
+
|
|
2356
|
+
def __init__(self, k: Kernable):
|
|
2357
|
+
"""Use `k.cache` of a `Kernable`."""
|
|
2358
|
+
self._k = k
|
|
2359
|
+
self._stub = rpc.CacheServiceStub
|
|
2360
|
+
|
|
2361
|
+
def set(self, domain: str, key: str, value: bytes | str, ttl_ms: int = 0, *,
|
|
2362
|
+
tenant: str | None = None, context: RequestContext | None = None) -> None:
|
|
2363
|
+
"""Stores `value` under `key` for `ttl_ms` milliseconds; `0` keeps it until it is deleted
|
|
2364
|
+
or the kernel restarts. A `str` is stored as UTF-8."""
|
|
2365
|
+
value = value.encode() if isinstance(value, str) else value
|
|
2366
|
+
self._call("set", "SetCache", tenant, context, domain, key, value=value, ttl_ms=ttl_ms)
|
|
2367
|
+
|
|
2368
|
+
def get(self, domain: str, key: str, *, tenant: str | None = None,
|
|
2369
|
+
context: RequestContext | None = None) -> bytes | None:
|
|
2370
|
+
"""The live value under `key` as `bytes` (decode a `str` with `.decode()`), or `None` if
|
|
2371
|
+
there is none or it expired."""
|
|
2372
|
+
res = self._call("get", "GetCache", tenant, context, domain, key)
|
|
2373
|
+
return res.value if res.found else None
|
|
2374
|
+
|
|
2375
|
+
def delete(self, domain: str, key: str, *, tenant: str | None = None,
|
|
2376
|
+
context: RequestContext | None = None) -> bool:
|
|
2377
|
+
"""Deletes the value under `key`; returns whether it had a live one."""
|
|
2378
|
+
return self._call("delete", "DeleteCache", tenant, context, domain, key).found
|
|
2379
|
+
|
|
2380
|
+
def _call(self, op, method, tenant, context, domain, key, **fields):
|
|
2381
|
+
r = pb.CacheRequest(tenant_id=tenant or self._k._tenant, domain=domain, key=key,
|
|
2382
|
+
issued_at=_issued_now(), protocol=PROTOCOL,
|
|
2383
|
+
context=_to_wire_context(context or self._k._context, ""), **fields)
|
|
2384
|
+
r.client_sig.CopyFrom(self._k._key.sign(cache_message(op, r)))
|
|
2385
|
+
if op == "get": # safe to repeat; a write is one-shot (the kernel's replay guard)
|
|
2386
|
+
return self._k._parts.route(r.tenant_id, lambda ch: _idempotent(
|
|
2387
|
+
self._k._timeout_s, lambda left: getattr(self._stub(ch), method)(r, timeout=left)))
|
|
2388
|
+
return self._k._parts.route(r.tenant_id, lambda ch: getattr(self._stub(ch), method)(r))
|
|
2389
|
+
|
|
2390
|
+
|
|
2391
|
+
#: Levels of `KERNABLE_LOG`; a line is printed if its level is at most the configured one.
|
|
2392
|
+
_LOG_LEVELS = {"off": 0, "error": 1, "warn": 2, "info": 3, "debug": 4}
|
|
2393
|
+
|
|
2394
|
+
|
|
2395
|
+
class _Log:
|
|
2396
|
+
"""Prints a worker's or listener's connection and activity lines to stderr, up to the level
|
|
2397
|
+
of `KERNABLE_LOG` (`off`, `error`, `warn`, `info` (default), `debug`) when the client was
|
|
2398
|
+
made."""
|
|
2399
|
+
|
|
2400
|
+
def __init__(self, url: str, key_path: str, key_id: str):
|
|
2401
|
+
self.level = _LOG_LEVELS.get(os.environ.get("KERNABLE_LOG", ""), _LOG_LEVELS["info"])
|
|
2402
|
+
#: `<url> · key <name> (<key id>…)`: the kernel and the key this process connects with.
|
|
2403
|
+
self.peer = f"{url} · key {os.path.basename(key_path).removesuffix('.key')} ({key_id[:12]}…)"
|
|
2404
|
+
|
|
2405
|
+
def enabled(self, level: str) -> bool:
|
|
2406
|
+
return _LOG_LEVELS[level] <= self.level
|
|
2407
|
+
|
|
2408
|
+
def style(self, code: int, text: str) -> str:
|
|
2409
|
+
"""`text` in an ANSI style (1 bold, 2 dim, 31 red, 33 yellow) on a terminal, unless
|
|
2410
|
+
`NO_COLOR` is set."""
|
|
2411
|
+
# Checked per line: stderr may be redirected after the client was made.
|
|
2412
|
+
color = sys.stderr.isatty() and not os.environ.get("NO_COLOR")
|
|
2413
|
+
return f"\x1b[{code}m{text}\x1b[0m" if color else text
|
|
2414
|
+
|
|
2415
|
+
def print(self, level: str, what: str, line: str) -> None:
|
|
2416
|
+
"""`12:04:11 listener carts <line>`, warnings in yellow and errors in red."""
|
|
2417
|
+
if not self.enabled(level):
|
|
2418
|
+
return
|
|
2419
|
+
text = self.style(31, line) if level == "error" else self.style(33, line) if level == "warn" else line
|
|
2420
|
+
print(f"{self.style(2, time.strftime('%H:%M:%S'))} {self.style(1, what)} {text}", file=sys.stderr)
|
|
2421
|
+
|
|
2422
|
+
def reconnect(self, what: str, reason: str, seconds: float, attempt: int, refused: bool,
|
|
2423
|
+
stopped: threading.Event) -> None:
|
|
2424
|
+
"""Logs the end of a session that was not stopped and waits before reconnecting:
|
|
2425
|
+
`seconds` after the first failed attempt, twice as long after each further one, at most
|
|
2426
|
+
30 s, give or take a fifth, so processes do not reconnect in lockstep after a kernel
|
|
2427
|
+
restart. A refusal that will not go away by itself is an error, not a warning. `stopped`
|
|
2428
|
+
cuts the wait short."""
|
|
2429
|
+
delay = _backoff(seconds, attempt)
|
|
2430
|
+
# A kernel of a pair that is not the primary right now may be in a moment.
|
|
2431
|
+
if _switching_roles(reason):
|
|
2432
|
+
delay = min(delay, _PAIR_RETRY_S)
|
|
2433
|
+
self.print("error" if refused else "warn", what,
|
|
2434
|
+
f"{'refused' if refused else 'connection lost'} ({reason}), reconnecting in {delay:.2f} s")
|
|
2435
|
+
stopped.wait(delay)
|
|
2436
|
+
|
|
2437
|
+
def store_failed(self, what: str, err: BaseException, seconds: float, attempt: int,
|
|
2438
|
+
stopped: threading.Event) -> None:
|
|
2439
|
+
"""Logs that a store could not be opened and waits before the next try, backing off
|
|
2440
|
+
like `reconnect`."""
|
|
2441
|
+
delay = _backoff(seconds, attempt)
|
|
2442
|
+
self.print("error", what, f"store: {err}; trying again in {delay:.2f} s")
|
|
2443
|
+
stopped.wait(delay)
|
|
2444
|
+
|
|
2445
|
+
|
|
2446
|
+
_PAIR_RETRY_S = 1.0
|
|
2447
|
+
"""The longest wait before trying a kernel of a pair again that was not the primary."""
|
|
2448
|
+
|
|
2449
|
+
|
|
2450
|
+
def _switching_roles(reason: str) -> bool:
|
|
2451
|
+
"""Whether `reason` says the kernel is one of a pair that is not the primary right now: a
|
|
2452
|
+
standby, one that lost its lease or hands its role over."""
|
|
2453
|
+
return (reason.startswith("not_primary") or "lost its lease at the witness" in reason
|
|
2454
|
+
or "hands its role over" in reason)
|
|
2455
|
+
|
|
2456
|
+
|
|
2457
|
+
def _backoff(seconds: float, attempt: int) -> float:
|
|
2458
|
+
"""The pause before try `attempt` (from 1): `seconds`, doubled each time up to 30 s, give or
|
|
2459
|
+
take a fifth."""
|
|
2460
|
+
return min(seconds * 2 ** (attempt - 1), 30.0) * random.uniform(0.8, 1.2)
|
|
2461
|
+
|
|
2462
|
+
|
|
2463
|
+
class _Watch:
|
|
2464
|
+
"""Logs `line` as an error if the block still runs after `seconds`: a handler cannot be
|
|
2465
|
+
stopped from outside, but a stuck one shows."""
|
|
2466
|
+
|
|
2467
|
+
def __init__(self, log: "_Log", what: str, seconds: float | None, line: str = ""):
|
|
2468
|
+
self._timer = None
|
|
2469
|
+
if seconds is not None:
|
|
2470
|
+
self._timer = threading.Timer(seconds, log.print, ("error", what, line))
|
|
2471
|
+
self._timer.daemon = True
|
|
2472
|
+
|
|
2473
|
+
def __enter__(self):
|
|
2474
|
+
if self._timer:
|
|
2475
|
+
self._timer.start()
|
|
2476
|
+
|
|
2477
|
+
def __exit__(self, *exc):
|
|
2478
|
+
if self._timer:
|
|
2479
|
+
self._timer.cancel()
|
|
2480
|
+
|
|
2481
|
+
|
|
2482
|
+
def _handling(msg) -> tuple[float | None, str]:
|
|
2483
|
+
"""After how long a listener's message is too long, and what to log then: a query or check
|
|
2484
|
+
after 5 s (the kernel's default `--query-timeout-ms` and `--middleware-timeout-ms`, after
|
|
2485
|
+
which the request has failed), a batch after 30 s (everything behind it waits)."""
|
|
2486
|
+
kind = msg.WhichOneof("msg")
|
|
2487
|
+
if kind == "batch":
|
|
2488
|
+
what, seconds = f"the batch up to {msg.batch.up_to}", 30
|
|
2489
|
+
elif kind == "query":
|
|
2490
|
+
what, seconds = f"the query {msg.query.query.domain}.{msg.query.query.query_type}", 5
|
|
2491
|
+
elif kind == "check":
|
|
2492
|
+
what, seconds = "a middleware check", 5
|
|
2493
|
+
else:
|
|
2494
|
+
return None, ""
|
|
2495
|
+
return seconds, f"still handling {what} after {seconds} s: everything behind it waits; is a handler stuck?"
|
|
2496
|
+
|
|
2497
|
+
|
|
2498
|
+
def _relative(type: str, domains: set[str]) -> str:
|
|
2499
|
+
"""`shop.ItemAdded` as `ItemAdded` when `shop` is the only domain in `domains`."""
|
|
2500
|
+
domain, _, name = type.partition(".")
|
|
2501
|
+
return name if len(domains) == 1 and domain in domains and name else type
|
|
2502
|
+
|
|
2503
|
+
|
|
2504
|
+
def _listener_details(l: "Listener", position: int, name: Callable[[str], str] = lambda t: t) -> list[str]:
|
|
2505
|
+
"""What a listener does, one part per entry: where it starts, what it reads, answers and
|
|
2506
|
+
checks. `name` shortens the types (see `_relative`)."""
|
|
2507
|
+
def listed(types: Iterable[str]) -> str:
|
|
2508
|
+
return ", ".join(name(t) for t in types)
|
|
2509
|
+
|
|
2510
|
+
details = [f"from {position} · reads {listed(l._handlers)}" if l._handlers else f"from {position}"]
|
|
2511
|
+
if l._queries:
|
|
2512
|
+
details.append(f"answers {listed(l._queries)}")
|
|
2513
|
+
for when, (spec, _) in l._middlewares.items():
|
|
2514
|
+
details.append(f"checks {listed(spec.types)} {when}")
|
|
2515
|
+
return details
|
|
2516
|
+
|
|
2517
|
+
|
|
2518
|
+
@dataclass
|
|
2519
|
+
class Manifest:
|
|
2520
|
+
"""What a service consists of, as `Kernable.apply` takes it: the membership an app applies
|
|
2521
|
+
(`app.manifest`, see `Kernable.app`)."""
|
|
2522
|
+
|
|
2523
|
+
#: The service the membership is for.
|
|
2524
|
+
service: str
|
|
2525
|
+
#: The aggregates the service runs; the kernel waits for a worker of each.
|
|
2526
|
+
workers: list[AggregateType] = field(default_factory=list)
|
|
2527
|
+
#: The listeners the kernel waits for.
|
|
2528
|
+
listeners: list["Listener"] = field(default_factory=list)
|
|
2529
|
+
#: The service's listeners that may check requests; they count like listeners.
|
|
2530
|
+
middlewares: list["Listener"] = field(default_factory=list)
|
|
2531
|
+
#: Free metadata, e.g. `{"version": "1.0.0"}`.
|
|
2532
|
+
metadata: dict[str, str] = field(default_factory=dict)
|
|
2533
|
+
#: The service's business processes; `Kernable.app` registers them.
|
|
2534
|
+
processes: list[Process] = field(default_factory=list)
|
|
2535
|
+
#: The deployment (code version) that serves the service; empty: every deployment.
|
|
2536
|
+
deployment: str = ""
|
|
2537
|
+
#: When it switches the service to its deployment while another serves (see `Kernable.apply`).
|
|
2538
|
+
switch: str = "caught-up"
|
|
2539
|
+
|
|
2540
|
+
|
|
2541
|
+
class _AllReady:
|
|
2542
|
+
"""Set once all of some streams are connected; waits like a `threading.Event`."""
|
|
2543
|
+
|
|
2544
|
+
def __init__(self, events: list[threading.Event]):
|
|
2545
|
+
self._events = events
|
|
2546
|
+
|
|
2547
|
+
def is_set(self) -> bool:
|
|
2548
|
+
return all(e.is_set() for e in self._events)
|
|
2549
|
+
|
|
2550
|
+
def wait(self, timeout: float | None = None) -> bool:
|
|
2551
|
+
"""Waits until every stream is connected, at most `timeout` seconds in all; returns whether
|
|
2552
|
+
they are."""
|
|
2553
|
+
deadline = None if timeout is None else time.monotonic() + timeout
|
|
2554
|
+
for e in self._events:
|
|
2555
|
+
if not e.wait(None if deadline is None else max(0.0, deadline - time.monotonic())):
|
|
2556
|
+
return False
|
|
2557
|
+
return True
|
|
2558
|
+
|
|
2559
|
+
|
|
2560
|
+
class App:
|
|
2561
|
+
"""An app made of whole domains, returned by `kernable.app(...)`: `run()` applies its
|
|
2562
|
+
membership (`manifest`), then runs its worker and listeners until `stop()`; `ready` is set
|
|
2563
|
+
once all of them are connected.
|
|
2564
|
+
|
|
2565
|
+
`run()` blocks the calling thread. To do something once the app is up (e.g. in a test),
|
|
2566
|
+
run it in a thread and wait: `threading.Thread(target=app.run, daemon=True).start()`, then
|
|
2567
|
+
`app.ready.wait(5)`, which returns whether everything connected within 5 seconds. Call
|
|
2568
|
+
`app.stop()` from another thread or a handler to end `run()`. Attributes: `manifest` (a
|
|
2569
|
+
`Manifest`), `worker` (a `Worker`, or `None` without aggregates), `listeners` (a
|
|
2570
|
+
`ListenerHandle` per read model and reactor, live ones too), `ready` and `processes` (a
|
|
2571
|
+
`RegisteredProcess` per process of its domains, once registered).
|
|
2572
|
+
|
|
2573
|
+
One service is one app: two processes with the same service would replace each other's
|
|
2574
|
+
membership."""
|
|
2575
|
+
|
|
2576
|
+
def __init__(self, k: Kernable, manifest: Manifest, worker: "Worker | None",
|
|
2577
|
+
listeners: list["ListenerHandle"], domains: Iterable[str] = ()):
|
|
2578
|
+
"""Use `Kernable.app`."""
|
|
2579
|
+
self._kernable = k
|
|
2580
|
+
self._log = k._log
|
|
2581
|
+
# The names of the app's domains.
|
|
2582
|
+
self._domains = set(domains)
|
|
2583
|
+
self._stopped = False
|
|
2584
|
+
#: The membership `run()` applies.
|
|
2585
|
+
self.manifest = manifest
|
|
2586
|
+
#: The worker of all the domains' aggregates, if they have any.
|
|
2587
|
+
self.worker = worker
|
|
2588
|
+
#: A running listener per read model and reactor, live ones too.
|
|
2589
|
+
self.listeners = listeners
|
|
2590
|
+
#: The `RegisteredProcess` per process (qualified name), once registered by `run()`.
|
|
2591
|
+
self.processes: dict[str, RegisteredProcess] = {}
|
|
2592
|
+
streams = [s for s in (worker, *listeners) if s is not None]
|
|
2593
|
+
#: Set once every worker and listener of the app is connected; while another deployment
|
|
2594
|
+
#: serves, they warm up beside it.
|
|
2595
|
+
self.connected = _AllReady([s.ready for s in streams])
|
|
2596
|
+
#: Set once the app's membership is applied: its deployment serves the service. Right at
|
|
2597
|
+
#: the start, unless another deployment serves; then once the app's deployment has caught
|
|
2598
|
+
#: up and taken over (`switch` of `Kernable.app`), or an operator switched to it.
|
|
2599
|
+
self.serving = threading.Event()
|
|
2600
|
+
#: Set once every worker and listener is connected and the app's deployment serves.
|
|
2601
|
+
self.ready = _AllReady([s.ready for s in streams] + [self.serving])
|
|
2602
|
+
self._streams = streams
|
|
2603
|
+
|
|
2604
|
+
def run(self) -> None:
|
|
2605
|
+
"""Applies the membership (see `Kernable.apply`), then runs the worker and the listeners,
|
|
2606
|
+
each in a thread of its own, until `stop()`; blocks until then. If another deployment
|
|
2607
|
+
serves the service, the kernel switches to this one only as the app's `switch` says: until
|
|
2608
|
+
then the app runs beside it (workers idle, read models building), applies its membership
|
|
2609
|
+
again every second, and registers its processes once it serves. A failing handler does
|
|
2610
|
+
not end it: a command handler's error rejects the command (`handler_error`), a listener
|
|
2611
|
+
handler's error makes the listener reconnect and get the event again. If a worker or
|
|
2612
|
+
listener itself fails, it stops the others and raises that error (a store that cannot
|
|
2613
|
+
be opened is tried again instead); Ctrl-C stops all of them too. Once all are connected, it prints an overview of the app (unless
|
|
2614
|
+
`KERNABLE_LOG` is below `info`). Raises `grpc.RpcError` if the kernel refuses the
|
|
2615
|
+
membership."""
|
|
2616
|
+
started = time.monotonic()
|
|
2617
|
+
m = self.manifest
|
|
2618
|
+
models = [p.model() for p in m.processes] # an invalid one stops the app before it starts
|
|
2619
|
+
self._stopped = False
|
|
2620
|
+
self.serving.clear()
|
|
2621
|
+
|
|
2622
|
+
def apply() -> None:
|
|
2623
|
+
self._kernable.apply(service=m.service, workers=m.workers, listeners=m.listeners,
|
|
2624
|
+
middlewares=m.middlewares, metadata=m.metadata, processes=m.processes,
|
|
2625
|
+
deployment=m.deployment, switch=m.switch)
|
|
2626
|
+
|
|
2627
|
+
try:
|
|
2628
|
+
apply()
|
|
2629
|
+
self.serving.set()
|
|
2630
|
+
except grpc.RpcError as err:
|
|
2631
|
+
if not waits_for_switch(err):
|
|
2632
|
+
raise
|
|
2633
|
+
self._log.print("info", "app", f"deployment {m.deployment} runs beside the one that serves "
|
|
2634
|
+
f"({err.details()})")
|
|
2635
|
+
|
|
2636
|
+
def switch_later() -> None:
|
|
2637
|
+
# Applies the membership every second until the kernel takes it.
|
|
2638
|
+
while not self._stopped:
|
|
2639
|
+
time.sleep(1.0)
|
|
2640
|
+
if self._stopped:
|
|
2641
|
+
return
|
|
2642
|
+
try:
|
|
2643
|
+
apply()
|
|
2644
|
+
except grpc.RpcError as err:
|
|
2645
|
+
if not waits_for_switch(err):
|
|
2646
|
+
self._log.print("warn", "app", f"applying the membership failed: {err.details()}")
|
|
2647
|
+
continue
|
|
2648
|
+
self._log.print("info", "app", f"deployment {m.deployment} serves {m.service} now")
|
|
2649
|
+
self.serving.set()
|
|
2650
|
+
return
|
|
2651
|
+
|
|
2652
|
+
if not self.serving.is_set():
|
|
2653
|
+
threading.Thread(target=switch_later, name="kernable switch", daemon=True).start()
|
|
2654
|
+
for s in self._streams:
|
|
2655
|
+
s.announce = False
|
|
2656
|
+
errors: list[BaseException] = []
|
|
2657
|
+
|
|
2658
|
+
def overview() -> None:
|
|
2659
|
+
while not self.ready.wait(0.1):
|
|
2660
|
+
if self._stopped:
|
|
2661
|
+
return
|
|
2662
|
+
self._overview(round((time.monotonic() - started) * 1000))
|
|
2663
|
+
|
|
2664
|
+
def run(stream: _Stream) -> None:
|
|
2665
|
+
try:
|
|
2666
|
+
stream.run()
|
|
2667
|
+
except BaseException as err:
|
|
2668
|
+
errors.append(err)
|
|
2669
|
+
self.stop()
|
|
2670
|
+
|
|
2671
|
+
def register() -> None:
|
|
2672
|
+
# Once the app serves and the worker is connected, so the kernel knows the commands
|
|
2673
|
+
# the models send.
|
|
2674
|
+
while not self.serving.wait(0.1) or (self.worker is not None and not self.worker.ready.wait(0.1)):
|
|
2675
|
+
if self._stopped:
|
|
2676
|
+
return
|
|
2677
|
+
try:
|
|
2678
|
+
for model in models:
|
|
2679
|
+
r = self._kernable.register_process(model)
|
|
2680
|
+
self.processes[r.name] = r
|
|
2681
|
+
except BaseException as err:
|
|
2682
|
+
errors.append(err)
|
|
2683
|
+
self.stop()
|
|
2684
|
+
|
|
2685
|
+
threads = [threading.Thread(target=run, args=(s,), name=f"kernable {s._what}", daemon=True)
|
|
2686
|
+
for s in self._streams]
|
|
2687
|
+
threading.Thread(target=overview, name="kernable overview", daemon=True).start()
|
|
2688
|
+
if models:
|
|
2689
|
+
threads.append(threading.Thread(target=register, name="kernable processes", daemon=True))
|
|
2690
|
+
for t in threads:
|
|
2691
|
+
t.start()
|
|
2692
|
+
try:
|
|
2693
|
+
for t in threads:
|
|
2694
|
+
t.join()
|
|
2695
|
+
except BaseException: # e.g. Ctrl-C: stop everything
|
|
2696
|
+
self.stop()
|
|
2697
|
+
raise
|
|
2698
|
+
if errors:
|
|
2699
|
+
raise errors[0]
|
|
2700
|
+
|
|
2701
|
+
def stop(self) -> None:
|
|
2702
|
+
"""Stops the worker and all listeners; `run()` returns once they are done."""
|
|
2703
|
+
self._stopped = True
|
|
2704
|
+
for s in self._streams:
|
|
2705
|
+
s.stop()
|
|
2706
|
+
|
|
2707
|
+
def _overview(self, ms: int) -> None:
|
|
2708
|
+
log = self._log
|
|
2709
|
+
if not log.enabled("info"):
|
|
2710
|
+
return
|
|
2711
|
+
m = self.manifest
|
|
2712
|
+
domains = self._domains
|
|
2713
|
+
|
|
2714
|
+
# A single domain's own names go without it; with several, every name is qualified.
|
|
2715
|
+
def short(domain: str, name: str) -> str:
|
|
2716
|
+
return name if len(domains) == 1 and domain in domains else f"{domain}.{name}"
|
|
2717
|
+
|
|
2718
|
+
def count(n: int, what: str) -> str:
|
|
2719
|
+
return f"{n} {what}{'' if n == 1 else 's'}"
|
|
2720
|
+
|
|
2721
|
+
listed = [count(len(m.workers), "worker"), count(len(m.listeners), "listener")]
|
|
2722
|
+
if m.middlewares:
|
|
2723
|
+
listed.append(count(len(m.middlewares), "middleware"))
|
|
2724
|
+
meta = m.metadata
|
|
2725
|
+
rows: list[tuple[str, str, list[str]]] = []
|
|
2726
|
+
for a in m.workers:
|
|
2727
|
+
rows.append(("worker", short(a.domain, a.name), [f"runs {', '.join(sorted(a.handlers))}"]))
|
|
2728
|
+
for h in self.listeners:
|
|
2729
|
+
l = h.listener
|
|
2730
|
+
rows.append(("live" if l.live else "listener", short(l.domain.name, l.name),
|
|
2731
|
+
_listener_details(l, h.started_at, lambda t: _relative(t, domains))))
|
|
2732
|
+
width = max((len(name) for _, name, _ in rows), default=0)
|
|
2733
|
+
header = " · ".join(p for p in ("kernable", " ".join(p for p in (m.service, meta.get("version")) if p),
|
|
2734
|
+
meta.get("description")) if p)
|
|
2735
|
+
lines = [
|
|
2736
|
+
log.style(1, header),
|
|
2737
|
+
f" {'kernel':<10} {log.peer}",
|
|
2738
|
+
f" {'membership':<10} applied: {', '.join(listed)}",
|
|
2739
|
+
"",
|
|
2740
|
+
]
|
|
2741
|
+
for kind, name, (first, *rest) in rows:
|
|
2742
|
+
lines.append(f" {kind:<10} {name:<{width}} {first}")
|
|
2743
|
+
lines.extend(f" {'':<10} {'':<{width}} {more}" for more in rest)
|
|
2744
|
+
lines += ["", f" {log.style(2, f'ready in {ms} ms')}", ""]
|
|
2745
|
+
# One write, so the overview's lines stay together.
|
|
2746
|
+
sys.stderr.write("\n".join(lines) + "\n")
|
|
2747
|
+
|
|
2748
|
+
|
|
2749
|
+
class _HandlerFailed(Exception):
|
|
2750
|
+
"""A handler's own error, carried out of a stream's loop: told apart from the stream's errors
|
|
2751
|
+
(a `grpc.RpcError` from a request the handler made is no refusal of the stream)."""
|
|
2752
|
+
|
|
2753
|
+
def __init__(self, err: Exception):
|
|
2754
|
+
super().__init__(str(err) or type(err).__name__)
|
|
2755
|
+
self.err = err
|
|
2756
|
+
|
|
2757
|
+
|
|
2758
|
+
def _handle(handler: Callable, *args):
|
|
2759
|
+
"""Runs a handler; its errors come out as `_HandlerFailed`, except `ClientClosed`."""
|
|
2760
|
+
try:
|
|
2761
|
+
return handler(*args)
|
|
2762
|
+
except ClientClosed:
|
|
2763
|
+
raise
|
|
2764
|
+
except Exception as err:
|
|
2765
|
+
raise _HandlerFailed(err) from err
|
|
2766
|
+
|
|
2767
|
+
|
|
2768
|
+
class _Stream:
|
|
2769
|
+
"""A bidirectional stream that reconnects until stopped."""
|
|
2770
|
+
|
|
2771
|
+
def __init__(self, what: str, reconnect_s: float, log: _Log):
|
|
2772
|
+
self.ready = threading.Event()
|
|
2773
|
+
#: Whether the first connect gets its own line; an `App` prints it in its overview instead.
|
|
2774
|
+
self.announce = True
|
|
2775
|
+
self._connects = 0
|
|
2776
|
+
self._log = log
|
|
2777
|
+
self._what = what
|
|
2778
|
+
self._reconnect_s = reconnect_s
|
|
2779
|
+
self._stopped = False
|
|
2780
|
+
#: Set by `stop()`: cuts the wait before a reconnect short.
|
|
2781
|
+
self._wake = threading.Event()
|
|
2782
|
+
self._call = None
|
|
2783
|
+
self._outbox: queue.Queue | None = None
|
|
2784
|
+
|
|
2785
|
+
@classmethod
|
|
2786
|
+
def _new(cls, *args, **kwargs):
|
|
2787
|
+
"""Made by the client (`Kernable.worker`, `.listener`, `.subscribe`), not by applications:
|
|
2788
|
+
the arguments are the client's internals. It bypasses `__init__`, which each public
|
|
2789
|
+
stream class makes raise `TypeError`, so a direct call never builds a broken object."""
|
|
2790
|
+
stream = cls.__new__(cls)
|
|
2791
|
+
stream._init(*args, **kwargs)
|
|
2792
|
+
return stream
|
|
2793
|
+
|
|
2794
|
+
def run(self) -> None:
|
|
2795
|
+
"""Processes messages until `stop()`, reconnecting when the stream drops."""
|
|
2796
|
+
attempt = 0
|
|
2797
|
+
while not self._stopped:
|
|
2798
|
+
connects = self._connects
|
|
2799
|
+
refused = False
|
|
2800
|
+
outbox: queue.Queue = queue.Queue()
|
|
2801
|
+
self._outbox = outbox
|
|
2802
|
+
try:
|
|
2803
|
+
self._call = None
|
|
2804
|
+
self._call = self._open(iter(outbox.get, None))
|
|
2805
|
+
if self._stopped: # stopped while it opened: stop() missed this call
|
|
2806
|
+
self._call.cancel()
|
|
2807
|
+
outbox.put(self._hello())
|
|
2808
|
+
for msg in self._call:
|
|
2809
|
+
reply = _handle(self._on, msg)
|
|
2810
|
+
if reply is not None:
|
|
2811
|
+
outbox.put(reply)
|
|
2812
|
+
reason = "stream ended"
|
|
2813
|
+
except ClientClosed:
|
|
2814
|
+
return
|
|
2815
|
+
except _HandlerFailed as failed: # a failing handler: reconnect and get the message again
|
|
2816
|
+
reason = str(failed)
|
|
2817
|
+
except grpc.RpcError as err:
|
|
2818
|
+
reason = err.details()
|
|
2819
|
+
refused = err.code() in _REFUSALS
|
|
2820
|
+
except Exception as err:
|
|
2821
|
+
reason = str(err) or type(err).__name__
|
|
2822
|
+
finally:
|
|
2823
|
+
outbox.put(None)
|
|
2824
|
+
# Whatever ended the loop, the call ends with it (a no-op once it ended by itself).
|
|
2825
|
+
if self._call is not None:
|
|
2826
|
+
self._call.cancel()
|
|
2827
|
+
attempt = 1 if self._connects > connects else attempt + 1
|
|
2828
|
+
if not self._stopped:
|
|
2829
|
+
self._log.reconnect(self._what, reason, self._reconnect_s, attempt, refused, self._wake)
|
|
2830
|
+
|
|
2831
|
+
def stop(self) -> None:
|
|
2832
|
+
self._stopped = True
|
|
2833
|
+
self._wake.set()
|
|
2834
|
+
if self._call is not None:
|
|
2835
|
+
self._call.cancel()
|
|
2836
|
+
for child in getattr(self, "_children", ()):
|
|
2837
|
+
child.stop()
|
|
2838
|
+
|
|
2839
|
+
def _partitions(self, source) -> "list[tuple[str, grpc.Channel]] | None":
|
|
2840
|
+
"""The partitions of `source`, read again, backing off like a reconnect, while no
|
|
2841
|
+
kernel answers; `None` once stopped."""
|
|
2842
|
+
if not isinstance(source, _Partitioned):
|
|
2843
|
+
return [("", source)]
|
|
2844
|
+
attempt = 0
|
|
2845
|
+
while not self._stopped:
|
|
2846
|
+
# Read in a thread of its own: `stop()` ends the wait for it at once.
|
|
2847
|
+
box: dict = {}
|
|
2848
|
+
|
|
2849
|
+
def read() -> None:
|
|
2850
|
+
try:
|
|
2851
|
+
box["parts"] = source.all()
|
|
2852
|
+
except BaseException as err:
|
|
2853
|
+
box["error"] = err
|
|
2854
|
+
|
|
2855
|
+
reader = threading.Thread(target=read, name=f"kernable {self._what} map", daemon=True)
|
|
2856
|
+
reader.start()
|
|
2857
|
+
while reader.is_alive():
|
|
2858
|
+
reader.join(0.05)
|
|
2859
|
+
if self._stopped:
|
|
2860
|
+
return None
|
|
2861
|
+
try:
|
|
2862
|
+
if "error" in box:
|
|
2863
|
+
raise box["error"]
|
|
2864
|
+
return box["parts"]
|
|
2865
|
+
except ClientClosed:
|
|
2866
|
+
return None
|
|
2867
|
+
except grpc.RpcError as err:
|
|
2868
|
+
attempt += 1
|
|
2869
|
+
self._log.reconnect(self._what, err.details(), self._reconnect_s, attempt,
|
|
2870
|
+
err.code() in _REFUSALS, self._wake)
|
|
2871
|
+
return None
|
|
2872
|
+
|
|
2873
|
+
def _run_children(self, children: "list[_Stream]") -> None:
|
|
2874
|
+
"""Runs a stream per partition, each in a thread, until `stop()`; `ready` once all are."""
|
|
2875
|
+
self._children = children
|
|
2876
|
+
for child in children:
|
|
2877
|
+
child.announce = self.announce
|
|
2878
|
+
errors: list[BaseException] = []
|
|
2879
|
+
|
|
2880
|
+
def run(child: "_Stream") -> None:
|
|
2881
|
+
try:
|
|
2882
|
+
child.run()
|
|
2883
|
+
except BaseException as err:
|
|
2884
|
+
errors.append(err)
|
|
2885
|
+
self.stop()
|
|
2886
|
+
|
|
2887
|
+
def ready() -> None:
|
|
2888
|
+
for child in children:
|
|
2889
|
+
while not child.ready.wait(0.1):
|
|
2890
|
+
if self._stopped:
|
|
2891
|
+
return
|
|
2892
|
+
self.ready.set()
|
|
2893
|
+
|
|
2894
|
+
threads = [threading.Thread(target=run, args=(c,), name=f"kernable {c._what}", daemon=True)
|
|
2895
|
+
for c in children]
|
|
2896
|
+
threading.Thread(target=ready, name=f"kernable {self._what} ready", daemon=True).start()
|
|
2897
|
+
if self._stopped:
|
|
2898
|
+
return
|
|
2899
|
+
for t in threads:
|
|
2900
|
+
t.start()
|
|
2901
|
+
for t in threads:
|
|
2902
|
+
t.join()
|
|
2903
|
+
if errors:
|
|
2904
|
+
raise errors[0]
|
|
2905
|
+
|
|
2906
|
+
def _open(self, requests):
|
|
2907
|
+
raise NotImplementedError
|
|
2908
|
+
|
|
2909
|
+
def _hello(self):
|
|
2910
|
+
raise NotImplementedError
|
|
2911
|
+
|
|
2912
|
+
def _on(self, msg):
|
|
2913
|
+
raise NotImplementedError
|
|
2914
|
+
|
|
2915
|
+
|
|
2916
|
+
class Worker(_Stream):
|
|
2917
|
+
"""Runs the commands of its aggregates, one at a time: the kernel sends each command with
|
|
2918
|
+
the aggregate's events (or a snapshot and the events after it), the worker folds the state,
|
|
2919
|
+
calls the command handler and sends the new events back, signed. Made with
|
|
2920
|
+
`Kernable.worker`.
|
|
2921
|
+
|
|
2922
|
+
`run()` processes commands until `stop()`, reconnecting when the session drops (after 1 s);
|
|
2923
|
+
it blocks the calling thread. `ready` (a `threading.Event`) is set once the kernel
|
|
2924
|
+
registered the worker: run it in a thread and wait with `worker.ready.wait(5)`, e.g.
|
|
2925
|
+
`threading.Thread(target=worker.run, daemon=True).start()`. `stop()` ends `run()`, also
|
|
2926
|
+
from another thread."""
|
|
2927
|
+
|
|
2928
|
+
def __init__(self, *args, **kwargs):
|
|
2929
|
+
"""Not for applications: `Kernable.worker` (or `Kernable.app`) makes workers."""
|
|
2930
|
+
raise TypeError("use Kernable.worker (or Kernable.app), not Worker(...)")
|
|
2931
|
+
|
|
2932
|
+
def _init(self, channel: "grpc.Channel | _Partitioned", key: Key, aggregates, log: _Log,
|
|
2933
|
+
service: str = "", deployment: str = ""):
|
|
2934
|
+
"""Use `Kernable.worker`."""
|
|
2935
|
+
super().__init__("worker", 1.0, log)
|
|
2936
|
+
# A channel, or the partitions: then a session per partition.
|
|
2937
|
+
self._source = channel
|
|
2938
|
+
self._stub = None
|
|
2939
|
+
self._key = key
|
|
2940
|
+
self._aggregates = list(aggregates)
|
|
2941
|
+
self._by_command = {f"{a.domain}.{t}": a for a in aggregates for t in a.handlers}
|
|
2942
|
+
# By stream: the kernel names the aggregate type of every command.
|
|
2943
|
+
self._by_stream = {(a.domain, a.name): a for a in aggregates}
|
|
2944
|
+
# The service this worker belongs to; its membership decides whether the kernel waits for it.
|
|
2945
|
+
self._service = service
|
|
2946
|
+
# The code version it declares (see `Kernable`).
|
|
2947
|
+
self._deployment = deployment
|
|
2948
|
+
# Executes whose history is still coming in parts, by lease; per session.
|
|
2949
|
+
self._partial = {}
|
|
2950
|
+
|
|
2951
|
+
def run(self) -> None:
|
|
2952
|
+
"""Processes commands until `stop()`, reconnecting when the session drops; with
|
|
2953
|
+
partitions, a session per partition."""
|
|
2954
|
+
parts = self._partitions(self._source)
|
|
2955
|
+
if parts is None:
|
|
2956
|
+
return
|
|
2957
|
+
if len(parts) > 1 or parts[0][0] != "":
|
|
2958
|
+
self._run_children([Worker._new(channel, self._key, self._aggregates, self._log, self._service,
|
|
2959
|
+
self._deployment)
|
|
2960
|
+
for _, channel in parts])
|
|
2961
|
+
return
|
|
2962
|
+
self._stub = rpc.WorkerGatewayStub(parts[0][1])
|
|
2963
|
+
super().run()
|
|
2964
|
+
|
|
2965
|
+
def _open(self, requests):
|
|
2966
|
+
return self._stub.Session(requests)
|
|
2967
|
+
|
|
2968
|
+
def _hello(self):
|
|
2969
|
+
self._partial = {}
|
|
2970
|
+
# Every event type the aggregates know, applied or declared by their domains: the kernel
|
|
2971
|
+
# refuses an outdated worker, and with a deployment, commands on aggregates holding an
|
|
2972
|
+
# event it does not know.
|
|
2973
|
+
schemas = {f"{a.domain}.{t}": a.schema_of(t) for a in self._aggregates
|
|
2974
|
+
for t in [*a.appliers, *a.events]}
|
|
2975
|
+
register = pb.Register(worker_id=f"py-{uuid.uuid4().hex[:8]}", key_id=self._key.id, protocol=PROTOCOL,
|
|
2976
|
+
max_concurrency=1,
|
|
2977
|
+
aggregates=[pb.WorkerAggregate(aggregate_type=f"{a.domain}.{a.name}",
|
|
2978
|
+
command_types=[f"{a.domain}.{t}" for t in a.handlers])
|
|
2979
|
+
for a in self._aggregates],
|
|
2980
|
+
event_schemas=[pb.EventSchema(event_type=t, schema=s) for t, s in schemas.items()],
|
|
2981
|
+
service=self._service, deployment=self._deployment)
|
|
2982
|
+
return pb.WorkerMessage(register=register)
|
|
2983
|
+
|
|
2984
|
+
def _on(self, msg):
|
|
2985
|
+
kind = msg.WhichOneof("msg")
|
|
2986
|
+
if kind == "challenge":
|
|
2987
|
+
sig = self._key.sign(challenge_message(msg.challenge.nonce)).sig
|
|
2988
|
+
return pb.WorkerMessage(challenge_response=pb.ChallengeResponse(sig=sig))
|
|
2989
|
+
if kind == "registered":
|
|
2990
|
+
if msg.registered.reader_of:
|
|
2991
|
+
self._log.print("info", self._what, "connected to a reader: commands run on the writer "
|
|
2992
|
+
f"({msg.registered.reader_of}); this worker gets none here")
|
|
2993
|
+
if self._connects > 0:
|
|
2994
|
+
self._log.print("info", self._what, "connected again")
|
|
2995
|
+
elif self.announce:
|
|
2996
|
+
self._log.print("info", self._what, f"connected to {self._log.peer} · runs {', '.join(sorted(self._by_command))}")
|
|
2997
|
+
self._connects += 1
|
|
2998
|
+
self.ready.set()
|
|
2999
|
+
elif kind in ("execute", "execute_more"):
|
|
3000
|
+
x = self._assembled(msg)
|
|
3001
|
+
if x is None:
|
|
3002
|
+
return None
|
|
3003
|
+
cmd = x.command
|
|
3004
|
+
stream = f"{cmd.domain}.{cmd.command_type} {cmd.aggregate_type}/{cmd.aggregate_id}"
|
|
3005
|
+
# A handler cannot be stopped from outside; one that runs past its lease at least shows.
|
|
3006
|
+
with _Watch(self._log, self._what, max(0.0, x.deadline / 1000 - time.time()),
|
|
3007
|
+
f"{stream} still running after its lease ended: the kernel gives it to another worker; "
|
|
3008
|
+
"is the handler stuck?"):
|
|
3009
|
+
reply = self._execute(x)
|
|
3010
|
+
if self._log.enabled("debug"):
|
|
3011
|
+
done = (f"rejected {reply.rejection.reason.code}" if reply.HasField("rejection")
|
|
3012
|
+
else f"committed {len(reply.result.events)} events")
|
|
3013
|
+
self._log.print("debug", self._what,
|
|
3014
|
+
f"{cmd.domain}.{cmd.command_type} {cmd.aggregate_type}/{cmd.aggregate_id} {done}")
|
|
3015
|
+
return reply
|
|
3016
|
+
return None
|
|
3017
|
+
|
|
3018
|
+
def _assembled(self, msg):
|
|
3019
|
+
"""The Execute to run once its last part is here (a long history comes in parts), else None."""
|
|
3020
|
+
if msg.WhichOneof("msg") == "execute":
|
|
3021
|
+
x = msg.execute
|
|
3022
|
+
else:
|
|
3023
|
+
rest = msg.execute_more
|
|
3024
|
+
x = self._partial.pop(rest.lease_id, None)
|
|
3025
|
+
if x is None:
|
|
3026
|
+
return None
|
|
3027
|
+
x.history.extend(rest.history)
|
|
3028
|
+
x.vault.extend(rest.vault)
|
|
3029
|
+
x.more = rest.more
|
|
3030
|
+
if x.more:
|
|
3031
|
+
self._partial[x.lease_id] = x
|
|
3032
|
+
return None
|
|
3033
|
+
return x
|
|
3034
|
+
|
|
3035
|
+
def _execute(self, x):
|
|
3036
|
+
cmd = x.command
|
|
3037
|
+
try:
|
|
3038
|
+
agg = self._by_stream.get((cmd.domain, cmd.aggregate_type))
|
|
3039
|
+
if agg is None or cmd.command_type not in agg.handlers:
|
|
3040
|
+
raise Rejected(f"no handler for {cmd.command_type} on {cmd.domain}.{cmd.aggregate_type}",
|
|
3041
|
+
"unknown_command")
|
|
3042
|
+
if x.HasField("snapshot") and x.snapshot.schema != agg.schema:
|
|
3043
|
+
raise Rejected(f"snapshot schema {x.snapshot.schema} is not {agg.schema}", "snapshot_stale")
|
|
3044
|
+
history = [(e.event_type, e.schema, _unseal(_decode(e.payload), x.vault)[0],
|
|
3045
|
+
_meta(e.event_type, e.version, cmd, e.command_id), ApplyContext(_from_wire_context(e.context)))
|
|
3046
|
+
for e in x.history]
|
|
3047
|
+
state = agg.fold(history, agg._from_json(_decode(x.snapshot.state)) if x.HasField("snapshot") else _INITIAL)
|
|
3048
|
+
meta = CommandMeta(cmd.command_id, cmd.command_type, cmd.domain, cmd.aggregate_type, cmd.aggregate_id,
|
|
3049
|
+
cmd.tenant_id)
|
|
3050
|
+
ctx = CommandContext(_from_wire_context(cmd.context), _time(x.deadline), _time(x.received_at), cmd.command_id)
|
|
3051
|
+
data = _unseal(_decode(cmd.payload), x.vault)[0]
|
|
3052
|
+
# A copy: a handler that changes it must not change the snapshot.
|
|
3053
|
+
produced = _call(agg.handlers[cmd.command_type], copy.deepcopy(state), data, meta, ctx)
|
|
3054
|
+
_check_personal(agg, cmd, produced)
|
|
3055
|
+
requires = self._requires(agg, cmd, data, state)
|
|
3056
|
+
events, vault = [], []
|
|
3057
|
+
for e in produced:
|
|
3058
|
+
spec = agg.events.get(e["type"])
|
|
3059
|
+
payload, refs, values = _seal(e.get("data"), spec.vault if spec else None, cmd.aggregate_id)
|
|
3060
|
+
vault.extend(values)
|
|
3061
|
+
erases = str(_call(spec.eraser, e.get("data"), cmd.aggregate_id) or "") if spec and spec.eraser else ""
|
|
3062
|
+
events.append(pb.NewEvent(event_type=e["type"], schema=agg.schema_of(e["type"]), payload=payload,
|
|
3063
|
+
vault_refs=refs, erases=erases))
|
|
3064
|
+
indexed = bool(agg.keys or agg.facts)
|
|
3065
|
+
after = (self._after(agg, cmd, x.expected_version, state, produced)
|
|
3066
|
+
if indexed or _snapshot_due(agg, x.expected_version, len(produced)) else None)
|
|
3067
|
+
snapshot = self._snapshot(agg, cmd, x.expected_version, after, produced)
|
|
3068
|
+
# Every command, also one without events: the set the aggregate holds now.
|
|
3069
|
+
index = agg.index(after) if indexed else Index()
|
|
3070
|
+
except SchemaError:
|
|
3071
|
+
raise # not the command's fault: this worker cannot read the aggregate; stop until updated
|
|
3072
|
+
except Exception as err:
|
|
3073
|
+
# An unexpected error's text may hold personal data or internals, and a rejection is
|
|
3074
|
+
# kept for good: it names only the error's type; the log has the rest.
|
|
3075
|
+
if not isinstance(err, Rejected):
|
|
3076
|
+
self._log.print("error", "worker", f"{cmd.domain}.{cmd.command_type} "
|
|
3077
|
+
f"{cmd.aggregate_type}/{cmd.aggregate_id} failed:\n{traceback.format_exc()}")
|
|
3078
|
+
reason = (pb.Rejected(code=err.code, message=str(err)) if isinstance(err, Rejected)
|
|
3079
|
+
else pb.Rejected(code="handler_error", message=type(err).__name__))
|
|
3080
|
+
sig = self._key.sign(rejection_message(cmd.command_id, reason))
|
|
3081
|
+
return pb.WorkerMessage(rejection=pb.Rejection(lease_id=x.lease_id, reason=reason, worker_sig=sig))
|
|
3082
|
+
sig = self._key.sign(result_message(cmd, x.expected_version, events))
|
|
3083
|
+
result = pb.Result(lease_id=x.lease_id, events=events, worker_sig=sig, snapshot=snapshot, vault=vault)
|
|
3084
|
+
# The full set of keys and facts after the events (none: holds nothing), and what the
|
|
3085
|
+
# command requires; signed apart from the events, only when there is any.
|
|
3086
|
+
result.keys.extend(pb.UniqueKey(name=k.name, value_hash=hashlib.sha256(k.value.encode()).digest(),
|
|
3087
|
+
subject=k.subject, **{"global": k.scope == "global"}) for k in index.keys)
|
|
3088
|
+
result.facts.extend(index.facts)
|
|
3089
|
+
result.requires.extend(requires)
|
|
3090
|
+
if result.keys or result.facts or result.requires:
|
|
3091
|
+
result.index_sig.CopyFrom(self._key.sign(index_message(cmd, x.expected_version, result.keys, result.facts,
|
|
3092
|
+
result.requires)))
|
|
3093
|
+
return pb.WorkerMessage(result=result)
|
|
3094
|
+
|
|
3095
|
+
@staticmethod
|
|
3096
|
+
def _requires(agg: AggregateType, cmd, data, state) -> list:
|
|
3097
|
+
"""The facts of other aggregates the command requires, from its data and the state before it."""
|
|
3098
|
+
fn = agg.requirements.get(cmd.command_type)
|
|
3099
|
+
if fn is None:
|
|
3100
|
+
return []
|
|
3101
|
+
wire = []
|
|
3102
|
+
for r in _call(fn, data, copy.deepcopy(state)) or []:
|
|
3103
|
+
if not isinstance(r, Requirement):
|
|
3104
|
+
raise TypeError(f"requires of {cmd.domain}.{cmd.command_type} returns Requirement, not {type(r).__name__}")
|
|
3105
|
+
wire.append(pb.Requirement(domain=r.domain or cmd.domain, aggregate_type=r.aggregate_type,
|
|
3106
|
+
aggregate_id=r.aggregate_id, fact=r.fact, key=r.key,
|
|
3107
|
+
value_hash=hashlib.sha256(r.value.encode()).digest() if r.key else b"",
|
|
3108
|
+
absent=r.absent))
|
|
3109
|
+
return wire
|
|
3110
|
+
|
|
3111
|
+
@staticmethod
|
|
3112
|
+
def _after(agg: AggregateType, cmd, before: int, state, events):
|
|
3113
|
+
"""The state after the produced `events`, applied as they will be read back (a JSON round
|
|
3114
|
+
trip), so it equals a replay."""
|
|
3115
|
+
if not events:
|
|
3116
|
+
return state
|
|
3117
|
+
apply_ctx = ApplyContext(_from_wire_context(cmd.context))
|
|
3118
|
+
read = [(e["type"], agg.schema_of(e["type"]), _decode(_encode(e.get("data"))),
|
|
3119
|
+
_meta(e["type"], before + i + 1, cmd, cmd.command_id), apply_ctx) for i, e in enumerate(events)]
|
|
3120
|
+
return agg.fold(read, state)
|
|
3121
|
+
|
|
3122
|
+
def _snapshot(self, agg: AggregateType, cmd, before: int, after, events):
|
|
3123
|
+
"""A snapshot of the state `after` the produced `events`, if they cross a multiple of
|
|
3124
|
+
`snapshot_every`."""
|
|
3125
|
+
if not _snapshot_due(agg, before, len(events)):
|
|
3126
|
+
return None
|
|
3127
|
+
version = before + len(events)
|
|
3128
|
+
state_bytes = _encode(agg._to_json(after))
|
|
3129
|
+
sig = self._key.sign(snapshot_message(cmd, version, agg.schema, state_bytes))
|
|
3130
|
+
return pb.Snapshot(version=version, schema=agg.schema, state=state_bytes, worker_sig=sig)
|
|
3131
|
+
|
|
3132
|
+
|
|
3133
|
+
def _snapshot_due(agg: AggregateType, before: int, produced: int) -> bool:
|
|
3134
|
+
"""Whether `produced` events after version `before` cross a multiple of `snapshot_every`."""
|
|
3135
|
+
every = agg.snapshot_every
|
|
3136
|
+
return every > 0 and produced > 0 and (before + produced) // every != before // every
|
|
3137
|
+
|
|
3138
|
+
|
|
3139
|
+
class Listener:
|
|
3140
|
+
"""A listener definition: a durable, named event listener of a domain, made with
|
|
3141
|
+
`Domain.read_model` or `Domain.reactor` and run with `kernable.listener(listener)`, which
|
|
3142
|
+
returns a `ListenerHandle`. Events arrive in order, one at a time, after every listener with a
|
|
3143
|
+
lower order has processed them. Delivery is at least once: if a handler raises, the listener
|
|
3144
|
+
reconnects and the event is delivered again, until a reactor gives up on it and parks it
|
|
3145
|
+
(`park_after` of `Domain.reactor`).
|
|
3146
|
+
|
|
3147
|
+
Type names without a domain (`ItemAdded`, `GetCart`) are the listener's own domain's; types of
|
|
3148
|
+
other domains are qualified (`identity.AccountRegistered`, `identity.*`).
|
|
3149
|
+
|
|
3150
|
+
Handlers get event payloads upcast to the versions of the listener's domain and of the domains
|
|
3151
|
+
it `use`s. An event with a newer schema than known (a newer deployment wrote it) never reaches
|
|
3152
|
+
a handler, so it is never misread: a reactor that parks (`park_after` above 0, not live, not
|
|
3153
|
+
one of several instances that each get every event) parks it at once with the code
|
|
3154
|
+
`unknown_event`; every other listener skips it, logs a warning once per type and schema, and
|
|
3155
|
+
counts it (`ListenerHandle.skipped`). Events of types without a handler are not affected.
|
|
3156
|
+
|
|
3157
|
+
`on`, `query` and `middleware` work as decorators (`@carts_readmodel.on("ItemAdded")`) or
|
|
3158
|
+
directly (`carts_readmodel.on("ItemAdded", fn)`); handlers take only the parameters they
|
|
3159
|
+
need. They run in the thread of the listener's `run()`, one at a time, so plain dicts are
|
|
3160
|
+
safe to use as state there; guard state that other threads also touch."""
|
|
3161
|
+
|
|
3162
|
+
def __init__(self, domain: Domain, name: str, order: int, start: int | None, reconnect_s: float,
|
|
3163
|
+
store: str | None = None, schema: str = "1", *, live: bool = False, park_after: int = 0,
|
|
3164
|
+
instance: str | None = None, single: bool = False, reactor: bool = False):
|
|
3165
|
+
"""Use `Domain.read_model` or `Domain.reactor`."""
|
|
3166
|
+
#: Defined with `Domain.reactor`: a reader (a read-only copy of the writer) lets it wait.
|
|
3167
|
+
self.reactor = reactor
|
|
3168
|
+
#: The instance ID it connects with (`None`: `KERNABLE_INSTANCE`, else none); see `Domain.read_model`.
|
|
3169
|
+
self.instance = instance
|
|
3170
|
+
#: Whether the instances take turns; see `Domain.reactor`.
|
|
3171
|
+
self.single = single
|
|
3172
|
+
#: Attempts before an event its handler keeps failing on is parked; 0: never.
|
|
3173
|
+
self.park_after = park_after
|
|
3174
|
+
#: The domain the listener belongs to: unqualified type names are its types.
|
|
3175
|
+
self.domain = domain
|
|
3176
|
+
self.name = name
|
|
3177
|
+
self.order = order
|
|
3178
|
+
self.start = start
|
|
3179
|
+
self.reconnect_s = reconnect_s
|
|
3180
|
+
#: Where the running listener keeps its state (`sqlite:<path>` or `memory:`), if anywhere.
|
|
3181
|
+
self.store_url = store
|
|
3182
|
+
self.schema = schema
|
|
3183
|
+
#: Not registered with the kernel: see `Domain.reactor`.
|
|
3184
|
+
self.live = live
|
|
3185
|
+
self._handlers: dict[str, Callable] = {}
|
|
3186
|
+
self._raw: set[str] = set()
|
|
3187
|
+
self._domains: dict[str, dict[str, EventSchema]] = {domain.name: domain.events}
|
|
3188
|
+
self._queries: dict[str, Callable] = {}
|
|
3189
|
+
#: The middleware handlers and their specs, by when they run: "before" and "after".
|
|
3190
|
+
self._middlewares: dict[str, tuple[pb.Middleware, Callable]] = {}
|
|
3191
|
+
|
|
3192
|
+
def _qualify(self, type: str) -> str:
|
|
3193
|
+
return type if "." in type else f"{self.domain.name}.{type}"
|
|
3194
|
+
|
|
3195
|
+
def use(self, *domains: Domain) -> "Listener":
|
|
3196
|
+
"""Upcasts the events of these other domains with their event schemas (see
|
|
3197
|
+
`Domain.event`); the listener's own domain's apply already. Returns the listener, so it
|
|
3198
|
+
chains: `shop_domain.read_model("orders").use(billing_domain)`."""
|
|
3199
|
+
for d in domains:
|
|
3200
|
+
self._domains[d.name] = d.events
|
|
3201
|
+
return self
|
|
3202
|
+
|
|
3203
|
+
def on(self, event_type: str, fn: Callable | None = None, *, raw: bool = False):
|
|
3204
|
+
"""Registers a handler for an event type (`ItemAdded`, or `orders.OrderPlaced` of another
|
|
3205
|
+
domain) or a whole domain (`orders.*`); works as a decorator too: `fn(data, evt, ctx)` gets
|
|
3206
|
+
the event's payload, a `StoredEventMeta` and a `ListenerContext` (both optional). A `raw`
|
|
3207
|
+
handler gets payloads as stored, without upcasting, whatever their schema (for archives
|
|
3208
|
+
and exports). Example: `@carts_readmodel.on("ItemAdded")` over `def _(data, evt):
|
|
3209
|
+
rows[evt.aggregate_id] = data["qty"]`. If the handler raises, the listener reconnects and
|
|
3210
|
+
the event is delivered again, so handlers must be idempotent. Raises `ValueError` if
|
|
3211
|
+
`event_type` is none of these."""
|
|
3212
|
+
if _is_message_class(event_type, Event):
|
|
3213
|
+
cls = event_type
|
|
3214
|
+
b = _binding(cls, "declared (an aggregate's apply, or Domain.event)")
|
|
3215
|
+
self._domains.setdefault(b.domain, b.schemas.events)
|
|
3216
|
+
qualified = f"{b.domain}.{cls.type_name()}"
|
|
3217
|
+
self._raw.discard(qualified)
|
|
3218
|
+
|
|
3219
|
+
def wrap(f: Callable) -> Callable:
|
|
3220
|
+
return lambda data, evt=None, ctx=None: _call(f, cls.from_wire(data), evt, ctx)
|
|
3221
|
+
|
|
3222
|
+
return _register_wrapped(self, self._handlers, qualified, fn, wrap)
|
|
3223
|
+
# A bare `*` is not the own domain (that is `<domain>.*`): in middleware it means everything.
|
|
3224
|
+
if not _UNQUALIFIED.match(event_type) and not _LISTENED.match(event_type):
|
|
3225
|
+
raise ValueError(f"listen to a type (Type), a qualified type (domain.Type) or a domain (domain.*), not {event_type!r}")
|
|
3226
|
+
qualified = self._qualify(event_type)
|
|
3227
|
+
(self._raw.add if raw else self._raw.discard)(qualified)
|
|
3228
|
+
return _register(self, self._handlers, qualified, fn)
|
|
3229
|
+
|
|
3230
|
+
def query(self, query_type: str, fn: Callable | None = None):
|
|
3231
|
+
"""Answers a query type (`GetCart`, or `orders.GetOrder` of another domain) from this
|
|
3232
|
+
listener's state; works as a decorator too: `fn(data, qry, ctx)` gets the query's data, a
|
|
3233
|
+
`QueryMeta` and a `QueryContext` (both optional). The kernel delivers queries in order with
|
|
3234
|
+
the events, so an answer reflects every event handled before it. Return the answer
|
|
3235
|
+
(JSON-like; `None` reaches the caller as `None`); raise `Rejected` to refuse (any other exception refuses with
|
|
3236
|
+
`handler_error`). A handler cannot query its own listener (the answer would wait behind
|
|
3237
|
+
itself): the kernel refuses it. Example: `@carts_readmodel.query("GetCart")` over
|
|
3238
|
+
`def _(data): return rows[data["id"]]`. Raises `ValueError` if `query_type` is invalid."""
|
|
3239
|
+
if _is_message_class(query_type, Query):
|
|
3240
|
+
cls = query_type
|
|
3241
|
+
_bind(cls, self.domain.name, self.domain)
|
|
3242
|
+
qualified = f"{self.domain.name}.{cls.type_name()}"
|
|
3243
|
+
# Its target counts also when the query is asked by name, by clients that `use` the domain.
|
|
3244
|
+
target = _query_target(cls)
|
|
3245
|
+
if target and cls.type_name() not in self.domain.queries:
|
|
3246
|
+
self.domain.query(cls.type_name()).target(*target)
|
|
3247
|
+
|
|
3248
|
+
def wrap(f: Callable) -> Callable:
|
|
3249
|
+
return lambda data, qry=None, ctx=None: _to_wire(_call(f, cls.from_wire(data), qry, ctx))
|
|
3250
|
+
|
|
3251
|
+
return _register_wrapped(self, self._queries, qualified, fn, wrap)
|
|
3252
|
+
qualified = self._qualify(query_type)
|
|
3253
|
+
if not _QUERY.match(qualified):
|
|
3254
|
+
raise ValueError(f"answer a query type (Query or domain.Query), not {query_type!r}")
|
|
3255
|
+
return _register(self, self._queries, qualified, fn)
|
|
3256
|
+
|
|
3257
|
+
def middleware(self, types: list[str], order: int = 0, fn: Callable | None = None, *, when: str = "before"):
|
|
3258
|
+
"""Makes this listener a middleware; works as a decorator too. The kernel asks `fn` about
|
|
3259
|
+
every command and query of `types` (`*` for all, `Type` of the own domain, `domain.*` or
|
|
3260
|
+
`domain.Type`) before accepting or routing it (or after, see `when`), in chain order (lower
|
|
3261
|
+
`order` first, then by listener name), and only once this listener has caught up with the
|
|
3262
|
+
events. `fn(check, outcome, ctx)` gets a `Check`, after the request what came of it, and a
|
|
3263
|
+
`MiddlewareContext` with the listener's store (both optional): an
|
|
3264
|
+
`Outcome` for a command, a `QueryOutcome` for a query (`check.kind` tells which); before,
|
|
3265
|
+
None. `types` cover file and cache requests too, before they run: `domain.PutData`,
|
|
3266
|
+
`.GetData`, `.DeleteData`, `.ListData`, `.SetCache`, `.GetCache` and `.DeleteCache` (and `domain.*` or
|
|
3267
|
+
`*` include them); `check.kind` is then `"data"` or `"cache"`, `check.key` the key and
|
|
3268
|
+
`check.data` None, and a refusal reaches the client as `PERMISSION_DENIED`. Return False or
|
|
3269
|
+
raise `Rejected` to refuse; any other error refuses too, and so does a return value other
|
|
3270
|
+
than None, True or False. The registration is stored:
|
|
3271
|
+
while this listener is away, the kernel refuses those requests (`kernable listeners
|
|
3272
|
+
remove` unregisters it). `fn` cannot submit or query what it checks itself (it would wait
|
|
3273
|
+
behind itself): the kernel refuses it. A listener has at most one middleware before and
|
|
3274
|
+
one after.
|
|
3275
|
+
`when`: "before" (default) checks a request before the kernel accepts it; raising refuses
|
|
3276
|
+
it. "after" gets the request's outcome too: raising refuses a query's response; a
|
|
3277
|
+
command's outcome, and a rejected query, it only observes, best effort (it misses what
|
|
3278
|
+
happens while it is away, and nobody waits for it).
|
|
3279
|
+
Negative orders are reserved for kernable's kits (a grant of `reserved_order`). A live listener cannot be a middleware.
|
|
3280
|
+
Example: `@guard_readmodel.middleware(["shop.*"])` over a `def _(check):` that raises
|
|
3281
|
+
`Rejected("not allowed", "forbidden")` unless `check.request.sender_identity` may send
|
|
3282
|
+
`check.type`.
|
|
3283
|
+
Raises `ValueError` for an invalid type or `when`, a second middleware at the same
|
|
3284
|
+
`when`, or a live listener."""
|
|
3285
|
+
if self.live:
|
|
3286
|
+
raise ValueError(f"{self.name} is live: a live listener cannot be a middleware")
|
|
3287
|
+
qualified = [t if t == "*" else self._qualify(t) for t in types]
|
|
3288
|
+
for t, q in zip(types, qualified):
|
|
3289
|
+
if not _CHECKED.match(q):
|
|
3290
|
+
raise ValueError(f"check `*`, a type (Type or domain.Type) or a domain (domain.*), not {t!r}")
|
|
3291
|
+
if when not in ("before", "after"):
|
|
3292
|
+
raise ValueError(f"run a middleware \"before\" or \"after\", not {when!r}")
|
|
3293
|
+
if when in self._middlewares:
|
|
3294
|
+
raise ValueError(f"{self.name} has a middleware {when} already")
|
|
3295
|
+
spec = pb.Middleware(order=order, types=qualified, when=pb.WHEN_AFTER if when == "after" else pb.WHEN_BEFORE)
|
|
3296
|
+
if fn is not None:
|
|
3297
|
+
self._middlewares[when] = (spec, fn)
|
|
3298
|
+
return self
|
|
3299
|
+
|
|
3300
|
+
def decorator(f: Callable) -> Callable:
|
|
3301
|
+
self._middlewares[when] = (spec, f)
|
|
3302
|
+
return f
|
|
3303
|
+
|
|
3304
|
+
return decorator
|
|
3305
|
+
|
|
3306
|
+
|
|
3307
|
+
class ListenerHandle(_Stream):
|
|
3308
|
+
"""A running `Listener`, returned by `kernable.listener(listener)`: `run()` processes events,
|
|
3309
|
+
queries and checks until `stop()`; `ready` is set once the kernel accepted it.
|
|
3310
|
+
|
|
3311
|
+
`run()` blocks the calling thread. To run several listeners in one process, run all but
|
|
3312
|
+
one in threads (`threading.Thread(target=handle.run, daemon=True).start()`), or use
|
|
3313
|
+
`Kernable.app`. `ready` is a `threading.Event`: `handle.ready.wait(5)` returns whether the
|
|
3314
|
+
kernel accepted the listener within 5 seconds. `stop()` ends `run()`, also from another
|
|
3315
|
+
thread. Attributes: `listener` (the definition), `position` (global position of the last
|
|
3316
|
+
handled event, or skipped, for other event types), `skipped` (how many events it skipped as
|
|
3317
|
+
newer than its code, see `Listener`) and `store` (the listener's `Store`, while `run()` has it
|
|
3318
|
+
open)."""
|
|
3319
|
+
|
|
3320
|
+
def __init__(self, *args, **kwargs):
|
|
3321
|
+
"""Not for applications: `Kernable.listener` (or `Kernable.app`) runs listeners."""
|
|
3322
|
+
raise TypeError("use Kernable.listener (or Kernable.app), not ListenerHandle(...)")
|
|
3323
|
+
|
|
3324
|
+
def _init(self, channel: "grpc.Channel | _Partitioned", key: Key, listener: Listener, log: _Log,
|
|
3325
|
+
service: str = "", kernable: "Kernable | None" = None, part: str = "",
|
|
3326
|
+
store: "Store | None" = None, parent: "ListenerHandle | None" = None):
|
|
3327
|
+
"""Use `Kernable.listener`. `part` and `store`: a partition's stream, sharing the
|
|
3328
|
+
store of the handle that runs it."""
|
|
3329
|
+
# For handlers' follow-up commands and queries.
|
|
3330
|
+
self._kernable = kernable
|
|
3331
|
+
# How often the handler failed on an event, by position, until it gets past it.
|
|
3332
|
+
self._attempts: dict[int, int] = {}
|
|
3333
|
+
super().__init__(f"listener {listener.name}", listener.reconnect_s, log)
|
|
3334
|
+
# The service this listener belongs to; its membership decides whether it counts.
|
|
3335
|
+
self._service = service
|
|
3336
|
+
#: The listener definition this runs.
|
|
3337
|
+
self.listener = listener
|
|
3338
|
+
#: The listener's store, while `run()` has it open (with `store=`).
|
|
3339
|
+
self.store: Store | None = store
|
|
3340
|
+
# A channel, or the partitions: then a stream per partition, sharing the store.
|
|
3341
|
+
self._source = channel
|
|
3342
|
+
self._part = part
|
|
3343
|
+
self._shared = store is not None
|
|
3344
|
+
# As a partition's stream: the handle that runs it. As that handle: how far each
|
|
3345
|
+
# partition's stream has handled, for the order across tenant moves.
|
|
3346
|
+
self._parent = parent
|
|
3347
|
+
self._progress: dict[str, int] = {}
|
|
3348
|
+
self._moves = threading.Condition()
|
|
3349
|
+
# Tenants this stream read the map again for.
|
|
3350
|
+
self._checked: set[str] = set()
|
|
3351
|
+
self._stub = None
|
|
3352
|
+
self._key = key
|
|
3353
|
+
self._start = listener.start
|
|
3354
|
+
#: Global position of the last handled event (or skipped, for other event types).
|
|
3355
|
+
self.position = 0
|
|
3356
|
+
#: How many events it skipped because their schema is newer than its code reads (a newer
|
|
3357
|
+
#: deployment wrote them); see `Listener`. Across all partitions' streams.
|
|
3358
|
+
self.skipped = 0
|
|
3359
|
+
# Newest event when the kernel accepted the listener; events up to it are replayed.
|
|
3360
|
+
self._replay_up_to = 0
|
|
3361
|
+
# The position sent in the last hello, if any.
|
|
3362
|
+
self._sent_start: int | None = None
|
|
3363
|
+
#: Position after which the kernel delivered on the first connect.
|
|
3364
|
+
self.started_at = 0
|
|
3365
|
+
|
|
3366
|
+
def run(self) -> None:
|
|
3367
|
+
"""Processes events, queries and checks until `stop()`, reconnecting when the stream
|
|
3368
|
+
drops; with a store, opens it first and continues after its position, and closes it when
|
|
3369
|
+
`run()` ends. A store that cannot be opened (a full disk) is tried again, backing off like
|
|
3370
|
+
a reconnect. With partitions, a stream per partition, each from its own position in the
|
|
3371
|
+
shared store."""
|
|
3372
|
+
parts = self._partitions(self._source)
|
|
3373
|
+
if parts is None:
|
|
3374
|
+
return
|
|
3375
|
+
partitioned = len(parts) > 1 or parts[0][0] != ""
|
|
3376
|
+
if self.listener.store_url and self.store is None:
|
|
3377
|
+
attempt = 0
|
|
3378
|
+
while self.store is None:
|
|
3379
|
+
try:
|
|
3380
|
+
self.store = Store(self.listener.store_url, self.listener.schema)
|
|
3381
|
+
except Exception as err: # sqlite3 and OS errors alike
|
|
3382
|
+
attempt += 1
|
|
3383
|
+
self._log.store_failed(self._what, err, self._reconnect_s, attempt, self._wake)
|
|
3384
|
+
if self._stopped:
|
|
3385
|
+
return
|
|
3386
|
+
if not partitioned:
|
|
3387
|
+
self.position = self.store.committed
|
|
3388
|
+
# A new (or cleared) store says nothing: the listener's own start applies (a
|
|
3389
|
+
# reactor starts at the newest event).
|
|
3390
|
+
if self.store.committed > 0:
|
|
3391
|
+
self._start = self.store.committed
|
|
3392
|
+
if self._shared:
|
|
3393
|
+
self.position = self.store.committed_for(self._part)
|
|
3394
|
+
if self.position > 0:
|
|
3395
|
+
self._start = self.position
|
|
3396
|
+
try:
|
|
3397
|
+
if partitioned:
|
|
3398
|
+
self._run_children([ListenerHandle._new(channel, self._key, self.listener, self._log, self._service,
|
|
3399
|
+
self._kernable, part=name, store=self.store,
|
|
3400
|
+
parent=self)
|
|
3401
|
+
for name, channel in parts])
|
|
3402
|
+
return
|
|
3403
|
+
self._stub = rpc.ListenerServiceStub(parts[0][1])
|
|
3404
|
+
super().run()
|
|
3405
|
+
finally:
|
|
3406
|
+
# Stopping releases the store; a later run opens it again. A partition's stream
|
|
3407
|
+
# leaves it to the handle that owns it.
|
|
3408
|
+
if self.store is not None and not self._shared:
|
|
3409
|
+
self.store.close()
|
|
3410
|
+
self.store = None
|
|
3411
|
+
|
|
3412
|
+
def _open(self, requests):
|
|
3413
|
+
return self._stub.Listen(requests)
|
|
3414
|
+
|
|
3415
|
+
def _hello(self):
|
|
3416
|
+
d = self.listener
|
|
3417
|
+
instance = d.instance if d.instance is not None else os.environ.get("KERNABLE_INSTANCE", "")
|
|
3418
|
+
hello = pb.Hello(name=d.name, key_id=self._key.id, protocol=PROTOCOL, order=d.order,
|
|
3419
|
+
event_types=list(d._handlers),
|
|
3420
|
+
query_types=list(d._queries),
|
|
3421
|
+
middlewares=[spec for spec, _ in d._middlewares.values()], live=d.live,
|
|
3422
|
+
service=self._service, instance=instance,
|
|
3423
|
+
single=bool(instance) and d.single and not d.live,
|
|
3424
|
+
deployment=self._kernable._deployment if self._kernable else "",
|
|
3425
|
+
reactor=d.reactor)
|
|
3426
|
+
# Reconnects resume after the last handled event, once the kernel sent one (an instance
|
|
3427
|
+
# that only waited has none).
|
|
3428
|
+
start = self.position if self._connects > 0 else self._start
|
|
3429
|
+
self._sent_start = start
|
|
3430
|
+
if start is not None:
|
|
3431
|
+
hello.from_position = start
|
|
3432
|
+
return pb.ListenerMessage(hello=hello)
|
|
3433
|
+
|
|
3434
|
+
def _on(self, msg):
|
|
3435
|
+
# Requests sent from the handlers name this listener, so the kernel never makes them
|
|
3436
|
+
# wait for it (it handles one message at a time).
|
|
3437
|
+
token = _HANDLING.set(self.listener.name)
|
|
3438
|
+
try:
|
|
3439
|
+
with _Watch(self._log, self._what, *_handling(msg)):
|
|
3440
|
+
return self._handle(msg)
|
|
3441
|
+
finally:
|
|
3442
|
+
_HANDLING.reset(token)
|
|
3443
|
+
|
|
3444
|
+
def _handle(self, msg):
|
|
3445
|
+
kind = msg.WhichOneof("msg")
|
|
3446
|
+
if kind == "challenge":
|
|
3447
|
+
sig = self._key.sign(challenge_message(msg.challenge.nonce)).sig
|
|
3448
|
+
return pb.ListenerMessage(challenge_response=pb.ChallengeResponse(sig=sig))
|
|
3449
|
+
if kind == "ready":
|
|
3450
|
+
self.position = msg.ready.after_position
|
|
3451
|
+
if self._parent is not None:
|
|
3452
|
+
self._parent._advanced(self._part, msg.ready.after_position)
|
|
3453
|
+
self._replay_up_to = msg.ready.head_position
|
|
3454
|
+
self._log_connected(msg.ready.after_position)
|
|
3455
|
+
self.ready.set()
|
|
3456
|
+
elif kind == "waiting":
|
|
3457
|
+
# Another instance runs the group, or (a reactor) another deployment serves: this one
|
|
3458
|
+
# is ready to take over.
|
|
3459
|
+
if msg.waiting.reader_of:
|
|
3460
|
+
self._log.print("info", self._what, "waiting: this kernel is a reader; reactors act on the "
|
|
3461
|
+
f"writer ({msg.waiting.reader_of})")
|
|
3462
|
+
elif msg.waiting.serving:
|
|
3463
|
+
self._log.print("info", self._what, f"waiting: deployment {msg.waiting.serving} serves; "
|
|
3464
|
+
"this reactor acts once its own deployment does")
|
|
3465
|
+
else:
|
|
3466
|
+
self._log.print("info", self._what, f"waiting: instance {msg.waiting.active} is active")
|
|
3467
|
+
self.ready.set()
|
|
3468
|
+
elif kind == "batch":
|
|
3469
|
+
# A kernel that goes back (a bug, or not the kernel) must not make a store apply
|
|
3470
|
+
# events twice: the session ends, and the next resumes after what was handled.
|
|
3471
|
+
if msg.batch.up_to < self.position:
|
|
3472
|
+
raise RuntimeError(f"the kernel sent a batch up to {msg.batch.up_to}, behind position {self.position}")
|
|
3473
|
+
self._in_order(msg.batch.events)
|
|
3474
|
+
# With a store, the batch's data and position are committed together, before the ack.
|
|
3475
|
+
if self.store:
|
|
3476
|
+
self.store._begin()
|
|
3477
|
+
try:
|
|
3478
|
+
parked, skipped = self._handle_batch(msg.batch)
|
|
3479
|
+
if self.store:
|
|
3480
|
+
self.store._commit(msg.batch.up_to, self._part)
|
|
3481
|
+
except BaseException:
|
|
3482
|
+
if self.store:
|
|
3483
|
+
self.store._rollback()
|
|
3484
|
+
self.position = self.store.committed_for(self._part)
|
|
3485
|
+
raise
|
|
3486
|
+
self.position = msg.batch.up_to
|
|
3487
|
+
if self._parent is not None:
|
|
3488
|
+
self._parent._advanced(self._part, msg.batch.up_to)
|
|
3489
|
+
# Values removed after an erasure must not stay behind in the WAL, outside the batch.
|
|
3490
|
+
if self.store and any(e.erases for e in [*msg.batch.events, *msg.batch.retries]):
|
|
3491
|
+
self.store._checkpoint()
|
|
3492
|
+
return pb.ListenerMessage(ack=pb.Ack(position=msg.batch.up_to, parked=parked, skipped=skipped))
|
|
3493
|
+
elif kind == "query":
|
|
3494
|
+
answer = self._answer(msg.query.call_id, msg.query.query)
|
|
3495
|
+
if self._log.enabled("debug"):
|
|
3496
|
+
q = msg.query.query
|
|
3497
|
+
done = (f"rejected {q.domain}.{q.query_type} {answer.rejected.code}" if answer.HasField("rejected")
|
|
3498
|
+
else f"answered {q.domain}.{q.query_type}")
|
|
3499
|
+
self._log.print("debug", self._what, done)
|
|
3500
|
+
return pb.ListenerMessage(answer=answer)
|
|
3501
|
+
elif kind == "check":
|
|
3502
|
+
return pb.ListenerMessage(verdict=self._verdict(msg.check))
|
|
3503
|
+
return None
|
|
3504
|
+
|
|
3505
|
+
def _advanced(self, part: str, position: int) -> None:
|
|
3506
|
+
"""On the parent: partition `part`'s stream has handled everything up to `position`."""
|
|
3507
|
+
with self._moves:
|
|
3508
|
+
if self._progress.get(part, -1) < position:
|
|
3509
|
+
self._progress[part] = position
|
|
3510
|
+
self._moves.notify_all()
|
|
3511
|
+
|
|
3512
|
+
def _in_order(self, events) -> None:
|
|
3513
|
+
"""Before a batch of a partition's stream: the events of a tenant that moved here after its move wait until the stream of the partition it came from has passed the
|
|
3514
|
+
tenant's last event there, so a tenant's events keep their order across the move. Waits
|
|
3515
|
+
at most 30 s (a partition that is down), then goes on with a warning."""
|
|
3516
|
+
parent = self._parent
|
|
3517
|
+
if parent is None or not isinstance(parent._source, _Partitioned) or not events:
|
|
3518
|
+
return
|
|
3519
|
+
source = parent._source
|
|
3520
|
+
m = source.current()
|
|
3521
|
+
if m is None:
|
|
3522
|
+
return
|
|
3523
|
+
|
|
3524
|
+
def known(m, tenant: str) -> bool:
|
|
3525
|
+
return _partition_of(m, tenant) == self._part or any(
|
|
3526
|
+
x.tenant == tenant and self._part in (x.to, getattr(x, "from")) for x in m.moved)
|
|
3527
|
+
|
|
3528
|
+
for e in events:
|
|
3529
|
+
if not known(m, e.tenant_id) and e.tenant_id not in self._checked:
|
|
3530
|
+
# The tenant came here after the map was read; another kernel may take a moment
|
|
3531
|
+
# to hold the new one.
|
|
3532
|
+
self._checked.add(e.tenant_id)
|
|
3533
|
+
for i in range(3):
|
|
3534
|
+
if known(m, e.tenant_id):
|
|
3535
|
+
break
|
|
3536
|
+
if i > 0:
|
|
3537
|
+
time.sleep(0.5)
|
|
3538
|
+
m = source.current(refresh=True) or m
|
|
3539
|
+
for x in m.moved:
|
|
3540
|
+
origin = getattr(x, "from")
|
|
3541
|
+
if x.tenant != e.tenant_id or x.to != self._part or e.global_position <= x.target_after:
|
|
3542
|
+
continue
|
|
3543
|
+
if not any(p.name == origin for p in m.partitions):
|
|
3544
|
+
continue
|
|
3545
|
+
parent._reached(self, origin, x.source_head, e.tenant_id)
|
|
3546
|
+
|
|
3547
|
+
def _reached(self, child: "ListenerHandle", part: str, position: int, tenant: str) -> None:
|
|
3548
|
+
"""On the parent: returns once partition `part`'s stream has handled up to `position`, or
|
|
3549
|
+
after 30 s."""
|
|
3550
|
+
deadline = time.monotonic() + 30
|
|
3551
|
+
with self._moves:
|
|
3552
|
+
while self._progress.get(part, -1) < position:
|
|
3553
|
+
if child._stopped:
|
|
3554
|
+
raise RuntimeError("stopped")
|
|
3555
|
+
left = deadline - time.monotonic()
|
|
3556
|
+
if left <= 0:
|
|
3557
|
+
self._log.print("warn", child._what,
|
|
3558
|
+
f"tenant {tenant} moved here from partition {part}, whose stream has not "
|
|
3559
|
+
f"passed position {position} in 30 s; going on without its order")
|
|
3560
|
+
return
|
|
3561
|
+
self._moves.wait(min(left, 0.1))
|
|
3562
|
+
|
|
3563
|
+
def _log_connected(self, after_position: int) -> None:
|
|
3564
|
+
"""Logs that the kernel accepted the listener; the position is the start sent (the
|
|
3565
|
+
kernel's, if none was)."""
|
|
3566
|
+
position = after_position if self._sent_start is None else self._sent_start
|
|
3567
|
+
if self._connects == 0:
|
|
3568
|
+
self.started_at = position
|
|
3569
|
+
if self.announce:
|
|
3570
|
+
self._log.print("info", self._what, " · ".join(
|
|
3571
|
+
[f"connected to {self._log.peer}", *_listener_details(self.listener, position)]))
|
|
3572
|
+
else:
|
|
3573
|
+
self._log.print("info", self._what, f"connected again at position {position}")
|
|
3574
|
+
self._connects += 1
|
|
3575
|
+
|
|
3576
|
+
def _handle_batch(self, batch) -> "tuple[list[pb.Parked], list[pb.Skipped]]":
|
|
3577
|
+
"""Handles a batch: first the retries, then the events after the position. Returns the
|
|
3578
|
+
events it parked and those it skipped as newer than its code; raises if a handler failed
|
|
3579
|
+
on one it does not park."""
|
|
3580
|
+
d = self.listener
|
|
3581
|
+
park_after = 0 if d.live else d.park_after
|
|
3582
|
+
instance = d.instance if d.instance is not None else os.environ.get("KERNABLE_INSTANCE", "")
|
|
3583
|
+
# Instances that each get every event cannot park; they skip what they cannot read.
|
|
3584
|
+
parks = park_after > 0 and not (instance and not d.single)
|
|
3585
|
+
parked, skipped = [], []
|
|
3586
|
+
for e, retry in [*((e, True) for e in batch.retries), *((e, False) for e in batch.events)]:
|
|
3587
|
+
if not retry and e.global_position <= self.position:
|
|
3588
|
+
continue # handled already
|
|
3589
|
+
name = f"{e.domain}.{e.event_type}"
|
|
3590
|
+
if name not in d._handlers:
|
|
3591
|
+
name = f"{e.domain}.*"
|
|
3592
|
+
handler = d._handlers.get(name)
|
|
3593
|
+
if handler:
|
|
3594
|
+
data, redacted = _unseal(_decode(e.payload), batch.vault)
|
|
3595
|
+
schema = e.schema
|
|
3596
|
+
if name not in d._raw:
|
|
3597
|
+
try:
|
|
3598
|
+
schema, data = _upcast(d._domains.get(e.domain), e.domain, e.event_type, schema, data)
|
|
3599
|
+
except SchemaError as err:
|
|
3600
|
+
if not err.newer:
|
|
3601
|
+
raise
|
|
3602
|
+
if parks:
|
|
3603
|
+
parked.append(self._unknown(err, e.global_position))
|
|
3604
|
+
else:
|
|
3605
|
+
skipped.append(self._skip(err, e.global_position))
|
|
3606
|
+
if not retry:
|
|
3607
|
+
self.position = e.global_position
|
|
3608
|
+
continue
|
|
3609
|
+
evt = StoredEventMeta(e.event_type, e.version, e.tenant_id, e.domain, e.aggregate_type, e.aggregate_id,
|
|
3610
|
+
e.command_id, e.global_position, schema, redacted, _time(e.created_at),
|
|
3611
|
+
e.deployment)
|
|
3612
|
+
ctx = ListenerContext(e.global_position <= self._replay_up_to, _from_wire_context(e.context),
|
|
3613
|
+
self.store, e.command_id, e.tenant_id, d.domain.name, self._kernable)
|
|
3614
|
+
saved = self.store is not None and park_after > 0
|
|
3615
|
+
if saved:
|
|
3616
|
+
self.store._savepoint()
|
|
3617
|
+
try:
|
|
3618
|
+
_call(handler, data, evt, ctx)
|
|
3619
|
+
except Exception as err:
|
|
3620
|
+
# An erasing event is never parked: its handler must get past it, or what it
|
|
3621
|
+
# removes would stay behind.
|
|
3622
|
+
parked.append(self._give_up(f"{e.domain}.{e.event_type}", e.global_position, err,
|
|
3623
|
+
0 if e.erases else park_after))
|
|
3624
|
+
if saved:
|
|
3625
|
+
self.store._rollback_event()
|
|
3626
|
+
else:
|
|
3627
|
+
if saved:
|
|
3628
|
+
self.store._release()
|
|
3629
|
+
self._attempts.pop(e.global_position, None)
|
|
3630
|
+
self._log.print("debug", self._what, f"{e.domain}.{e.event_type} at {e.global_position}")
|
|
3631
|
+
if not retry:
|
|
3632
|
+
self.position = e.global_position
|
|
3633
|
+
return parked, skipped
|
|
3634
|
+
|
|
3635
|
+
def _unknown(self, err: SchemaError, position: int) -> "pb.Parked":
|
|
3636
|
+
"""Parks an event newer than this code reads, at once: no attempt could handle it."""
|
|
3637
|
+
message = (f"{err.type} schema {err.schema}: this code reads it up to {err.known}; "
|
|
3638
|
+
"a newer deployment wrote it")
|
|
3639
|
+
self._attempts.pop(position, None)
|
|
3640
|
+
self._log.print("warn", self._what, f"{err.type} at {position} parked: {message}")
|
|
3641
|
+
return pb.Parked(position=position, code="unknown_event", message=message, attempts=1)
|
|
3642
|
+
|
|
3643
|
+
def _skip(self, err: SchemaError, position: int) -> "pb.Skipped":
|
|
3644
|
+
"""Skips an event newer than this code reads: counts it, warns once per type and schema."""
|
|
3645
|
+
for h in (self, self._parent):
|
|
3646
|
+
if h is not None:
|
|
3647
|
+
h.skipped += 1
|
|
3648
|
+
with _SKIP_LOCK:
|
|
3649
|
+
first = (err.type, err.schema) not in _SKIP_WARNED
|
|
3650
|
+
_SKIP_WARNED.add((err.type, err.schema))
|
|
3651
|
+
if first:
|
|
3652
|
+
self._log.print("warn", self._what, f"skipped {err.type} schema {err.schema}: this code reads it up "
|
|
3653
|
+
f"to {err.known}; a newer deployment wrote it")
|
|
3654
|
+
return pb.Skipped(position=position, event_type=err.type, schema=err.schema)
|
|
3655
|
+
|
|
3656
|
+
def _give_up(self, type: str, position: int, err: Exception, park_after: int) -> "pb.Parked":
|
|
3657
|
+
"""What to do after the handler failed on the event at `position`: park it (a rejection
|
|
3658
|
+
at once, another error after `park_after` attempts), or raise `err`, so the listener gets
|
|
3659
|
+
the event again (the kernel or the store away, or attempts left)."""
|
|
3660
|
+
if park_after <= 0 or _transient(err):
|
|
3661
|
+
raise err
|
|
3662
|
+
attempts = self._attempts.get(position, 0) + 1
|
|
3663
|
+
self._attempts[position] = attempts
|
|
3664
|
+
if not isinstance(err, Rejected) and attempts < park_after:
|
|
3665
|
+
raise err
|
|
3666
|
+
del self._attempts[position]
|
|
3667
|
+
if isinstance(err, Rejected):
|
|
3668
|
+
self._log.print("warn", self._what, f"{type} at {position} rejected ({err.code}), parked: {err}")
|
|
3669
|
+
return pb.Parked(position=position, code=err.code, message=str(err), attempts=attempts)
|
|
3670
|
+
self._log.print("error", self._what, f"{type} at {position} failed {attempts} times, parked: {err!r}")
|
|
3671
|
+
return pb.Parked(position=position, code="handler_error", message=_type_name(err), attempts=attempts)
|
|
3672
|
+
|
|
3673
|
+
def _verdict(self, call) -> pb.Verdict:
|
|
3674
|
+
try:
|
|
3675
|
+
when = "after" if call.when == pb.WHEN_AFTER else "before"
|
|
3676
|
+
middleware = self.listener._middlewares.get(when)
|
|
3677
|
+
if middleware is None:
|
|
3678
|
+
raise Rejected(f"no middleware {when}", "not_a_middleware")
|
|
3679
|
+
verdict = _call(middleware[1], _to_check(call), _to_outcome(call) if when == "after" else None, MiddlewareContext(self.store))
|
|
3680
|
+
if verdict is False:
|
|
3681
|
+
raise Rejected(f"refused by {self.listener.name}", "refused")
|
|
3682
|
+
if verdict is not None and verdict is not True:
|
|
3683
|
+
# Fails closed: a middleware that returns something else is a mistake.
|
|
3684
|
+
raise Rejected(f"{self.listener.name} returned {type(verdict).__name__}, not True, False or None",
|
|
3685
|
+
"middleware_error")
|
|
3686
|
+
return pb.Verdict(call_id=call.call_id)
|
|
3687
|
+
except Exception as err:
|
|
3688
|
+
reason = (pb.Rejected(code=err.code, message=str(err)) if isinstance(err, Rejected)
|
|
3689
|
+
else pb.Rejected(code="middleware_error", message=repr(err)))
|
|
3690
|
+
return pb.Verdict(call_id=call.call_id, rejected=reason)
|
|
3691
|
+
|
|
3692
|
+
def _answer(self, call_id: str, q) -> pb.QueryAnswer:
|
|
3693
|
+
try:
|
|
3694
|
+
handler = self.listener._queries.get(f"{q.domain}.{q.query_type}")
|
|
3695
|
+
if handler is None:
|
|
3696
|
+
raise Rejected(f"no handler for {q.domain}.{q.query_type}", "unknown_query")
|
|
3697
|
+
meta = QueryMeta(q.query_id, q.query_type, q.domain, q.tenant_id)
|
|
3698
|
+
result = _call(handler, _decode(q.payload), meta, QueryContext(_from_wire_context(q.context), self.store))
|
|
3699
|
+
# An answer of None is JSON `null`, not `{}`: "nothing" must not look like an empty object.
|
|
3700
|
+
return pb.QueryAnswer(call_id=call_id, response=b"null" if result is None else _encode(result))
|
|
3701
|
+
except Exception as err:
|
|
3702
|
+
reason = (pb.Rejected(code=err.code, message=str(err)) if isinstance(err, Rejected)
|
|
3703
|
+
else pb.Rejected(code="handler_error", message=repr(err)))
|
|
3704
|
+
return pb.QueryAnswer(call_id=call_id, rejected=reason)
|
|
3705
|
+
|
|
3706
|
+
|
|
3707
|
+
class Subscription(_Stream):
|
|
3708
|
+
"""A running subscription, from `Kernable.subscribe`: it streams until `stop()`, reconnecting
|
|
3709
|
+
after `position` when the stream drops. Attributes: `position` (global position of the last
|
|
3710
|
+
handled event) and `done` (a `threading.Event`, set once the subscription has ended:
|
|
3711
|
+
`sub.stop(); sub.done.wait(5)`)."""
|
|
3712
|
+
|
|
3713
|
+
def __init__(self, *args, **kwargs):
|
|
3714
|
+
"""Not for applications: `Kernable.subscribe` starts subscriptions."""
|
|
3715
|
+
raise TypeError("use Kernable.subscribe, not Subscription(...)")
|
|
3716
|
+
|
|
3717
|
+
def _init(self, source: "grpc.Channel | _Partitioned", key: Key, handler: Callable, after: int,
|
|
3718
|
+
types: list[str], reconnect_s: float, log: _Log, part: str = "",
|
|
3719
|
+
turns: "threading.Lock | None" = None):
|
|
3720
|
+
"""Use `Kernable.subscribe`. `part` and `turns`: a partition's stream, taking turns with
|
|
3721
|
+
the other partitions' streams for the handler."""
|
|
3722
|
+
super().__init__(f"subscription {part}" if part else "subscription", reconnect_s, log)
|
|
3723
|
+
#: Global position of the last handled event.
|
|
3724
|
+
self.position = after
|
|
3725
|
+
#: Set once the subscription has ended.
|
|
3726
|
+
self.done = threading.Event()
|
|
3727
|
+
self._source = source
|
|
3728
|
+
self._key = key
|
|
3729
|
+
self._handler = handler
|
|
3730
|
+
self._types = types
|
|
3731
|
+
self._turns = turns
|
|
3732
|
+
|
|
3733
|
+
def run(self) -> None:
|
|
3734
|
+
"""Streams events to the handler until `stop()`; `Kernable.subscribe` runs it in a thread."""
|
|
3735
|
+
try:
|
|
3736
|
+
parts = self._partitions(self._source)
|
|
3737
|
+
if parts is None:
|
|
3738
|
+
return
|
|
3739
|
+
if len(parts) == 1 and parts[0][0] == "":
|
|
3740
|
+
self._read(parts[0][1])
|
|
3741
|
+
return
|
|
3742
|
+
turns = threading.Lock()
|
|
3743
|
+
self._run_children([Subscription._new(channel, self._key, self._handler, 0, self._types, self._reconnect_s,
|
|
3744
|
+
self._log, part=name, turns=turns)
|
|
3745
|
+
for name, channel in parts])
|
|
3746
|
+
finally:
|
|
3747
|
+
self.done.set()
|
|
3748
|
+
|
|
3749
|
+
def _read(self, channel: grpc.Channel) -> None:
|
|
3750
|
+
stub = rpc.EventServiceStub(channel)
|
|
3751
|
+
attempt = 0
|
|
3752
|
+
while not self._stopped:
|
|
3753
|
+
start = self.position
|
|
3754
|
+
refused = False
|
|
3755
|
+
try:
|
|
3756
|
+
issued_at = _issued_now()
|
|
3757
|
+
request = pb.SubscribeRequest(after_position=self.position, event_types=self._types,
|
|
3758
|
+
issued_at=issued_at, protocol=PROTOCOL)
|
|
3759
|
+
request.client_sig.CopyFrom(self._key.sign(subscribe_message(self.position, self._types, issued_at)))
|
|
3760
|
+
self._call = None
|
|
3761
|
+
self._call = stub.Subscribe(request)
|
|
3762
|
+
if self._stopped: # stopped while it opened: stop() missed this call
|
|
3763
|
+
self._call.cancel()
|
|
3764
|
+
self.ready.set()
|
|
3765
|
+
for e in self._call:
|
|
3766
|
+
event = _to_event(e)
|
|
3767
|
+
if self._turns is None:
|
|
3768
|
+
_handle(self._handler, event)
|
|
3769
|
+
else:
|
|
3770
|
+
with self._turns:
|
|
3771
|
+
_handle(self._handler, event)
|
|
3772
|
+
self.position = e.global_position
|
|
3773
|
+
reason = "stream ended"
|
|
3774
|
+
except ClientClosed:
|
|
3775
|
+
return
|
|
3776
|
+
except _HandlerFailed as failed: # a failing handler: reconnect and get the event again
|
|
3777
|
+
reason = str(failed)
|
|
3778
|
+
except grpc.RpcError as err:
|
|
3779
|
+
reason = err.details() or str(err.code())
|
|
3780
|
+
refused = err.code() in _REFUSALS
|
|
3781
|
+
except Exception as err:
|
|
3782
|
+
reason = str(err) or type(err).__name__
|
|
3783
|
+
finally:
|
|
3784
|
+
# Whatever ended the loop, the call ends with it (a no-op once it ended by itself).
|
|
3785
|
+
if self._call is not None:
|
|
3786
|
+
self._call.cancel()
|
|
3787
|
+
attempt = 1 if self.position > start else attempt + 1
|
|
3788
|
+
if not self._stopped:
|
|
3789
|
+
self._log.reconnect(self._what, reason, self._reconnect_s, attempt, refused, self._wake)
|
|
3790
|
+
|
|
3791
|
+
|
|
3792
|
+
def _encode(data: Any) -> bytes:
|
|
3793
|
+
# NaN and Infinity are not JSON: other SDKs could not read them back.
|
|
3794
|
+
return json.dumps({} if data is None else data, allow_nan=False).encode()
|
|
3795
|
+
|
|
3796
|
+
|
|
3797
|
+
def _decode(payload: bytes) -> Any:
|
|
3798
|
+
return json.loads(payload) if payload else {}
|
|
3799
|
+
|
|
3800
|
+
|
|
3801
|
+
def _time(ms: int) -> datetime:
|
|
3802
|
+
return datetime.fromtimestamp(ms / 1000, timezone.utc)
|
|
3803
|
+
|
|
3804
|
+
|
|
3805
|
+
def _meta(event_type: str, version: int, cmd, command_id: str) -> EventMeta:
|
|
3806
|
+
"""What `apply` gets about an event of `cmd`'s aggregate, caused by command `command_id`."""
|
|
3807
|
+
return EventMeta(event_type, version, cmd.tenant_id, cmd.domain, cmd.aggregate_type, cmd.aggregate_id,
|
|
3808
|
+
command_id)
|
|
3809
|
+
|
|
3810
|
+
|
|
3811
|
+
def _transient(err: Exception) -> bool:
|
|
3812
|
+
"""Whether a failed handler should simply try again: the kernel was unreachable or busy, a
|
|
3813
|
+
submit did not settle in time, or the store failed. Never parks."""
|
|
3814
|
+
if isinstance(err, (NotSettled, sqlite3.Error)):
|
|
3815
|
+
return True
|
|
3816
|
+
return isinstance(err, grpc.RpcError) and callable(getattr(err, "code", None)) and err.code() in _BUSY
|
|
3817
|
+
|
|
3818
|
+
|
|
3819
|
+
def _type_name(err: BaseException) -> str:
|
|
3820
|
+
"""The error's type, all a parked event keeps of a handler error."""
|
|
3821
|
+
return type(err).__name__
|
|
3822
|
+
|
|
3823
|
+
|
|
3824
|
+
def _call(fn: Callable, *args):
|
|
3825
|
+
"""Calls `fn` with as many of `args` as it takes, so the trailing parameters are optional."""
|
|
3826
|
+
params = inspect.signature(fn).parameters.values()
|
|
3827
|
+
if any(p.kind == p.VAR_POSITIONAL for p in params):
|
|
3828
|
+
return fn(*args)
|
|
3829
|
+
positional = [p for p in params if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD)]
|
|
3830
|
+
return fn(*args[:len(positional)])
|
|
3831
|
+
|
|
3832
|
+
|
|
3833
|
+
def _tls_from_env() -> Tls | None:
|
|
3834
|
+
"""Mutual TLS from `KERNABLE_TLS_CA`, `KERNABLE_TLS_CERT` and `KERNABLE_TLS_KEY`: all or none."""
|
|
3835
|
+
files = [os.environ.get(f"KERNABLE_TLS_{n}") for n in ("CA", "CERT", "KEY")]
|
|
3836
|
+
if not any(files):
|
|
3837
|
+
return None
|
|
3838
|
+
if not all(files):
|
|
3839
|
+
raise ValueError("set all of KERNABLE_TLS_CA, KERNABLE_TLS_CERT and KERNABLE_TLS_KEY, or none")
|
|
3840
|
+
return Tls(*files)
|
|
3841
|
+
|
|
3842
|
+
|
|
3843
|
+
def _to_wire_context(c: RequestContext, command_id: str) -> pb.RequestContext:
|
|
3844
|
+
wire = pb.RequestContext(
|
|
3845
|
+
sender_tenant=c.sender_tenant, sender_identity=c.sender_identity, sender_account=c.sender_account,
|
|
3846
|
+
sender_session=c.sender_session, on_behalf_of=c.on_behalf_of,
|
|
3847
|
+
correlation_id=c.correlation_id or command_id, causation_id=c.causation_id, trace_id=c.trace_id,
|
|
3848
|
+
bypass_authorization=c.bypass_authorization,
|
|
3849
|
+
attributes=[pb.Attribute(key=k, value=v) for k, v in c.attributes.items()])
|
|
3850
|
+
if c.expected_version is not None:
|
|
3851
|
+
wire.expected_version = c.expected_version
|
|
3852
|
+
return wire
|
|
3853
|
+
|
|
3854
|
+
|
|
3855
|
+
def _from_wire_context(c) -> RequestContext:
|
|
3856
|
+
return RequestContext(
|
|
3857
|
+
c.sender_tenant, c.sender_identity, c.sender_account, c.sender_session, c.on_behalf_of,
|
|
3858
|
+
c.expected_version if c.HasField("expected_version") else None, c.correlation_id, c.causation_id,
|
|
3859
|
+
c.trace_id, c.bypass_authorization, {a.key: a.value for a in c.attributes})
|
|
3860
|
+
|
|
3861
|
+
|
|
3862
|
+
def _ms(when: "datetime | int | float | None") -> int:
|
|
3863
|
+
"""A time as ms since epoch; 0 for None. A naive `datetime` is local time, as Python reads it."""
|
|
3864
|
+
if when is None:
|
|
3865
|
+
return 0
|
|
3866
|
+
if isinstance(when, datetime):
|
|
3867
|
+
return int(when.timestamp() * 1000)
|
|
3868
|
+
if isinstance(when, bool) or not isinstance(when, (int, float)):
|
|
3869
|
+
raise TypeError(f"a time is a datetime or ms since epoch, not {when!r}")
|
|
3870
|
+
return int(when)
|
|
3871
|
+
|
|
3872
|
+
|
|
3873
|
+
def _to_check(call) -> Check:
|
|
3874
|
+
if call.WhichOneof("request") == "command":
|
|
3875
|
+
c = call.command
|
|
3876
|
+
return Check("command", c.command_id, c.tenant_id, c.domain, c.command_type,
|
|
3877
|
+
_unseal(_decode(c.payload), call.vault)[0],
|
|
3878
|
+
_from_wire_context(c.context), c.aggregate_type, c.aggregate_id,
|
|
3879
|
+
target=Target(f"{c.domain}.{c.aggregate_type}", c.aggregate_id))
|
|
3880
|
+
kind = call.WhichOneof("request")
|
|
3881
|
+
if kind == "cancel":
|
|
3882
|
+
# A cancel of a scheduled command: its type, aggregate and data are the scheduled command's.
|
|
3883
|
+
x = call.scheduled
|
|
3884
|
+
return Check("cancel", x.command_id, call.cancel.tenant_id, x.domain, x.command_type, _unseal(_decode(x.payload), None)[0],
|
|
3885
|
+
_from_wire_context(call.cancel.context), x.aggregate_type, x.aggregate_id,
|
|
3886
|
+
target=Target(f"{x.domain}.{x.aggregate_type}", x.aggregate_id))
|
|
3887
|
+
if kind in ("data", "cache"):
|
|
3888
|
+
r = getattr(call, kind)
|
|
3889
|
+
type = call.type.rsplit(".", 1)[-1] # qualified, e.g. `shop.PutData`
|
|
3890
|
+
return Check(kind, "", r.tenant_id, r.domain, type, None, _from_wire_context(r.context), key=r.key)
|
|
3891
|
+
q = call.query
|
|
3892
|
+
target = Target(q.target.aggregate_type, q.target.aggregate_id) if q.target.aggregate_id else None
|
|
3893
|
+
return Check("query", q.query_id, q.tenant_id, q.domain, q.query_type, _decode(q.payload),
|
|
3894
|
+
_from_wire_context(q.context), target=target)
|
|
3895
|
+
|
|
3896
|
+
|
|
3897
|
+
def _rejected_outcome(r) -> Outcome:
|
|
3898
|
+
"""The outcome of a rejected command, with the key or requirement the kernel named."""
|
|
3899
|
+
q = r.requirement if r.HasField("requirement") else None
|
|
3900
|
+
return Outcome(committed=False, code=r.code, message=r.message, key=r.key,
|
|
3901
|
+
requirement=Requirement(q.domain, q.aggregate_type, q.aggregate_id, q.fact, key=q.key,
|
|
3902
|
+
absent=q.absent) if q else None)
|
|
3903
|
+
|
|
3904
|
+
|
|
3905
|
+
def _to_outcome(call) -> Outcome | QueryOutcome:
|
|
3906
|
+
"""What came of the request a middleware after it checks."""
|
|
3907
|
+
if call.WhichOneof("outcome") == "command_result":
|
|
3908
|
+
r = call.command_result
|
|
3909
|
+
if r.WhichOneof("outcome") == "committed":
|
|
3910
|
+
return Outcome(committed=True, events=[_to_event(e) for e in r.committed.events])
|
|
3911
|
+
return _rejected_outcome(r.rejected)
|
|
3912
|
+
r = call.query_result
|
|
3913
|
+
if r.WhichOneof("outcome") == "response":
|
|
3914
|
+
return QueryOutcome(answered=True, data=_decode(r.response))
|
|
3915
|
+
return QueryOutcome(answered=False, code=r.rejected.code, message=r.rejected.message)
|
|
3916
|
+
|
|
3917
|
+
|
|
3918
|
+
def _to_event(e, vault=None) -> StoredEvent:
|
|
3919
|
+
data, redacted = _unseal(_decode(e.payload), vault)
|
|
3920
|
+
return StoredEvent(position=e.global_position, tenant=e.tenant_id, domain=e.domain, aggregate_type=e.aggregate_type,
|
|
3921
|
+
aggregate_id=e.aggregate_id, version=e.version, type=e.event_type,
|
|
3922
|
+
data=data, command_id=e.command_id, request=_from_wire_context(e.context),
|
|
3923
|
+
schema=e.schema, redacted=redacted, created_at=_time(e.created_at), deployment=e.deployment)
|