kernable-sdk 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
kernable/__init__.py ADDED
@@ -0,0 +1,3923 @@
1
+ """kernable SDK for Python: write event-sourced business logic in Python; the kernel does the rest.
2
+
3
+ The API mirrors the TypeScript SDK:
4
+
5
+ account_aggregate = domain("identity").aggregate("account", {"status": "none"})
6
+
7
+ @account_aggregate.apply("AccountRegistered")
8
+ def _(state, data):
9
+ return {**state, "status": "active", "email": data["email"]}
10
+
11
+ @account_aggregate.command("RegisterAccount")
12
+ def _(state, data):
13
+ if state["status"] != "none":
14
+ raise Rejected("Account already exists")
15
+ return [{"type": "AccountRegistered", "data": {"email": data["email"]}}]
16
+
17
+ k = Kernable(url, key_path)
18
+ k.worker(account_aggregate).run()
19
+
20
+ Listeners are defined on a domain too and run through the connection:
21
+
22
+ accounts_readmodel = domain("identity").read_model("accounts")
23
+ k.listener(accounts_readmodel).run()
24
+
25
+ Stubs: generate them first (`make py-stubs`), see sdks/python/README.md.
26
+ """
27
+
28
+ import concurrent.futures
29
+ import contextvars
30
+ import copy
31
+ import dataclasses
32
+ import hashlib
33
+ import inspect
34
+ import json
35
+ import os
36
+ import queue
37
+ import random
38
+ import re
39
+ import sqlite3
40
+ import sys
41
+ import threading
42
+ import time
43
+ import traceback
44
+ import uuid
45
+ from dataclasses import dataclass, field
46
+ from datetime import datetime, timedelta, timezone
47
+ from typing import ClassVar, Any, Callable, Iterable
48
+
49
+ import grpc
50
+
51
+ from kernable.signing import (Key, cache_message, cancel_message, challenge_message, command_message, data_message, derived_id,
52
+ index_message, membership_message, process_act_message, process_message, query_message, vault_ref, rejection_message,
53
+ result_message, snapshot_message, subscribe_message)
54
+ from kernable.store import Store
55
+ from kernable.v1 import kernable_pb2 as pb
56
+ from kernable.v1 import kernable_pb2_grpc as rpc
57
+
58
+ from .messages import ( # noqa: E402
59
+ Command, Event, Query, bind as _bind, binding as _binding, declare_command as _declare_command,
60
+ declare_event as _declare_event, from_wire as _from_wire, is_message_class as _is_message_class,
61
+ query_target as _query_target, to_wire as _to_wire,
62
+ )
63
+ from .processes import Process, Value, value # noqa: E402
64
+
65
+ __all__ = ["PROTOCOL", "waits_for_switch", "Event", "Command", "Query", "StoredEvent", "Aggregate", "AggregateType", "MiddlewareContext", "App", "ApplyContext", "CacheClient", "CancelOutcome", "Check", "ClientClosed", "CommandContext", "CommandMeta",
66
+ "CommandSchema", "QuerySchema", "Target",
67
+ "Data", "DataClient", "Domain", "Listed", "Listing",
68
+ "EventMeta", "EventSchema", "HeldKey", "Index", "Kernable", "Key", "Listener", "ListenerContext", "ListenerHandle", "Manifest", "NotDue", "NotSettled", "Outcome",
69
+ "Process", "QueryContext", "QueryMeta", "QueryOutcome", "RegisteredProcess", "Rejected", "RequestContext", "Requirement", "Store", "StoredEventMeta", "Subscription", "Tls", "Worker",
70
+ "Value", "domain", "reindex_type", "value"]
71
+
72
+ #: The key of a channel in `Kernable._seen`: the builtin `id`, which a parameter named `id` (a
73
+ #: command ID) would hide inside the method that keeps it.
74
+ _channel_id = id
75
+
76
+ #: Time zero (UTC): the default of times not known.
77
+ _EPOCH = datetime.fromtimestamp(0, timezone.utc)
78
+
79
+
80
+ class Rejected(Exception):
81
+ """A business reason to refuse a command or query: raise it from a command handler, a query
82
+ handler or a middleware, e.g. `raise Rejected("Cart is checked out", "checked_out")`.
83
+
84
+ `code` (default `rejected`) is what callers act on, the message (`str(err)`) what they show.
85
+ `Kernable.query` raises it when the read model or a middleware refuses the query; a rejected
86
+ command does not raise: `Kernable.submit` returns an `Outcome` with `committed=False`. Any
87
+ other exception from a command or query handler rejects too, with the code `handler_error`
88
+ (`middleware_error` from a middleware)."""
89
+
90
+ def __init__(self, message: str, code: str = "rejected"):
91
+ super().__init__(message)
92
+ self.code = code
93
+ #: For a rejected follow-up command (`ListenerContext.submit`): the key the kernel named
94
+ #: (`unique_taken`).
95
+ self.key = ""
96
+ #: For a rejected follow-up command: the requirement the kernel named (`requirement_missing`).
97
+ self.requirement: "Requirement | None" = None
98
+
99
+
100
+ @dataclass
101
+ class RequestContext:
102
+ """Who sends a request and in which context: signed with the command and stored with every
103
+ event it causes. `attributes` are business data only, never security decisions.
104
+
105
+ Set a default for a client with `Kernable(context=...)`, or one per request with `context=`
106
+ of `submit`, `enqueue` and `query`. Inside a listener's handler, submit follow-up commands
107
+ with `ctx.follow()`. Example: `RequestContext(sender_identity="customer-7",
108
+ expected_version=3)`."""
109
+
110
+ #: The tenant the sender belongs to.
111
+ sender_tenant: str = ""
112
+ #: Who sends the request, e.g. a user or service ID; middlewares check permissions on it.
113
+ sender_identity: str = ""
114
+ sender_account: str = ""
115
+ sender_session: str = ""
116
+ #: Authorize as this identity of the target tenant; the sender stays the real principal.
117
+ on_behalf_of: str = ""
118
+ #: Reject the command (`version_mismatch`) unless the aggregate is at this version.
119
+ expected_version: int | None = None
120
+ #: Shared by everything one original request causes; defaults to the command ID.
121
+ correlation_id: str = ""
122
+ #: The command that caused this one.
123
+ causation_id: str = ""
124
+ trace_id: str = ""
125
+ #: Privileged: only keys granted `privileged` for it may set it.
126
+ bypass_authorization: bool = False
127
+ attributes: dict[str, str] = field(default_factory=dict)
128
+
129
+
130
+ @dataclass
131
+ class CommandMeta:
132
+ """What a command handler gets about the command besides state and data (as an optional third
133
+ parameter)."""
134
+
135
+ #: The command's ID: given as `id` to `submit`, else a new UUID.
136
+ command_id: str
137
+ #: The command type within the domain.
138
+ type: str
139
+ domain: str = ""
140
+ aggregate_type: str = ""
141
+ aggregate_id: str = ""
142
+ tenant: str = ""
143
+
144
+
145
+ @dataclass
146
+ class CommandContext:
147
+ """What a command handler gets besides state, data and the command (as an optional fourth
148
+ parameter)."""
149
+
150
+ #: Who sends the command: check permissions here.
151
+ request: RequestContext
152
+ #: When the kernel hands the command to another worker if this one has not answered.
153
+ deadline: datetime
154
+ #: When the kernel received the command (UTC); `now()` returns it.
155
+ received_at: datetime
156
+ #: The command whose IDs `id()` derives.
157
+ _command_id: str = field(default="", repr=False)
158
+ _ids: int = field(default=0, repr=False)
159
+
160
+ @classmethod
161
+ def new(cls, command_id: str, request: RequestContext | None = None,
162
+ received_at: datetime = _EPOCH) -> "CommandContext":
163
+ """A context for calling a command handler directly, e.g. in a unit test with
164
+ `Aggregate.handle`: `id()` derives from `command_id` as in a real command, `now()` returns
165
+ `received_at` (default: the epoch), and its deadline is 30 s after that."""
166
+ return cls(request or RequestContext(), received_at + timedelta(seconds=30), received_at, command_id)
167
+
168
+ def now(self) -> datetime:
169
+ """When the kernel received the command: the same every time the command runs.
170
+ Use it instead of `datetime.now()`."""
171
+ return self.received_at
172
+
173
+ def id(self) -> str:
174
+ """A new ID that is the same every time the command runs (1st call, 2nd call, ...).
175
+ Use it instead of random IDs."""
176
+ n, self._ids = self._ids, self._ids + 1
177
+ return derived_id(self._command_id, n)
178
+
179
+
180
+ @dataclass
181
+ class EventMeta:
182
+ """What `apply` gets besides state and data (as an optional third parameter): the event
183
+ without its payload. Only fields known before the kernel stores the event, so a snapshot and
184
+ a replay compute the same state; put a time into the data (`ctx.now()`) if the state needs one."""
185
+
186
+ type: str
187
+ #: The aggregate's version after this event: 1 for its first event.
188
+ version: int = 0
189
+ tenant: str = ""
190
+ domain: str = ""
191
+ aggregate_type: str = ""
192
+ aggregate_id: str = ""
193
+ command_id: str = ""
194
+
195
+
196
+ @dataclass
197
+ class ApplyContext:
198
+ """What `apply` gets besides state, data and the event (as an optional fourth parameter).
199
+ No clock and no IDs: `apply` must compute the same state on every replay."""
200
+
201
+ #: The request context of the command that caused the event.
202
+ request: RequestContext = field(default_factory=RequestContext)
203
+
204
+
205
+ @dataclass
206
+ class QueryMeta:
207
+ """What a query handler gets about the query besides its data (as an optional second
208
+ parameter)."""
209
+
210
+ query_id: str
211
+ #: The query type within the domain.
212
+ type: str
213
+ domain: str = ""
214
+ tenant: str = ""
215
+
216
+
217
+ @dataclass
218
+ class QueryContext:
219
+ """What a query handler gets besides the query's data and meta (as an optional third
220
+ parameter)."""
221
+
222
+ #: Who asks: check permissions here.
223
+ request: RequestContext
224
+ #: The listener's store, if it has one (`store=` of `Domain.read_model` and `Domain.reactor`).
225
+ store: "Store | None" = None
226
+
227
+
228
+ @dataclass
229
+ class MiddlewareContext:
230
+ """What a middleware handler gets besides the check (an optional third parameter)."""
231
+
232
+ #: The listener's store, if it has one.
233
+ store: "Store | None" = None
234
+
235
+
236
+ @dataclass(frozen=True)
237
+ class Target:
238
+ """The aggregate a request is about, as `Check.target` holds it."""
239
+
240
+ #: Qualified: `shop.order`.
241
+ type: str
242
+ id: str
243
+
244
+
245
+ @dataclass
246
+ class Check:
247
+ """A command, query, file or cache request a middleware checks, before the kernel accepts,
248
+ routes or runs it (or, for commands and queries, after): the first parameter of a
249
+ `Listener.middleware` handler."""
250
+
251
+ #: `"command"`, `"query"`, `"data"` (a file request of `k.data`), `"cache"` (a cache
252
+ #: request of `k.cache`) or `"cancel"` (`Kernable.cancel` of a scheduled command: `id`,
253
+ #: `domain`, `type`, the aggregate, `target` and `data` are the scheduled command's,
254
+ #: `request` the cancel's sender; decide it as the command).
255
+ kind: str
256
+ #: The command or query ID; empty for a file or cache request.
257
+ id: str
258
+ tenant: str
259
+ domain: str
260
+ #: The type within the domain: the command or query type, or for a file request `PutData`,
261
+ #: `GetData`, `DeleteData` or `ListData`, for a cache request `SetCache`, `GetCache` or
262
+ #: `DeleteCache`.
263
+ type: str
264
+ #: The request's payload; vault fields are `None` unless the key is granted `vault_read`.
265
+ #: Always `None` for a file or cache request: the middleware decides on who asks for which
266
+ #: key, never sees the file or value.
267
+ data: Any
268
+ #: Who sends it: check permissions here.
269
+ request: RequestContext
270
+ #: Commands only; empty otherwise.
271
+ aggregate_type: str = ""
272
+ #: Commands only; empty otherwise.
273
+ aggregate_id: str = ""
274
+ #: File and cache requests only: the file or cache key; empty otherwise.
275
+ key: str = ""
276
+ #: The aggregate the request is about: a command's aggregate (`Target("shop.order", "o-17")`),
277
+ #: or the one a query declares (`Query.target`, `QuerySchema.target`). None for queries that
278
+ #: declare none, and for file and cache requests.
279
+ target: "Target | None" = None
280
+
281
+
282
+ @dataclass
283
+ class StoredEvent:
284
+ """An event as stored by the kernel, as `Outcome.events` holds it."""
285
+
286
+ #: Global position of the event in the kernel's log.
287
+ position: int
288
+ tenant: str
289
+ #: Always the domain of the command that caused the event.
290
+ domain: str
291
+ aggregate_type: str
292
+ aggregate_id: str
293
+ #: The aggregate's version after this event: 1 for its first event.
294
+ version: int
295
+ #: The event type within the domain, e.g. `ItemAdded`.
296
+ type: str
297
+ #: The payload.
298
+ data: Any
299
+ #: The command that caused the event.
300
+ command_id: str
301
+ #: The request context of the command that caused the event.
302
+ request: RequestContext = field(default_factory=RequestContext)
303
+ #: Version of the payload format, as stored (see `Domain.event`).
304
+ schema: int = 1
305
+ #: Fields kept in the vault whose value is not here (None in `data`): erased, or not readable
306
+ #: with this key (granted `vault_read`).
307
+ redacted: list[str] = field(default_factory=list)
308
+ #: When the kernel committed the event (UTC).
309
+ created_at: datetime = _EPOCH
310
+ #: The code version (deployment) of the worker that wrote it; empty if none.
311
+ deployment: str = ""
312
+
313
+
314
+ @dataclass
315
+ class StoredEventMeta(EventMeta):
316
+ """What a listener handler gets about the event besides its data (as an optional second
317
+ parameter): the stored event without its payload and request context. It has every field
318
+ of `EventMeta` (`type`, `version`, `tenant`, `domain`, `aggregate_type`, `aggregate_id`,
319
+ `command_id`), and these. Unlike `EventMeta`, it says when the event was committed: a read
320
+ model may show that time, `apply` may not depend on it."""
321
+
322
+ #: Global position of the event in the kernel's log.
323
+ position: int = 0
324
+ #: Version of the payload format: the version the listener knows (see `Domain.event`), or the
325
+ #: stored one for a `raw` handler.
326
+ schema: int = 1
327
+ #: Fields kept in the vault whose value is not in the data (None there): erased, or not
328
+ #: readable with this key (granted `vault_read`).
329
+ redacted: list[str] = field(default_factory=list)
330
+ #: When the kernel committed the event (UTC).
331
+ created_at: datetime = _EPOCH
332
+ #: The code version (deployment) of the worker that wrote it; empty if none.
333
+ deployment: str = ""
334
+
335
+
336
+ @dataclass
337
+ class ListenerContext:
338
+ """What a listener handler gets besides the event's data and meta (as an optional third
339
+ parameter)."""
340
+
341
+ #: The event was committed before this connection began: a read model rebuilding from the
342
+ #: start, and equally the backlog a reactor catches up on after a restart or a lost
343
+ #: connection. A reactor must not skip side effects because of it (those events are new to
344
+ #: it); submitting with fixed command IDs makes them safe to repeat instead. Use it to skip
345
+ #: work that only matters live, e.g. a notification about something long past, in a listener
346
+ #: that starts at 0.
347
+ replaying: bool
348
+ #: The request context of the command that caused the event.
349
+ request: RequestContext = field(default_factory=RequestContext)
350
+ #: The listener's store, if it has one (`store=` of `Domain.read_model` and `Domain.reactor`).
351
+ store: "Store | None" = None
352
+ #: The command that caused the event, for `follow()`.
353
+ _command_id: str = field(default="", repr=False)
354
+ #: The event's tenant: the default for follow-up commands and queries.
355
+ _tenant: str = field(default="", repr=False)
356
+ #: The listener's domain: the default for follow-up commands and queries.
357
+ _domain: str = field(default="", repr=False)
358
+ #: The listener's client, for follow-up commands and queries.
359
+ _kernable: "Kernable | None" = field(default=None, repr=False)
360
+
361
+ def submit(self, aggregate_type, aggregate_id: str | None = None, type: str | None = None, data: Any = None, *, id: str,
362
+ domain: str | None = None, tenant: str | None = None, context: "RequestContext | None" = None,
363
+ wait: str = "settled", timeout_s: float | None = None) -> "list":
364
+ """Submits a command this event causes, on the listener's client, and returns its events.
365
+ It needs a fixed `id` (`f"confirm-{evt.aggregate_id}"`), so a redelivered event runs it
366
+ once. `domain` defaults to the listener's, `context` to `follow()`, `tenant` to the
367
+ event's; the rest as for `Kernable.submit`.
368
+
369
+ Raises `Rejected` (with the kernel's `key` or `requirement`, if any) if the command is
370
+ rejected: a reactor then parks the event at once (see `park_after` of
371
+ `Domain.reactor`); catch it to accept a rejection. Raises like `Kernable.submit` otherwise.
372
+ Example: `ctx.submit("order", evt.aggregate_id, "Confirm", id=f"confirm-{evt.aggregate_id}")`."""
373
+ if not id:
374
+ raise ValueError(f"{type or aggregate_type}: a follow-up command needs a fixed id, so a redelivered event runs it once")
375
+ assert self._kernable is not None, "only a running listener's handlers submit follow-ups"
376
+ if isinstance(aggregate_type, Command):
377
+ domain, aggregate_type, type, data = self._kernable._command_of(aggregate_type)
378
+ outcome = self._kernable.submit(domain or self._domain, aggregate_type, aggregate_id, type, data,
379
+ tenant=tenant or self._tenant, id=id, wait=wait,
380
+ context=context or self.follow(), timeout_s=timeout_s)
381
+ if outcome.committed:
382
+ return outcome.events
383
+ rejected = Rejected(outcome.message, outcome.code)
384
+ rejected.key = outcome.key
385
+ rejected.requirement = outcome.requirement
386
+ raise rejected
387
+
388
+ def query(self, type, data: Any = None, *, tenant: str | None = None,
389
+ context: "RequestContext | None" = None, min_position: int = 0,
390
+ timeout_s: float | None = None, all_partitions: bool = False) -> Any:
391
+ """Asks a query on the listener's client, like `Kernable.query`: `type` is of the
392
+ listener's domain (`GetOrder`) or qualified (`crm.GetCustomer`). `tenant` defaults to the
393
+ event's. It is the listener's own read: it keeps the client's context, not the event's
394
+ sender."""
395
+ assert self._kernable is not None, "only a running listener's handlers ask queries"
396
+ if isinstance(type, Query):
397
+ return self._kernable.query(type, tenant=tenant or self._tenant, context=context, min_position=min_position,
398
+ timeout_s=timeout_s, all_partitions=all_partitions)
399
+ domain, _, name = type.rpartition(".")
400
+ return self._kernable.query(domain or self._domain, name, data, tenant=tenant or self._tenant,
401
+ context=context, min_position=min_position,
402
+ timeout_s=timeout_s)
403
+
404
+ def process_act(self, name: str, context: RequestContext) -> bool:
405
+ """Sets as whom the process `name` acts, with the listener's client; see
406
+ `Kernable.process_act`. Returns whether it changed."""
407
+ assert self._kernable is not None, "only a running listener's handlers set as whom a process acts"
408
+ return self._kernable.process_act(name, context)
409
+
410
+ def follow(self) -> RequestContext:
411
+ """A request context for commands caused by this event: same sender and correlation."""
412
+ r = self.request
413
+ return RequestContext(r.sender_tenant, r.sender_identity, r.sender_account, r.sender_session,
414
+ r.on_behalf_of, None, r.correlation_id or self._command_id,
415
+ self._command_id, r.trace_id, False, dict(r.attributes))
416
+
417
+
418
+ @dataclass
419
+ class Outcome:
420
+ """What became of a command, returned by `Kernable.submit`: committed with its events, or
421
+ rejected with a code and a message. Example: `Outcome(committed=False, events=[],
422
+ code='rejected', message='Cart is checked out')`."""
423
+
424
+ #: Whether the command's events were stored; `False` if it was rejected.
425
+ committed: bool
426
+ #: The committed events, in order; empty if the command was rejected. Vault fields in them
427
+ #: are `None` (see `StoredEvent.redacted`).
428
+ events: list[StoredEvent] = field(default_factory=list)
429
+ #: The rejection code (see `Rejected`), if rejected: a handler's, or the kernel's, e.g.
430
+ #: `unknown_history` when the aggregate holds an event the deployment of the workers that run
431
+ #: the command does not know (a rollback after a newer deployment wrote it).
432
+ code: str = ""
433
+ #: The rejection message, if rejected.
434
+ message: str = ""
435
+ #: For `unique_taken`: the name of the key another aggregate holds (never its value). Set by
436
+ #: the kernel only, and stored with the rejection.
437
+ key: str = ""
438
+ #: For `requirement_missing`: the first requirement that does not hold. Set by the kernel only.
439
+ requirement: "Requirement | None" = None
440
+
441
+
442
+ @dataclass
443
+ class Requirement:
444
+ """A fact another aggregate of the command's tenant must hold (or, `absent`, must not hold)
445
+ when the command commits, as `requires=` of `Aggregate.command` returns them:
446
+ `Requirement("funds", "vehicle", data["vehicleId"], "open")`. The kernel checks it atomically
447
+ with the commit; if it does not hold, the command is rejected with `requirement_missing`.
448
+
449
+ The aggregate is named by its ID, or by a unique key of its type (`Requirement.by_key`, not a
450
+ personal key), which the kernel resolves at commit: no aggregate holds the key, none holds
451
+ the fact; an aggregate without events holds none either. An empty `domain` is the command's
452
+ own (`Requirement.of`)."""
453
+
454
+ #: Empty: the domain of the command's aggregate.
455
+ domain: str
456
+ #: Unqualified, e.g. `vehicle`.
457
+ aggregate_type: str
458
+ #: Empty when the aggregate is named by `key`.
459
+ aggregate_id: str
460
+ #: A fact the aggregate declares with `Aggregate.fact`.
461
+ fact: str
462
+ #: A unique key of the aggregate type that names the aggregate instead of its ID.
463
+ key: str = ""
464
+ #: The key's value, normalised as the aggregate type's key function does: the kernel gets only
465
+ #: its SHA-256. Empty in a rejection, which never carries the value.
466
+ value: str = ""
467
+ #: The aggregate must not hold the fact.
468
+ absent: bool = False
469
+
470
+ @classmethod
471
+ def of(cls, aggregate_type: str, aggregate_id: str, fact: str, *, absent: bool = False) -> "Requirement":
472
+ """A requirement of an aggregate type of the command's own domain:
473
+ `Requirement.of("vehicle", data["vehicleId"], "woundUp", absent=True)`."""
474
+ return cls("", aggregate_type, aggregate_id, fact, absent=absent)
475
+
476
+ @classmethod
477
+ def by_key(cls, aggregate_type: str, key: str, value: str, fact: str, *, domain: str = "",
478
+ absent: bool = False) -> "Requirement":
479
+ """The fact of the aggregate that holds the unique key `key` with `value`, normalised as
480
+ its key function does: `Requirement.by_key("vehicle", "isin", data["vehicle"], "open")`."""
481
+ return cls(domain, aggregate_type, "", fact, key=key, value=value, absent=absent)
482
+
483
+
484
+ @dataclass
485
+ class HeldKey:
486
+ """A unique key an aggregate holds for a state, as `Aggregate.index` returns it: the value in
487
+ clear (the worker sends only its SHA-256 hash)."""
488
+
489
+ #: The name declared with `Aggregate.unique`.
490
+ name: str
491
+ value: str
492
+ #: `"tenant"` or `"global"`; see `Aggregate.unique`.
493
+ scope: str = "tenant"
494
+ #: The person the value belongs to, for a personal key; empty otherwise.
495
+ subject: str = ""
496
+
497
+
498
+ @dataclass
499
+ class Index:
500
+ """The unique keys and facts an aggregate holds for a state, returned by `Aggregate.index`."""
501
+
502
+ keys: list[HeldKey] = field(default_factory=list)
503
+ #: The names of the facts that are true, in the order they were declared.
504
+ facts: list[str] = field(default_factory=list)
505
+
506
+
507
+ @dataclass
508
+ class RegisteredProcess:
509
+ """What became of `Kernable.register_process`: the process's version."""
510
+
511
+ #: Qualified: `shop.fulfilment`.
512
+ name: str
513
+ #: From 1; the newest version, whether registered now or before.
514
+ version: int
515
+ #: False if the model equals the newest version: nothing was registered.
516
+ new: bool
517
+
518
+
519
+ @dataclass
520
+ class CancelOutcome:
521
+ """What became of `Kernable.cancel`: cancelled, or rejected with a code and a message
522
+ (`too_late`, `not_found`, or a middleware's code)."""
523
+
524
+ #: Whether the scheduled command was cancelled (or had been already): it completes as
525
+ #: rejected `cancelled`.
526
+ cancelled: bool
527
+ #: The rejection code, if not cancelled.
528
+ code: str = ""
529
+ #: The rejection message, if not cancelled.
530
+ message: str = ""
531
+
532
+
533
+ @dataclass
534
+ class QueryOutcome:
535
+ """What came of a query, as a middleware that runs after requests (`when="after"`) gets it:
536
+ its response data or why it was rejected."""
537
+
538
+ #: Whether the read model answered; `False` if the query was rejected.
539
+ answered: bool
540
+ #: The answer, if answered.
541
+ data: Any = None
542
+ #: The rejection code, if rejected.
543
+ code: str = ""
544
+ #: The rejection message, if rejected.
545
+ message: str = ""
546
+
547
+
548
+ @dataclass
549
+ class Tls:
550
+ """Paths of PEM files for mutual TLS with a kernel at `host:port`: the CA that signed the
551
+ kernel's certificate, and this process's own certificate and key. Pass it as
552
+ `Kernable(tls=Tls(ca="ca.pem", cert="app.pem", key="app-key.pem"))`, or set
553
+ `KERNABLE_TLS_CA`, `KERNABLE_TLS_CERT` and `KERNABLE_TLS_KEY`."""
554
+
555
+ ca: str
556
+ cert: str
557
+ key: str
558
+
559
+
560
+ @dataclass
561
+ class Data:
562
+ """A file as stored by the kernel, returned by `k.data.get`."""
563
+
564
+ data: bytes
565
+ #: The media type given when the file was put, e.g. `image/png`.
566
+ content_type: str
567
+ #: The SHA-256 of `data`, hex, as the kernel computed it.
568
+ sha256: str = ""
569
+
570
+
571
+ @dataclass
572
+ class Listed:
573
+ """A file as `k.data.list` names it."""
574
+
575
+ key: str
576
+ content_type: str
577
+ #: The file's length in bytes.
578
+ size: int
579
+
580
+
581
+ @dataclass
582
+ class Listing:
583
+ """One page of `k.data.list`: up to 1,000 files, and `next` (to pass as `after=`) if there
584
+ may be more."""
585
+
586
+ files: list[Listed]
587
+ next: str | None
588
+
589
+
590
+ class ClientClosed(RuntimeError):
591
+ """Raised by a request on a client after `Kernable.close()`. Workers and listeners of a
592
+ closed client end their `run()`."""
593
+
594
+ def __init__(self):
595
+ super().__init__("the kernable client is closed")
596
+
597
+
598
+ class NotSettled(Exception):
599
+ """Raised by a settling `Kernable.submit` (the default `wait="settled"`): the command was
600
+ committed, but not every listener that counts processed its events within the kernel's
601
+ `--settle-timeout-ms`.
602
+
603
+ Read models may not show the change yet; `outcome` (an `Outcome`) has the committed events,
604
+ e.g. `except NotSettled as err: events = err.outcome.events`. Submitting again with the
605
+ same `id` returns the same outcome, without running the command again.
606
+ """
607
+
608
+ def __init__(self, outcome: Outcome):
609
+ super().__init__("command committed, but not all listeners have processed its events yet")
610
+ self.outcome = outcome
611
+
612
+
613
+ class NotDue(Exception):
614
+ """Raised by `Kernable.submit` for the `id` of a scheduled command (`enqueue` with `at`) that
615
+ is not due yet: it has no outcome to wait for. Submit its `id` again after its time;
616
+ `command_id` is the scheduled command's ID."""
617
+
618
+ def __init__(self, command_id: str):
619
+ super().__init__(f"command {command_id} is scheduled and not due yet: ask again after its time")
620
+ self.command_id = command_id
621
+
622
+
623
+ #: The protocol version this SDK speaks; see "Protocol versions" in `kernable.proto`.
624
+ PROTOCOL = 1
625
+
626
+ #: Attempts before a reactor parks an event its handler keeps failing on.
627
+ _PARK_AFTER = 5
628
+ # The event types and schemas a listener of this process skipped already: each is logged once.
629
+ _SKIP_WARNED: set[tuple[str, int]] = set()
630
+ _SKIP_LOCK = threading.Lock()
631
+
632
+ #: gRPC codes of a kernel that is away or busy.
633
+ _BUSY = frozenset({grpc.StatusCode.CANCELLED, grpc.StatusCode.DEADLINE_EXCEEDED, grpc.StatusCode.RESOURCE_EXHAUSTED,
634
+ grpc.StatusCode.ABORTED, grpc.StatusCode.INTERNAL, grpc.StatusCode.UNAVAILABLE})
635
+
636
+ #: The name of the listener whose handler is running, for requests sent from it.
637
+ _HANDLING: contextvars.ContextVar[str] = contextvars.ContextVar("kernable_listener", default="")
638
+
639
+ _INITIAL = object()
640
+ #: Room for the largest file a kernel takes (`--max-data-bytes`, at most 128 MiB) and the rest of
641
+ #: its message; grpcio receives 4 MiB by default.
642
+ _MAX_MESSAGE = 129 * 1024 * 1024
643
+ #: Pings on every connection, also idle ones: a connection the network dropped without a word (a
644
+ #: dead host, a NAT that forgot it) ends within 40 s, and the workers and listeners on it reconnect.
645
+ _CHANNEL_OPTIONS = [("grpc.max_send_message_length", _MAX_MESSAGE),
646
+ ("grpc.max_receive_message_length", _MAX_MESSAGE),
647
+ ("grpc.keepalive_time_ms", 30_000),
648
+ ("grpc.keepalive_timeout_ms", 10_000),
649
+ # grpcio times a keepalive ping out by this, not by the keepalive timeout
650
+ # (default 60 s).
651
+ ("grpc.http2.ping_timeout_ms", 10_000),
652
+ ("grpc.keepalive_permit_without_calls", 1)]
653
+ #: A client of several kernels (a pair) pings every 2 s and waits 2 s for the answer: a connection
654
+ #: the network cut off (a partition) ends within 4 s, and the client turns to the other kernel.
655
+ _PAIR_CHANNEL_OPTIONS = [o for o in _CHANNEL_OPTIONS
656
+ if o[0] not in ("grpc.keepalive_time_ms", "grpc.keepalive_timeout_ms",
657
+ "grpc.http2.ping_timeout_ms")] + [
658
+ ("grpc.keepalive_time_ms", 2_000),
659
+ ("grpc.keepalive_timeout_ms", 2_000),
660
+ ("grpc.http2.ping_timeout_ms", 2_000),
661
+ ]
662
+ #: gRPC codes that a retry does not change.
663
+ _REFUSALS = {grpc.StatusCode.INVALID_ARGUMENT, grpc.StatusCode.PERMISSION_DENIED, grpc.StatusCode.FAILED_PRECONDITION,
664
+ grpc.StatusCode.UNIMPLEMENTED, grpc.StatusCode.UNAUTHENTICATED}
665
+ #: How long a request waits, for the connection and the answer, unless told otherwise: above the
666
+ #: kernel's own submit and settle timeouts.
667
+ DEFAULT_TIMEOUT_S = 60.0
668
+ _DOMAIN = re.compile(r"^[A-Za-z][A-Za-z0-9_-]*$")
669
+ _UNQUALIFIED = re.compile(r"^[^.*]+$")
670
+ _LISTENED = re.compile(r"^[A-Za-z][A-Za-z0-9_-]*\.([^.]+|\*)$")
671
+ _QUERY = re.compile(r"^[A-Za-z][A-Za-z0-9_-]*\.[^.*]+$")
672
+ _CHECKED = re.compile(r"^(\*|[A-Za-z][A-Za-z0-9_-]*\.([^.]+|\*))$")
673
+ _DEPLOYMENT = re.compile(r"^[A-Za-z0-9_.-]{1,128}$")
674
+
675
+
676
+ def _deployment(value: str) -> str:
677
+ """`value` if it names a deployment (or is empty); raises `ValueError` otherwise."""
678
+ if value and not _DEPLOYMENT.match(value):
679
+ raise ValueError(f"a deployment is 1 to 128 letters, digits, `-`, `_` and `.`, not {value!r}")
680
+ return value
681
+ def reindex_type(aggregate_type: str) -> str:
682
+ """The type of the built-in command an aggregate with unique keys or facts runs without
683
+ events, `_reindex_<aggregate_type>` (see `Aggregate.unique` and `Kernable.reindex`): one per
684
+ aggregate, as command types are unique within a domain. `reindex_type("fund")` is
685
+ `"_reindex_fund"`."""
686
+ return f"_reindex_{aggregate_type}"
687
+
688
+
689
+ class EventSchema:
690
+ """The payload format of one event type: its current version and the steps that bring older
691
+ payloads up to it. Events are stored as written and never changed; readers upcast them.
692
+ Made with `Domain.event`; its methods return it, so they chain:
693
+ `shop_domain.event("CustomerRegistered", 1).personal(["email"], lambda d, id: id)`."""
694
+
695
+ def __init__(self, type: str, schema: int):
696
+ """Use `Domain.event`. Raises `ValueError` if `type` is qualified or `schema` is below 1."""
697
+ if not re.match(r"^[^.*]+$", type):
698
+ raise ValueError(f"event types are unqualified within their domain, not {type!r}")
699
+ if not isinstance(schema, int) or schema < 1:
700
+ raise ValueError(f"a schema is a version from 1, not {schema!r}")
701
+ self.type = type
702
+ self.schema = schema
703
+ self.steps: dict[int, Callable] = {}
704
+ #: The fields kept in the vault; see `personal`.
705
+ self.vault = _VaultFields()
706
+ #: The subject this event erases, if it is an erasing event; see `erases`.
707
+ self.eraser: Callable | None = None
708
+
709
+ def personal(self, fields: list[str], subject: Callable) -> "EventSchema":
710
+ """Keeps these top-level fields in the vault, as personal data of the subject `subject`
711
+ returns from the payload and, optionally, the aggregate's ID (e.g.
712
+ `lambda d: d["customerId"]`, or `lambda d, id: id` for the aggregate itself). Events
713
+ then hold only references; the kernel hands the values to workers and to listeners
714
+ allowed to read them, until an event erases the subject."""
715
+ self.vault.personal.append((fields, subject))
716
+ return self
717
+
718
+ def erases(self, subject: Callable) -> "EventSchema":
719
+ """Makes this an erasing event: it deletes every vault value of the subject `subject`
720
+ returns, in every stream, and the snapshots that may hold them. Read models should drop
721
+ the subject's data when they see it; rebuilt, they read None."""
722
+ self.eraser = subject
723
+ return self
724
+
725
+ def upcast(self, from_version: int, fn: Callable | None = None):
726
+ """Turns a payload of version `from_version` into one of the next version: `fn(data)`
727
+ returns the new payload. Works as a decorator too: `@item_added.upcast(1)` over
728
+ `def _(data): return {**data, "qty": data.get("qty", 1)}`. Declare one step per older
729
+ version. Raises `ValueError` if `from_version` is not between 1 and the current version
730
+ minus 1."""
731
+ if not isinstance(from_version, int) or not 1 <= from_version < self.schema:
732
+ raise ValueError(f"upcast {self.type} from a version between 1 and {self.schema - 1}, not {from_version!r}")
733
+ return _register(self, self.steps, from_version, fn)
734
+
735
+
736
+ class _VaultFields:
737
+ """Payload fields kept in the vault instead of the signed log."""
738
+
739
+ def __init__(self):
740
+ self.personal: list[tuple[list[str], Callable]] = []
741
+ self.secret: list[str] = []
742
+
743
+
744
+ class CommandSchema:
745
+ """How a command's payload is kept: which fields go to the vault. Made with
746
+ `Domain.command`; its methods return it, so they chain:
747
+ `shop_domain.command("Register").personal(["email"], lambda d, id: id).secret(["password"])`.
748
+ Clients that submit these commands need the domain: `k.use(shop_domain)`."""
749
+
750
+ def __init__(self, type: str):
751
+ """Use `Domain.command`. Raises `ValueError` if `type` is qualified."""
752
+ if not re.match(r"^[^.*]+$", type):
753
+ raise ValueError(f"command types are unqualified within their domain, not {type!r}")
754
+ self.type = type
755
+ self.vault = _VaultFields()
756
+
757
+ def personal(self, fields: list[str], subject: Callable) -> "CommandSchema":
758
+ """Keeps these top-level fields in the vault as personal data of `subject`; see
759
+ `EventSchema.personal`. Declare them on the events that carry them too: a worker rejects
760
+ the command (`personal_in_clear`) if one of its events holds such a field, under the same
761
+ name, without declaring it."""
762
+ self.vault.personal.append((fields, subject))
763
+ return self
764
+
765
+ def secret(self, fields: list[str]) -> "CommandSchema":
766
+ """Keeps these top-level fields (passwords, one-time tokens) in the vault only until the
767
+ command completed. The handler gets them in clear; the stored command keeps a reference."""
768
+ self.vault.secret.extend(fields)
769
+ return self
770
+
771
+
772
+ class QuerySchema:
773
+ """What a domain declares about a query type given by name: the aggregate it is about. Made
774
+ with `Domain.query`; query classes declare it as `target` instead (see `Query`):
775
+ `shop_domain.query("GetOrder").target("order", "id")` (`GetOrder {"id": "o-17"}` is about
776
+ order o-17)."""
777
+
778
+ def __init__(self, type: str):
779
+ """Use `Domain.query`. Raises `ValueError` if `type` is qualified."""
780
+ if not re.match(r"^[^.*]+$", type):
781
+ raise ValueError(f"query types are unqualified within their domain, not {type!r}")
782
+ self.type = type
783
+ #: The aggregate the query is about, `(aggregate_type, field)`; see `target`.
784
+ self.target_of: tuple[str, str] | None = None
785
+
786
+ def target(self, aggregate_type: str, field: str) -> "QuerySchema":
787
+ """The query is about one aggregate of `aggregate_type` (`order`, or qualified
788
+ `shop.order`), whose ID is in the data's `field` (as sent: `orderId`). The SDK sends it
789
+ with the query, signed, and a middleware decides on it as on a command's aggregate.
790
+ Clients that ask it need the domain: `kernable.use(domain)`."""
791
+ self.target_of = (aggregate_type, field)
792
+ return self
793
+
794
+
795
+ def _query_target_of(declared: tuple[str, str] | None, domain: str, data: Any) -> "pb.Target | None":
796
+ """The wire target of a query: the declared aggregate type, qualified, and the ID from `data`."""
797
+ if declared is None:
798
+ return None
799
+ aggregate_type, field_ = declared
800
+ id = data.get(field_) if isinstance(data, dict) else None
801
+ if not isinstance(id, str) or not id:
802
+ return None # no ID: no target, so a rule that needs one refuses
803
+ return pb.Target(aggregate_type=aggregate_type if "." in aggregate_type else f"{domain}.{aggregate_type}",
804
+ aggregate_id=id)
805
+
806
+
807
+ _issued_lock = threading.Lock()
808
+ _issued_last = 0
809
+
810
+
811
+ def _issued_now() -> int:
812
+ """A request's issued_at: now, but later than every one this process issued before. Two calls in the
813
+ same millisecond would otherwise carry the same signature, and the kernel refuses the second as a replay."""
814
+ global _issued_last
815
+ with _issued_lock:
816
+ _issued_last = max(int(time.time() * 1000), _issued_last + 1)
817
+ return _issued_last
818
+
819
+
820
+ def _seal(data: Any, vault: _VaultFields | None, aggregate_id: str) -> tuple[bytes, list[str], list]:
821
+ """Moves the vault fields of `data` into vault values; the payload keeps `{"$vault": ref}`.
822
+ Absent and None fields stay as they are."""
823
+ if vault is None or not (vault.personal or vault.secret) or not isinstance(data, dict):
824
+ return _encode(data), [], []
825
+ sealed = dict(data)
826
+ values = []
827
+
828
+ def keep(f: str, subject: str, secret: bool):
829
+ if sealed.get(f) is None:
830
+ return
831
+ salt = os.urandom(16)
832
+ value = json.dumps(sealed[f], separators=(",", ":")).encode()
833
+ ref = vault_ref(salt, subject, secret, value)
834
+ values.append(pb.VaultValue(ref=ref, subject=subject, secret=secret, salt=salt, value=value))
835
+ sealed[f] = {"$vault": ref}
836
+
837
+ for fields, subject in vault.personal:
838
+ who = str(_call(subject, data, aggregate_id) or "")
839
+ if not who:
840
+ raise ValueError(f"no subject for the personal fields {', '.join(fields)}")
841
+ for f in fields:
842
+ keep(f, who, False)
843
+ for f in vault.secret:
844
+ keep(f, "", True)
845
+ return _encode(sealed), [v.ref for v in values], values
846
+
847
+
848
+ def _unseal(data: Any, vault) -> tuple[Any, list[str]]:
849
+ """Puts vault values back into `data`; references without a value become None."""
850
+ values = {v.ref: v.value for v in vault or []}
851
+ redacted: list[str] = []
852
+
853
+ def walk(v, path: str):
854
+ if isinstance(v, list):
855
+ return [walk(x, f"{path}[{i}]") for i, x in enumerate(v)]
856
+ if not isinstance(v, dict):
857
+ return v
858
+ if len(v) == 1 and isinstance(v.get("$vault"), str):
859
+ if v["$vault"] not in values:
860
+ redacted.append(path)
861
+ return None
862
+ return json.loads(values[v["$vault"]])
863
+ return {k: walk(x, f"{path}.{k}" if path else k) for k, x in v.items()}
864
+
865
+ return walk(data, ""), redacted
866
+
867
+
868
+ class SchemaError(Exception):
869
+ """An event a process cannot read: its schema is newer than known, or a step is missing."""
870
+
871
+ #: The event's qualified type (`shop.ItemAdded`).
872
+ type: str = ""
873
+ #: The event's schema, as stored.
874
+ schema: int = 0
875
+ #: The newest schema this process reads; `newer` is set when the event's is above it (a
876
+ #: newer deployment wrote it), unset when an upcast step is missing.
877
+ known: int = 0
878
+ newer: bool = False
879
+
880
+
881
+ def _upcast(events: dict | None, domain: str, type: str, schema: int, data: Any) -> tuple[int, Any]:
882
+ """`data` of a `type` event stored at `schema`, upcast to the current version in `events`."""
883
+ spec = (events or {}).get(type)
884
+ current = spec.schema if spec else 1
885
+ if schema > current:
886
+ err = SchemaError(f"{domain}.{type} has schema {schema}, but this process knows it only up to {current}: "
887
+ "update it")
888
+ err.type, err.schema, err.known, err.newer = f"{domain}.{type}", schema, current, True
889
+ raise err
890
+ for v in range(schema, current):
891
+ if v not in spec.steps:
892
+ raise SchemaError(f"no upcast for {domain}.{type} from schema {v}")
893
+ data = spec.steps[v](data)
894
+ return current, data
895
+
896
+
897
+ class Domain:
898
+ """A domain: a name that scopes its aggregates, commands and events, lowercase by convention
899
+ (`orders`) like aggregates, while message types are capitalized (`orders.PlaceOrder`). Both
900
+ cases are valid; names are case-sensitive. Types are unique within a domain; elsewhere they
901
+ are named `domain.Type`.
902
+
903
+ Make it with `domain("shop")`. It remembers the aggregates, read models and reactors defined
904
+ on it, so `Kernable.app` runs all of them."""
905
+
906
+ def __init__(self, name: str):
907
+ """Use `domain(name)`. Raises `ValueError` unless `name` starts with a letter and holds
908
+ only letters, digits, `-` and `_`."""
909
+ if not _DOMAIN.match(name):
910
+ raise ValueError(f"invalid domain name {name!r}")
911
+ self.name = name
912
+ #: Payload versions and upcasts per event type; types not listed are at version 1.
913
+ self.events: dict[str, EventSchema] = {}
914
+ #: Vault fields per command type; see `command`.
915
+ self.commands: dict[str, CommandSchema] = {}
916
+ #: What query types given by name are about; see `query`.
917
+ self.queries: dict[str, QuerySchema] = {}
918
+ #: The aggregates defined on it, which `Kernable.app` runs.
919
+ self.aggregates: list[AggregateType] = []
920
+ #: The read models and reactors defined on it, which `Kernable.app` runs.
921
+ self.listeners: list[Listener] = []
922
+ #: The business processes defined on it, which `Kernable.app` registers.
923
+ self.processes: list[Process] = []
924
+
925
+ def command(self, type: str) -> CommandSchema:
926
+ """Declares which fields of `type` commands the vault keeps (`personal`, `secret`); see
927
+ `CommandSchema`. Clients that submit them need the domain: `kernable.use(domain)`."""
928
+ spec = CommandSchema(type)
929
+ self.commands[type] = spec
930
+ return spec
931
+
932
+ def process(self, name: str) -> Process:
933
+ """Defines a business process `name` of this domain (`shop.fulfilment` for `fulfilment`):
934
+ returns the `Process` builder to chain its steps on (`start`, `command`, `wait`, `timer`,
935
+ `decide`, `query`, `end`). `Kernable.app` registers the domain's processes; see
936
+ `kernable.processes` for an example. Raises `ValueError` if the domain has a process of
937
+ that name already."""
938
+ if any(p.name == name for p in self.processes):
939
+ raise ValueError(f"{self.name} has a process {name!r} already")
940
+ p = Process(self.name, name)
941
+ self.processes.append(p)
942
+ return p
943
+
944
+ def query(self, type: str) -> QuerySchema:
945
+ """Declares what `type` queries (unqualified) are about: returns the `QuerySchema` to chain
946
+ `target` on. Query classes declare it as `target` instead. Clients that ask such queries
947
+ by name need the domain: `kernable.use(domain)`."""
948
+ spec = QuerySchema(type)
949
+ self.queries[type] = spec
950
+ return spec
951
+
952
+ def event(self, type, schema: int | None = None, *more) -> EventSchema | None:
953
+ """Declares that this domain writes `type` events at version `schema` (from 1), with an
954
+ `upcast` per older version, and which of their fields the vault keeps; see
955
+ `EventSchema`. The domain's aggregates and listeners use it; listeners of other domains
956
+ get it with `Listener.use(domain)`. Types not declared are at version 1. Example:
957
+ `shop_domain.event("ItemAdded", 2).upcast(1, lambda d: {**d, "qty": 1})`.
958
+
959
+ With event classes instead (`shop_domain.event(ItemAdded, CartCleared)`), declares them
960
+ from their class attributes, e.g. in a process that only reads them; an aggregate's
961
+ `apply` declares them too."""
962
+ if _is_message_class(type):
963
+ schemas = self
964
+ for cls in (type, schema, *more):
965
+ if cls is None:
966
+ continue
967
+ _declare_event(cls, self.events, EventSchema)
968
+ _bind(cls, self.name, schemas)
969
+ return None
970
+ spec = EventSchema(type, schema)
971
+ self.events[type] = spec
972
+ return spec
973
+
974
+ def aggregate(self, name: str, initial: Any, snapshot_every: int = 100, schema: str = "1") -> "AggregateType":
975
+ """An aggregate of this domain named `name` (lowercase by convention, like `cart`), whose
976
+ state starts as `initial` (a JSON-like value, usually a dict); see `Aggregate` for
977
+ `snapshot_every` and `schema`. Run it with `kernable.worker(aggregate)` or in an app.
978
+ Example: `cart_aggregate = shop_domain.aggregate("cart", {"items": 0})`. Raises
979
+ `ValueError` if the domain has an aggregate of that name already."""
980
+ if any(a.name == name for a in self.aggregates):
981
+ raise ValueError(f"{self.name} has an aggregate {name} already")
982
+ aggregate = AggregateType(self.name, name, initial, snapshot_every, schema, self.events, self.commands)
983
+ self.aggregates.append(aggregate)
984
+ return aggregate
985
+
986
+ def read_model(self, name: str, *, order: int = 0, start: int | None = 0, reconnect_s: float = 1.0,
987
+ store: str | None = None, schema: str = "1", live: bool = False,
988
+ park_after: int = 0, instance: str | None = None, single: bool = False) -> "Listener":
989
+ """A read model of this domain named `name`: builds a view from events and answers
990
+ queries from it (`on`, `query`; see `Listener`). Kept in memory and rebuilt from the
991
+ first event on every start (default). Run it with `kernable.listener(read_model)` or in
992
+ an app. Example: `carts_readmodel = shop_domain.read_model("carts")`.
993
+
994
+ `order` (default 0): listeners with a lower order process each event first. `start`:
995
+ the position to start after on the first connect (default 0, from the beginning; `None`:
996
+ the kernel's stored position, or the newest event for a new listener). `reconnect_s`:
997
+ seconds to wait before reconnecting after the stream drops (default 1), twice as long
998
+ after each further failed attempt, at most 30.
999
+
1000
+ With `store` (`sqlite:<path>` or `memory:`), its state lives in the running listener's
1001
+ `store`: data and position are written together per batch, and a restart continues after
1002
+ the stored position. Change `schema` when the handlers store data differently: the store is
1003
+ cleared and the read model rebuilds from the first event. `live` and `park_after` (default
1004
+ 0: never; a read model that parks misses the event's change): see `reactor`.
1005
+
1006
+ `instance`: runs this process's read model as one instance of a group under its name,
1007
+ e.g. one per node of a pair (1 to 128 letters, digits, `-`, `_` and `.`; default
1008
+ `KERNABLE_INSTANCE`, which `kernable node` sets; without either, the name has one
1009
+ connection). Every instance gets every event, from its own position, and answers
1010
+ queries; the group is through an event once one instance has handled it. `single`: see
1011
+ `reactor`. Raises `ValueError` if the domain has a listener (read model or reactor) of
1012
+ that name already."""
1013
+ return self._listener(Listener(self, name, order, start, reconnect_s, store, schema, live=live,
1014
+ park_after=park_after, instance=instance, single=single))
1015
+
1016
+ def reactor(self, name: str, *, order: int = 10, start: int | None = None, reconnect_s: float = 1.0,
1017
+ store: str | None = None, schema: str = "1", live: bool = False,
1018
+ park_after: int = _PARK_AFTER, instance: str | None = None, single: bool = True) -> "Listener":
1019
+ """A reactor of this domain named `name`: triggers side effects and follow-up commands
1020
+ (`on`; see `Listener`). It starts at the newest event when it is new, then resumes after
1021
+ the last acknowledged event (`start=None`, default), so it never reacts to what happened
1022
+ before it existed. Run it with `kernable.listener(reactor)` or in an app.
1023
+
1024
+ A settling submit from its handler waits for the listeners before it (the read models),
1025
+ not for the reactor itself, which handles one event at a time. Submit follow-ups with
1026
+ `ctx.submit` and a fixed `id`, so a redelivered event does not run the command twice.
1027
+ `order` (default 10), `start`, `reconnect_s`, `store` and `schema`: see `read_model`.
1028
+
1029
+ `park_after` (default 5): how often a handler may fail on one event before the reactor
1030
+ parks it and moves on (0: never, it keeps getting the event and everything behind it
1031
+ waits). A `Rejected` (also from `ctx.submit`) parks at once; the kernel being unreachable
1032
+ or busy (`UNAVAILABLE`, `DEADLINE_EXCEEDED`, `RESOURCE_EXHAUSTED`), `NotSettled` or the
1033
+ store failing never does: the reactor waits and tries again. The kernel keeps parked
1034
+ events with the rejection's code and message (for other errors `handler_error` and the
1035
+ exception's type; the log has the whole error) until an operator retries or dismisses
1036
+ them (`kernable listeners retry`). With a store, a parked event's writes are dropped, the
1037
+ rest of its batch kept. Live listeners never park, and no listener parks an event that
1038
+ erases a subject because its handler failed: the handler must get past it, so no personal
1039
+ data stays behind. An event newer than the code (`unknown_event`) is parked, erasing or
1040
+ not: no handler ran on it.
1041
+
1042
+ `live`: not registered with the kernel: no stored position, always starts at the newest
1043
+ event, nobody waits for it, and it is forgotten when it disconnects. For reactions that
1044
+ only matter while the process runs, e.g. pushing to connected browsers. Cannot be a
1045
+ middleware.
1046
+
1047
+ `instance`: see `read_model`. `single` (default `True` for reactors): the instances take
1048
+ turns: one gets events, the others wait and the first of them takes over when it leaves,
1049
+ so a side effect happens once. `False`: every instance gets every event; such a listener
1050
+ cannot park events. Live listeners are never single. Raises `ValueError` if the domain
1051
+ has a listener of that name already."""
1052
+ return self._listener(Listener(self, name, order, start, reconnect_s, store, schema, live=live,
1053
+ park_after=park_after, instance=instance, single=single,
1054
+ reactor=True))
1055
+
1056
+ def _listener(self, listener: "Listener") -> "Listener":
1057
+ if any(l.name == listener.name for l in self.listeners):
1058
+ raise ValueError(f"{self.name} has a listener {listener.name} already")
1059
+ self.listeners.append(listener)
1060
+ return listener
1061
+
1062
+
1063
+ def domain(name: str) -> Domain:
1064
+ """A domain named `name`, on which you define its aggregates, read models, reactors and
1065
+ event and command schemas: `shop_domain = domain("shop")`. Raises `ValueError` unless `name`
1066
+ starts with a letter and holds only letters, digits, `-` and `_`."""
1067
+ return Domain(name)
1068
+
1069
+
1070
+ class AggregateType:
1071
+ """How events change an aggregate's state (`apply`) and which events a command produces
1072
+ (`command`). Handlers must be deterministic and free of side effects; they may run again.
1073
+
1074
+ Make it with `Domain.aggregate`. Both work as decorators (`@agg.apply("Type")`) or directly
1075
+ (`agg.apply("Type", fn)`). A command handler returns a list of `{"type": ..., "data": ...}`
1076
+ and refuses with `raise Rejected(...)`. Handlers take only the parameters they need:
1077
+ `def _(state, data)` is enough, `def _(state, data, cmd, ctx)` gets the rest.
1078
+
1079
+ Every `snapshot_every` events (0 turns it off) the worker sends a snapshot of the state, so
1080
+ later commands fold only the events after it. The state must survive a JSON round trip.
1081
+ Change `schema` when `apply` produces differently shaped state: older snapshots are then
1082
+ ignored and the state is rebuilt from the events.
1083
+
1084
+ Uniqueness and conditions across aggregates: `unique` declares keys no other aggregate may
1085
+ hold at the same time (an email, an order number), `fact` what other aggregates' commands may
1086
+ require of this one (`requires=` of `command`). The kernel checks both atomically with the
1087
+ commit.
1088
+ """
1089
+
1090
+ def __init__(self, domain: str, name: str, initial: Any, snapshot_every: int = 100, schema: str = "1",
1091
+ events: dict[str, EventSchema] | None = None, commands: dict[str, CommandSchema] | None = None):
1092
+ """Use `Domain.aggregate`, which passes the domain's event and command schemas."""
1093
+ self.domain = domain
1094
+ #: The domain's event schemas.
1095
+ self.events = {} if events is None else events
1096
+ #: The domain's command schemas: which command fields are personal.
1097
+ self.commands = {} if commands is None else commands
1098
+ self.name = name
1099
+ self.initial = initial
1100
+ self.snapshot_every = snapshot_every
1101
+ self.schema = schema
1102
+ self.appliers: dict[str, Callable] = {}
1103
+ self.handlers: dict[str, Callable] = {}
1104
+ #: Per command type, the facts of other aggregates it requires; see `command`.
1105
+ self.requirements: dict[str, Callable] = {}
1106
+ #: The unique keys, by name: `(fn, scope, personal)`; see `unique`.
1107
+ self.keys: dict[str, tuple[Callable, str, Callable | None]] = {}
1108
+ #: The facts, by name; see `fact`.
1109
+ self.facts: dict[str, Callable] = {}
1110
+ #: How a state becomes the JSON of a snapshot, and back (an `Aggregate` class: its fields).
1111
+ self._to_json: Callable[[Any], Any] = lambda state: state
1112
+ self._from_json: Callable[[Any], Any] = lambda data: data
1113
+
1114
+ def apply(self, event_type: str, fn: Callable | None = None):
1115
+ """How an event of `event_type` changes the state: `fn(state, data, evt, ctx)` changes
1116
+ `state` in place and returns nothing, or returns a new state that replaces it. `state` is a
1117
+ copy, so changing it never touches the state it came from; `data` is the payload, upcast
1118
+ to the current version; `evt` (an `EventMeta`) and `ctx` (an `ApplyContext`) are
1119
+ optional. Works as a decorator too: `@cart_aggregate.apply("ItemAdded")` over
1120
+ `def _(state, data): state["items"] += data["qty"]`. Returns the decorated function, or
1121
+ the aggregate when `fn` is given.
1122
+
1123
+ With an event class (`@cart_aggregate.apply(ItemAdded)`), `data` is an instance of it."""
1124
+ if _is_message_class(event_type, Event):
1125
+ cls = event_type
1126
+ name = _declare_event(cls, self.events, EventSchema)
1127
+ _bind(cls, self.domain, self._schemas())
1128
+
1129
+ def wrap(f: Callable) -> Callable:
1130
+ return lambda state, data, evt=None, ctx=None: _call(f, state, cls.from_wire(data), evt, ctx)
1131
+
1132
+ return _register_wrapped(self, self.appliers, name, fn, wrap)
1133
+ return _register(self, self.appliers, event_type, fn)
1134
+
1135
+ def _schemas(self):
1136
+ """This aggregate's domain's schemas, as a class binding carries them."""
1137
+ from types import SimpleNamespace
1138
+ return SimpleNamespace(name=self.domain, events=self.events, commands=self.commands)
1139
+
1140
+ def _plain(self, produced: Any) -> Any:
1141
+ """Event class instances a handler returned, as `{"type": ..., "data": ...}`."""
1142
+ if not isinstance(produced, list):
1143
+ return produced
1144
+ out = []
1145
+ for e in produced:
1146
+ if isinstance(e, Event):
1147
+ name = _declare_event(type(e), self.events, EventSchema)
1148
+ _bind(type(e), self.domain, self._schemas())
1149
+ out.append({"type": name, "data": e.to_wire()})
1150
+ else:
1151
+ out.append(e)
1152
+ return out
1153
+
1154
+ def command(self, command_type: str, fn: Callable | None = None, *, requires: Callable | None = None):
1155
+ """Which events a `command_type` command produces: `fn(state, data, cmd, ctx)` returns
1156
+ them as a list of `{"type": ..., "data": ...}`, or raises
1157
+ `Rejected`; `cmd` (a `CommandMeta`) and `ctx` (a `CommandContext`) are optional. `state`
1158
+ is the state after every earlier event; do not change it. Works as a decorator too:
1159
+ `@cart_aggregate.command("AddItem")` over `def _(state, data): return [{"type":
1160
+ "ItemAdded", "data": data}]`. Returns the decorated function, or the aggregate when `fn`
1161
+ is given.
1162
+
1163
+ `requires(data, state)` returns the facts other aggregates of the command's tenant must
1164
+ hold when it commits, as a list of `Requirement` (`state` is the state before the
1165
+ command): `requires=lambda data, state: [Requirement("funds", "vehicle", data["vehicleId"],
1166
+ "open")]`. The kernel checks them atomically with the commit and rejects the command with
1167
+ `requirement_missing` if one is not held, so no other command can change them in between.
1168
+ Raises `ValueError` for a type that starts with `_reindex_`: aggregates with keys or
1169
+ facts run `_reindex_<aggregate_type>` themselves (see `unique`), and one of another aggregate
1170
+ of the domain could clash."""
1171
+ if _is_message_class(command_type, Command):
1172
+ cls = command_type
1173
+ name = _declare_command(cls, self.commands, CommandSchema)
1174
+ _bind(cls, self.domain, self._schemas(), self.name)
1175
+ if requires is not None:
1176
+ given = requires
1177
+ requires = lambda data, state: given(cls.from_wire(data), state) # noqa: E731
1178
+
1179
+ def wrap(f: Callable) -> Callable:
1180
+ return lambda state, data, meta=None, ctx=None: self._plain(_call(f, state, cls.from_wire(data), meta, ctx))
1181
+
1182
+ if requires is not None:
1183
+ self.requirements[name] = requires
1184
+ return _register_wrapped(self, self.handlers, name, fn, wrap)
1185
+ if command_type.startswith("_reindex_"):
1186
+ raise ValueError(f"{command_type}: command types starting with _reindex_ are the built-in commands "
1187
+ "of aggregates with keys or facts; name yours differently")
1188
+ if requires is not None:
1189
+ self.requirements[command_type] = requires
1190
+ return _register(self, self.handlers, command_type, fn)
1191
+
1192
+ def unique(self, name: str, fn: Callable | None = None, *, scope: str = "tenant",
1193
+ personal: Callable | None = None):
1194
+ """A unique key: `fn(state)` returns its value (a `str`), several values (a list of `str`), or
1195
+ `None` for none. After every command the aggregate holds exactly the values of its new
1196
+ state, and no two aggregates of this type hold the same value of `name` at the same time:
1197
+ a command that would take a value another one holds is rejected with `unique_taken`
1198
+ (the message names the key, never the value). Normalise the value in `fn` (e.g.
1199
+ `.lower()` for an email); the kernel compares its SHA-256 hash.
1200
+
1201
+ `scope`: `"tenant"` (default) for unique within the command's tenant, `"global"` for
1202
+ across tenants. `personal(state)`: the person the value belongs to, for personal data
1203
+ such as an email; the kernel keeps the key in the vault, and erasing that subject frees
1204
+ it. A personal key whose subject is empty or `None` is not held.
1205
+
1206
+ Example: `account_aggregate.unique("email", lambda s: s.get("email"), scope="global")`.
1207
+ Works as a decorator too (`@account_aggregate.unique("email")` over `def _(state):`) and
1208
+ returns the decorated function, or the aggregate when `fn` is given, so it chains.
1209
+
1210
+ Keys come from the state, so aggregates that existed before a key was declared hold it
1211
+ only after their next command: `Kernable.reindex` runs the built-in command
1212
+ `_reindex_<aggregate_type>` (`reindex_type`, e.g. `_reindex_account`; no events) on them.
1213
+ Workers of an aggregate with keys or facts run it too, so their key needs `run` for
1214
+ `<domain>._reindex_<aggregate_type>` (`<domain>.*` covers it). An aggregate whose value another
1215
+ one already holds (a duplicate from before) is rejected with `unique_taken` until its
1216
+ value changes or becomes `None`. Raises `ValueError` if the name is declared twice, is
1217
+ empty, holds a dot or is longer than 128 bytes, or `scope` is neither of the two."""
1218
+ _check_index_name("key", name, self.keys, self)
1219
+ if scope not in ("tenant", "global"):
1220
+ raise ValueError(f'the scope of a key is "tenant" or "global", not {scope!r}')
1221
+
1222
+ def define(f: Callable) -> None:
1223
+ self.keys[name] = (f, scope, personal)
1224
+ self.handlers.setdefault(reindex_type(self.name), _no_events)
1225
+
1226
+ return _define(self, define, fn)
1227
+
1228
+ def fact(self, name: str, fn: Callable | None = None):
1229
+ """A fact: `fn(state)` returns whether the aggregate holds it. After every command the
1230
+ aggregate holds the facts that are true for its new state, and commands of other
1231
+ aggregates may require them (`requires=` of `command`), checked atomically with their
1232
+ commit (`requirement_missing` if not held). Example: `vehicle_aggregate.fact("open",
1233
+ lambda s: s["status"] == "open")`. Works as a decorator too and chains like `unique`;
1234
+ aggregates that existed before a fact was declared hold it after `Kernable.reindex`.
1235
+ Raises `ValueError` if the name is declared twice, is empty, holds a dot or is longer
1236
+ than 128 bytes."""
1237
+ _check_index_name("fact", name, self.facts, self)
1238
+
1239
+ def define(f: Callable) -> None:
1240
+ self.facts[name] = f
1241
+ self.handlers.setdefault(reindex_type(self.name), _no_events)
1242
+
1243
+ return _define(self, define, fn)
1244
+
1245
+ def index(self, state: Any) -> Index:
1246
+ """The unique keys and facts this aggregate holds for `state`, as the worker computes them
1247
+ after a command; for unit tests next to `handle`: `account_aggregate.index(state).keys`.
1248
+ Raises the exception of a key's or fact's function, `TypeError` if a key's value is not a
1249
+ string."""
1250
+ keys: list[HeldKey] = []
1251
+ for name, (fn, scope, personal) in self.keys.items():
1252
+ values = fn(state)
1253
+ if values is None:
1254
+ continue
1255
+ for value in values if isinstance(values, (list, tuple)) else [values]:
1256
+ if not isinstance(value, str):
1257
+ raise TypeError(f"the key {name} of {self.domain}.{self.name} is a string, not {type(value).__name__}")
1258
+ subject = ""
1259
+ if personal is not None:
1260
+ subject = personal(state) or ""
1261
+ if not isinstance(subject, str):
1262
+ raise TypeError(f"the subject of the key {name} is a string, not {type(subject).__name__}")
1263
+ if not subject:
1264
+ continue
1265
+ held = HeldKey(name, value, scope, subject)
1266
+ if held not in keys:
1267
+ keys.append(held)
1268
+ return Index(keys, [name for name, fn in self.facts.items() if fn(state)])
1269
+
1270
+ def schema_of(self, event_type: str) -> int:
1271
+ """The payload version this aggregate writes `event_type` events at."""
1272
+ spec = self.events.get(event_type)
1273
+ return spec.schema if spec else 1
1274
+
1275
+ def fold(self, history: list[tuple], state: Any = _INITIAL) -> Any:
1276
+ """State after `history`, a list of tuples `(type, schema, data)`, optionally followed by
1277
+ an `EventMeta` and an `ApplyContext`, starting from `state` (default: `initial`). Events
1278
+ are upcast first; events without an `apply` are skipped. For unit tests without a kernel:
1279
+ `cart_aggregate.fold([("ItemAdded", 1, {"qty": 2})])`. Raises the handler's exception,
1280
+ or an error if an event's schema is newer than this process knows."""
1281
+ if state is _INITIAL:
1282
+ state = self.initial
1283
+ for event_type, schema, data, *rest in history:
1284
+ fn = self.appliers.get(event_type)
1285
+ if fn:
1286
+ meta = rest[0] if rest else EventMeta(event_type, domain=self.domain, aggregate_type=self.name)
1287
+ ctx = rest[1] if len(rest) > 1 else ApplyContext()
1288
+ draft = copy.deepcopy(state)
1289
+ result = _call(fn, draft, _upcast(self.events, self.domain, event_type, schema, data)[1], meta, ctx)
1290
+ state = draft if result is None else result
1291
+ return state
1292
+
1293
+ def handle(self, state: Any, command_type: str, data: Any, ctx: CommandContext, *, aggregate_id: str = "", tenant: str = "") -> list[dict]:
1294
+ """Runs the `command_type` handler on `state`, e.g. in a unit test, and returns its events.
1295
+ `data` reaches it as from a client (after a JSON round trip; `None` as `{}`), `state` as a
1296
+ copy. The handler's `CommandMeta` has the command ID of `ctx` (see `CommandContext.new`),
1297
+ this aggregate's domain and name, and an empty aggregate ID and tenant: `cart_aggregate.handle(
1298
+ state, "AddItem", {"qty": 2}, CommandContext.new("add-1"))`. Raises the handler's `Rejected`
1299
+ (or other exception), `ValueError` if there is no handler for `command_type`."""
1300
+ fn = self.handlers.get(command_type)
1301
+ if fn is None:
1302
+ raise ValueError(f"no handler for {command_type} on {self.domain}.{self.name}")
1303
+ meta = CommandMeta(ctx._command_id, command_type, self.domain, self.name, aggregate_id, tenant)
1304
+ return _call(fn, copy.deepcopy(state), _decode(_encode(data)), meta, ctx)
1305
+
1306
+
1307
+ def _no_events(state, data) -> list[dict]:
1308
+ """The built-in `_reindex_<aggregate_type>` command: no events, so the worker sends the keys and facts of the
1309
+ current state."""
1310
+ return []
1311
+
1312
+
1313
+ def _check_index_name(kind: str, name: str, declared: dict, agg: AggregateType) -> None:
1314
+ if not isinstance(name, str) or not name or "." in name or len(name.encode()) > 128:
1315
+ raise ValueError(f"a {kind} name is not empty, has no dots and at most 128 bytes, not {name!r}")
1316
+ if name in declared:
1317
+ raise ValueError(f"{agg.domain}.{agg.name} has a {kind} {name} already")
1318
+
1319
+
1320
+ def _define(owner, define: Callable, fn: Callable | None):
1321
+ """Defines `fn` with `define` and returns `owner`, or, without `fn`, a decorator that does."""
1322
+ if fn is not None:
1323
+ define(fn)
1324
+ return owner
1325
+
1326
+ def decorator(f: Callable) -> Callable:
1327
+ define(f)
1328
+ return f
1329
+
1330
+ return decorator
1331
+
1332
+
1333
+ def _personal(vault: _VaultFields) -> set[str]:
1334
+ return {f for fields, _ in vault.personal for f in fields}
1335
+
1336
+
1337
+ def _check_personal(agg: AggregateType, cmd, produced: list[dict]) -> None:
1338
+ """Rejects (`personal_in_clear`) an event that holds a personal field of its command, under the
1339
+ same name, but does not declare it personal: the value would go into the log in clear, for good."""
1340
+ spec = agg.commands.get(cmd.command_type)
1341
+ personal = _personal(spec.vault) if spec else set()
1342
+ if not personal:
1343
+ return
1344
+ for e in produced:
1345
+ data, event = e.get("data"), agg.events.get(e["type"])
1346
+ if not isinstance(data, dict):
1347
+ continue
1348
+ declared = _personal(event.vault) if event else set()
1349
+ for f in sorted(personal - declared):
1350
+ if data.get(f) is not None:
1351
+ raise Rejected(f"{cmd.command_type} declares {f} personal, but {e['type']} does not: it would be "
1352
+ f"stored in clear (declare it with Domain.event({e['type']!r}, …).personal)",
1353
+ "personal_in_clear")
1354
+
1355
+
1356
+ class Aggregate:
1357
+ """An aggregate defined as a class: its fields are the state, and the class is registered as an
1358
+ aggregate of `domain` (its name: the class name in lowercase, or `name`). The state is a
1359
+ dataclass (no `@dataclass`); mutable defaults (`[]`, `{}`) are copied for every aggregate, and
1360
+ snapshots hold its fields as JSON. Handlers get and change an instance: `s.status = "active"`.
1361
+
1362
+ class Cart(Aggregate, domain=shop_domain):
1363
+ items: int = 0
1364
+
1365
+ @Cart.apply(ItemAdded)
1366
+ def _(s: Cart, d: ItemAdded):
1367
+ s.items += d.count
1368
+
1369
+ `Cart.command`, `Cart.unique`, `Cart.fact`, `Cart.fold`, `Cart.handle` and `Cart.index` work
1370
+ as on `AggregateType` (what `domain.aggregate` returns, which keeps working with a dict)."""
1371
+
1372
+ __aggregate__: "ClassVar[AggregateType]"
1373
+
1374
+ def __init_subclass__(cls, *, domain: "Domain | None" = None, name: str | None = None, snapshot_every: int = 100,
1375
+ schema: str = "1", **kwargs: Any) -> None:
1376
+ super().__init_subclass__(**kwargs)
1377
+ if domain is None:
1378
+ return
1379
+ for field_name in list(cls.__dict__.get("__annotations__", {})):
1380
+ default = cls.__dict__.get(field_name)
1381
+ if isinstance(default, (list, dict, set)):
1382
+ setattr(cls, field_name, dataclasses.field(default_factory=lambda v=default: copy.deepcopy(v)))
1383
+ dataclasses.dataclass(cls)
1384
+ definition = domain.aggregate(name or cls.__name__.lower(), cls(), snapshot_every, schema)
1385
+ definition._to_json = lambda state: dataclasses.asdict(state) if isinstance(state, cls) else state
1386
+ names = {f.name for f in dataclasses.fields(cls)}
1387
+ definition._from_json = lambda data: cls(**{k: v for k, v in data.items() if k in names}) if isinstance(data, dict) else data
1388
+ cls.__aggregate__ = definition
1389
+
1390
+ def update(self, **values: Any) -> None:
1391
+ """Sets several fields at once: `s.update(status="active", slug=d.slug)`."""
1392
+ for key, value in values.items():
1393
+ setattr(self, key, value)
1394
+
1395
+ @classmethod
1396
+ def apply(cls, event_type, fn: Callable | None = None):
1397
+ return cls.__aggregate__.apply(event_type, fn)
1398
+
1399
+ @classmethod
1400
+ def command(cls, command_type, fn: Callable | None = None, *, requires: Callable | None = None):
1401
+ return cls.__aggregate__.command(command_type, fn, requires=requires)
1402
+
1403
+ @classmethod
1404
+ def unique(cls, name: str, fn: Callable | None = None, *, scope: str = "tenant", personal: Callable | None = None):
1405
+ return cls.__aggregate__.unique(name, fn, scope=scope, personal=personal)
1406
+
1407
+ @classmethod
1408
+ def fact(cls, name: str, fn: Callable | None = None):
1409
+ return cls.__aggregate__.fact(name, fn)
1410
+
1411
+ @classmethod
1412
+ def fold(cls, history: list[tuple], state: Any = _INITIAL) -> Any:
1413
+ return cls.__aggregate__.fold(history, state)
1414
+
1415
+ @classmethod
1416
+ def handle(cls, state: Any, command_type: str, data: Any, ctx: "CommandContext", *, aggregate_id: str = "", tenant: str = "") -> list[dict]:
1417
+ return cls.__aggregate__.handle(state, command_type, data, ctx, aggregate_id=aggregate_id, tenant=tenant)
1418
+
1419
+ @classmethod
1420
+ def index(cls, state: Any) -> "Index":
1421
+ return cls.__aggregate__.index(state)
1422
+
1423
+
1424
+ def _register_wrapped(owner, table: dict, name: str, fn: Callable | None, wrap: Callable[[Callable], Callable]):
1425
+ """`_register` for a message class: the handler is stored wrapped, the decorator returns it unwrapped."""
1426
+ if fn is not None:
1427
+ table[name] = wrap(fn)
1428
+ return owner
1429
+
1430
+ def decorator(f: Callable) -> Callable:
1431
+ table[name] = wrap(f)
1432
+ return f
1433
+
1434
+ return decorator
1435
+
1436
+
1437
+ def _register(owner, table: dict, name: str, fn: Callable | None):
1438
+ if fn is not None:
1439
+ table[name] = fn
1440
+ return owner
1441
+
1442
+ def decorator(f: Callable) -> Callable:
1443
+ table[name] = f
1444
+ return f
1445
+
1446
+ return decorator
1447
+
1448
+
1449
+ class _WithGrant(grpc.UnaryUnaryClientInterceptor, grpc.UnaryStreamClientInterceptor,
1450
+ grpc.StreamUnaryClientInterceptor, grpc.StreamStreamClientInterceptor):
1451
+ """Sends the key's grant with every call, in the `kernable-grant` metadata entry."""
1452
+
1453
+ def __init__(self, grant: str):
1454
+ self._grant = grant
1455
+
1456
+ def _details(self, details):
1457
+ metadata = [*(details.metadata or []), ("kernable-grant", self._grant)]
1458
+ return details._replace(metadata=metadata)
1459
+
1460
+ def intercept_unary_unary(self, continuation, details, request):
1461
+ return continuation(self._details(details), request)
1462
+
1463
+ def intercept_unary_stream(self, continuation, details, request):
1464
+ return continuation(self._details(details), request)
1465
+
1466
+ def intercept_stream_unary(self, continuation, details, requests):
1467
+ return continuation(self._details(details), requests)
1468
+
1469
+ def intercept_stream_stream(self, continuation, details, requests):
1470
+ return continuation(self._details(details), requests)
1471
+
1472
+
1473
+ class _Open(grpc.UnaryUnaryClientInterceptor, grpc.UnaryStreamClientInterceptor,
1474
+ grpc.StreamUnaryClientInterceptor, grpc.StreamStreamClientInterceptor):
1475
+ """Refuses every call once the client is closed: grpcio crashes the process on a call over a
1476
+ closed channel."""
1477
+
1478
+ def __init__(self):
1479
+ self.closed = False
1480
+
1481
+ def _check(self):
1482
+ if self.closed:
1483
+ raise ClientClosed()
1484
+
1485
+ def intercept_unary_unary(self, continuation, details, request):
1486
+ self._check()
1487
+ return continuation(details, request)
1488
+
1489
+ def intercept_unary_stream(self, continuation, details, request):
1490
+ self._check()
1491
+ return continuation(details, request)
1492
+
1493
+ def intercept_stream_unary(self, continuation, details, requests):
1494
+ self._check()
1495
+ return continuation(details, requests)
1496
+
1497
+ def intercept_stream_stream(self, continuation, details, requests):
1498
+ self._check()
1499
+ return continuation(details, requests)
1500
+
1501
+
1502
+ class _Deadline(grpc.UnaryUnaryClientInterceptor):
1503
+ """Lets every request wait for the connection (e.g. while the kernel restarts) and gives it a
1504
+ deadline, unless the call has its own."""
1505
+
1506
+ def __init__(self, timeout_s: float):
1507
+ self._timeout_s = timeout_s
1508
+
1509
+ def intercept_unary_unary(self, continuation, details, request):
1510
+ timeout = self._timeout_s if details.timeout is None else details.timeout
1511
+ return continuation(details._replace(timeout=timeout, wait_for_ready=True), request)
1512
+
1513
+
1514
+ class _Failover(grpc.Channel):
1515
+ """The channels to the kernels of a pair (`url` with several addresses): every call goes to
1516
+ the current one; on `UNAVAILABLE` (a standby answers `not_primary`, a kernel is gone or
1517
+ shutting down) the next one becomes current. A request tries the next address within its
1518
+ timeout, waiting at most a second for each to connect; a stream that ends with
1519
+ `UNAVAILABLE` reconnects to the next."""
1520
+
1521
+ _PER_ADDRESS_S = 1.0
1522
+
1523
+ def __init__(self, channels: list, raw: list, timeout_s: float):
1524
+ self._channels, self._raw, self._timeout_s = channels, raw, timeout_s
1525
+ self._current = 0
1526
+ self._lock = threading.Lock()
1527
+
1528
+ def current(self) -> int:
1529
+ return self._current
1530
+
1531
+ def next(self, from_: int) -> None:
1532
+ with self._lock:
1533
+ if self._current == from_:
1534
+ self._current = (from_ + 1) % len(self._channels)
1535
+
1536
+ def _unary(self, kind: str, method: str, *args, **kwargs):
1537
+ calls = [getattr(c, kind)(method, *args, **kwargs) for c in self._channels]
1538
+ failover = self
1539
+
1540
+ def attempt(how: str, request, timeout, kw):
1541
+ deadline = time.monotonic() + (failover._timeout_s if timeout is None else timeout)
1542
+ while True:
1543
+ at = failover._current
1544
+ left = deadline - time.monotonic()
1545
+ try:
1546
+ grpc.channel_ready_future(failover._raw[at]).result(
1547
+ timeout=max(0.0, min(left, failover._PER_ADDRESS_S)))
1548
+ except grpc.FutureTimeoutError:
1549
+ if time.monotonic() < deadline:
1550
+ failover.next(at)
1551
+ continue
1552
+ try:
1553
+ return getattr(calls[at], how)(request, timeout=max(0.001, deadline - time.monotonic()), **kw)
1554
+ except grpc.RpcError as e:
1555
+ if e.code() != grpc.StatusCode.UNAVAILABLE or time.monotonic() >= deadline:
1556
+ raise
1557
+ failover.next(at)
1558
+
1559
+ class _Call:
1560
+ def __call__(self, request, timeout=None, **kw):
1561
+ return attempt("__call__", request, timeout, kw)
1562
+
1563
+ def with_call(self, request, timeout=None, **kw):
1564
+ return attempt("with_call", request, timeout, kw)
1565
+
1566
+ def future(self, request, timeout=None, **kw):
1567
+ return calls[failover._current].future(request, timeout=timeout, **kw)
1568
+
1569
+ return _Call()
1570
+
1571
+ def _stream(self, kind: str, method: str, *args, **kwargs):
1572
+ calls = [getattr(c, kind)(method, *args, **kwargs) for c in self._channels]
1573
+ failover = self
1574
+
1575
+ class _Call:
1576
+ def __init__(self, at, inner):
1577
+ self._at, self._inner = at, inner
1578
+
1579
+ def __iter__(self):
1580
+ return self
1581
+
1582
+ def __next__(self):
1583
+ try:
1584
+ return next(self._inner)
1585
+ except grpc.RpcError as e:
1586
+ if e.code() == grpc.StatusCode.UNAVAILABLE:
1587
+ failover.next(self._at)
1588
+ raise
1589
+
1590
+ def __getattr__(self, name):
1591
+ return getattr(self._inner, name)
1592
+
1593
+ def call(request, *a, **kw):
1594
+ at = failover._current
1595
+ return _Call(at, calls[at](request, *a, **kw))
1596
+
1597
+ return call
1598
+
1599
+ def unary_unary(self, method, *args, **kwargs):
1600
+ return self._unary("unary_unary", method, *args, **kwargs)
1601
+
1602
+ def stream_unary(self, method, *args, **kwargs):
1603
+ return self._unary("stream_unary", method, *args, **kwargs)
1604
+
1605
+ def unary_stream(self, method, *args, **kwargs):
1606
+ return self._stream("unary_stream", method, *args, **kwargs)
1607
+
1608
+ def stream_stream(self, method, *args, **kwargs):
1609
+ return self._stream("stream_stream", method, *args, **kwargs)
1610
+
1611
+ def subscribe(self, callback, try_to_connect=False):
1612
+ self._raw[self._current].subscribe(callback, try_to_connect)
1613
+
1614
+ def unsubscribe(self, callback):
1615
+ for c in self._raw:
1616
+ c.unsubscribe(callback)
1617
+
1618
+ def close(self):
1619
+ for c in self._channels:
1620
+ c.close()
1621
+
1622
+
1623
+ def _local(url: str) -> bool:
1624
+ """Whether `url` (every address of a comma-separated list) is a Unix socket or a loopback
1625
+ address: plain text never leaves the machine."""
1626
+ if "," in url:
1627
+ return all(_local(u.strip()) for u in url.split(","))
1628
+ if url.startswith("unix:"):
1629
+ return True
1630
+ host = re.sub(r":\d+/?$", "", re.sub(r"^https?://", "", url)).strip("[]")
1631
+ return host in ("localhost", "::1") or re.fullmatch(r"127\.\d+\.\d+\.\d+", host) is not None
1632
+
1633
+
1634
+ def _moving(err: grpc.RpcError) -> bool:
1635
+ """Whether a kernel refused a request because its tenant is moving to another partition."""
1636
+ return err.code() == grpc.StatusCode.UNAVAILABLE and (err.details() or "").startswith("moving")
1637
+
1638
+
1639
+ # How long a request waits for its tenant's move to end.
1640
+ _MOVE_WAIT_S = 30.0
1641
+
1642
+
1643
+ def _switch(switch: str) -> str:
1644
+ """`switch` if it is one of `caught-up`, `now` and `never`."""
1645
+ if switch not in ("caught-up", "now", "never"):
1646
+ raise ValueError(f"switch {switch!r}: caught-up, now or never")
1647
+ return switch
1648
+
1649
+
1650
+ def waits_for_switch(err: BaseException) -> bool:
1651
+ """Whether the kernel refused a membership because it would switch its service to a
1652
+ deployment that has not caught up yet (`not_caught_up`), back to one that served before with
1653
+ `switch="caught-up"` (`served_before`), or at all with `switch="never"` (`not_serving`):
1654
+ applying it again may succeed later, once the deployment caught up or an operator switched
1655
+ to it."""
1656
+ return (isinstance(err, grpc.RpcError) and err.code() == grpc.StatusCode.FAILED_PRECONDITION
1657
+ and (err.details() or "").startswith(("not_caught_up", "not_serving", "served_before")))
1658
+
1659
+ # The first pause before an idempotent call asks a kernel that answered UNAVAILABLE again; it
1660
+ # doubles up to the second.
1661
+ _RETRY_PAUSE_S = 0.05
1662
+ _MAX_RETRY_PAUSE_S = 0.2
1663
+ # How long an idempotent request whose connection dropped under way is tried again: long enough
1664
+ # for a kernel handing its role over, short enough that a crash reaches the caller.
1665
+ _DROPPED_RETRY_S = 5.0
1666
+ _DROPPED = re.compile(r"connection dropped|socket closed|ECONNRESET|EPIPE|RST_STREAM|GOAWAY|connection reset",
1667
+ re.IGNORECASE)
1668
+
1669
+
1670
+ def _connection_dropped(err: grpc.RpcError) -> bool:
1671
+ """Whether `err` is the client's own UNAVAILABLE for a connection that went away under the
1672
+ request, not a kernel's answer."""
1673
+ return bool(_DROPPED.search(err.details() or ""))
1674
+
1675
+
1676
+ def _idempotent(timeout_s: float, call: Callable[[float], Any]) -> Any:
1677
+ """`call(left)` (a request with `left` seconds to go) for a call that is safe to repeat: a
1678
+ submit, by its command ID (the kernel answers a resubmission with the stored result); a
1679
+ cancel; a query; a data or cache read. A kernel that answers `UNAVAILABLE` (shutting down,
1680
+ handing over, a read model or middleware not connected yet, a connection that dropped) is
1681
+ asked again with backoff until `timeout_s` passed, at a single address too; then the last error is raised. A tenant
1682
+ that moves is waited for by `_Partitioned.route`."""
1683
+ deadline = time.monotonic() + timeout_s
1684
+ pause = _RETRY_PAUSE_S
1685
+ # Since when the request may have reached a kernel that then went away: from then on it is
1686
+ # tried again for a short while only (a handover), not until the deadline (a crash the
1687
+ # caller should hear of).
1688
+ dropped: "tuple[float, grpc.RpcError] | None" = None
1689
+ while True:
1690
+ until = deadline if dropped is None else min(deadline, dropped[0] + _DROPPED_RETRY_S)
1691
+ try:
1692
+ return call(max(0.001, until - time.monotonic()))
1693
+ except grpc.RpcError as e:
1694
+ if dropped is not None and until < deadline and time.monotonic() >= until:
1695
+ raise dropped[1] from e
1696
+ if (e.code() != grpc.StatusCode.UNAVAILABLE or _moving(e)
1697
+ or time.monotonic() + pause >= deadline):
1698
+ raise
1699
+ if _connection_dropped(e):
1700
+ if dropped is None:
1701
+ dropped = (time.monotonic(), e)
1702
+ elif time.monotonic() - dropped[0] >= _DROPPED_RETRY_S:
1703
+ raise
1704
+ time.sleep(pause)
1705
+ pause = min(pause * 2, _MAX_RETRY_PAUSE_S)
1706
+
1707
+
1708
+ def _partition_of(m, tenant: str) -> str:
1709
+ """The partition map `m` assigns `tenant` to."""
1710
+ return next((t.partition for t in m.tenants if t.tenant == tenant), m.default)
1711
+
1712
+
1713
+ def _wrong_partition(err: grpc.RpcError) -> bool:
1714
+ """Whether a kernel refused a request as one for a tenant of another partition."""
1715
+ return err.code() == grpc.StatusCode.FAILED_PRECONDITION and (err.details() or "").startswith("wrong_partition")
1716
+
1717
+
1718
+ class _Partitioned:
1719
+ """The channels of every partition, found by the map the first kernel holds: a request
1720
+ goes to its tenant's partition; refused there as `wrong_partition` (the map changed), the
1721
+ map is read again and the request sent anew. A kernel without partitions holds no map:
1722
+ everything goes to the first kernel."""
1723
+
1724
+ _UNREAD = object()
1725
+
1726
+ def __init__(self, entry: grpc.Channel, channel_for: Callable[[str], grpc.Channel], timeout_s: float):
1727
+ self.entry = entry
1728
+ self._timeout_s = timeout_s
1729
+ self._channel_for = channel_for
1730
+ self._map: Any = self._UNREAD
1731
+ self._parts: dict[str, grpc.Channel] = {}
1732
+ self._lock = threading.Lock()
1733
+
1734
+ def current(self, refresh: bool = False, timeout_s: float | None = None):
1735
+ """The map (a `PartitionMap`), read once and again after `refresh`; `None` without
1736
+ partitions. `timeout_s`: how long to wait for the kernel (default: the client's)."""
1737
+ with self._lock:
1738
+ if refresh:
1739
+ self._map = self._UNREAD
1740
+ if self._map is self._UNREAD:
1741
+ try:
1742
+ self._map = _idempotent(timeout_s or self._timeout_s, lambda left: rpc.PartitionServiceStub(
1743
+ self.entry).Map(pb.MapRequest(protocol=PROTOCOL), timeout=left))
1744
+ except grpc.RpcError as e:
1745
+ if e.code() not in (grpc.StatusCode.NOT_FOUND, grpc.StatusCode.UNIMPLEMENTED):
1746
+ raise
1747
+ self._map = None
1748
+ return self._map
1749
+
1750
+ def _channel(self, m, name: str) -> grpc.Channel:
1751
+ partition = next((p for p in m.partitions if p.name == name), None)
1752
+ if partition is None:
1753
+ raise ValueError(f"the partition map names no partition {name}")
1754
+ # `relay:<relay>/<copy>` is how partitions at other sites reach this one: not a client's way.
1755
+ own = [a for a in partition.addresses if not a.startswith("relay:")]
1756
+ if not own:
1757
+ raise ValueError(f"partition {name} is reached only through a relay: a client needs one of its own addresses")
1758
+ addresses = ",".join(own)
1759
+ with self._lock:
1760
+ if addresses not in self._parts:
1761
+ self._parts[addresses] = self._channel_for(addresses)
1762
+ return self._parts[addresses]
1763
+
1764
+ def for_tenant(self, tenant: str) -> grpc.Channel:
1765
+ m = self.current()
1766
+ if m is None:
1767
+ return self.entry
1768
+ return self._channel(m, _partition_of(m, tenant))
1769
+
1770
+ def control(self) -> grpc.Channel:
1771
+ """The control partition's channel, which takes memberships."""
1772
+ m = self.current()
1773
+ return self.entry if m is None else self._channel(m, m.control)
1774
+
1775
+ def all(self, timeout_s: float | None = None) -> "list[tuple[str, grpc.Channel]]":
1776
+ """Every partition with its channel; one, named `""`, without partitions."""
1777
+ m = self.current(timeout_s=timeout_s)
1778
+ if m is None:
1779
+ return [("", self.entry)]
1780
+ return [(p.name, self._channel(m, p.name)) for p in m.partitions]
1781
+
1782
+ def route(self, tenant: str, call: Callable[[grpc.Channel], Any]) -> Any:
1783
+ """`call` on `tenant`'s partition, sent anew after reading the map again on
1784
+ `wrong_partition`, and while the tenant moves (`moving`, refused before it took effect)
1785
+ for up to 30 s."""
1786
+ deadline = time.monotonic() + _MOVE_WAIT_S
1787
+ pause = 0.05
1788
+ attempt = 0
1789
+ while True:
1790
+ try:
1791
+ return call(self.for_tenant(tenant))
1792
+ except grpc.RpcError as e:
1793
+ if _moving(e) and time.monotonic() + pause < deadline:
1794
+ time.sleep(pause)
1795
+ pause = min(pause * 2, 1.0)
1796
+ continue
1797
+ if attempt == 3 or not _wrong_partition(e):
1798
+ raise
1799
+ attempt += 1
1800
+ self.current(refresh=True)
1801
+
1802
+ def close(self) -> None:
1803
+ self.entry.close()
1804
+ for c in self._parts.values():
1805
+ c.close()
1806
+
1807
+
1808
+ class Kernable:
1809
+ """A connection to the kernel: submits commands (`submit`, `enqueue`), asks queries
1810
+ (`query`), runs workers, listeners and apps, and applies memberships. All calls block the
1811
+ calling thread; one client may be shared by threads.
1812
+
1813
+ `k.data` (a `DataClient`) holds files of a domain, `k.cache` (a `CacheClient`) its shared
1814
+ cache. `Kernable()` without arguments takes everything from the environment or the
1815
+ defaults, which fit a kernel started with `kernable start --dev` in the current directory."""
1816
+
1817
+ def __init__(self, url: str | None = None, key: str | None = None, tenant: str = "default",
1818
+ tls: Tls | None = None, context: RequestContext | None = None,
1819
+ grant: str | None = None, service: str | None = None, insecure: bool | None = None,
1820
+ timeout_s: float = DEFAULT_TIMEOUT_S, deployment: str | None = None):
1821
+ """A client for the kernel at `url`, signing with the key in the file `key`. It
1822
+ connects on first use and again after the connection drops.
1823
+
1824
+ `url`: `http://host:port` (`host:port` works too) or, on Unix, `unix:/path/to.sock`;
1825
+ for a pair of kernels with a standby both, comma-separated (`http://a:7233,http://b:7233`):
1826
+ the client turns to the other one when a kernel answers `UNAVAILABLE` (a standby says
1827
+ `not_primary`) or cannot be reached. Default `KERNABLE_URL`, else
1828
+ `http://127.0.0.1:7233`, where `kernable start` listens.
1829
+ `key`: path to this process's `.key` file; default `KERNABLE_KEY`, else `app.key` in the
1830
+ current directory. `tenant`: default tenant of submits, queries, files and the cache
1831
+ (default `default`). `tls`: mutual TLS for `host:port`; default `KERNABLE_TLS_CA`,
1832
+ `KERNABLE_TLS_CERT` and `KERNABLE_TLS_KEY`, if set (all three or none). `context`:
1833
+ default request context for submits, queries, files and the cache, e.g. this service's
1834
+ identity, replaced by a request's own. `grant`: path to the key's grant (`kernable grant`), sent with every
1835
+ call; default `KERNABLE_GRANT`, else the `.grant` file next to the key file, if there is
1836
+ one. `service`: the service this process's workers and listeners belong to. The
1837
+ service's membership (see `apply`) decides what the kernel waits for: what it lists
1838
+ counts, the service's other workers and listeners are dynamic, and only listeners it lists
1839
+ as middlewares may check requests. Default `KERNABLE_SERVICE`, else none (everything
1840
+ registered counts). `insecure`: allows plain text to a kernel that is not on this machine;
1841
+ without TLS, grants, signed requests and vault values would cross the network readable,
1842
+ so such a URL raises `ValueError` unless this is set; default `KERNABLE_INSECURE=1`, else
1843
+ False. `timeout_s`: how long a request (submit, query, files, cache, apply) waits in all,
1844
+ for the connection (e.g. while the kernel restarts) and its answer; default 60 s. A
1845
+ connection the network dropped silently is noticed within 40 s by keepalive pings (within
1846
+ 4 s with several kernels, a pair, whose client then turns to the other). A
1847
+ submit (by its command ID), a cancel or a query that a kernel answers `UNAVAILABLE`
1848
+ (shutting down, handing over, a middleware or read model not connected) is asked again
1849
+ with backoff within this time; any other request may have run, and fails at once.
1850
+ `deployment`: the code version this process runs (1 to 128 letters, digits, `-`, `_` and
1851
+ `.`, e.g. `2026-11-11.1`); default `KERNABLE_DEPLOYMENT`, else none. Its workers and
1852
+ listeners declare it, the events its workers write carry it, its queries prefer read
1853
+ models of it, and `apply` names it in the membership: the deployment a membership names
1854
+ runs the service's commands and answers its queries, while the others stay connected
1855
+ for a rollback.
1856
+
1857
+ Workers and listeners print connection lines to stderr up to the level `KERNABLE_LOG`
1858
+ names (`off`, `error`, `warn`, `info` (default), `debug`). Raises `OSError` if the key
1859
+ or grant file cannot be read, `ValueError` if only some of the TLS variables are set or
1860
+ `deployment` is invalid."""
1861
+ self._deployment = _deployment(deployment or os.environ.get("KERNABLE_DEPLOYMENT") or "")
1862
+ key = key or os.environ.get("KERNABLE_KEY") or "app.key"
1863
+ tls = tls or _tls_from_env()
1864
+ self._context = context or RequestContext()
1865
+ url = url or os.environ.get("KERNABLE_URL") or "http://127.0.0.1:7233"
1866
+ given_url = url
1867
+ url = re.sub(r"^https?://", "", url).rstrip("/")
1868
+ if insecure is None:
1869
+ insecure = os.environ.get("KERNABLE_INSECURE") == "1"
1870
+ if not tls and not insecure and not _local(url):
1871
+ raise ValueError(f"{given_url} is not on this machine: connect with TLS (tls=...), or pass "
1872
+ "insecure=True to send in plain text")
1873
+ def open_channel(url: str, several: bool = False):
1874
+ options = _PAIR_CHANNEL_OPTIONS if several else _CHANNEL_OPTIONS
1875
+ if tls:
1876
+ def read(path: str) -> bytes:
1877
+ with open(path, "rb") as f:
1878
+ return f.read()
1879
+
1880
+ creds = grpc.ssl_channel_credentials(read(tls.ca), read(tls.key), read(tls.cert))
1881
+ return grpc.secure_channel(url, creds, options=options)
1882
+ # grpcio sends the socket path as :authority for unix: URLs, which the kernel's
1883
+ # HTTP/2 server rejects; any valid authority works.
1884
+ return grpc.insecure_channel(url, options=[("grpc.default_authority", "localhost"),
1885
+ *options])
1886
+
1887
+ grant = grant or os.environ.get("KERNABLE_GRANT")
1888
+ if grant is None and os.path.exists(default := key.removesuffix(".key") + ".grant"):
1889
+ grant = default
1890
+ granted = None
1891
+ if grant is not None:
1892
+ with open(grant) as f:
1893
+ granted = f.read().strip()
1894
+ # The outermost interceptor of every channel: checked before anything touches it.
1895
+ self._open = _Open()
1896
+
1897
+ def channel_for(addresses: str) -> grpc.Channel:
1898
+ """A channel to the kernels at `addresses`, comma-separated (a pair: see `_Failover`)."""
1899
+ urls = [re.sub(r"^https?://", "", u.strip()).rstrip("/") for u in addresses.split(",") if u.strip()]
1900
+ raw = [open_channel(u, len(urls) > 1) for u in urls]
1901
+ channels = []
1902
+ for c in raw:
1903
+ if granted is not None:
1904
+ c = grpc.intercept_channel(c, _WithGrant(granted))
1905
+ channels.append(grpc.intercept_channel(c, _Deadline(timeout_s)))
1906
+ channel = channels[0] if len(channels) == 1 else _Failover(channels, raw, timeout_s)
1907
+ return grpc.intercept_channel(channel, self._open)
1908
+
1909
+ self._timeout_s = timeout_s
1910
+ self._channel = channel_for(url)
1911
+ # With partitions, each tenant's requests go to its partition.
1912
+ self._parts = _Partitioned(self._channel, channel_for, timeout_s)
1913
+ # The newest position this client's own commands committed at, per partition's channel:
1914
+ # its queries wait for it, on whichever copy (writer or reader) answers.
1915
+ self._seen: dict[int, int] = {}
1916
+ self._key = Key.from_file(key)
1917
+ self._log = _Log(given_url, key, self._key.id)
1918
+ # Whoever can read the key file can sign as this process.
1919
+ mode = os.stat(key).st_mode & 0o777
1920
+ if os.name == "posix" and mode & 0o077:
1921
+ self._log.print("warn", "key", f"{key} can be read by other users (mode {mode:o}): chmod 600 {key}")
1922
+ self._tenant = tenant
1923
+ self._service = service or os.environ.get("KERNABLE_SERVICE") or ""
1924
+ self._domains: dict[str, Domain] = {}
1925
+ #: Files of a domain: `k.data.put(domain, key, data)`, `get`, `delete`.
1926
+ self.data = DataClient(self)
1927
+ #: A domain's shared cache: `k.cache.set(domain, key, value, ttl_ms)`, `get`, `delete`.
1928
+ self.cache = CacheClient(self)
1929
+
1930
+ def use(self, *domains: Domain) -> "Kernable":
1931
+ """Seals the vault fields of these domains' commands when submitting them (see
1932
+ `Domain.command`), and sends the targets of their queries given by name (see
1933
+ `Domain.query`). Returns this client, so it chains: `k = Kernable().use(shop_domain)`."""
1934
+ for d in domains:
1935
+ self._domains[d.name] = d
1936
+ return self
1937
+
1938
+ def worker(self, *aggregates: "AggregateType | type[Aggregate]") -> "Worker":
1939
+ """A worker that runs these aggregates' commands, in this client's service; see
1940
+ `Worker`. `run()` on it blocks until `stop()`: `k.worker(cart_aggregate).run()`."""
1941
+ aggregates = tuple(getattr(a, "__aggregate__", a) for a in aggregates)
1942
+ return Worker._new(self._parts, self._key, aggregates, self._log, self._service, self._deployment)
1943
+
1944
+ def app(self, *, domains: Iterable[Domain], service: str | None = None, version: str | None = None,
1945
+ description: str | None = None, metadata: dict[str, str] | None = None,
1946
+ switch: str | None = None) -> "App":
1947
+ """An app made of whole domains: every aggregate, read model and reactor defined on them.
1948
+ `run()` on it applies the service's membership (the domains' aggregates as workers, their
1949
+ read models and reactors as listeners, those with a middleware as middlewares, their
1950
+ processes, and this client's deployment; live listeners are not listed), then runs all of them, and registers the
1951
+ domains' processes once its worker is connected (see `register_process`). `service` defaults to this client's; `version`
1952
+ and `description` go into the membership's metadata, followed by `metadata`.
1953
+
1954
+ Example: `k.app(service="shop", version="1.0.0", domains=[shop_domain]).run()`. Every
1955
+ start applies the membership, so the key needs the `manage` grant for the service. One
1956
+ service is one app: two processes with the same service would replace each other's
1957
+ membership. `switch` says when the app's deployment takes the service over from another
1958
+ one that serves it (see `apply`): by default (`"caught-up"`) once its read models have
1959
+ caught up, if the deployment never served before; meanwhile it runs beside the serving
1960
+ deployment, its workers idle and its read models building. One that served before (an
1961
+ older deployment's app restarting) runs beside until an operator rolls back. Default: `KERNABLE_SWITCH`, else `"caught-up"`. Raises `ValueError` if
1962
+ neither `service` nor the client names a service, or for an unknown `switch`."""
1963
+ switch = _switch(switch if switch is not None else os.environ.get("KERNABLE_SWITCH") or "caught-up")
1964
+ service = service or self._service
1965
+ if not service:
1966
+ raise ValueError("an app needs a service: pass `service` or set it on the client (KERNABLE_SERVICE)")
1967
+ domains = list(domains)
1968
+ aggregates = [a for d in domains for a in d.aggregates]
1969
+ listeners = [l for d in domains for l in d.listeners]
1970
+ durable = [l for l in listeners if not l.live]
1971
+ meta = {k: v for k, v in (("version", version), ("description", description)) if v is not None}
1972
+ manifest = Manifest(service=service, workers=aggregates,
1973
+ listeners=[l for l in durable if not l._middlewares],
1974
+ middlewares=[l for l in durable if l._middlewares],
1975
+ metadata={**meta, **(metadata or {})},
1976
+ processes=[p for d in domains for p in d.processes],
1977
+ deployment=self._deployment, switch=switch)
1978
+ worker = Worker._new(self._parts, self._key, aggregates, self._log, service, self._deployment) if aggregates else None
1979
+ handles = [ListenerHandle._new(self._parts, self._key, l, self._log, service, self) for l in listeners]
1980
+ return App(self, manifest, worker, handles, [d.name for d in domains])
1981
+
1982
+ def apply(self, *, service: str | None = None, workers: Iterable[AggregateType] = (),
1983
+ listeners: Iterable["Listener"] = (), middlewares: Iterable["Listener"] = (),
1984
+ metadata: dict[str, str] | None = None, processes: Iterable["Process | str"] = (),
1985
+ deployment: str | None = None, switch: str = "caught-up") -> None:
1986
+ """Replaces the membership of a service (default: this client's `service`): the workers,
1987
+ listeners and middlewares the kernel waits for, plus free metadata such as
1988
+ `{"version": "1.0.0"}`. `workers` are the aggregates the service runs (the kernel waits
1989
+ for a worker of each), `listeners` the listeners it waits for (the service's others count
1990
+ for nobody), `middlewares` the service's listeners that may check requests (they count like
1991
+ listeners), `processes` its business processes (`Process`es or qualified names): one it no
1992
+ longer lists starts no new cases, its running cases finish. `deployment` (default: this
1993
+ client's) names the code version that serves the service: its workers run the commands,
1994
+ its listeners count and its read models answer queries first; the service's workers and
1995
+ listeners of other deployments stay connected, idle or catching up. A rollback applies a
1996
+ membership naming the earlier deployment. Empty: every deployment serves. `switch` says
1997
+ when a membership that names another deployment than the serving one switches to it:
1998
+ `"caught-up"` (default) once that deployment has caught up (its listed workers connected,
1999
+ its read models within the kernel's `--deployment-switch-lag` of the newest event),
2000
+ refused before as `not_caught_up`, and never back to a deployment that served before (it
2001
+ wrote events), refused as `served_before`: switching back is a rollback, which an
2002
+ operator does (`kernable apply --deployment`); `"now"` at once;
2003
+ `"never"`: applied only while its deployment serves already (or the service has no
2004
+ membership yet), refused otherwise as `not_serving`. See `waits_for_switch`.
2005
+
2006
+ Apply it once per deploy, e.g. from a deploy script, not from every process: the newest
2007
+ apply wins, and an older one that arrives late is refused. The key needs the `manage` grant for the service. A middleware the new
2008
+ membership no longer lists stops checking. Example: `k.apply(service="shop",
2009
+ workers=[cart_aggregate], listeners=[carts_readmodel], metadata={"version": "1.0.0"})`.
2010
+ Raises `ValueError` without a service or for an invalid `deployment`, `grpc.RpcError` if
2011
+ the kernel refuses it."""
2012
+ service = service or self._service
2013
+ deployment = self._deployment if deployment is None else _deployment(deployment)
2014
+ if not service:
2015
+ raise ValueError("apply needs a service: pass `service` or set it on the client (KERNABLE_SERVICE)")
2016
+ membership = pb.Membership(service=service, workers=[f"{a.domain}.{a.name}" for a in workers],
2017
+ listeners=[l.name for l in listeners],
2018
+ middlewares=[l.name for l in middlewares],
2019
+ metadata=[pb.Attribute(key=k, value=v) for k, v in (metadata or {}).items()],
2020
+ issued_at=_issued_now(),
2021
+ processes=[p.qualified if isinstance(p, Process) else p for p in processes],
2022
+ deployment=deployment)
2023
+ switch = _switch(switch)
2024
+ request = pb.ApplyRequest(membership=membership, sig=self._key.sign(membership_message(membership)),
2025
+ protocol=PROTOCOL, force=switch == "now", no_switch=switch == "never",
2026
+ no_rollback=switch == "caught-up")
2027
+ # The same membership again is fine (the kernel takes a retried deploy): a dropped
2028
+ # connection, or a kernel starting, is asked again.
2029
+ _idempotent(self._timeout_s, lambda left: rpc.MembershipServiceStub(self._parts.control()).Apply(
2030
+ request, timeout=left))
2031
+
2032
+ def process_act(self, name: str, context: RequestContext) -> bool:
2033
+ """Sets as whom the process `name` (qualified: `shop.fulfilment`) acts: the request context
2034
+ its commands and queries carry (sender tenant, identity, session), signed with this
2035
+ client's key, which needs the grant `process` for it. Without it, a process acts as
2036
+ `process:<name>`. Returns whether it changed (False: it acted so already). The kit's
2037
+ identity reactor calls it for every process it gives an identity. Raises `grpc.RpcError`
2038
+ if the kernel refuses it (`NOT_FOUND` for a process never registered)."""
2039
+ req = pb.ProcessActing(name=name, context=_to_wire_context(context, ""), issued_at=_issued_now(), protocol=PROTOCOL)
2040
+ req.sig.CopyFrom(self._key.sign(process_act_message(req.name, req.context, req.issued_at)))
2041
+ return rpc.ProcessServiceStub(self._parts.control()).Act(req).changed
2042
+
2043
+ def register_process(self, process: "Process | dict") -> RegisteredProcess:
2044
+ """Registers a business process (a `Process`, or its model as a dict), signed with this
2045
+ client's key: a model equal to the newest version registers nothing (`new` is False),
2046
+ a different one becomes the next version; running cases stay on theirs. `Kernable.app`
2047
+ does this for its domains' processes. The key needs the grant `process` for the
2048
+ process, and `submit`, `query` and `listen` for the commands, queries and events the
2049
+ model names. Raises `ValueError` if the builder finds the model invalid (see
2050
+ `Process.model`), `grpc.RpcError` if the kernel refuses it (`INVALID_ARGUMENT` names the
2051
+ step and why)."""
2052
+ model = process.model() if isinstance(process, Process) else process
2053
+ raw = json.dumps(model, separators=(",", ":"), ensure_ascii=False).encode()
2054
+ req = pb.ProcessRegistration(model=raw, issued_at=_issued_now(), protocol=PROTOCOL)
2055
+ req.sig.CopyFrom(self._key.sign(process_message(req.model, req.issued_at)))
2056
+ res = rpc.ProcessServiceStub(self._parts.control()).Register(req)
2057
+ return RegisteredProcess(name=res.name, version=res.version, new=res.new)
2058
+
2059
+ def _command_of(self, command: Command) -> tuple:
2060
+ """Domain, aggregate type, type and payload of a command class instance; its domain's vault fields known."""
2061
+ b = _binding(type(command), "handled by an aggregate")
2062
+ self._domains.setdefault(b.domain, b.schemas)
2063
+ return b.domain, b.aggregate_type, type(command).type_name(), command.to_wire()
2064
+
2065
+ def submit(self, domain, aggregate_type: str | None = None, aggregate_id: str | None = None, type: str | None = None, data: Any = None, *,
2066
+ tenant: str | None = None, id: str | None = None, wait: str = "settled",
2067
+ context: RequestContext | None = None, timeout_s: float | None = None, at: Any = None) -> Outcome:
2068
+ """Runs a `type` command of `domain` on the aggregate `aggregate_type` with ID
2069
+ `aggregate_id` and returns its `Outcome`. `data` is the command's payload (JSON-like; `None` sends `{}`).
2070
+
2071
+ `wait="settled"` (default) responds once every listener that counts has processed the
2072
+ events, so read models already show them, and raises `NotSettled` on timeout; from a
2073
+ listener's handler, only the listeners with a lower order count. `wait="committed"`
2074
+ responds once the events are stored; then query with `min_position`. `id` is an
2075
+ idempotency key (default: a new UUID): submitting the same `id` again returns the
2076
+ original outcome. `tenant` replaces the client's tenant. `context` replaces the client's
2077
+ default context; inside a listener, pass `ctx.follow()`. `timeout_s` replaces the
2078
+ client's deadline for this call.
2079
+
2080
+ A rejected command returns `Outcome(committed=False, code=..., message=...)`; it does not
2081
+ raise. Raises `grpc.RpcError` if the kernel refuses the call (no grant, unknown aggregate
2082
+ type, timeout). Example: `k.submit("shop", "cart", "c1", "AddItem", {"sku": "apple",
2083
+ "qty": 2})`.
2084
+
2085
+ With a command class instance: `k.submit(AddItem(sku="apple", qty=2), aggregate_id="c1")`;
2086
+ its class says the domain, aggregate type, type and vault fields (no `use` needed).
2087
+
2088
+ A scheduled command has no outcome yet: schedule it with `enqueue(..., at=...)` (`at`
2089
+ here raises `ValueError`). Submitted again by its `id`, it waits for the scheduled
2090
+ command's outcome, as long as the kernel holds the call; raises `NotDue` if the kernel
2091
+ answers that it is not due yet.
2092
+ """
2093
+ if at is not None:
2094
+ raise ValueError("a scheduled command (`at`) has no outcome yet: use `enqueue`")
2095
+ if isinstance(domain, Command):
2096
+ domain, aggregate_type, type, data = self._command_of(domain)
2097
+ mode = pb.WAIT_COMMITTED if wait == "committed" else pb.WAIT_SETTLED
2098
+ res = self._send(domain, aggregate_type, aggregate_id, type, data, tenant, id, mode, context, timeout_s)
2099
+ if res.WhichOneof("outcome") == "rejected":
2100
+ return _rejected_outcome(res.rejected)
2101
+ if res.WhichOneof("outcome") != "committed":
2102
+ raise NotDue(res.command_id)
2103
+ outcome = Outcome(committed=True, events=[_to_event(e) for e in res.committed.events])
2104
+ if mode == pb.WAIT_SETTLED and not res.committed.settled:
2105
+ raise NotSettled(outcome)
2106
+ return outcome
2107
+
2108
+ def enqueue(self, domain, aggregate_type: str | None = None, aggregate_id: str | None = None, type: str | None = None, data: Any = None, *,
2109
+ tenant: str | None = None, id: str | None = None, context: RequestContext | None = None,
2110
+ timeout_s: float | None = None, at: "datetime | int | None" = None,
2111
+ until: "datetime | int | None" = None) -> str:
2112
+ """Stores the command in the kernel's queue and returns its ID without waiting for it to
2113
+ run (`wait="accepted"`). Submit it again with the same `id` to get the outcome. The
2114
+ parameters are those of `submit`.
2115
+
2116
+ With `at` (a `datetime`, or ms since epoch), a scheduled command: it runs at that time and
2117
+ not before, e.g. a reminder in 24 hours, `k.enqueue(Remind(), aggregate_id="o1",
2118
+ at=datetime.now(timezone.utc) + timedelta(days=1))`. The kernel stores it at once; the
2119
+ aggregate's other commands pass meanwhile, and middlewares check it now and again when it
2120
+ is due. `until`: if it cannot run by then (after an outage), it is rejected `expired`.
2121
+ Both are signed with the command. Cancel it with `cancel`."""
2122
+ if until is not None and at is None:
2123
+ raise ValueError("`until` needs `at`")
2124
+ if isinstance(domain, Command):
2125
+ domain, aggregate_type, type, data = self._command_of(domain)
2126
+ return self._send(domain, aggregate_type, aggregate_id, type, data, tenant, id, pb.WAIT_ACCEPTED,
2127
+ context, timeout_s, _ms(at), _ms(until)).command_id
2128
+
2129
+ def cancel(self, command_id: str, *, tenant: str | None = None, context: RequestContext | None = None,
2130
+ timeout_s: float | None = None) -> CancelOutcome:
2131
+ """Cancels a scheduled command (`enqueue` with `at`) before it is due: it completes as
2132
+ rejected `cancelled`, and cancelling it again answers the same. Needs `submit` for the
2133
+ command's type; middlewares check it as a cancel of that type (`Check.kind == "cancel"`).
2134
+ Returns a `CancelOutcome`: not cancelled with `too_late` (due, running or done already),
2135
+ `not_found`, or a middleware's code. `tenant` and `context` replace the client's;
2136
+ `timeout_s` its deadline for this call. Example: `k.cancel(reminder_id)`."""
2137
+ req = pb.CancelRequest(command_id=command_id, tenant_id=tenant or self._tenant,
2138
+ context=_to_wire_context(context or self._context, command_id),
2139
+ issued_at=_issued_now(), protocol=PROTOCOL)
2140
+ req.client_sig.CopyFrom(self._key.sign(cancel_message(req)))
2141
+ res = self._parts.route(req.tenant_id, lambda ch: _idempotent(
2142
+ timeout_s or self._timeout_s, lambda left: rpc.CommandServiceStub(ch).Cancel(req, timeout=left)))
2143
+ if res.WhichOneof("outcome") == "cancelled":
2144
+ return CancelOutcome(cancelled=True)
2145
+ return CancelOutcome(cancelled=False, code=res.rejected.code, message=res.rejected.message)
2146
+
2147
+ def _caught_up_everywhere(self, q: "pb.QueryRequest", timeout_s: float | None) -> None:
2148
+ """Returns once a read model that answers `q` has processed every partition's head, as
2149
+ of the call: a wait-only query to each partition, at the same time."""
2150
+ wait = pb.QueryRequest()
2151
+ wait.CopyFrom(q)
2152
+ wait.payload, wait.min_position, wait.wait_only, wait.listener = b"", 0, True, ""
2153
+ wait.client_sig.CopyFrom(self._key.sign(query_message(wait)))
2154
+ wait.listener = q.listener
2155
+ parts = self._parts.all(timeout_s)
2156
+ with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, len(parts))) as pool:
2157
+ waits = [pool.submit(lambda ch=ch: _idempotent(
2158
+ timeout_s or self._timeout_s, lambda left: rpc.QueryServiceStub(ch).Query(wait, timeout=left)))
2159
+ for _, ch in parts]
2160
+ for w in waits:
2161
+ w.result()
2162
+
2163
+ def query(self, domain, type: str | None = None, data: Any = None, *, tenant: str | None = None,
2164
+ context: RequestContext | None = None, min_position: int = 0, timeout_s: float | None = None,
2165
+ all_partitions: bool = False) -> Any:
2166
+ """Asks the read model that answers `domain.type` with `data` and returns its answer,
2167
+ decoded from JSON. Queries always go through the kernel, also within one process.
2168
+ `min_position`: answer only after the read model has processed this global position
2169
+ (e.g. of an event from a `wait="committed"` submit). `tenant` and `context` replace the
2170
+ client's; `timeout_s` its deadline for this call. `all_partitions`, for a read model
2171
+ across tenants with partitions: answer only once the read model has processed every
2172
+ partition's events up to the query, not only those of the tenant's partition (a
2173
+ wait-only query to each partition first).
2174
+
2175
+ Raises `Rejected` if the read model or a middleware rejects it, `grpc.RpcError` if the
2176
+ kernel refuses the call (e.g. `UNAVAILABLE` when no read model answers it). Example:
2177
+ `cart = k.query("shop", "GetCart", {"id": "c1"})`."""
2178
+ if isinstance(domain, Query):
2179
+ asked = domain
2180
+ kind = asked.__class__
2181
+ b = _binding(kind, "answered by a read model")
2182
+ return _from_wire(self._ask(b.domain, kind.type_name(), asked.to_wire(), _query_target(kind), tenant,
2183
+ context, min_position, timeout_s, all_partitions))
2184
+ spec = self._domains.get(domain)
2185
+ # A domain known only from a command class it submitted has no queries: none declared.
2186
+ declared = getattr(spec, "queries", {}).get(type) if spec else None
2187
+ return self._ask(domain, type, data, declared.target_of if declared else None, tenant, context,
2188
+ min_position, timeout_s, all_partitions)
2189
+
2190
+ def _ask(self, domain: str, type: str, data: Any, declared: tuple[str, str] | None, tenant: str | None,
2191
+ context: RequestContext | None, min_position: int, timeout_s: float | None, all_partitions: bool) -> Any:
2192
+ """`query`, with the target the query type declares: `(aggregate_type, field)`."""
2193
+ query_id = str(uuid.uuid4())
2194
+ q = pb.QueryRequest(query_id=query_id, tenant_id=tenant or self._tenant, domain=domain, query_type=type,
2195
+ payload=_encode(data), context=_to_wire_context(context or self._context, query_id),
2196
+ min_position=min_position, issued_at=_issued_now(), protocol=PROTOCOL,
2197
+ listener=_HANDLING.get(), target=_query_target_of(declared, domain, data),
2198
+ deployment=self._deployment)
2199
+ if all_partitions:
2200
+ self._caught_up_everywhere(q, timeout_s)
2201
+ q.issued_at = _issued_now()
2202
+ def ask(ch):
2203
+ # Read your writes: wait for this client's own newest command, on whichever copy
2204
+ # answers. Not from a handler, which must not wait behind its own listener.
2205
+ if not q.listener:
2206
+ q.min_position = max(min_position, self._seen.get(_channel_id(ch), 0))
2207
+ q.client_sig.CopyFrom(self._key.sign(query_message(q)))
2208
+ return _idempotent(timeout_s or self._timeout_s, lambda left: rpc.QueryServiceStub(ch).Query(q, timeout=left))
2209
+ res = self._parts.route(q.tenant_id, ask)
2210
+ if res.WhichOneof("outcome") == "rejected":
2211
+ raise Rejected(res.rejected.message, res.rejected.code)
2212
+ return _decode(res.response)
2213
+
2214
+ def reindex(self, domain: str, aggregate_type: str, aggregate_ids: Iterable[str], *,
2215
+ tenant: str | None = None, context: RequestContext | None = None) -> dict[str, Outcome]:
2216
+ """Runs the built-in command `_reindex_<aggregate_type>` on these aggregates, one after
2217
+ the other, so they hold the unique keys and facts of their current state, e.g. after
2218
+ declaring a key (`Aggregate.unique`) for aggregates that existed before. It produces no events. Returns
2219
+ the `Outcome` per ID: committed, or rejected, e.g. with `unique_taken` for an aggregate
2220
+ whose value another one holds already. An ID without events holds nothing. `tenant` and
2221
+ `context` replace the client's. Raises `grpc.RpcError` if the kernel refuses a call
2222
+ (e.g. the aggregate's worker declares no keys or facts). Example:
2223
+ `k.reindex("identity", "account", ["a1", "a2"])`."""
2224
+ command_type = reindex_type(aggregate_type)
2225
+ return {id: self.submit(domain, aggregate_type, id, command_type, None, tenant=tenant, wait="committed",
2226
+ context=context) for id in aggregate_ids}
2227
+
2228
+ def listener(self, listener: "Listener") -> "ListenerHandle":
2229
+ """A handle that runs a listener defined with `Domain.read_model` or `Domain.reactor`, in
2230
+ this client's service; see `ListenerHandle`. `run()` on it blocks until `stop()`:
2231
+ `k.listener(carts_readmodel).run()`. Each call gives a new handle."""
2232
+ return ListenerHandle._new(self._parts, self._key, listener, self._log, self._service, self)
2233
+
2234
+ def subscribe(self, handler: Callable[["StoredEvent"], None], *, after: int = 0,
2235
+ types: Iterable[str] = (), reconnect_s: float = 1.0) -> "Subscription":
2236
+ """Streams committed events after `after` (backlog first, then live) to `handler`, one at
2237
+ a time, in a thread of its own from the call on, e.g. to export them:
2238
+ `k.subscribe(export, types=["shop.*"])`. Returns a `Subscription`; `stop()` ends it.
2239
+
2240
+ `after`: global position of the last event already handled (0: from the beginning).
2241
+ `types`: qualified event types (`orders.OrderPlaced`) or domains (`orders.*`); empty: all
2242
+ the key may read. Reconnects and resumes after the last handled event, waiting
2243
+ `reconnect_s` after the stream ends, twice as long after each further failed attempt, at
2244
+ most 30 s; if the handler raises, that event is delivered again. Persist `position`
2245
+ yourself to resume across restarts. Unlike a listener, a subscription is not registered:
2246
+ the kernel keeps no position, nobody waits for it, and events arrive as stored (not
2247
+ upcast, vault fields `None` and listed in `redacted`). The key needs `subscribe` for the
2248
+ types. With partitions, a stream per partition, each from the beginning (`after` names a
2249
+ position of one kernel); events are in order within a partition, not across them."""
2250
+ sub = Subscription._new(self._parts, self._key, handler, after, list(types), reconnect_s, self._log)
2251
+ threading.Thread(target=sub.run, name="kernable subscription", daemon=True).start()
2252
+ return sub
2253
+
2254
+ def close(self) -> None:
2255
+ """Closes the connection now; only needed in a long-running process that discards the
2256
+ client. Requests afterwards raise `ClientClosed`, and its workers and listeners end."""
2257
+ self._open.closed = True
2258
+ self._parts.close()
2259
+
2260
+ def _send(self, domain, aggregate_type, aggregate_id, command_type, data, tenant, id, wait, context, timeout_s,
2261
+ deliver_at: int = 0, deliver_until: int = 0):
2262
+ command_id = id or str(uuid.uuid4())
2263
+ context = context or self._context
2264
+ spec = self._domains[domain].commands.get(command_type) if domain in self._domains else None
2265
+ payload, refs, values = _seal(data, spec.vault if spec else None, aggregate_id)
2266
+ cmd = pb.Command(command_id=command_id, tenant_id=tenant or self._tenant, domain=domain,
2267
+ aggregate_type=aggregate_type, aggregate_id=aggregate_id,
2268
+ command_type=command_type, payload=payload,
2269
+ context=_to_wire_context(context, command_id), vault_refs=refs,
2270
+ deliver_at=deliver_at, deliver_until=deliver_until)
2271
+ cmd.client_sig.CopyFrom(self._key.sign(command_message(cmd)))
2272
+ request = pb.SubmitRequest(command=cmd, wait=wait, protocol=PROTOCOL, listener=_HANDLING.get(),
2273
+ vault=values)
2274
+ def send(ch):
2275
+ res = _idempotent(timeout_s or self._timeout_s,
2276
+ lambda left: rpc.CommandServiceStub(ch).Submit(request, timeout=left))
2277
+ if res.WhichOneof("outcome") == "committed":
2278
+ events = res.committed.events
2279
+ position = max(res.committed.position, events[-1].global_position if events else 0)
2280
+ self._seen[_channel_id(ch)] = max(self._seen.get(_channel_id(ch), 0), position)
2281
+ return res
2282
+ return self._parts.route(cmd.tenant_id, send)
2283
+
2284
+
2285
+ class DataClient:
2286
+ """Files of a domain, kept by the kernel under (tenant, domain, key): invoices, uploads,
2287
+ exports. They are not part of the signed event log and not erased with the vault; keep
2288
+ personal data in vault fields instead. Needs a grant of `data_read` (get, list) or `data_write`
2289
+ (put, delete) for the domain; a refused or invalid request raises `grpc.RpcError`. Each
2290
+ request runs the middleware chain first, as `<domain>.PutData`, `.GetData`, `.DeleteData` or `.ListData`
2291
+ (the middleware sees the key and the request context, never the file); a middleware's
2292
+ refusal raises `grpc.RpcError` with code `PERMISSION_DENIED` and details `"<code>:
2293
+ <message>"`. Keys are 1 to 512 bytes, files at most 16 MiB (the kernel's `--max-data-bytes`).
2294
+ Use it as `k.data`:
2295
+ `k.data.put("shop", "invoices/c1.pdf", pdf_bytes, "application/pdf")`. `tenant=` replaces the
2296
+ client's tenant, `context=` the client's default request context."""
2297
+
2298
+ def __init__(self, k: Kernable):
2299
+ """Use `k.data` of a `Kernable`."""
2300
+ self._k = k
2301
+ self._stub = rpc.DataServiceStub
2302
+
2303
+ def put(self, domain: str, key: str, data: bytes | str, content_type: str | None = None, *,
2304
+ tenant: str | None = None, context: RequestContext | None = None) -> str:
2305
+ """Stores `data` under `key`, replacing a file stored there. A `str` is stored as UTF-8;
2306
+ `content_type` defaults to `text/plain; charset=utf-8` for a `str` and
2307
+ `application/octet-stream` for `bytes`. Returns the SHA-256 of the data (hex) as the
2308
+ kernel computed it: for an event to name."""
2309
+ if isinstance(data, str):
2310
+ data, content_type = data.encode(), content_type or "text/plain; charset=utf-8"
2311
+ return self._call("put", "PutData", tenant, context, domain, key, data=data,
2312
+ content_type=content_type or "application/octet-stream").sha256.hex()
2313
+
2314
+ def get(self, domain: str, key: str, *, tenant: str | None = None,
2315
+ context: RequestContext | None = None) -> Data | None:
2316
+ """The file stored under `key`, or `None` if there is none."""
2317
+ res = self._call("get", "GetData", tenant, context, domain, key)
2318
+ return Data(res.data, res.content_type, res.sha256.hex()) if res.found else None
2319
+
2320
+ def list(self, domain: str, prefix: str = "", *, after: str = "", tenant: str | None = None,
2321
+ context: RequestContext | None = None) -> Listing:
2322
+ """The files whose keys start with `prefix` (granted `data_read`), in the order of their
2323
+ keys' bytes, up to 1,000 after the key `after`; `next` of the result, if set, is the
2324
+ `after` of the next page. To find files no event names, or what a tenant stores."""
2325
+ res = self._call("list", "ListData", tenant, context, domain, prefix, after=after)
2326
+ return Listing([Listed(f.key, f.content_type, f.size) for f in res.files], res.next or None)
2327
+
2328
+ def delete(self, domain: str, key: str, *, tenant: str | None = None,
2329
+ context: RequestContext | None = None) -> bool:
2330
+ """Deletes the file stored under `key`; returns whether there was one."""
2331
+ return self._call("delete", "DeleteData", tenant, context, domain, key).found
2332
+
2333
+ def _call(self, op, method, tenant, context, domain, key, **fields):
2334
+ r = pb.DataRequest(tenant_id=tenant or self._k._tenant, domain=domain, key=key,
2335
+ issued_at=_issued_now(), protocol=PROTOCOL,
2336
+ context=_to_wire_context(context or self._k._context, ""), **fields)
2337
+ r.client_sig.CopyFrom(self._k._key.sign(data_message(op, r)))
2338
+ if op in ("get", "list"): # safe to repeat; a write is one-shot (the kernel's replay guard)
2339
+ return self._k._parts.route(r.tenant_id, lambda ch: _idempotent(
2340
+ self._k._timeout_s, lambda left: getattr(self._stub(ch), method)(r, timeout=left)))
2341
+ return self._k._parts.route(r.tenant_id, lambda ch: getattr(self._stub(ch), method)(r))
2342
+
2343
+
2344
+ class CacheClient:
2345
+ """A domain's shared cache, kept in the kernel's memory under (tenant, domain, key): rate
2346
+ limits, short-lived tokens, computed values. Not durable: a kernel restart empties it. Values
2347
+ are not part of the signed event log and not erased with the vault. Needs a grant of `cache`
2348
+ for the domain; a refused or invalid request raises `grpc.RpcError`. Each request runs the
2349
+ middleware chain first, as `<domain>.SetCache`, `.GetCache` or `.DeleteCache` (the middleware
2350
+ sees the key and the request context, never the value); a middleware's refusal raises
2351
+ `grpc.RpcError` with code `PERMISSION_DENIED` and details `"<code>: <message>"`. Keys are 1 to
2352
+ 512 bytes, values at most 1 MiB. Use it as `k.cache`: `k.cache.set("shop", "rate:c1", "3",
2353
+ ttl_ms=60_000)`, then `k.cache.get("shop", "rate:c1")` returns `b"3"`. `tenant=` replaces the
2354
+ client's tenant, `context=` the client's default request context."""
2355
+
2356
+ def __init__(self, k: Kernable):
2357
+ """Use `k.cache` of a `Kernable`."""
2358
+ self._k = k
2359
+ self._stub = rpc.CacheServiceStub
2360
+
2361
+ def set(self, domain: str, key: str, value: bytes | str, ttl_ms: int = 0, *,
2362
+ tenant: str | None = None, context: RequestContext | None = None) -> None:
2363
+ """Stores `value` under `key` for `ttl_ms` milliseconds; `0` keeps it until it is deleted
2364
+ or the kernel restarts. A `str` is stored as UTF-8."""
2365
+ value = value.encode() if isinstance(value, str) else value
2366
+ self._call("set", "SetCache", tenant, context, domain, key, value=value, ttl_ms=ttl_ms)
2367
+
2368
+ def get(self, domain: str, key: str, *, tenant: str | None = None,
2369
+ context: RequestContext | None = None) -> bytes | None:
2370
+ """The live value under `key` as `bytes` (decode a `str` with `.decode()`), or `None` if
2371
+ there is none or it expired."""
2372
+ res = self._call("get", "GetCache", tenant, context, domain, key)
2373
+ return res.value if res.found else None
2374
+
2375
+ def delete(self, domain: str, key: str, *, tenant: str | None = None,
2376
+ context: RequestContext | None = None) -> bool:
2377
+ """Deletes the value under `key`; returns whether it had a live one."""
2378
+ return self._call("delete", "DeleteCache", tenant, context, domain, key).found
2379
+
2380
+ def _call(self, op, method, tenant, context, domain, key, **fields):
2381
+ r = pb.CacheRequest(tenant_id=tenant or self._k._tenant, domain=domain, key=key,
2382
+ issued_at=_issued_now(), protocol=PROTOCOL,
2383
+ context=_to_wire_context(context or self._k._context, ""), **fields)
2384
+ r.client_sig.CopyFrom(self._k._key.sign(cache_message(op, r)))
2385
+ if op == "get": # safe to repeat; a write is one-shot (the kernel's replay guard)
2386
+ return self._k._parts.route(r.tenant_id, lambda ch: _idempotent(
2387
+ self._k._timeout_s, lambda left: getattr(self._stub(ch), method)(r, timeout=left)))
2388
+ return self._k._parts.route(r.tenant_id, lambda ch: getattr(self._stub(ch), method)(r))
2389
+
2390
+
2391
+ #: Levels of `KERNABLE_LOG`; a line is printed if its level is at most the configured one.
2392
+ _LOG_LEVELS = {"off": 0, "error": 1, "warn": 2, "info": 3, "debug": 4}
2393
+
2394
+
2395
+ class _Log:
2396
+ """Prints a worker's or listener's connection and activity lines to stderr, up to the level
2397
+ of `KERNABLE_LOG` (`off`, `error`, `warn`, `info` (default), `debug`) when the client was
2398
+ made."""
2399
+
2400
+ def __init__(self, url: str, key_path: str, key_id: str):
2401
+ self.level = _LOG_LEVELS.get(os.environ.get("KERNABLE_LOG", ""), _LOG_LEVELS["info"])
2402
+ #: `<url> · key <name> (<key id>…)`: the kernel and the key this process connects with.
2403
+ self.peer = f"{url} · key {os.path.basename(key_path).removesuffix('.key')} ({key_id[:12]}…)"
2404
+
2405
+ def enabled(self, level: str) -> bool:
2406
+ return _LOG_LEVELS[level] <= self.level
2407
+
2408
+ def style(self, code: int, text: str) -> str:
2409
+ """`text` in an ANSI style (1 bold, 2 dim, 31 red, 33 yellow) on a terminal, unless
2410
+ `NO_COLOR` is set."""
2411
+ # Checked per line: stderr may be redirected after the client was made.
2412
+ color = sys.stderr.isatty() and not os.environ.get("NO_COLOR")
2413
+ return f"\x1b[{code}m{text}\x1b[0m" if color else text
2414
+
2415
+ def print(self, level: str, what: str, line: str) -> None:
2416
+ """`12:04:11 listener carts <line>`, warnings in yellow and errors in red."""
2417
+ if not self.enabled(level):
2418
+ return
2419
+ text = self.style(31, line) if level == "error" else self.style(33, line) if level == "warn" else line
2420
+ print(f"{self.style(2, time.strftime('%H:%M:%S'))} {self.style(1, what)} {text}", file=sys.stderr)
2421
+
2422
+ def reconnect(self, what: str, reason: str, seconds: float, attempt: int, refused: bool,
2423
+ stopped: threading.Event) -> None:
2424
+ """Logs the end of a session that was not stopped and waits before reconnecting:
2425
+ `seconds` after the first failed attempt, twice as long after each further one, at most
2426
+ 30 s, give or take a fifth, so processes do not reconnect in lockstep after a kernel
2427
+ restart. A refusal that will not go away by itself is an error, not a warning. `stopped`
2428
+ cuts the wait short."""
2429
+ delay = _backoff(seconds, attempt)
2430
+ # A kernel of a pair that is not the primary right now may be in a moment.
2431
+ if _switching_roles(reason):
2432
+ delay = min(delay, _PAIR_RETRY_S)
2433
+ self.print("error" if refused else "warn", what,
2434
+ f"{'refused' if refused else 'connection lost'} ({reason}), reconnecting in {delay:.2f} s")
2435
+ stopped.wait(delay)
2436
+
2437
+ def store_failed(self, what: str, err: BaseException, seconds: float, attempt: int,
2438
+ stopped: threading.Event) -> None:
2439
+ """Logs that a store could not be opened and waits before the next try, backing off
2440
+ like `reconnect`."""
2441
+ delay = _backoff(seconds, attempt)
2442
+ self.print("error", what, f"store: {err}; trying again in {delay:.2f} s")
2443
+ stopped.wait(delay)
2444
+
2445
+
2446
+ _PAIR_RETRY_S = 1.0
2447
+ """The longest wait before trying a kernel of a pair again that was not the primary."""
2448
+
2449
+
2450
+ def _switching_roles(reason: str) -> bool:
2451
+ """Whether `reason` says the kernel is one of a pair that is not the primary right now: a
2452
+ standby, one that lost its lease or hands its role over."""
2453
+ return (reason.startswith("not_primary") or "lost its lease at the witness" in reason
2454
+ or "hands its role over" in reason)
2455
+
2456
+
2457
+ def _backoff(seconds: float, attempt: int) -> float:
2458
+ """The pause before try `attempt` (from 1): `seconds`, doubled each time up to 30 s, give or
2459
+ take a fifth."""
2460
+ return min(seconds * 2 ** (attempt - 1), 30.0) * random.uniform(0.8, 1.2)
2461
+
2462
+
2463
+ class _Watch:
2464
+ """Logs `line` as an error if the block still runs after `seconds`: a handler cannot be
2465
+ stopped from outside, but a stuck one shows."""
2466
+
2467
+ def __init__(self, log: "_Log", what: str, seconds: float | None, line: str = ""):
2468
+ self._timer = None
2469
+ if seconds is not None:
2470
+ self._timer = threading.Timer(seconds, log.print, ("error", what, line))
2471
+ self._timer.daemon = True
2472
+
2473
+ def __enter__(self):
2474
+ if self._timer:
2475
+ self._timer.start()
2476
+
2477
+ def __exit__(self, *exc):
2478
+ if self._timer:
2479
+ self._timer.cancel()
2480
+
2481
+
2482
+ def _handling(msg) -> tuple[float | None, str]:
2483
+ """After how long a listener's message is too long, and what to log then: a query or check
2484
+ after 5 s (the kernel's default `--query-timeout-ms` and `--middleware-timeout-ms`, after
2485
+ which the request has failed), a batch after 30 s (everything behind it waits)."""
2486
+ kind = msg.WhichOneof("msg")
2487
+ if kind == "batch":
2488
+ what, seconds = f"the batch up to {msg.batch.up_to}", 30
2489
+ elif kind == "query":
2490
+ what, seconds = f"the query {msg.query.query.domain}.{msg.query.query.query_type}", 5
2491
+ elif kind == "check":
2492
+ what, seconds = "a middleware check", 5
2493
+ else:
2494
+ return None, ""
2495
+ return seconds, f"still handling {what} after {seconds} s: everything behind it waits; is a handler stuck?"
2496
+
2497
+
2498
+ def _relative(type: str, domains: set[str]) -> str:
2499
+ """`shop.ItemAdded` as `ItemAdded` when `shop` is the only domain in `domains`."""
2500
+ domain, _, name = type.partition(".")
2501
+ return name if len(domains) == 1 and domain in domains and name else type
2502
+
2503
+
2504
+ def _listener_details(l: "Listener", position: int, name: Callable[[str], str] = lambda t: t) -> list[str]:
2505
+ """What a listener does, one part per entry: where it starts, what it reads, answers and
2506
+ checks. `name` shortens the types (see `_relative`)."""
2507
+ def listed(types: Iterable[str]) -> str:
2508
+ return ", ".join(name(t) for t in types)
2509
+
2510
+ details = [f"from {position} · reads {listed(l._handlers)}" if l._handlers else f"from {position}"]
2511
+ if l._queries:
2512
+ details.append(f"answers {listed(l._queries)}")
2513
+ for when, (spec, _) in l._middlewares.items():
2514
+ details.append(f"checks {listed(spec.types)} {when}")
2515
+ return details
2516
+
2517
+
2518
+ @dataclass
2519
+ class Manifest:
2520
+ """What a service consists of, as `Kernable.apply` takes it: the membership an app applies
2521
+ (`app.manifest`, see `Kernable.app`)."""
2522
+
2523
+ #: The service the membership is for.
2524
+ service: str
2525
+ #: The aggregates the service runs; the kernel waits for a worker of each.
2526
+ workers: list[AggregateType] = field(default_factory=list)
2527
+ #: The listeners the kernel waits for.
2528
+ listeners: list["Listener"] = field(default_factory=list)
2529
+ #: The service's listeners that may check requests; they count like listeners.
2530
+ middlewares: list["Listener"] = field(default_factory=list)
2531
+ #: Free metadata, e.g. `{"version": "1.0.0"}`.
2532
+ metadata: dict[str, str] = field(default_factory=dict)
2533
+ #: The service's business processes; `Kernable.app` registers them.
2534
+ processes: list[Process] = field(default_factory=list)
2535
+ #: The deployment (code version) that serves the service; empty: every deployment.
2536
+ deployment: str = ""
2537
+ #: When it switches the service to its deployment while another serves (see `Kernable.apply`).
2538
+ switch: str = "caught-up"
2539
+
2540
+
2541
+ class _AllReady:
2542
+ """Set once all of some streams are connected; waits like a `threading.Event`."""
2543
+
2544
+ def __init__(self, events: list[threading.Event]):
2545
+ self._events = events
2546
+
2547
+ def is_set(self) -> bool:
2548
+ return all(e.is_set() for e in self._events)
2549
+
2550
+ def wait(self, timeout: float | None = None) -> bool:
2551
+ """Waits until every stream is connected, at most `timeout` seconds in all; returns whether
2552
+ they are."""
2553
+ deadline = None if timeout is None else time.monotonic() + timeout
2554
+ for e in self._events:
2555
+ if not e.wait(None if deadline is None else max(0.0, deadline - time.monotonic())):
2556
+ return False
2557
+ return True
2558
+
2559
+
2560
+ class App:
2561
+ """An app made of whole domains, returned by `kernable.app(...)`: `run()` applies its
2562
+ membership (`manifest`), then runs its worker and listeners until `stop()`; `ready` is set
2563
+ once all of them are connected.
2564
+
2565
+ `run()` blocks the calling thread. To do something once the app is up (e.g. in a test),
2566
+ run it in a thread and wait: `threading.Thread(target=app.run, daemon=True).start()`, then
2567
+ `app.ready.wait(5)`, which returns whether everything connected within 5 seconds. Call
2568
+ `app.stop()` from another thread or a handler to end `run()`. Attributes: `manifest` (a
2569
+ `Manifest`), `worker` (a `Worker`, or `None` without aggregates), `listeners` (a
2570
+ `ListenerHandle` per read model and reactor, live ones too), `ready` and `processes` (a
2571
+ `RegisteredProcess` per process of its domains, once registered).
2572
+
2573
+ One service is one app: two processes with the same service would replace each other's
2574
+ membership."""
2575
+
2576
+ def __init__(self, k: Kernable, manifest: Manifest, worker: "Worker | None",
2577
+ listeners: list["ListenerHandle"], domains: Iterable[str] = ()):
2578
+ """Use `Kernable.app`."""
2579
+ self._kernable = k
2580
+ self._log = k._log
2581
+ # The names of the app's domains.
2582
+ self._domains = set(domains)
2583
+ self._stopped = False
2584
+ #: The membership `run()` applies.
2585
+ self.manifest = manifest
2586
+ #: The worker of all the domains' aggregates, if they have any.
2587
+ self.worker = worker
2588
+ #: A running listener per read model and reactor, live ones too.
2589
+ self.listeners = listeners
2590
+ #: The `RegisteredProcess` per process (qualified name), once registered by `run()`.
2591
+ self.processes: dict[str, RegisteredProcess] = {}
2592
+ streams = [s for s in (worker, *listeners) if s is not None]
2593
+ #: Set once every worker and listener of the app is connected; while another deployment
2594
+ #: serves, they warm up beside it.
2595
+ self.connected = _AllReady([s.ready for s in streams])
2596
+ #: Set once the app's membership is applied: its deployment serves the service. Right at
2597
+ #: the start, unless another deployment serves; then once the app's deployment has caught
2598
+ #: up and taken over (`switch` of `Kernable.app`), or an operator switched to it.
2599
+ self.serving = threading.Event()
2600
+ #: Set once every worker and listener is connected and the app's deployment serves.
2601
+ self.ready = _AllReady([s.ready for s in streams] + [self.serving])
2602
+ self._streams = streams
2603
+
2604
+ def run(self) -> None:
2605
+ """Applies the membership (see `Kernable.apply`), then runs the worker and the listeners,
2606
+ each in a thread of its own, until `stop()`; blocks until then. If another deployment
2607
+ serves the service, the kernel switches to this one only as the app's `switch` says: until
2608
+ then the app runs beside it (workers idle, read models building), applies its membership
2609
+ again every second, and registers its processes once it serves. A failing handler does
2610
+ not end it: a command handler's error rejects the command (`handler_error`), a listener
2611
+ handler's error makes the listener reconnect and get the event again. If a worker or
2612
+ listener itself fails, it stops the others and raises that error (a store that cannot
2613
+ be opened is tried again instead); Ctrl-C stops all of them too. Once all are connected, it prints an overview of the app (unless
2614
+ `KERNABLE_LOG` is below `info`). Raises `grpc.RpcError` if the kernel refuses the
2615
+ membership."""
2616
+ started = time.monotonic()
2617
+ m = self.manifest
2618
+ models = [p.model() for p in m.processes] # an invalid one stops the app before it starts
2619
+ self._stopped = False
2620
+ self.serving.clear()
2621
+
2622
+ def apply() -> None:
2623
+ self._kernable.apply(service=m.service, workers=m.workers, listeners=m.listeners,
2624
+ middlewares=m.middlewares, metadata=m.metadata, processes=m.processes,
2625
+ deployment=m.deployment, switch=m.switch)
2626
+
2627
+ try:
2628
+ apply()
2629
+ self.serving.set()
2630
+ except grpc.RpcError as err:
2631
+ if not waits_for_switch(err):
2632
+ raise
2633
+ self._log.print("info", "app", f"deployment {m.deployment} runs beside the one that serves "
2634
+ f"({err.details()})")
2635
+
2636
+ def switch_later() -> None:
2637
+ # Applies the membership every second until the kernel takes it.
2638
+ while not self._stopped:
2639
+ time.sleep(1.0)
2640
+ if self._stopped:
2641
+ return
2642
+ try:
2643
+ apply()
2644
+ except grpc.RpcError as err:
2645
+ if not waits_for_switch(err):
2646
+ self._log.print("warn", "app", f"applying the membership failed: {err.details()}")
2647
+ continue
2648
+ self._log.print("info", "app", f"deployment {m.deployment} serves {m.service} now")
2649
+ self.serving.set()
2650
+ return
2651
+
2652
+ if not self.serving.is_set():
2653
+ threading.Thread(target=switch_later, name="kernable switch", daemon=True).start()
2654
+ for s in self._streams:
2655
+ s.announce = False
2656
+ errors: list[BaseException] = []
2657
+
2658
+ def overview() -> None:
2659
+ while not self.ready.wait(0.1):
2660
+ if self._stopped:
2661
+ return
2662
+ self._overview(round((time.monotonic() - started) * 1000))
2663
+
2664
+ def run(stream: _Stream) -> None:
2665
+ try:
2666
+ stream.run()
2667
+ except BaseException as err:
2668
+ errors.append(err)
2669
+ self.stop()
2670
+
2671
+ def register() -> None:
2672
+ # Once the app serves and the worker is connected, so the kernel knows the commands
2673
+ # the models send.
2674
+ while not self.serving.wait(0.1) or (self.worker is not None and not self.worker.ready.wait(0.1)):
2675
+ if self._stopped:
2676
+ return
2677
+ try:
2678
+ for model in models:
2679
+ r = self._kernable.register_process(model)
2680
+ self.processes[r.name] = r
2681
+ except BaseException as err:
2682
+ errors.append(err)
2683
+ self.stop()
2684
+
2685
+ threads = [threading.Thread(target=run, args=(s,), name=f"kernable {s._what}", daemon=True)
2686
+ for s in self._streams]
2687
+ threading.Thread(target=overview, name="kernable overview", daemon=True).start()
2688
+ if models:
2689
+ threads.append(threading.Thread(target=register, name="kernable processes", daemon=True))
2690
+ for t in threads:
2691
+ t.start()
2692
+ try:
2693
+ for t in threads:
2694
+ t.join()
2695
+ except BaseException: # e.g. Ctrl-C: stop everything
2696
+ self.stop()
2697
+ raise
2698
+ if errors:
2699
+ raise errors[0]
2700
+
2701
+ def stop(self) -> None:
2702
+ """Stops the worker and all listeners; `run()` returns once they are done."""
2703
+ self._stopped = True
2704
+ for s in self._streams:
2705
+ s.stop()
2706
+
2707
+ def _overview(self, ms: int) -> None:
2708
+ log = self._log
2709
+ if not log.enabled("info"):
2710
+ return
2711
+ m = self.manifest
2712
+ domains = self._domains
2713
+
2714
+ # A single domain's own names go without it; with several, every name is qualified.
2715
+ def short(domain: str, name: str) -> str:
2716
+ return name if len(domains) == 1 and domain in domains else f"{domain}.{name}"
2717
+
2718
+ def count(n: int, what: str) -> str:
2719
+ return f"{n} {what}{'' if n == 1 else 's'}"
2720
+
2721
+ listed = [count(len(m.workers), "worker"), count(len(m.listeners), "listener")]
2722
+ if m.middlewares:
2723
+ listed.append(count(len(m.middlewares), "middleware"))
2724
+ meta = m.metadata
2725
+ rows: list[tuple[str, str, list[str]]] = []
2726
+ for a in m.workers:
2727
+ rows.append(("worker", short(a.domain, a.name), [f"runs {', '.join(sorted(a.handlers))}"]))
2728
+ for h in self.listeners:
2729
+ l = h.listener
2730
+ rows.append(("live" if l.live else "listener", short(l.domain.name, l.name),
2731
+ _listener_details(l, h.started_at, lambda t: _relative(t, domains))))
2732
+ width = max((len(name) for _, name, _ in rows), default=0)
2733
+ header = " · ".join(p for p in ("kernable", " ".join(p for p in (m.service, meta.get("version")) if p),
2734
+ meta.get("description")) if p)
2735
+ lines = [
2736
+ log.style(1, header),
2737
+ f" {'kernel':<10} {log.peer}",
2738
+ f" {'membership':<10} applied: {', '.join(listed)}",
2739
+ "",
2740
+ ]
2741
+ for kind, name, (first, *rest) in rows:
2742
+ lines.append(f" {kind:<10} {name:<{width}} {first}")
2743
+ lines.extend(f" {'':<10} {'':<{width}} {more}" for more in rest)
2744
+ lines += ["", f" {log.style(2, f'ready in {ms} ms')}", ""]
2745
+ # One write, so the overview's lines stay together.
2746
+ sys.stderr.write("\n".join(lines) + "\n")
2747
+
2748
+
2749
+ class _HandlerFailed(Exception):
2750
+ """A handler's own error, carried out of a stream's loop: told apart from the stream's errors
2751
+ (a `grpc.RpcError` from a request the handler made is no refusal of the stream)."""
2752
+
2753
+ def __init__(self, err: Exception):
2754
+ super().__init__(str(err) or type(err).__name__)
2755
+ self.err = err
2756
+
2757
+
2758
+ def _handle(handler: Callable, *args):
2759
+ """Runs a handler; its errors come out as `_HandlerFailed`, except `ClientClosed`."""
2760
+ try:
2761
+ return handler(*args)
2762
+ except ClientClosed:
2763
+ raise
2764
+ except Exception as err:
2765
+ raise _HandlerFailed(err) from err
2766
+
2767
+
2768
+ class _Stream:
2769
+ """A bidirectional stream that reconnects until stopped."""
2770
+
2771
+ def __init__(self, what: str, reconnect_s: float, log: _Log):
2772
+ self.ready = threading.Event()
2773
+ #: Whether the first connect gets its own line; an `App` prints it in its overview instead.
2774
+ self.announce = True
2775
+ self._connects = 0
2776
+ self._log = log
2777
+ self._what = what
2778
+ self._reconnect_s = reconnect_s
2779
+ self._stopped = False
2780
+ #: Set by `stop()`: cuts the wait before a reconnect short.
2781
+ self._wake = threading.Event()
2782
+ self._call = None
2783
+ self._outbox: queue.Queue | None = None
2784
+
2785
+ @classmethod
2786
+ def _new(cls, *args, **kwargs):
2787
+ """Made by the client (`Kernable.worker`, `.listener`, `.subscribe`), not by applications:
2788
+ the arguments are the client's internals. It bypasses `__init__`, which each public
2789
+ stream class makes raise `TypeError`, so a direct call never builds a broken object."""
2790
+ stream = cls.__new__(cls)
2791
+ stream._init(*args, **kwargs)
2792
+ return stream
2793
+
2794
+ def run(self) -> None:
2795
+ """Processes messages until `stop()`, reconnecting when the stream drops."""
2796
+ attempt = 0
2797
+ while not self._stopped:
2798
+ connects = self._connects
2799
+ refused = False
2800
+ outbox: queue.Queue = queue.Queue()
2801
+ self._outbox = outbox
2802
+ try:
2803
+ self._call = None
2804
+ self._call = self._open(iter(outbox.get, None))
2805
+ if self._stopped: # stopped while it opened: stop() missed this call
2806
+ self._call.cancel()
2807
+ outbox.put(self._hello())
2808
+ for msg in self._call:
2809
+ reply = _handle(self._on, msg)
2810
+ if reply is not None:
2811
+ outbox.put(reply)
2812
+ reason = "stream ended"
2813
+ except ClientClosed:
2814
+ return
2815
+ except _HandlerFailed as failed: # a failing handler: reconnect and get the message again
2816
+ reason = str(failed)
2817
+ except grpc.RpcError as err:
2818
+ reason = err.details()
2819
+ refused = err.code() in _REFUSALS
2820
+ except Exception as err:
2821
+ reason = str(err) or type(err).__name__
2822
+ finally:
2823
+ outbox.put(None)
2824
+ # Whatever ended the loop, the call ends with it (a no-op once it ended by itself).
2825
+ if self._call is not None:
2826
+ self._call.cancel()
2827
+ attempt = 1 if self._connects > connects else attempt + 1
2828
+ if not self._stopped:
2829
+ self._log.reconnect(self._what, reason, self._reconnect_s, attempt, refused, self._wake)
2830
+
2831
+ def stop(self) -> None:
2832
+ self._stopped = True
2833
+ self._wake.set()
2834
+ if self._call is not None:
2835
+ self._call.cancel()
2836
+ for child in getattr(self, "_children", ()):
2837
+ child.stop()
2838
+
2839
+ def _partitions(self, source) -> "list[tuple[str, grpc.Channel]] | None":
2840
+ """The partitions of `source`, read again, backing off like a reconnect, while no
2841
+ kernel answers; `None` once stopped."""
2842
+ if not isinstance(source, _Partitioned):
2843
+ return [("", source)]
2844
+ attempt = 0
2845
+ while not self._stopped:
2846
+ # Read in a thread of its own: `stop()` ends the wait for it at once.
2847
+ box: dict = {}
2848
+
2849
+ def read() -> None:
2850
+ try:
2851
+ box["parts"] = source.all()
2852
+ except BaseException as err:
2853
+ box["error"] = err
2854
+
2855
+ reader = threading.Thread(target=read, name=f"kernable {self._what} map", daemon=True)
2856
+ reader.start()
2857
+ while reader.is_alive():
2858
+ reader.join(0.05)
2859
+ if self._stopped:
2860
+ return None
2861
+ try:
2862
+ if "error" in box:
2863
+ raise box["error"]
2864
+ return box["parts"]
2865
+ except ClientClosed:
2866
+ return None
2867
+ except grpc.RpcError as err:
2868
+ attempt += 1
2869
+ self._log.reconnect(self._what, err.details(), self._reconnect_s, attempt,
2870
+ err.code() in _REFUSALS, self._wake)
2871
+ return None
2872
+
2873
+ def _run_children(self, children: "list[_Stream]") -> None:
2874
+ """Runs a stream per partition, each in a thread, until `stop()`; `ready` once all are."""
2875
+ self._children = children
2876
+ for child in children:
2877
+ child.announce = self.announce
2878
+ errors: list[BaseException] = []
2879
+
2880
+ def run(child: "_Stream") -> None:
2881
+ try:
2882
+ child.run()
2883
+ except BaseException as err:
2884
+ errors.append(err)
2885
+ self.stop()
2886
+
2887
+ def ready() -> None:
2888
+ for child in children:
2889
+ while not child.ready.wait(0.1):
2890
+ if self._stopped:
2891
+ return
2892
+ self.ready.set()
2893
+
2894
+ threads = [threading.Thread(target=run, args=(c,), name=f"kernable {c._what}", daemon=True)
2895
+ for c in children]
2896
+ threading.Thread(target=ready, name=f"kernable {self._what} ready", daemon=True).start()
2897
+ if self._stopped:
2898
+ return
2899
+ for t in threads:
2900
+ t.start()
2901
+ for t in threads:
2902
+ t.join()
2903
+ if errors:
2904
+ raise errors[0]
2905
+
2906
+ def _open(self, requests):
2907
+ raise NotImplementedError
2908
+
2909
+ def _hello(self):
2910
+ raise NotImplementedError
2911
+
2912
+ def _on(self, msg):
2913
+ raise NotImplementedError
2914
+
2915
+
2916
+ class Worker(_Stream):
2917
+ """Runs the commands of its aggregates, one at a time: the kernel sends each command with
2918
+ the aggregate's events (or a snapshot and the events after it), the worker folds the state,
2919
+ calls the command handler and sends the new events back, signed. Made with
2920
+ `Kernable.worker`.
2921
+
2922
+ `run()` processes commands until `stop()`, reconnecting when the session drops (after 1 s);
2923
+ it blocks the calling thread. `ready` (a `threading.Event`) is set once the kernel
2924
+ registered the worker: run it in a thread and wait with `worker.ready.wait(5)`, e.g.
2925
+ `threading.Thread(target=worker.run, daemon=True).start()`. `stop()` ends `run()`, also
2926
+ from another thread."""
2927
+
2928
+ def __init__(self, *args, **kwargs):
2929
+ """Not for applications: `Kernable.worker` (or `Kernable.app`) makes workers."""
2930
+ raise TypeError("use Kernable.worker (or Kernable.app), not Worker(...)")
2931
+
2932
+ def _init(self, channel: "grpc.Channel | _Partitioned", key: Key, aggregates, log: _Log,
2933
+ service: str = "", deployment: str = ""):
2934
+ """Use `Kernable.worker`."""
2935
+ super().__init__("worker", 1.0, log)
2936
+ # A channel, or the partitions: then a session per partition.
2937
+ self._source = channel
2938
+ self._stub = None
2939
+ self._key = key
2940
+ self._aggregates = list(aggregates)
2941
+ self._by_command = {f"{a.domain}.{t}": a for a in aggregates for t in a.handlers}
2942
+ # By stream: the kernel names the aggregate type of every command.
2943
+ self._by_stream = {(a.domain, a.name): a for a in aggregates}
2944
+ # The service this worker belongs to; its membership decides whether the kernel waits for it.
2945
+ self._service = service
2946
+ # The code version it declares (see `Kernable`).
2947
+ self._deployment = deployment
2948
+ # Executes whose history is still coming in parts, by lease; per session.
2949
+ self._partial = {}
2950
+
2951
+ def run(self) -> None:
2952
+ """Processes commands until `stop()`, reconnecting when the session drops; with
2953
+ partitions, a session per partition."""
2954
+ parts = self._partitions(self._source)
2955
+ if parts is None:
2956
+ return
2957
+ if len(parts) > 1 or parts[0][0] != "":
2958
+ self._run_children([Worker._new(channel, self._key, self._aggregates, self._log, self._service,
2959
+ self._deployment)
2960
+ for _, channel in parts])
2961
+ return
2962
+ self._stub = rpc.WorkerGatewayStub(parts[0][1])
2963
+ super().run()
2964
+
2965
+ def _open(self, requests):
2966
+ return self._stub.Session(requests)
2967
+
2968
+ def _hello(self):
2969
+ self._partial = {}
2970
+ # Every event type the aggregates know, applied or declared by their domains: the kernel
2971
+ # refuses an outdated worker, and with a deployment, commands on aggregates holding an
2972
+ # event it does not know.
2973
+ schemas = {f"{a.domain}.{t}": a.schema_of(t) for a in self._aggregates
2974
+ for t in [*a.appliers, *a.events]}
2975
+ register = pb.Register(worker_id=f"py-{uuid.uuid4().hex[:8]}", key_id=self._key.id, protocol=PROTOCOL,
2976
+ max_concurrency=1,
2977
+ aggregates=[pb.WorkerAggregate(aggregate_type=f"{a.domain}.{a.name}",
2978
+ command_types=[f"{a.domain}.{t}" for t in a.handlers])
2979
+ for a in self._aggregates],
2980
+ event_schemas=[pb.EventSchema(event_type=t, schema=s) for t, s in schemas.items()],
2981
+ service=self._service, deployment=self._deployment)
2982
+ return pb.WorkerMessage(register=register)
2983
+
2984
+ def _on(self, msg):
2985
+ kind = msg.WhichOneof("msg")
2986
+ if kind == "challenge":
2987
+ sig = self._key.sign(challenge_message(msg.challenge.nonce)).sig
2988
+ return pb.WorkerMessage(challenge_response=pb.ChallengeResponse(sig=sig))
2989
+ if kind == "registered":
2990
+ if msg.registered.reader_of:
2991
+ self._log.print("info", self._what, "connected to a reader: commands run on the writer "
2992
+ f"({msg.registered.reader_of}); this worker gets none here")
2993
+ if self._connects > 0:
2994
+ self._log.print("info", self._what, "connected again")
2995
+ elif self.announce:
2996
+ self._log.print("info", self._what, f"connected to {self._log.peer} · runs {', '.join(sorted(self._by_command))}")
2997
+ self._connects += 1
2998
+ self.ready.set()
2999
+ elif kind in ("execute", "execute_more"):
3000
+ x = self._assembled(msg)
3001
+ if x is None:
3002
+ return None
3003
+ cmd = x.command
3004
+ stream = f"{cmd.domain}.{cmd.command_type} {cmd.aggregate_type}/{cmd.aggregate_id}"
3005
+ # A handler cannot be stopped from outside; one that runs past its lease at least shows.
3006
+ with _Watch(self._log, self._what, max(0.0, x.deadline / 1000 - time.time()),
3007
+ f"{stream} still running after its lease ended: the kernel gives it to another worker; "
3008
+ "is the handler stuck?"):
3009
+ reply = self._execute(x)
3010
+ if self._log.enabled("debug"):
3011
+ done = (f"rejected {reply.rejection.reason.code}" if reply.HasField("rejection")
3012
+ else f"committed {len(reply.result.events)} events")
3013
+ self._log.print("debug", self._what,
3014
+ f"{cmd.domain}.{cmd.command_type} {cmd.aggregate_type}/{cmd.aggregate_id} {done}")
3015
+ return reply
3016
+ return None
3017
+
3018
+ def _assembled(self, msg):
3019
+ """The Execute to run once its last part is here (a long history comes in parts), else None."""
3020
+ if msg.WhichOneof("msg") == "execute":
3021
+ x = msg.execute
3022
+ else:
3023
+ rest = msg.execute_more
3024
+ x = self._partial.pop(rest.lease_id, None)
3025
+ if x is None:
3026
+ return None
3027
+ x.history.extend(rest.history)
3028
+ x.vault.extend(rest.vault)
3029
+ x.more = rest.more
3030
+ if x.more:
3031
+ self._partial[x.lease_id] = x
3032
+ return None
3033
+ return x
3034
+
3035
+ def _execute(self, x):
3036
+ cmd = x.command
3037
+ try:
3038
+ agg = self._by_stream.get((cmd.domain, cmd.aggregate_type))
3039
+ if agg is None or cmd.command_type not in agg.handlers:
3040
+ raise Rejected(f"no handler for {cmd.command_type} on {cmd.domain}.{cmd.aggregate_type}",
3041
+ "unknown_command")
3042
+ if x.HasField("snapshot") and x.snapshot.schema != agg.schema:
3043
+ raise Rejected(f"snapshot schema {x.snapshot.schema} is not {agg.schema}", "snapshot_stale")
3044
+ history = [(e.event_type, e.schema, _unseal(_decode(e.payload), x.vault)[0],
3045
+ _meta(e.event_type, e.version, cmd, e.command_id), ApplyContext(_from_wire_context(e.context)))
3046
+ for e in x.history]
3047
+ state = agg.fold(history, agg._from_json(_decode(x.snapshot.state)) if x.HasField("snapshot") else _INITIAL)
3048
+ meta = CommandMeta(cmd.command_id, cmd.command_type, cmd.domain, cmd.aggregate_type, cmd.aggregate_id,
3049
+ cmd.tenant_id)
3050
+ ctx = CommandContext(_from_wire_context(cmd.context), _time(x.deadline), _time(x.received_at), cmd.command_id)
3051
+ data = _unseal(_decode(cmd.payload), x.vault)[0]
3052
+ # A copy: a handler that changes it must not change the snapshot.
3053
+ produced = _call(agg.handlers[cmd.command_type], copy.deepcopy(state), data, meta, ctx)
3054
+ _check_personal(agg, cmd, produced)
3055
+ requires = self._requires(agg, cmd, data, state)
3056
+ events, vault = [], []
3057
+ for e in produced:
3058
+ spec = agg.events.get(e["type"])
3059
+ payload, refs, values = _seal(e.get("data"), spec.vault if spec else None, cmd.aggregate_id)
3060
+ vault.extend(values)
3061
+ erases = str(_call(spec.eraser, e.get("data"), cmd.aggregate_id) or "") if spec and spec.eraser else ""
3062
+ events.append(pb.NewEvent(event_type=e["type"], schema=agg.schema_of(e["type"]), payload=payload,
3063
+ vault_refs=refs, erases=erases))
3064
+ indexed = bool(agg.keys or agg.facts)
3065
+ after = (self._after(agg, cmd, x.expected_version, state, produced)
3066
+ if indexed or _snapshot_due(agg, x.expected_version, len(produced)) else None)
3067
+ snapshot = self._snapshot(agg, cmd, x.expected_version, after, produced)
3068
+ # Every command, also one without events: the set the aggregate holds now.
3069
+ index = agg.index(after) if indexed else Index()
3070
+ except SchemaError:
3071
+ raise # not the command's fault: this worker cannot read the aggregate; stop until updated
3072
+ except Exception as err:
3073
+ # An unexpected error's text may hold personal data or internals, and a rejection is
3074
+ # kept for good: it names only the error's type; the log has the rest.
3075
+ if not isinstance(err, Rejected):
3076
+ self._log.print("error", "worker", f"{cmd.domain}.{cmd.command_type} "
3077
+ f"{cmd.aggregate_type}/{cmd.aggregate_id} failed:\n{traceback.format_exc()}")
3078
+ reason = (pb.Rejected(code=err.code, message=str(err)) if isinstance(err, Rejected)
3079
+ else pb.Rejected(code="handler_error", message=type(err).__name__))
3080
+ sig = self._key.sign(rejection_message(cmd.command_id, reason))
3081
+ return pb.WorkerMessage(rejection=pb.Rejection(lease_id=x.lease_id, reason=reason, worker_sig=sig))
3082
+ sig = self._key.sign(result_message(cmd, x.expected_version, events))
3083
+ result = pb.Result(lease_id=x.lease_id, events=events, worker_sig=sig, snapshot=snapshot, vault=vault)
3084
+ # The full set of keys and facts after the events (none: holds nothing), and what the
3085
+ # command requires; signed apart from the events, only when there is any.
3086
+ result.keys.extend(pb.UniqueKey(name=k.name, value_hash=hashlib.sha256(k.value.encode()).digest(),
3087
+ subject=k.subject, **{"global": k.scope == "global"}) for k in index.keys)
3088
+ result.facts.extend(index.facts)
3089
+ result.requires.extend(requires)
3090
+ if result.keys or result.facts or result.requires:
3091
+ result.index_sig.CopyFrom(self._key.sign(index_message(cmd, x.expected_version, result.keys, result.facts,
3092
+ result.requires)))
3093
+ return pb.WorkerMessage(result=result)
3094
+
3095
+ @staticmethod
3096
+ def _requires(agg: AggregateType, cmd, data, state) -> list:
3097
+ """The facts of other aggregates the command requires, from its data and the state before it."""
3098
+ fn = agg.requirements.get(cmd.command_type)
3099
+ if fn is None:
3100
+ return []
3101
+ wire = []
3102
+ for r in _call(fn, data, copy.deepcopy(state)) or []:
3103
+ if not isinstance(r, Requirement):
3104
+ raise TypeError(f"requires of {cmd.domain}.{cmd.command_type} returns Requirement, not {type(r).__name__}")
3105
+ wire.append(pb.Requirement(domain=r.domain or cmd.domain, aggregate_type=r.aggregate_type,
3106
+ aggregate_id=r.aggregate_id, fact=r.fact, key=r.key,
3107
+ value_hash=hashlib.sha256(r.value.encode()).digest() if r.key else b"",
3108
+ absent=r.absent))
3109
+ return wire
3110
+
3111
+ @staticmethod
3112
+ def _after(agg: AggregateType, cmd, before: int, state, events):
3113
+ """The state after the produced `events`, applied as they will be read back (a JSON round
3114
+ trip), so it equals a replay."""
3115
+ if not events:
3116
+ return state
3117
+ apply_ctx = ApplyContext(_from_wire_context(cmd.context))
3118
+ read = [(e["type"], agg.schema_of(e["type"]), _decode(_encode(e.get("data"))),
3119
+ _meta(e["type"], before + i + 1, cmd, cmd.command_id), apply_ctx) for i, e in enumerate(events)]
3120
+ return agg.fold(read, state)
3121
+
3122
+ def _snapshot(self, agg: AggregateType, cmd, before: int, after, events):
3123
+ """A snapshot of the state `after` the produced `events`, if they cross a multiple of
3124
+ `snapshot_every`."""
3125
+ if not _snapshot_due(agg, before, len(events)):
3126
+ return None
3127
+ version = before + len(events)
3128
+ state_bytes = _encode(agg._to_json(after))
3129
+ sig = self._key.sign(snapshot_message(cmd, version, agg.schema, state_bytes))
3130
+ return pb.Snapshot(version=version, schema=agg.schema, state=state_bytes, worker_sig=sig)
3131
+
3132
+
3133
+ def _snapshot_due(agg: AggregateType, before: int, produced: int) -> bool:
3134
+ """Whether `produced` events after version `before` cross a multiple of `snapshot_every`."""
3135
+ every = agg.snapshot_every
3136
+ return every > 0 and produced > 0 and (before + produced) // every != before // every
3137
+
3138
+
3139
+ class Listener:
3140
+ """A listener definition: a durable, named event listener of a domain, made with
3141
+ `Domain.read_model` or `Domain.reactor` and run with `kernable.listener(listener)`, which
3142
+ returns a `ListenerHandle`. Events arrive in order, one at a time, after every listener with a
3143
+ lower order has processed them. Delivery is at least once: if a handler raises, the listener
3144
+ reconnects and the event is delivered again, until a reactor gives up on it and parks it
3145
+ (`park_after` of `Domain.reactor`).
3146
+
3147
+ Type names without a domain (`ItemAdded`, `GetCart`) are the listener's own domain's; types of
3148
+ other domains are qualified (`identity.AccountRegistered`, `identity.*`).
3149
+
3150
+ Handlers get event payloads upcast to the versions of the listener's domain and of the domains
3151
+ it `use`s. An event with a newer schema than known (a newer deployment wrote it) never reaches
3152
+ a handler, so it is never misread: a reactor that parks (`park_after` above 0, not live, not
3153
+ one of several instances that each get every event) parks it at once with the code
3154
+ `unknown_event`; every other listener skips it, logs a warning once per type and schema, and
3155
+ counts it (`ListenerHandle.skipped`). Events of types without a handler are not affected.
3156
+
3157
+ `on`, `query` and `middleware` work as decorators (`@carts_readmodel.on("ItemAdded")`) or
3158
+ directly (`carts_readmodel.on("ItemAdded", fn)`); handlers take only the parameters they
3159
+ need. They run in the thread of the listener's `run()`, one at a time, so plain dicts are
3160
+ safe to use as state there; guard state that other threads also touch."""
3161
+
3162
+ def __init__(self, domain: Domain, name: str, order: int, start: int | None, reconnect_s: float,
3163
+ store: str | None = None, schema: str = "1", *, live: bool = False, park_after: int = 0,
3164
+ instance: str | None = None, single: bool = False, reactor: bool = False):
3165
+ """Use `Domain.read_model` or `Domain.reactor`."""
3166
+ #: Defined with `Domain.reactor`: a reader (a read-only copy of the writer) lets it wait.
3167
+ self.reactor = reactor
3168
+ #: The instance ID it connects with (`None`: `KERNABLE_INSTANCE`, else none); see `Domain.read_model`.
3169
+ self.instance = instance
3170
+ #: Whether the instances take turns; see `Domain.reactor`.
3171
+ self.single = single
3172
+ #: Attempts before an event its handler keeps failing on is parked; 0: never.
3173
+ self.park_after = park_after
3174
+ #: The domain the listener belongs to: unqualified type names are its types.
3175
+ self.domain = domain
3176
+ self.name = name
3177
+ self.order = order
3178
+ self.start = start
3179
+ self.reconnect_s = reconnect_s
3180
+ #: Where the running listener keeps its state (`sqlite:<path>` or `memory:`), if anywhere.
3181
+ self.store_url = store
3182
+ self.schema = schema
3183
+ #: Not registered with the kernel: see `Domain.reactor`.
3184
+ self.live = live
3185
+ self._handlers: dict[str, Callable] = {}
3186
+ self._raw: set[str] = set()
3187
+ self._domains: dict[str, dict[str, EventSchema]] = {domain.name: domain.events}
3188
+ self._queries: dict[str, Callable] = {}
3189
+ #: The middleware handlers and their specs, by when they run: "before" and "after".
3190
+ self._middlewares: dict[str, tuple[pb.Middleware, Callable]] = {}
3191
+
3192
+ def _qualify(self, type: str) -> str:
3193
+ return type if "." in type else f"{self.domain.name}.{type}"
3194
+
3195
+ def use(self, *domains: Domain) -> "Listener":
3196
+ """Upcasts the events of these other domains with their event schemas (see
3197
+ `Domain.event`); the listener's own domain's apply already. Returns the listener, so it
3198
+ chains: `shop_domain.read_model("orders").use(billing_domain)`."""
3199
+ for d in domains:
3200
+ self._domains[d.name] = d.events
3201
+ return self
3202
+
3203
+ def on(self, event_type: str, fn: Callable | None = None, *, raw: bool = False):
3204
+ """Registers a handler for an event type (`ItemAdded`, or `orders.OrderPlaced` of another
3205
+ domain) or a whole domain (`orders.*`); works as a decorator too: `fn(data, evt, ctx)` gets
3206
+ the event's payload, a `StoredEventMeta` and a `ListenerContext` (both optional). A `raw`
3207
+ handler gets payloads as stored, without upcasting, whatever their schema (for archives
3208
+ and exports). Example: `@carts_readmodel.on("ItemAdded")` over `def _(data, evt):
3209
+ rows[evt.aggregate_id] = data["qty"]`. If the handler raises, the listener reconnects and
3210
+ the event is delivered again, so handlers must be idempotent. Raises `ValueError` if
3211
+ `event_type` is none of these."""
3212
+ if _is_message_class(event_type, Event):
3213
+ cls = event_type
3214
+ b = _binding(cls, "declared (an aggregate's apply, or Domain.event)")
3215
+ self._domains.setdefault(b.domain, b.schemas.events)
3216
+ qualified = f"{b.domain}.{cls.type_name()}"
3217
+ self._raw.discard(qualified)
3218
+
3219
+ def wrap(f: Callable) -> Callable:
3220
+ return lambda data, evt=None, ctx=None: _call(f, cls.from_wire(data), evt, ctx)
3221
+
3222
+ return _register_wrapped(self, self._handlers, qualified, fn, wrap)
3223
+ # A bare `*` is not the own domain (that is `<domain>.*`): in middleware it means everything.
3224
+ if not _UNQUALIFIED.match(event_type) and not _LISTENED.match(event_type):
3225
+ raise ValueError(f"listen to a type (Type), a qualified type (domain.Type) or a domain (domain.*), not {event_type!r}")
3226
+ qualified = self._qualify(event_type)
3227
+ (self._raw.add if raw else self._raw.discard)(qualified)
3228
+ return _register(self, self._handlers, qualified, fn)
3229
+
3230
+ def query(self, query_type: str, fn: Callable | None = None):
3231
+ """Answers a query type (`GetCart`, or `orders.GetOrder` of another domain) from this
3232
+ listener's state; works as a decorator too: `fn(data, qry, ctx)` gets the query's data, a
3233
+ `QueryMeta` and a `QueryContext` (both optional). The kernel delivers queries in order with
3234
+ the events, so an answer reflects every event handled before it. Return the answer
3235
+ (JSON-like; `None` reaches the caller as `None`); raise `Rejected` to refuse (any other exception refuses with
3236
+ `handler_error`). A handler cannot query its own listener (the answer would wait behind
3237
+ itself): the kernel refuses it. Example: `@carts_readmodel.query("GetCart")` over
3238
+ `def _(data): return rows[data["id"]]`. Raises `ValueError` if `query_type` is invalid."""
3239
+ if _is_message_class(query_type, Query):
3240
+ cls = query_type
3241
+ _bind(cls, self.domain.name, self.domain)
3242
+ qualified = f"{self.domain.name}.{cls.type_name()}"
3243
+ # Its target counts also when the query is asked by name, by clients that `use` the domain.
3244
+ target = _query_target(cls)
3245
+ if target and cls.type_name() not in self.domain.queries:
3246
+ self.domain.query(cls.type_name()).target(*target)
3247
+
3248
+ def wrap(f: Callable) -> Callable:
3249
+ return lambda data, qry=None, ctx=None: _to_wire(_call(f, cls.from_wire(data), qry, ctx))
3250
+
3251
+ return _register_wrapped(self, self._queries, qualified, fn, wrap)
3252
+ qualified = self._qualify(query_type)
3253
+ if not _QUERY.match(qualified):
3254
+ raise ValueError(f"answer a query type (Query or domain.Query), not {query_type!r}")
3255
+ return _register(self, self._queries, qualified, fn)
3256
+
3257
+ def middleware(self, types: list[str], order: int = 0, fn: Callable | None = None, *, when: str = "before"):
3258
+ """Makes this listener a middleware; works as a decorator too. The kernel asks `fn` about
3259
+ every command and query of `types` (`*` for all, `Type` of the own domain, `domain.*` or
3260
+ `domain.Type`) before accepting or routing it (or after, see `when`), in chain order (lower
3261
+ `order` first, then by listener name), and only once this listener has caught up with the
3262
+ events. `fn(check, outcome, ctx)` gets a `Check`, after the request what came of it, and a
3263
+ `MiddlewareContext` with the listener's store (both optional): an
3264
+ `Outcome` for a command, a `QueryOutcome` for a query (`check.kind` tells which); before,
3265
+ None. `types` cover file and cache requests too, before they run: `domain.PutData`,
3266
+ `.GetData`, `.DeleteData`, `.ListData`, `.SetCache`, `.GetCache` and `.DeleteCache` (and `domain.*` or
3267
+ `*` include them); `check.kind` is then `"data"` or `"cache"`, `check.key` the key and
3268
+ `check.data` None, and a refusal reaches the client as `PERMISSION_DENIED`. Return False or
3269
+ raise `Rejected` to refuse; any other error refuses too, and so does a return value other
3270
+ than None, True or False. The registration is stored:
3271
+ while this listener is away, the kernel refuses those requests (`kernable listeners
3272
+ remove` unregisters it). `fn` cannot submit or query what it checks itself (it would wait
3273
+ behind itself): the kernel refuses it. A listener has at most one middleware before and
3274
+ one after.
3275
+ `when`: "before" (default) checks a request before the kernel accepts it; raising refuses
3276
+ it. "after" gets the request's outcome too: raising refuses a query's response; a
3277
+ command's outcome, and a rejected query, it only observes, best effort (it misses what
3278
+ happens while it is away, and nobody waits for it).
3279
+ Negative orders are reserved for kernable's kits (a grant of `reserved_order`). A live listener cannot be a middleware.
3280
+ Example: `@guard_readmodel.middleware(["shop.*"])` over a `def _(check):` that raises
3281
+ `Rejected("not allowed", "forbidden")` unless `check.request.sender_identity` may send
3282
+ `check.type`.
3283
+ Raises `ValueError` for an invalid type or `when`, a second middleware at the same
3284
+ `when`, or a live listener."""
3285
+ if self.live:
3286
+ raise ValueError(f"{self.name} is live: a live listener cannot be a middleware")
3287
+ qualified = [t if t == "*" else self._qualify(t) for t in types]
3288
+ for t, q in zip(types, qualified):
3289
+ if not _CHECKED.match(q):
3290
+ raise ValueError(f"check `*`, a type (Type or domain.Type) or a domain (domain.*), not {t!r}")
3291
+ if when not in ("before", "after"):
3292
+ raise ValueError(f"run a middleware \"before\" or \"after\", not {when!r}")
3293
+ if when in self._middlewares:
3294
+ raise ValueError(f"{self.name} has a middleware {when} already")
3295
+ spec = pb.Middleware(order=order, types=qualified, when=pb.WHEN_AFTER if when == "after" else pb.WHEN_BEFORE)
3296
+ if fn is not None:
3297
+ self._middlewares[when] = (spec, fn)
3298
+ return self
3299
+
3300
+ def decorator(f: Callable) -> Callable:
3301
+ self._middlewares[when] = (spec, f)
3302
+ return f
3303
+
3304
+ return decorator
3305
+
3306
+
3307
+ class ListenerHandle(_Stream):
3308
+ """A running `Listener`, returned by `kernable.listener(listener)`: `run()` processes events,
3309
+ queries and checks until `stop()`; `ready` is set once the kernel accepted it.
3310
+
3311
+ `run()` blocks the calling thread. To run several listeners in one process, run all but
3312
+ one in threads (`threading.Thread(target=handle.run, daemon=True).start()`), or use
3313
+ `Kernable.app`. `ready` is a `threading.Event`: `handle.ready.wait(5)` returns whether the
3314
+ kernel accepted the listener within 5 seconds. `stop()` ends `run()`, also from another
3315
+ thread. Attributes: `listener` (the definition), `position` (global position of the last
3316
+ handled event, or skipped, for other event types), `skipped` (how many events it skipped as
3317
+ newer than its code, see `Listener`) and `store` (the listener's `Store`, while `run()` has it
3318
+ open)."""
3319
+
3320
+ def __init__(self, *args, **kwargs):
3321
+ """Not for applications: `Kernable.listener` (or `Kernable.app`) runs listeners."""
3322
+ raise TypeError("use Kernable.listener (or Kernable.app), not ListenerHandle(...)")
3323
+
3324
+ def _init(self, channel: "grpc.Channel | _Partitioned", key: Key, listener: Listener, log: _Log,
3325
+ service: str = "", kernable: "Kernable | None" = None, part: str = "",
3326
+ store: "Store | None" = None, parent: "ListenerHandle | None" = None):
3327
+ """Use `Kernable.listener`. `part` and `store`: a partition's stream, sharing the
3328
+ store of the handle that runs it."""
3329
+ # For handlers' follow-up commands and queries.
3330
+ self._kernable = kernable
3331
+ # How often the handler failed on an event, by position, until it gets past it.
3332
+ self._attempts: dict[int, int] = {}
3333
+ super().__init__(f"listener {listener.name}", listener.reconnect_s, log)
3334
+ # The service this listener belongs to; its membership decides whether it counts.
3335
+ self._service = service
3336
+ #: The listener definition this runs.
3337
+ self.listener = listener
3338
+ #: The listener's store, while `run()` has it open (with `store=`).
3339
+ self.store: Store | None = store
3340
+ # A channel, or the partitions: then a stream per partition, sharing the store.
3341
+ self._source = channel
3342
+ self._part = part
3343
+ self._shared = store is not None
3344
+ # As a partition's stream: the handle that runs it. As that handle: how far each
3345
+ # partition's stream has handled, for the order across tenant moves.
3346
+ self._parent = parent
3347
+ self._progress: dict[str, int] = {}
3348
+ self._moves = threading.Condition()
3349
+ # Tenants this stream read the map again for.
3350
+ self._checked: set[str] = set()
3351
+ self._stub = None
3352
+ self._key = key
3353
+ self._start = listener.start
3354
+ #: Global position of the last handled event (or skipped, for other event types).
3355
+ self.position = 0
3356
+ #: How many events it skipped because their schema is newer than its code reads (a newer
3357
+ #: deployment wrote them); see `Listener`. Across all partitions' streams.
3358
+ self.skipped = 0
3359
+ # Newest event when the kernel accepted the listener; events up to it are replayed.
3360
+ self._replay_up_to = 0
3361
+ # The position sent in the last hello, if any.
3362
+ self._sent_start: int | None = None
3363
+ #: Position after which the kernel delivered on the first connect.
3364
+ self.started_at = 0
3365
+
3366
+ def run(self) -> None:
3367
+ """Processes events, queries and checks until `stop()`, reconnecting when the stream
3368
+ drops; with a store, opens it first and continues after its position, and closes it when
3369
+ `run()` ends. A store that cannot be opened (a full disk) is tried again, backing off like
3370
+ a reconnect. With partitions, a stream per partition, each from its own position in the
3371
+ shared store."""
3372
+ parts = self._partitions(self._source)
3373
+ if parts is None:
3374
+ return
3375
+ partitioned = len(parts) > 1 or parts[0][0] != ""
3376
+ if self.listener.store_url and self.store is None:
3377
+ attempt = 0
3378
+ while self.store is None:
3379
+ try:
3380
+ self.store = Store(self.listener.store_url, self.listener.schema)
3381
+ except Exception as err: # sqlite3 and OS errors alike
3382
+ attempt += 1
3383
+ self._log.store_failed(self._what, err, self._reconnect_s, attempt, self._wake)
3384
+ if self._stopped:
3385
+ return
3386
+ if not partitioned:
3387
+ self.position = self.store.committed
3388
+ # A new (or cleared) store says nothing: the listener's own start applies (a
3389
+ # reactor starts at the newest event).
3390
+ if self.store.committed > 0:
3391
+ self._start = self.store.committed
3392
+ if self._shared:
3393
+ self.position = self.store.committed_for(self._part)
3394
+ if self.position > 0:
3395
+ self._start = self.position
3396
+ try:
3397
+ if partitioned:
3398
+ self._run_children([ListenerHandle._new(channel, self._key, self.listener, self._log, self._service,
3399
+ self._kernable, part=name, store=self.store,
3400
+ parent=self)
3401
+ for name, channel in parts])
3402
+ return
3403
+ self._stub = rpc.ListenerServiceStub(parts[0][1])
3404
+ super().run()
3405
+ finally:
3406
+ # Stopping releases the store; a later run opens it again. A partition's stream
3407
+ # leaves it to the handle that owns it.
3408
+ if self.store is not None and not self._shared:
3409
+ self.store.close()
3410
+ self.store = None
3411
+
3412
+ def _open(self, requests):
3413
+ return self._stub.Listen(requests)
3414
+
3415
+ def _hello(self):
3416
+ d = self.listener
3417
+ instance = d.instance if d.instance is not None else os.environ.get("KERNABLE_INSTANCE", "")
3418
+ hello = pb.Hello(name=d.name, key_id=self._key.id, protocol=PROTOCOL, order=d.order,
3419
+ event_types=list(d._handlers),
3420
+ query_types=list(d._queries),
3421
+ middlewares=[spec for spec, _ in d._middlewares.values()], live=d.live,
3422
+ service=self._service, instance=instance,
3423
+ single=bool(instance) and d.single and not d.live,
3424
+ deployment=self._kernable._deployment if self._kernable else "",
3425
+ reactor=d.reactor)
3426
+ # Reconnects resume after the last handled event, once the kernel sent one (an instance
3427
+ # that only waited has none).
3428
+ start = self.position if self._connects > 0 else self._start
3429
+ self._sent_start = start
3430
+ if start is not None:
3431
+ hello.from_position = start
3432
+ return pb.ListenerMessage(hello=hello)
3433
+
3434
+ def _on(self, msg):
3435
+ # Requests sent from the handlers name this listener, so the kernel never makes them
3436
+ # wait for it (it handles one message at a time).
3437
+ token = _HANDLING.set(self.listener.name)
3438
+ try:
3439
+ with _Watch(self._log, self._what, *_handling(msg)):
3440
+ return self._handle(msg)
3441
+ finally:
3442
+ _HANDLING.reset(token)
3443
+
3444
+ def _handle(self, msg):
3445
+ kind = msg.WhichOneof("msg")
3446
+ if kind == "challenge":
3447
+ sig = self._key.sign(challenge_message(msg.challenge.nonce)).sig
3448
+ return pb.ListenerMessage(challenge_response=pb.ChallengeResponse(sig=sig))
3449
+ if kind == "ready":
3450
+ self.position = msg.ready.after_position
3451
+ if self._parent is not None:
3452
+ self._parent._advanced(self._part, msg.ready.after_position)
3453
+ self._replay_up_to = msg.ready.head_position
3454
+ self._log_connected(msg.ready.after_position)
3455
+ self.ready.set()
3456
+ elif kind == "waiting":
3457
+ # Another instance runs the group, or (a reactor) another deployment serves: this one
3458
+ # is ready to take over.
3459
+ if msg.waiting.reader_of:
3460
+ self._log.print("info", self._what, "waiting: this kernel is a reader; reactors act on the "
3461
+ f"writer ({msg.waiting.reader_of})")
3462
+ elif msg.waiting.serving:
3463
+ self._log.print("info", self._what, f"waiting: deployment {msg.waiting.serving} serves; "
3464
+ "this reactor acts once its own deployment does")
3465
+ else:
3466
+ self._log.print("info", self._what, f"waiting: instance {msg.waiting.active} is active")
3467
+ self.ready.set()
3468
+ elif kind == "batch":
3469
+ # A kernel that goes back (a bug, or not the kernel) must not make a store apply
3470
+ # events twice: the session ends, and the next resumes after what was handled.
3471
+ if msg.batch.up_to < self.position:
3472
+ raise RuntimeError(f"the kernel sent a batch up to {msg.batch.up_to}, behind position {self.position}")
3473
+ self._in_order(msg.batch.events)
3474
+ # With a store, the batch's data and position are committed together, before the ack.
3475
+ if self.store:
3476
+ self.store._begin()
3477
+ try:
3478
+ parked, skipped = self._handle_batch(msg.batch)
3479
+ if self.store:
3480
+ self.store._commit(msg.batch.up_to, self._part)
3481
+ except BaseException:
3482
+ if self.store:
3483
+ self.store._rollback()
3484
+ self.position = self.store.committed_for(self._part)
3485
+ raise
3486
+ self.position = msg.batch.up_to
3487
+ if self._parent is not None:
3488
+ self._parent._advanced(self._part, msg.batch.up_to)
3489
+ # Values removed after an erasure must not stay behind in the WAL, outside the batch.
3490
+ if self.store and any(e.erases for e in [*msg.batch.events, *msg.batch.retries]):
3491
+ self.store._checkpoint()
3492
+ return pb.ListenerMessage(ack=pb.Ack(position=msg.batch.up_to, parked=parked, skipped=skipped))
3493
+ elif kind == "query":
3494
+ answer = self._answer(msg.query.call_id, msg.query.query)
3495
+ if self._log.enabled("debug"):
3496
+ q = msg.query.query
3497
+ done = (f"rejected {q.domain}.{q.query_type} {answer.rejected.code}" if answer.HasField("rejected")
3498
+ else f"answered {q.domain}.{q.query_type}")
3499
+ self._log.print("debug", self._what, done)
3500
+ return pb.ListenerMessage(answer=answer)
3501
+ elif kind == "check":
3502
+ return pb.ListenerMessage(verdict=self._verdict(msg.check))
3503
+ return None
3504
+
3505
+ def _advanced(self, part: str, position: int) -> None:
3506
+ """On the parent: partition `part`'s stream has handled everything up to `position`."""
3507
+ with self._moves:
3508
+ if self._progress.get(part, -1) < position:
3509
+ self._progress[part] = position
3510
+ self._moves.notify_all()
3511
+
3512
+ def _in_order(self, events) -> None:
3513
+ """Before a batch of a partition's stream: the events of a tenant that moved here after its move wait until the stream of the partition it came from has passed the
3514
+ tenant's last event there, so a tenant's events keep their order across the move. Waits
3515
+ at most 30 s (a partition that is down), then goes on with a warning."""
3516
+ parent = self._parent
3517
+ if parent is None or not isinstance(parent._source, _Partitioned) or not events:
3518
+ return
3519
+ source = parent._source
3520
+ m = source.current()
3521
+ if m is None:
3522
+ return
3523
+
3524
+ def known(m, tenant: str) -> bool:
3525
+ return _partition_of(m, tenant) == self._part or any(
3526
+ x.tenant == tenant and self._part in (x.to, getattr(x, "from")) for x in m.moved)
3527
+
3528
+ for e in events:
3529
+ if not known(m, e.tenant_id) and e.tenant_id not in self._checked:
3530
+ # The tenant came here after the map was read; another kernel may take a moment
3531
+ # to hold the new one.
3532
+ self._checked.add(e.tenant_id)
3533
+ for i in range(3):
3534
+ if known(m, e.tenant_id):
3535
+ break
3536
+ if i > 0:
3537
+ time.sleep(0.5)
3538
+ m = source.current(refresh=True) or m
3539
+ for x in m.moved:
3540
+ origin = getattr(x, "from")
3541
+ if x.tenant != e.tenant_id or x.to != self._part or e.global_position <= x.target_after:
3542
+ continue
3543
+ if not any(p.name == origin for p in m.partitions):
3544
+ continue
3545
+ parent._reached(self, origin, x.source_head, e.tenant_id)
3546
+
3547
+ def _reached(self, child: "ListenerHandle", part: str, position: int, tenant: str) -> None:
3548
+ """On the parent: returns once partition `part`'s stream has handled up to `position`, or
3549
+ after 30 s."""
3550
+ deadline = time.monotonic() + 30
3551
+ with self._moves:
3552
+ while self._progress.get(part, -1) < position:
3553
+ if child._stopped:
3554
+ raise RuntimeError("stopped")
3555
+ left = deadline - time.monotonic()
3556
+ if left <= 0:
3557
+ self._log.print("warn", child._what,
3558
+ f"tenant {tenant} moved here from partition {part}, whose stream has not "
3559
+ f"passed position {position} in 30 s; going on without its order")
3560
+ return
3561
+ self._moves.wait(min(left, 0.1))
3562
+
3563
+ def _log_connected(self, after_position: int) -> None:
3564
+ """Logs that the kernel accepted the listener; the position is the start sent (the
3565
+ kernel's, if none was)."""
3566
+ position = after_position if self._sent_start is None else self._sent_start
3567
+ if self._connects == 0:
3568
+ self.started_at = position
3569
+ if self.announce:
3570
+ self._log.print("info", self._what, " · ".join(
3571
+ [f"connected to {self._log.peer}", *_listener_details(self.listener, position)]))
3572
+ else:
3573
+ self._log.print("info", self._what, f"connected again at position {position}")
3574
+ self._connects += 1
3575
+
3576
+ def _handle_batch(self, batch) -> "tuple[list[pb.Parked], list[pb.Skipped]]":
3577
+ """Handles a batch: first the retries, then the events after the position. Returns the
3578
+ events it parked and those it skipped as newer than its code; raises if a handler failed
3579
+ on one it does not park."""
3580
+ d = self.listener
3581
+ park_after = 0 if d.live else d.park_after
3582
+ instance = d.instance if d.instance is not None else os.environ.get("KERNABLE_INSTANCE", "")
3583
+ # Instances that each get every event cannot park; they skip what they cannot read.
3584
+ parks = park_after > 0 and not (instance and not d.single)
3585
+ parked, skipped = [], []
3586
+ for e, retry in [*((e, True) for e in batch.retries), *((e, False) for e in batch.events)]:
3587
+ if not retry and e.global_position <= self.position:
3588
+ continue # handled already
3589
+ name = f"{e.domain}.{e.event_type}"
3590
+ if name not in d._handlers:
3591
+ name = f"{e.domain}.*"
3592
+ handler = d._handlers.get(name)
3593
+ if handler:
3594
+ data, redacted = _unseal(_decode(e.payload), batch.vault)
3595
+ schema = e.schema
3596
+ if name not in d._raw:
3597
+ try:
3598
+ schema, data = _upcast(d._domains.get(e.domain), e.domain, e.event_type, schema, data)
3599
+ except SchemaError as err:
3600
+ if not err.newer:
3601
+ raise
3602
+ if parks:
3603
+ parked.append(self._unknown(err, e.global_position))
3604
+ else:
3605
+ skipped.append(self._skip(err, e.global_position))
3606
+ if not retry:
3607
+ self.position = e.global_position
3608
+ continue
3609
+ evt = StoredEventMeta(e.event_type, e.version, e.tenant_id, e.domain, e.aggregate_type, e.aggregate_id,
3610
+ e.command_id, e.global_position, schema, redacted, _time(e.created_at),
3611
+ e.deployment)
3612
+ ctx = ListenerContext(e.global_position <= self._replay_up_to, _from_wire_context(e.context),
3613
+ self.store, e.command_id, e.tenant_id, d.domain.name, self._kernable)
3614
+ saved = self.store is not None and park_after > 0
3615
+ if saved:
3616
+ self.store._savepoint()
3617
+ try:
3618
+ _call(handler, data, evt, ctx)
3619
+ except Exception as err:
3620
+ # An erasing event is never parked: its handler must get past it, or what it
3621
+ # removes would stay behind.
3622
+ parked.append(self._give_up(f"{e.domain}.{e.event_type}", e.global_position, err,
3623
+ 0 if e.erases else park_after))
3624
+ if saved:
3625
+ self.store._rollback_event()
3626
+ else:
3627
+ if saved:
3628
+ self.store._release()
3629
+ self._attempts.pop(e.global_position, None)
3630
+ self._log.print("debug", self._what, f"{e.domain}.{e.event_type} at {e.global_position}")
3631
+ if not retry:
3632
+ self.position = e.global_position
3633
+ return parked, skipped
3634
+
3635
+ def _unknown(self, err: SchemaError, position: int) -> "pb.Parked":
3636
+ """Parks an event newer than this code reads, at once: no attempt could handle it."""
3637
+ message = (f"{err.type} schema {err.schema}: this code reads it up to {err.known}; "
3638
+ "a newer deployment wrote it")
3639
+ self._attempts.pop(position, None)
3640
+ self._log.print("warn", self._what, f"{err.type} at {position} parked: {message}")
3641
+ return pb.Parked(position=position, code="unknown_event", message=message, attempts=1)
3642
+
3643
+ def _skip(self, err: SchemaError, position: int) -> "pb.Skipped":
3644
+ """Skips an event newer than this code reads: counts it, warns once per type and schema."""
3645
+ for h in (self, self._parent):
3646
+ if h is not None:
3647
+ h.skipped += 1
3648
+ with _SKIP_LOCK:
3649
+ first = (err.type, err.schema) not in _SKIP_WARNED
3650
+ _SKIP_WARNED.add((err.type, err.schema))
3651
+ if first:
3652
+ self._log.print("warn", self._what, f"skipped {err.type} schema {err.schema}: this code reads it up "
3653
+ f"to {err.known}; a newer deployment wrote it")
3654
+ return pb.Skipped(position=position, event_type=err.type, schema=err.schema)
3655
+
3656
+ def _give_up(self, type: str, position: int, err: Exception, park_after: int) -> "pb.Parked":
3657
+ """What to do after the handler failed on the event at `position`: park it (a rejection
3658
+ at once, another error after `park_after` attempts), or raise `err`, so the listener gets
3659
+ the event again (the kernel or the store away, or attempts left)."""
3660
+ if park_after <= 0 or _transient(err):
3661
+ raise err
3662
+ attempts = self._attempts.get(position, 0) + 1
3663
+ self._attempts[position] = attempts
3664
+ if not isinstance(err, Rejected) and attempts < park_after:
3665
+ raise err
3666
+ del self._attempts[position]
3667
+ if isinstance(err, Rejected):
3668
+ self._log.print("warn", self._what, f"{type} at {position} rejected ({err.code}), parked: {err}")
3669
+ return pb.Parked(position=position, code=err.code, message=str(err), attempts=attempts)
3670
+ self._log.print("error", self._what, f"{type} at {position} failed {attempts} times, parked: {err!r}")
3671
+ return pb.Parked(position=position, code="handler_error", message=_type_name(err), attempts=attempts)
3672
+
3673
+ def _verdict(self, call) -> pb.Verdict:
3674
+ try:
3675
+ when = "after" if call.when == pb.WHEN_AFTER else "before"
3676
+ middleware = self.listener._middlewares.get(when)
3677
+ if middleware is None:
3678
+ raise Rejected(f"no middleware {when}", "not_a_middleware")
3679
+ verdict = _call(middleware[1], _to_check(call), _to_outcome(call) if when == "after" else None, MiddlewareContext(self.store))
3680
+ if verdict is False:
3681
+ raise Rejected(f"refused by {self.listener.name}", "refused")
3682
+ if verdict is not None and verdict is not True:
3683
+ # Fails closed: a middleware that returns something else is a mistake.
3684
+ raise Rejected(f"{self.listener.name} returned {type(verdict).__name__}, not True, False or None",
3685
+ "middleware_error")
3686
+ return pb.Verdict(call_id=call.call_id)
3687
+ except Exception as err:
3688
+ reason = (pb.Rejected(code=err.code, message=str(err)) if isinstance(err, Rejected)
3689
+ else pb.Rejected(code="middleware_error", message=repr(err)))
3690
+ return pb.Verdict(call_id=call.call_id, rejected=reason)
3691
+
3692
+ def _answer(self, call_id: str, q) -> pb.QueryAnswer:
3693
+ try:
3694
+ handler = self.listener._queries.get(f"{q.domain}.{q.query_type}")
3695
+ if handler is None:
3696
+ raise Rejected(f"no handler for {q.domain}.{q.query_type}", "unknown_query")
3697
+ meta = QueryMeta(q.query_id, q.query_type, q.domain, q.tenant_id)
3698
+ result = _call(handler, _decode(q.payload), meta, QueryContext(_from_wire_context(q.context), self.store))
3699
+ # An answer of None is JSON `null`, not `{}`: "nothing" must not look like an empty object.
3700
+ return pb.QueryAnswer(call_id=call_id, response=b"null" if result is None else _encode(result))
3701
+ except Exception as err:
3702
+ reason = (pb.Rejected(code=err.code, message=str(err)) if isinstance(err, Rejected)
3703
+ else pb.Rejected(code="handler_error", message=repr(err)))
3704
+ return pb.QueryAnswer(call_id=call_id, rejected=reason)
3705
+
3706
+
3707
+ class Subscription(_Stream):
3708
+ """A running subscription, from `Kernable.subscribe`: it streams until `stop()`, reconnecting
3709
+ after `position` when the stream drops. Attributes: `position` (global position of the last
3710
+ handled event) and `done` (a `threading.Event`, set once the subscription has ended:
3711
+ `sub.stop(); sub.done.wait(5)`)."""
3712
+
3713
+ def __init__(self, *args, **kwargs):
3714
+ """Not for applications: `Kernable.subscribe` starts subscriptions."""
3715
+ raise TypeError("use Kernable.subscribe, not Subscription(...)")
3716
+
3717
+ def _init(self, source: "grpc.Channel | _Partitioned", key: Key, handler: Callable, after: int,
3718
+ types: list[str], reconnect_s: float, log: _Log, part: str = "",
3719
+ turns: "threading.Lock | None" = None):
3720
+ """Use `Kernable.subscribe`. `part` and `turns`: a partition's stream, taking turns with
3721
+ the other partitions' streams for the handler."""
3722
+ super().__init__(f"subscription {part}" if part else "subscription", reconnect_s, log)
3723
+ #: Global position of the last handled event.
3724
+ self.position = after
3725
+ #: Set once the subscription has ended.
3726
+ self.done = threading.Event()
3727
+ self._source = source
3728
+ self._key = key
3729
+ self._handler = handler
3730
+ self._types = types
3731
+ self._turns = turns
3732
+
3733
+ def run(self) -> None:
3734
+ """Streams events to the handler until `stop()`; `Kernable.subscribe` runs it in a thread."""
3735
+ try:
3736
+ parts = self._partitions(self._source)
3737
+ if parts is None:
3738
+ return
3739
+ if len(parts) == 1 and parts[0][0] == "":
3740
+ self._read(parts[0][1])
3741
+ return
3742
+ turns = threading.Lock()
3743
+ self._run_children([Subscription._new(channel, self._key, self._handler, 0, self._types, self._reconnect_s,
3744
+ self._log, part=name, turns=turns)
3745
+ for name, channel in parts])
3746
+ finally:
3747
+ self.done.set()
3748
+
3749
+ def _read(self, channel: grpc.Channel) -> None:
3750
+ stub = rpc.EventServiceStub(channel)
3751
+ attempt = 0
3752
+ while not self._stopped:
3753
+ start = self.position
3754
+ refused = False
3755
+ try:
3756
+ issued_at = _issued_now()
3757
+ request = pb.SubscribeRequest(after_position=self.position, event_types=self._types,
3758
+ issued_at=issued_at, protocol=PROTOCOL)
3759
+ request.client_sig.CopyFrom(self._key.sign(subscribe_message(self.position, self._types, issued_at)))
3760
+ self._call = None
3761
+ self._call = stub.Subscribe(request)
3762
+ if self._stopped: # stopped while it opened: stop() missed this call
3763
+ self._call.cancel()
3764
+ self.ready.set()
3765
+ for e in self._call:
3766
+ event = _to_event(e)
3767
+ if self._turns is None:
3768
+ _handle(self._handler, event)
3769
+ else:
3770
+ with self._turns:
3771
+ _handle(self._handler, event)
3772
+ self.position = e.global_position
3773
+ reason = "stream ended"
3774
+ except ClientClosed:
3775
+ return
3776
+ except _HandlerFailed as failed: # a failing handler: reconnect and get the event again
3777
+ reason = str(failed)
3778
+ except grpc.RpcError as err:
3779
+ reason = err.details() or str(err.code())
3780
+ refused = err.code() in _REFUSALS
3781
+ except Exception as err:
3782
+ reason = str(err) or type(err).__name__
3783
+ finally:
3784
+ # Whatever ended the loop, the call ends with it (a no-op once it ended by itself).
3785
+ if self._call is not None:
3786
+ self._call.cancel()
3787
+ attempt = 1 if self.position > start else attempt + 1
3788
+ if not self._stopped:
3789
+ self._log.reconnect(self._what, reason, self._reconnect_s, attempt, refused, self._wake)
3790
+
3791
+
3792
+ def _encode(data: Any) -> bytes:
3793
+ # NaN and Infinity are not JSON: other SDKs could not read them back.
3794
+ return json.dumps({} if data is None else data, allow_nan=False).encode()
3795
+
3796
+
3797
+ def _decode(payload: bytes) -> Any:
3798
+ return json.loads(payload) if payload else {}
3799
+
3800
+
3801
+ def _time(ms: int) -> datetime:
3802
+ return datetime.fromtimestamp(ms / 1000, timezone.utc)
3803
+
3804
+
3805
+ def _meta(event_type: str, version: int, cmd, command_id: str) -> EventMeta:
3806
+ """What `apply` gets about an event of `cmd`'s aggregate, caused by command `command_id`."""
3807
+ return EventMeta(event_type, version, cmd.tenant_id, cmd.domain, cmd.aggregate_type, cmd.aggregate_id,
3808
+ command_id)
3809
+
3810
+
3811
+ def _transient(err: Exception) -> bool:
3812
+ """Whether a failed handler should simply try again: the kernel was unreachable or busy, a
3813
+ submit did not settle in time, or the store failed. Never parks."""
3814
+ if isinstance(err, (NotSettled, sqlite3.Error)):
3815
+ return True
3816
+ return isinstance(err, grpc.RpcError) and callable(getattr(err, "code", None)) and err.code() in _BUSY
3817
+
3818
+
3819
+ def _type_name(err: BaseException) -> str:
3820
+ """The error's type, all a parked event keeps of a handler error."""
3821
+ return type(err).__name__
3822
+
3823
+
3824
+ def _call(fn: Callable, *args):
3825
+ """Calls `fn` with as many of `args` as it takes, so the trailing parameters are optional."""
3826
+ params = inspect.signature(fn).parameters.values()
3827
+ if any(p.kind == p.VAR_POSITIONAL for p in params):
3828
+ return fn(*args)
3829
+ positional = [p for p in params if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD)]
3830
+ return fn(*args[:len(positional)])
3831
+
3832
+
3833
+ def _tls_from_env() -> Tls | None:
3834
+ """Mutual TLS from `KERNABLE_TLS_CA`, `KERNABLE_TLS_CERT` and `KERNABLE_TLS_KEY`: all or none."""
3835
+ files = [os.environ.get(f"KERNABLE_TLS_{n}") for n in ("CA", "CERT", "KEY")]
3836
+ if not any(files):
3837
+ return None
3838
+ if not all(files):
3839
+ raise ValueError("set all of KERNABLE_TLS_CA, KERNABLE_TLS_CERT and KERNABLE_TLS_KEY, or none")
3840
+ return Tls(*files)
3841
+
3842
+
3843
+ def _to_wire_context(c: RequestContext, command_id: str) -> pb.RequestContext:
3844
+ wire = pb.RequestContext(
3845
+ sender_tenant=c.sender_tenant, sender_identity=c.sender_identity, sender_account=c.sender_account,
3846
+ sender_session=c.sender_session, on_behalf_of=c.on_behalf_of,
3847
+ correlation_id=c.correlation_id or command_id, causation_id=c.causation_id, trace_id=c.trace_id,
3848
+ bypass_authorization=c.bypass_authorization,
3849
+ attributes=[pb.Attribute(key=k, value=v) for k, v in c.attributes.items()])
3850
+ if c.expected_version is not None:
3851
+ wire.expected_version = c.expected_version
3852
+ return wire
3853
+
3854
+
3855
+ def _from_wire_context(c) -> RequestContext:
3856
+ return RequestContext(
3857
+ c.sender_tenant, c.sender_identity, c.sender_account, c.sender_session, c.on_behalf_of,
3858
+ c.expected_version if c.HasField("expected_version") else None, c.correlation_id, c.causation_id,
3859
+ c.trace_id, c.bypass_authorization, {a.key: a.value for a in c.attributes})
3860
+
3861
+
3862
+ def _ms(when: "datetime | int | float | None") -> int:
3863
+ """A time as ms since epoch; 0 for None. A naive `datetime` is local time, as Python reads it."""
3864
+ if when is None:
3865
+ return 0
3866
+ if isinstance(when, datetime):
3867
+ return int(when.timestamp() * 1000)
3868
+ if isinstance(when, bool) or not isinstance(when, (int, float)):
3869
+ raise TypeError(f"a time is a datetime or ms since epoch, not {when!r}")
3870
+ return int(when)
3871
+
3872
+
3873
+ def _to_check(call) -> Check:
3874
+ if call.WhichOneof("request") == "command":
3875
+ c = call.command
3876
+ return Check("command", c.command_id, c.tenant_id, c.domain, c.command_type,
3877
+ _unseal(_decode(c.payload), call.vault)[0],
3878
+ _from_wire_context(c.context), c.aggregate_type, c.aggregate_id,
3879
+ target=Target(f"{c.domain}.{c.aggregate_type}", c.aggregate_id))
3880
+ kind = call.WhichOneof("request")
3881
+ if kind == "cancel":
3882
+ # A cancel of a scheduled command: its type, aggregate and data are the scheduled command's.
3883
+ x = call.scheduled
3884
+ return Check("cancel", x.command_id, call.cancel.tenant_id, x.domain, x.command_type, _unseal(_decode(x.payload), None)[0],
3885
+ _from_wire_context(call.cancel.context), x.aggregate_type, x.aggregate_id,
3886
+ target=Target(f"{x.domain}.{x.aggregate_type}", x.aggregate_id))
3887
+ if kind in ("data", "cache"):
3888
+ r = getattr(call, kind)
3889
+ type = call.type.rsplit(".", 1)[-1] # qualified, e.g. `shop.PutData`
3890
+ return Check(kind, "", r.tenant_id, r.domain, type, None, _from_wire_context(r.context), key=r.key)
3891
+ q = call.query
3892
+ target = Target(q.target.aggregate_type, q.target.aggregate_id) if q.target.aggregate_id else None
3893
+ return Check("query", q.query_id, q.tenant_id, q.domain, q.query_type, _decode(q.payload),
3894
+ _from_wire_context(q.context), target=target)
3895
+
3896
+
3897
+ def _rejected_outcome(r) -> Outcome:
3898
+ """The outcome of a rejected command, with the key or requirement the kernel named."""
3899
+ q = r.requirement if r.HasField("requirement") else None
3900
+ return Outcome(committed=False, code=r.code, message=r.message, key=r.key,
3901
+ requirement=Requirement(q.domain, q.aggregate_type, q.aggregate_id, q.fact, key=q.key,
3902
+ absent=q.absent) if q else None)
3903
+
3904
+
3905
+ def _to_outcome(call) -> Outcome | QueryOutcome:
3906
+ """What came of the request a middleware after it checks."""
3907
+ if call.WhichOneof("outcome") == "command_result":
3908
+ r = call.command_result
3909
+ if r.WhichOneof("outcome") == "committed":
3910
+ return Outcome(committed=True, events=[_to_event(e) for e in r.committed.events])
3911
+ return _rejected_outcome(r.rejected)
3912
+ r = call.query_result
3913
+ if r.WhichOneof("outcome") == "response":
3914
+ return QueryOutcome(answered=True, data=_decode(r.response))
3915
+ return QueryOutcome(answered=False, code=r.rejected.code, message=r.rejected.message)
3916
+
3917
+
3918
+ def _to_event(e, vault=None) -> StoredEvent:
3919
+ data, redacted = _unseal(_decode(e.payload), vault)
3920
+ return StoredEvent(position=e.global_position, tenant=e.tenant_id, domain=e.domain, aggregate_type=e.aggregate_type,
3921
+ aggregate_id=e.aggregate_id, version=e.version, type=e.event_type,
3922
+ data=data, command_id=e.command_id, request=_from_wire_context(e.context),
3923
+ schema=e.schema, redacted=redacted, created_at=_time(e.created_at), deployment=e.deployment)