pyfect-dev 0.1.0__py3-none-any.whl

pyfect/cluster/sharding/_storage_loop.py

@@ -0,0 +1,421 @@
+"""The storage inbox: reads unprocessed messages from storage and hands
+them to the entity managers, in order, a batch at a time. It shuts down
+before shard acquisition, so no message is processed once the shards are
+released, and after the entity managers, so interrupts and acks still
+reach them. Entities whose mailbox filled up resume from storage on
+their own fiber."""
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+from typing import Any, Never
+
+from pyfect import cause as Cause
+from pyfect._core import Effect
+from pyfect.clock import sleep
+from pyfect.cluster.cluster_error import (
+    AlreadyProcessingMessage,
+    EntityNotAssignedToRunner,
+    MailboxFull,
+    PersistenceError,
+)
+from pyfect.cluster.entity_address import EntityAddress
+from pyfect.cluster.envelope import Interrupt
+from pyfect.cluster.message import Incoming, IncomingEnvelope, IncomingRequest
+from pyfect.cluster.reply import ReplyWithContext
+from pyfect.cluster.runner_address import RunnerAddress
+from pyfect.cluster.snowflake import Snowflake
+from pyfect.effect import catch_tag, fail, gen, succeed, suspend, sync
+from pyfect.logger import log_debug, log_warning
+from pyfect.schedule import spaced
+from pyfect.scope import add_finalizer, fork_in, scoped
+
+from ._state import PendingNotification, ResumptionState, ShardingState, entity_not_registered
+
+_VOID: Effect[None] = succeed(None)
+
+
+class _Read:
+    """The state of one pass of the storage loop."""
+
+    __slots__ = (
+        "capped",
+        "delivered",
+        "index",
+        "messages",
+        "removable",
+        "reset_addresses",
+        "reset_request_ids",
+    )
+
+    def __init__(self) -> None:
+        self.index = 0
+        self.messages: list[Incoming] = []
+        self.delivered = False
+        self.removable: dict[int, PendingNotification] = {}
+        self.reset_addresses: dict[EntityAddress, None] = {}
+        self.reset_request_ids: list[Snowflake] = []
+        self.capped: dict[EntityAddress, None] = {}
+
+
+class StorageInbox:
+    __slots__ = ("_self_address", "_state")
+
+    def __init__(self, state: ShardingState, self_address: RunnerAddress) -> None:
+        self._state = state
+        self._self_address = self_address
+
+    # --- one message ----------------------------------------------------------------------
+
+    def _release_completed_claim(self, read: _Read, message: Incoming) -> None:
+        """A completed request may stay deduplicated after storage claims
+        its reset: release the claim so the reset is redelivered at once."""
+        found = self._state.entity_managers.get(message.envelope.address.entity_type)
+        if found is None or not found.manager.is_processing_for(message, exclude_completed=True):
+            read.reset_request_ids.append(message.envelope.request_id)
+
+    def _send(self, read: _Read) -> Effect[None]:
+        state = self._state
+
+        def mark_delivered(_: None) -> None:
+            read.delivered = True
+
+        def body() -> Effect[None, Any]:  # noqa: PLR0911
+            message = read.messages[read.index]
+            address = message.envelope.address
+            if address.shard_id not in state.acquired_shards:
+                return _VOID
+            found = state.entity_managers.get(address.entity_type)
+            if found is None:
+                if state.registration_time_remaining() > 0:
+                    # Reset the address in case the entity is slow to register.
+                    read.reset_addresses[address] = None
+                    return _VOID
+                # The entity did not register in time: save a defect reply.
+                return entity_not_registered(address.entity_type)
+            if found.status == "closed":
+                return _VOID
+            manager = found.manager
+            processing = manager.is_processing_for(message)
+            if isinstance(message, IncomingEnvelope) and processing:
+                # The message may affect a request being processed.
+                return manager.send(message).map(mark_delivered)
+            if processing or found.status == "closing":
+                # Already processing, or the entity is closing: skip it.
+                if processing:
+                    self._release_completed_claim(read, message)
+                return _VOID
+            if isinstance(message, IncomingRequest):
+                pending = state.pending_notifications.pop(message.envelope.request_id, None)
+                if pending is not None:
+                    read.removable.pop(id(pending), None)
+                    pending.resume(_VOID)
+            # At entity capacity, messages for entities not yet resident are
+            # skipped; their claims are released after the read.
+            if state.residency_at_capacity() and not manager.is_resident_unsafe(address):
+                read.capped[address] = None
+                return _VOID
+            # An entity resuming on another fiber takes the message id.
+            resuming = state.entity_resumption.get(address)
+            if resuming is not None:
+                resuming.unprocessed[message.envelope.request_id] = None
+                if isinstance(message, IncomingEnvelope) and isinstance(
+                    message.envelope, Interrupt
+                ):
+                    resuming.interrupts[message.envelope.request_id] = message
+                return _VOID
+            return manager.send(message).map(mark_delivered)
+
+        def failed(cause: Cause.Cause[Any]) -> Effect[None]:
+            message = read.messages[read.index]
+            found = cause.find_recoverable_error()
+            if not isinstance(found, Cause.Fail):
+                if cause.has_interrupts:
+                    return _VOID
+                defect = ReplyWithContext.from_defect(
+                    id=state.snowflakes.next_unsafe(),
+                    request_id=message.envelope.request_id,
+                    defect=cause.raisable(),
+                )
+                return state.storage.save_reply(defect).ignore()
+            error = found.error
+            if isinstance(error, AlreadyProcessingMessage):
+                # Completion raced with decoding.
+                self._release_completed_claim(read, message)
+                return _VOID
+            if isinstance(error, MailboxFull):
+                address = message.envelope.address
+                manager = state.entity_managers.get(address.entity_type)
+                # A resident entity has a full mailbox; its messages resume
+                # from storage once there is room.
+                if (
+                    isinstance(message, IncomingRequest)
+                    and manager is not None
+                    and manager.manager.is_resident_unsafe(address)
+                ):
+                    return self.resume_from_storage(message)
+                # Otherwise the runner ran out of entity slots meanwhile.
+                read.capped[address] = None
+            return _VOID
+
+        return suspend(body).catch_cause(failed)
+
+    # --- one pass -------------------------------------------------------------------------
+
+    def _read_and_process(
+        self,
+        read: _Read,
+        *,
+        limit: int | None = None,
+        addresses: list[EntityAddress] | None = None,
+    ) -> Effect[int, Any]:
+        state = self._state
+
+        @gen
+        def run():
+            read.messages = yield from state.storage.unprocessed_messages(
+                list(state.acquired_shards), limit=limit, addresses=addresses
+            )
+            read.index = 0
+            while read.index < len(read.messages):
+                yield from self._send(read)
+                read.index += 1
+            return len(read.messages)
+
+        return run
+
+    def _pass(self, read: _Read, batch_size: int) -> Effect[None, Any]:
+        state = self._state
+
+        @gen
+        def run():  # noqa: PLR0912
+            # Wait for the next poll interval, or a notification.
+            yield from state.storage_read_latch.wait()
+            # A notification from now on starts the next read at once.
+            state.storage_read_latch.close_unsafe()
+            # While the lock is held, resuming entities know no more ids are
+            # added to their unprocessed set.
+            yield from state.storage_read_lock.take(1)
+            for found in list(state.entity_managers.values()):
+                found.manager.clear_processed()
+            for pending in state.pending_notifications.values():
+                read.removable[id(pending)] = pending
+
+            read.delivered = False
+            read_count = 0
+            full_batch = False
+            exhaustive = False
+
+            # First deliver to resident entities, so they keep making
+            # progress at the entity cap.
+            if state.residency_at_capacity():
+                resident: list[EntityAddress] = []
+                for found in list(state.entity_managers.values()):
+                    if found.status == "closed":
+                        continue
+                    resident.extend(found.manager.resident_addresses_unsafe())
+                if resident:
+                    read_count = yield from self._read_and_process(
+                        read, limit=batch_size, addresses=resident
+                    )
+                    full_batch = read_count >= batch_size
+
+            # Then the rest, spawning entities while slots are left.
+            if not state.residency_at_capacity() and read_count < batch_size:
+                limit = batch_size - read_count
+                count = yield from self._read_and_process(read, limit=limit)
+                if count >= limit:
+                    full_batch = True
+                elif not read.capped:
+                    exhaustive = True
+
+            if read.removable:
+                # On an exhaustive read, a message not seen is no longer
+                # processable here. At the cap, the messages are persisted
+                # and delivered once a slot frees up, so senders succeed.
+                # Otherwise the read was cut by the batch size and the
+                # notifications stay for the next read.
+                capped = bool(read.capped) or state.residency_at_capacity()
+                if exhaustive or capped:
+                    for pending in list(read.removable.values()):
+                        state.pending_notifications.pop(pending.message.envelope.request_id, None)
+                        pending.resume(
+                            fail(
+                                EntityNotAssignedToRunner(address=pending.message.envelope.address)
+                            )
+                            if exhaustive
+                            else _VOID
+                        )
+                read.removable.clear()
+            for address in list(read.reset_addresses):
+                yield from log_warning(
+                    "Could not find entity manager for address, retrying"
+                ).annotate_logs("address", address)
+            # Claims for unregistered entity types, and those skipped at the
+            # entity cap, are reset in one storage call after the lock is
+            # released and before the next read.
+            to_reset: list[EntityAddress] | None = None
+            if read.reset_addresses or read.capped:
+                to_reset = [*read.reset_addresses, *read.capped]
+                read.reset_addresses.clear()
+                read.capped.clear()
+
+            # Let the resuming entities check whether they are done.
+            yield from state.storage_read_lock.release(1)
+
+            if read.reset_request_ids:
+                yield from state.storage.reset_requests(list(read.reset_request_ids)).ignore()
+                read.reset_request_ids.clear()
+            if to_reset is not None:
+                yield from state.storage.reset_addresses(to_reset).ignore()
+
+            # A full batch means more could be waiting: read again at once,
+            # if this read made progress (a full batch of skipped messages
+            # must not spin the loop).
+            if full_batch and read.delivered:
+                state.storage_read_latch.open_unsafe()
+
+        return run
+
+    def loop(self) -> Effect[Never]:
+        state = self._state
+        batch_size = max(1, state.config.unprocessed_message_batch_size)
+
+        @gen
+        def run():
+            yield from log_debug("Starting")
+            yield from add_finalizer(lambda _: log_debug("Shutting down"))
+            read = _Read()
+            while True:
+                yield from self._pass(read, batch_size)
+
+        def failed(cause: Cause.Cause[Any]) -> Effect[None]:
+            return log_warning("Could not read messages from storage", cause)
+
+        return (
+            scoped(run)
+            .ensuring(state.storage_read_lock.release_all().as_void())
+            .catch_cause(failed)
+            .forever()
+            .annotate_logs(
+                {
+                    "module": "effect/cluster/Sharding",
+                    "fiber": "Storage read loop",
+                    "runner": self._self_address,
+                }
+            )
+        )
+
+    # --- resuming from storage ------------------------------------------------------------
+
+    def resume_from_storage(self, last_received: IncomingRequest) -> Effect[None]:
+        """Resume the unprocessed messages of an entity whose mailbox was
+        full."""
+        state = self._state
+        address = last_received.envelope.address
+        found = state.entity_resumption.get(address)
+        if found is not None:
+            found.unprocessed[last_received.envelope.request_id] = None
+            return _VOID
+        state.entity_resumption[address] = ResumptionState(
+            {last_received.envelope.request_id: None}
+        )
+        return fork_in(self._resume(address), state.sharding_scope).as_void()
+
+    def _resume(self, address: EntityAddress) -> Effect[None]:
+        state = self._state
+
+        @gen
+        def run():
+            found = state.entity_managers.get(address.entity_type)
+            if found is None:
+                state.entity_resumption.pop(address, None)
+                return
+            manager = found.manager
+            resumption = state.entity_resumption[address]
+            done = False
+            while not done:
+                # The shard moved away: stop.
+                if address.shard_id not in state.acquired_shards:
+                    return
+                ids = list(resumption.unprocessed)[:1024]
+                messages = yield from state.storage.unprocessed_messages_by_id(ids)
+                # Should not happen, but handled just in case.
+                if not messages:
+                    yield from sleep(state.config.entity_message_poll_interval)
+                    continue
+                for message in messages:
+                    yield from self._send_with_retry(manager, resumption, address, message)
+                for request_id in ids:
+                    resumption.unprocessed.pop(request_id, None)
+                if resumption.unprocessed:
+                    continue
+
+                # Caught up with the storage loop: let it take over.
+                def finish() -> bool:
+                    if resumption.unprocessed:
+                        return False
+                    state.entity_resumption.pop(address, None)
+                    return True
+
+                done = yield from state.storage_read_lock.with_permits(sync(finish))
+
+        def failed(cause: Cause.Cause[Any]) -> Effect[None]:
+            return log_debug("Could not resume unprocessed messages", cause)
+
+        def forget() -> None:
+            state.entity_resumption.pop(address, None)
+
+        return (
+            run.retry(
+                while_=lambda error: isinstance(error, PersistenceError),
+                schedule=spaced(state.config.entity_message_poll_interval),
+            )
+            .catch_cause(failed)
+            .annotate_logs(
+                {
+                    "module": "effect/cluster/Sharding",
+                    "fiber": "Resuming unprocessed messages",
+                    "runner": self._self_address,
+                    "entity": address,
+                }
+            )
+            .ensuring(sync(forget))
+        )
+
+    def _send_with_retry(
+        self, manager: Any, resumption: ResumptionState, address: EntityAddress, message: Incoming
+    ) -> Effect[None, EntityNotAssignedToRunner]:
+        state = self._state
+
+        def attempt() -> Effect[None, Any]:
+            if address.shard_id not in state.acquired_shards:
+                return fail(EntityNotAssignedToRunner(address=address))
+            # A request that was interrupted gets its interrupt after it.
+            interrupt = (
+                resumption.interrupts.get(message.envelope.request_id)
+                if isinstance(message, IncomingRequest)
+                else None
+            )
+            if interrupt is None:
+                return manager.send(message)
+
+            def then(_: None) -> Effect[None, Any]:
+                resumption.interrupts.pop(message.envelope.request_id, None)
+                return manager.send(interrupt)
+
+            return manager.send(message).flat_map(then)
+
+        def retry(_: MailboxFull) -> Effect[None, EntityNotAssignedToRunner]:
+            return self._send_with_retry(manager, resumption, address, message).delay(
+                state.config.send_retry_interval
+            )
+
+        return (
+            suspend(attempt)
+            .pipe(catch_tag(MailboxFull, retry))
+            .pipe(catch_tag(AlreadyProcessingMessage, lambda _: _VOID))
+        )
+
+
+__all__ = ["StorageInbox"]

pyfect/cluster/sharding_config.py

@@ -0,0 +1,230 @@
+"""How a runner shards entities (Effect's ``ShardingConfig``).
+
+``DEFAULTS`` holds Effect's defaults; ``layer(dataclasses.replace(DEFAULTS,
+shards_per_group=30))`` provides a variation, and ``config`` /
+``layer_from_env`` read one (``HOST``, ``PORT``, ``SHARDS_PER_GROUP``, ...
+in the environment). Durations are ``Seconds``. ``"unbounded"`` for the
+mailbox capacity and resident entities is only available in code.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Literal
+
+from pyfect import config as config_
+from pyfect import layer as layer_
+from pyfect import schema as S
+from pyfect.cluster.runner_address import RunnerAddress
+from pyfect.config import Config, ConfigError
+from pyfect.config_provider import from_env
+from pyfect.context import Tag
+from pyfect.duration import Seconds
+
+type Unbounded = Literal["unbounded"]
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ShardingConfigService:
+    """A runner's sharding settings. ``runner_address`` ``None`` makes a
+    client-only runner; ``runner_listen_address`` is where it binds, when
+    not at ``runner_address``."""
+
+    runner_address: RunnerAddress | None
+    runner_listen_address: RunnerAddress | None
+    runner_shard_weight: int
+    available_shard_groups: tuple[str, ...]
+    assigned_shard_groups: tuple[str, ...]
+    shards_per_group: int
+    shard_lock_refresh_interval: Seconds
+    shard_lock_expiration: Seconds
+    shard_lock_disable_advisory: bool
+    preemptive_shutdown: bool
+    entity_mailbox_capacity: int | Unbounded
+    entity_mailbox_metrics: bool
+    max_resident_entities: int | Unbounded
+    unprocessed_message_batch_size: int
+    entity_max_idle_time: Seconds
+    entity_registration_timeout: Seconds
+    entity_termination_timeout: Seconds
+    entity_message_poll_interval: Seconds
+    entity_reply_poll_interval: Seconds
+    refresh_assignments_interval: Seconds
+    send_retry_interval: Seconds
+    runner_health_check_interval: Seconds
+    simulate_remote_serialization: bool
+
+
+class ShardingConfig(Tag[ShardingConfigService], key="effect/cluster/ShardingConfig"):
+    """The sharding settings of this runner."""
+
+
+_DEFAULT_ADDRESS = RunnerAddress(host="localhost", port=34431)
+
+DEFAULTS = ShardingConfigService(
+    runner_address=_DEFAULT_ADDRESS,
+    runner_listen_address=None,
+    runner_shard_weight=1,
+    shards_per_group=300,
+    available_shard_groups=("default",),
+    assigned_shard_groups=("default",),
+    preemptive_shutdown=True,
+    shard_lock_refresh_interval=Seconds(10),
+    shard_lock_expiration=Seconds(35),
+    shard_lock_disable_advisory=False,
+    entity_mailbox_capacity=4096,
+    entity_mailbox_metrics=False,
+    max_resident_entities=10_000,
+    unprocessed_message_batch_size=1024,
+    entity_max_idle_time=Seconds(60),
+    entity_registration_timeout=Seconds(60),
+    entity_termination_timeout=Seconds(15),
+    entity_message_poll_interval=Seconds(10),
+    entity_reply_poll_interval=Seconds(0.2),
+    send_retry_interval=Seconds(0.1),
+    refresh_assignments_interval=Seconds(3),
+    runner_health_check_interval=Seconds(60),
+    simulate_remote_serialization=True,
+)
+"""Effect's defaults."""
+
+
+def layer(settings: ShardingConfigService = DEFAULTS) -> layer_.Layer[ShardingConfig]:
+    return layer_.succeed(ShardingConfig, settings)
+
+
+layer_defaults: layer_.Layer[ShardingConfig] = layer()
+
+_POSITIVE_INT = S.Int.check(S.is_greater_than(0))
+
+
+def _seconds(name: str, default: Seconds) -> Config[Seconds]:
+    return config_.duration(name).with_default(default)
+
+
+def _bool(name: str, default: bool) -> Config[bool]:
+    return config_.boolean(name).with_default(default)
+
+
+def _int(name: str, default: int) -> Config[int]:
+    return config_.integer(name).with_default(default)
+
+
+def _groups(name: str) -> Config[tuple[str, ...]]:
+    return config_.list_of(S.String, name).map(tuple).with_default(("default",))
+
+
+def _listen_address() -> Config[RunnerAddress | None]:
+    host = config_.option(config_.string("listenHost"))
+    port = _int("listenPort", _DEFAULT_ADDRESS.port)
+    return config_.all(host, port).map(
+        lambda parts: (
+            None if parts[0] is None else RunnerAddress(host=parts[0].value, port=parts[1])
+        )
+    )
+
+
+def _make(d: ShardingConfigService) -> Config[ShardingConfigService]:
+    @config_.gen
+    def read():
+        host = yield from config_.string("host").with_default(_DEFAULT_ADDRESS.host)
+        port = yield from _int("port", _DEFAULT_ADDRESS.port)
+        return ShardingConfigService(
+            runner_address=RunnerAddress(host=host, port=port),
+            runner_listen_address=(yield from _listen_address()),
+            runner_shard_weight=(yield from _int("runnerShardWeight", d.runner_shard_weight)),
+            available_shard_groups=(yield from _groups("availableShardGroups")),
+            assigned_shard_groups=(yield from _groups("shardGroups")),
+            shards_per_group=(yield from _int("shardsPerGroup", d.shards_per_group)),
+            shard_lock_refresh_interval=(
+                yield from _seconds("shardLockRefreshInterval", d.shard_lock_refresh_interval)
+            ),
+            shard_lock_expiration=(
+                yield from _seconds("shardLockExpiration", d.shard_lock_expiration)
+            ),
+            shard_lock_disable_advisory=(
+                yield from _bool("shardLockDisableAdvisory", d.shard_lock_disable_advisory)
+            ),
+            preemptive_shutdown=(yield from _bool("preemptiveShutdown", d.preemptive_shutdown)),
+            entity_mailbox_capacity=(yield from _int("entityMailboxCapacity", 4096)),
+            entity_mailbox_metrics=(
+                yield from _bool("entityMailboxMetrics", d.entity_mailbox_metrics)
+            ),
+            max_resident_entities=(
+                yield from config_.schema(_POSITIVE_INT, "maxResidentEntities").with_default(10_000)
+            ),
+            unprocessed_message_batch_size=(
+                yield from config_.schema(
+                    _POSITIVE_INT, "unprocessedMessageBatchSize"
+                ).with_default(d.unprocessed_message_batch_size)
+            ),
+            entity_max_idle_time=(yield from _seconds("entityMaxIdleTime", d.entity_max_idle_time)),
+            entity_registration_timeout=(
+                yield from _seconds("entityRegistrationTimeout", d.entity_registration_timeout)
+            ),
+            entity_termination_timeout=(
+                yield from _seconds("entityTerminationTimeout", d.entity_termination_timeout)
+            ),
+            entity_message_poll_interval=(
+                yield from _seconds("entityMessagePollInterval", d.entity_message_poll_interval)
+            ),
+            entity_reply_poll_interval=(
+                yield from _seconds("entityReplyPollInterval", d.entity_reply_poll_interval)
+            ),
+            send_retry_interval=(yield from _seconds("sendRetryInterval", d.send_retry_interval)),
+            refresh_assignments_interval=(
+                yield from _seconds("refreshAssignmentsInterval", d.refresh_assignments_interval)
+            ),
+            runner_health_check_interval=(
+                yield from _seconds("runnerHealthCheckInterval", d.runner_health_check_interval)
+            ),
+            simulate_remote_serialization=(
+                yield from _bool("simulateRemoteSerialization", d.simulate_remote_serialization)
+            ),
+        )
+
+    return read
+
+
+config: Config[ShardingConfigService] = _make(DEFAULTS)
+"""The settings read from the current config provider, Effect's defaults
+for what is absent. Keys are camelCase (``shardsPerGroup``)."""
+
+
+def layer_from_env(
+    update: Callable[[ShardingConfigService], ShardingConfigService] | None = None,
+) -> layer_.Layer[ShardingConfig, ConfigError]:
+    """The settings read from the environment in CONSTANT_CASE
+    (``SHARDS_PER_GROUP``), then passed through ``update``."""
+    read = config.parse(from_env().constant_case())
+    return layer_.effect(ShardingConfig, read if update is None else read.map(update))
+
+
+@dataclass(frozen=True, slots=True)
+class ShardGroups:
+    available: tuple[str, ...]
+    """Sorted, without duplicates."""
+    assigned: tuple[str, ...]
+    """The available groups this runner serves, in the same order."""
+
+
+def shard_group_config(settings: ShardingConfigService) -> ShardGroups:
+    """The available groups, and those of them this runner is assigned."""
+    available = tuple(sorted(set(settings.available_shard_groups)))
+    assigned = tuple(g for g in available if g in settings.assigned_shard_groups)
+    return ShardGroups(available, assigned)
+
+
+__all__ = [
+    "DEFAULTS",
+    "ShardGroups",
+    "ShardingConfig",
+    "ShardingConfigService",
+    "Unbounded",
+    "config",
+    "layer",
+    "layer_defaults",
+    "layer_from_env",
+    "shard_group_config",
+]

pyfect/cluster/sharding_registration_event.py

@@ -0,0 +1,28 @@
+"""What ``Sharding.get_registration_events`` streams (Effect's
+``ShardingRegistrationEvent``): an entity or a singleton was registered on
+this runner."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from pyfect.cluster.singleton_address import SingletonAddress
+
+if TYPE_CHECKING:
+    from pyfect.cluster.entity import Entity
+
+
+@dataclass(frozen=True, slots=True)
+class EntityRegistered:
+    entity: type[Entity]
+
+
+@dataclass(frozen=True, slots=True)
+class SingletonRegistered:
+    address: SingletonAddress
+
+
+type ShardingRegistrationEvent = EntityRegistered | SingletonRegistered
+
+__all__ = ["EntityRegistered", "ShardingRegistrationEvent", "SingletonRegistered"]