pyfect-dev 0.1.0__py3-none-any.whl

pyfect/cluster/_resource_ref.py

@@ -0,0 +1,266 @@
+"""A resource that can be rebuilt in place (Effect's
+``internal/resourceRef``): SQL runner storage keeps its reserved lock
+connection in one, replacing it when a statement on it fails.
+
+``await_`` waits for the current resource; ``rebuild_unsafe`` releases it
+(in the background, so a stalled release cannot block) and acquires a new
+one; a failed acquisition leaves the ref ``Failed`` until the next
+rebuild. Closing the parent scope closes the ref and wakes its waiters
+with an interruption.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Generator
+from dataclasses import dataclass, field
+from typing import Any, overload
+
+from pyfect._core import Effect
+from pyfect._fiber import Fiber
+from pyfect._unsubscripted import Unsubscripted
+from pyfect.cluster._interruptors import around_entity
+from pyfect.cluster.entity_address import EntityAddress
+from pyfect.deferred import Deferred
+from pyfect.effect import gen, interrupt, succeed, suspend, sync, void
+from pyfect.exit import Exit, Failure, Success
+from pyfect.exit import interrupt as interrupt_exit
+from pyfect.scope import CloseableScope, fork_in
+
+
+@dataclass(eq=False, slots=True)
+class _Pending[A]:
+    scope: CloseableScope
+    releases: CloseableScope | Fiber[None, Any] | None
+    ready: Deferred[A]
+    releasing: bool = False
+    value: A | None = None
+    work: Fiber[None, Any] | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class _Ready[A]:
+    scope: CloseableScope
+    value: A
+
+
+@dataclass(frozen=True, slots=True)
+class _Failed[A]:
+    ready: Deferred[A]
+    releasing: Fiber[None, Any] | None
+
+
+@dataclass(frozen=True, slots=True)
+class _Closed:
+    pass
+
+
+type _State[A] = _Closed | _Ready[A] | _Pending[A] | _Failed[A]
+
+_INTERRUPTED: Exit[Any, Any] = interrupt_exit()
+
+
+@dataclass(eq=False, slots=True)
+class ResourceRef[A = Unsubscripted]:
+    _release_scope: CloseableScope
+    _acquire: Callable[[CloseableScope], Effect[A]]
+    _teardown_address: EntityAddress | None
+    _state: _State[A] = field(default_factory=_Closed)
+
+    @overload
+    @classmethod
+    def make[B](
+        cls: type[ResourceRef[Unsubscripted]],
+        parent: CloseableScope,
+        acquire: Callable[[CloseableScope], Effect[B]],
+        teardown_address: EntityAddress | None = None,
+    ) -> Effect[ResourceRef[B]]: ...
+    @overload
+    @classmethod
+    def make(
+        cls,
+        parent: CloseableScope,
+        acquire: Callable[[CloseableScope], Effect[A]],
+        teardown_address: EntityAddress | None = None,
+    ) -> Effect[ResourceRef[A]]: ...
+    @classmethod
+    def make[B](
+        cls,
+        parent: CloseableScope,
+        acquire: Callable[[CloseableScope], Effect[B]],
+        teardown_address: EntityAddress | None = None,
+    ) -> Effect[ResourceRef[B]]:
+        """A ref holding what ``acquire`` gives (it gets the resource's own
+        scope), closed with ``parent``."""
+
+        def body() -> Generator[Effect[Any, Any, Any], Any, ResourceRef[B]]:
+            release_scope = yield from parent.fork()
+            ref = ResourceRef[B](release_scope, acquire, teardown_address)
+            yield from parent.add_finalizer(ref._close)
+            pending = ref._pending(None, Deferred[B]())
+            value = yield from acquire(pending.scope)
+            if ref._state is pending:
+                ref._publish(pending, value)
+            return ref
+
+        return gen(body)
+
+    def get_unsafe(self) -> A | None:
+        """The resource, when one is ready."""
+        state = self._state
+        return state.value if isinstance(state, _Ready) else None
+
+    @property
+    def await_(self) -> Effect[A]:
+        """The resource, once ready (interrupted when the ref is closed)."""
+
+        def run() -> Effect[A]:
+            state = self._state
+            match state:
+                case _Closed():
+                    return interrupt()
+                case _Ready():
+                    return succeed(state.value)
+                case _Pending() | _Failed():
+                    return state.ready.wait()
+
+        return suspend(run)
+
+    def rebuild_unsafe(
+        self,
+        *,
+        from_: A | None = None,
+        prepare: Callable[[A], Effect[None]] | None = None,
+    ) -> Effect[None] | None:
+        """Replace the resource, unless it is no longer ``from_`` (the same
+        object, when given) or the ref is closed; ``None`` when nothing is
+        rebuilt."""
+        state = self._state
+        match state:
+            case _Closed():
+                return None
+            case _Ready():
+                if from_ is not None and from_ is not state.value:
+                    return None
+                pending = self._pending(state.scope, Deferred[A]())
+            case _Failed():
+                if from_ is not None:
+                    return None
+                pending = self._pending(state.releasing, state.ready)
+            case _Pending():
+                if from_ is None or state.value is not from_:
+                    return None
+                if state.work is not None:
+                    state.work.interrupt_as_unsafe(None)
+                pending = self._pending(state.scope, state.ready)
+        return self._run(pending, prepare)
+
+    def _pending(
+        self, releases: CloseableScope | Fiber[None, Any] | None, ready: Deferred[A]
+    ) -> _Pending[A]:
+        pending = _Pending(CloseableScope(), releases, ready)
+        self._state = pending
+        return pending
+
+    def _publish(self, pending: _Pending[A], value: A) -> None:
+        self._state = _Ready(pending.scope, value)
+        pending.ready.done_unsafe(Success(value))
+
+    def _run(
+        self, pending: _Pending[A], prepare: Callable[[A], Effect[None]] | None
+    ) -> Effect[None]:
+        def release_previous() -> Effect[None]:
+            if self._state is not pending:
+                return interrupt()
+            releases = pending.releases
+            if releases is None:
+                return void
+            pending.releasing = True
+            waited = (
+                releases.join()
+                if isinstance(releases, Fiber)
+                else self._release(releases).flat_map(lambda fiber: fiber.join())
+            )
+
+            def released(_: None) -> None:
+                pending.releasing = False
+
+            return waited.map(released)
+
+        def acquired(value: A) -> Effect[None]:
+            if self._state is not pending:
+                return interrupt()
+            pending.value = value
+            prepared = void if prepare is None else prepare(value)
+
+            def publish(_: None) -> Effect[None]:
+                if self._state is not pending:
+                    return interrupt()
+                self._publish(pending, value)
+                return void
+
+            return prepared.flat_map(publish)
+
+        def ended(exit: Exit[None, Any]) -> Effect[None]:
+            if isinstance(exit, Success) or self._state is not pending:
+                return void
+            if pending.releasing:
+                # Nothing was acquired: carry the waiters on without
+                # rejoining the stalled release.
+                self._state = _Failed(pending.ready, None)
+                return void
+            cause = exit.cause
+
+            def failed(releasing: Fiber[None, Any]) -> None:
+                if self._state is not pending:
+                    return
+                self._state = _Failed(Deferred[A](), releasing)
+                pending.ready.done_unsafe(Failure(cause))
+
+            return self._release(pending.scope).map(failed)
+
+        work = (
+            suspend(release_previous)
+            .flat_map(lambda _: suspend(lambda: self._acquire(pending.scope)))
+            .flat_map(acquired)
+            .on_exit(ended)
+        )
+
+        def started(fiber: Fiber[None, Any]) -> Effect[None]:
+            pending.work = fiber
+            # Replaced or closed before the fiber existed.
+            if self._state is not pending:
+                fiber.interrupt_as_unsafe(None)
+            return fiber.join()
+
+        return work.fork(start_immediately=True).flat_map(started)
+
+    def _release(self, scope: CloseableScope) -> Effect[Fiber[None, Any]]:
+        """Close ``scope`` in the background (a stalled release must not
+        block a rebuild)."""
+        close = scope.close(Success(None))
+        if self._teardown_address is not None:
+            close = around_entity(self._teardown_address, close)
+        return fork_in(close, self._release_scope, start_immediately=True)
+
+    def _close(self, exit: Exit[Any, Any]) -> Effect[None]:
+        state = self._state
+        self._state = _Closed()
+        match state:
+            case _Closed():
+                return void
+            case _Ready():
+                return state.scope.close(exit)
+            case _Failed():
+                return sync(lambda: state.ready.done_unsafe(_INTERRUPTED)).as_void()
+            case _Pending():
+                state.ready.done_unsafe(_INTERRUPTED)
+                if state.work is not None:
+                    state.work.interrupt_as_unsafe(None)
+                close = state.scope.close(exit)
+                releases = state.releases
+                if isinstance(releases, CloseableScope):
+                    return releases.close(exit).flat_map(lambda _: close)
+                return close
+
+
+__all__ = ["ResourceRef"]

pyfect/cluster/_shard_lock.py

@@ -0,0 +1,14 @@
+"""How often shard locks are refreshed (Effect's ``internal/shardLock``)."""
+
+from __future__ import annotations
+
+from pyfect.cluster.sharding_config import ShardingConfigService
+from pyfect.duration import Seconds
+
+
+def effective_interval(config: ShardingConfigService) -> Seconds:
+    """The refresh interval, at most a third of the lock expiration."""
+    return Seconds(min(config.shard_lock_refresh_interval, config.shard_lock_expiration / 3))
+
+
+__all__ = ["effective_interval"]

pyfect/cluster/cluster_cron.py

@@ -0,0 +1,230 @@
+"""Cron jobs owned by the cluster (Effect's ``ClusterCron``).
+
+``make`` turns a ``Cron`` into a layer that runs one recurring job across
+the cluster: a singleton schedules the first run, and each run is a
+persisted message to an entity, delivered at its instant, that schedules
+the next. With ``calculate_next_run_from_previous`` every run goes to one
+executor entity, so runs never overlap and missed runs are caught up in
+order. A run reads its instant from ``CurrentScheduledTime``."""
+
+from __future__ import annotations
+
+import builtins
+from collections.abc import Callable, Generator
+from datetime import UTC, datetime, timedelta
+from typing import Any, Never, cast, overload
+
+from pyfect import layer as layer_
+from pyfect._core import Effect
+from pyfect._effect_args import require_effect
+from pyfect.cause import Cause
+from pyfect.cluster import singleton
+from pyfect.cluster.cluster_schema import (
+    ClientTracingEnabled,
+    Persisted,
+    ShardGroup,
+    Uninterruptible,
+)
+from pyfect.cluster.entity import CurrentAddress, Entity, Request
+from pyfect.cluster.entity_id import EntityId
+from pyfect.cluster.sharding import Sharding
+from pyfect.context import Context, Tag
+from pyfect.cron import Cron
+from pyfect.cron import next as cron_next
+from pyfect.date_time import format_iso, now
+from pyfect.duration import Duration, to_seconds
+from pyfect.effect import fail_cause, gen, provide_service, void
+from pyfect.exit import Exit, Failure, Success
+from pyfect.layer import Layer
+from pyfect.logger import log_warning
+from pyfect.rpc import make as make_rpc
+from pyfect.schedule import exponential, min, spaced
+from pyfect.schema import Boolean, Class, DateTimeUtc, Union, field
+from pyfect.schema._undefined import Undefined
+from pyfect.scope import Scope
+
+
+class CurrentScheduledTime(Tag[datetime], key="effect/cluster/ClusterCron/CurrentScheduledTime"):
+    """The instant the running cron job was scheduled for (a UTC
+    ``datetime``), inside ``make``'s ``execute``."""
+
+
+class CronPayload(Class, identifier="effect/cluster/ClusterCron/CronPayload"):
+    """A run of a cron job, due at ``date_time``. Runs sent to the executor
+    (``executor``) share one address, so their instant is their key."""
+
+    date_time: datetime = field(DateTimeUtc, key="dateTime")
+    executor: bool | None = field(Union(Boolean, Undefined), default=None)
+
+    def primary_key(self) -> str:
+        return format_iso(self.date_time) if self.executor else ""
+
+    def deliver_at(self) -> datetime:
+        return self.date_time
+
+
+_EXECUTOR_ID = "executor"
+
+_retry_policy = min(exponential(0.2, 1.5), spaced(60))
+
+
+def _with_retry[A](message: str, effect: Effect[A, Any, Any]) -> Effect[A, Never, Any]:
+    def warn(cause: Cause[Any]) -> Effect[None]:
+        return log_warning(message, cause)
+
+    return effect.tap_cause(warn).sandbox().retry(_retry_policy).or_die()
+
+
+@overload
+def make[E](
+    *,
+    name: str,
+    cron: Cron,
+    execute: Effect[None, E, CurrentScheduledTime | Scope],
+    shard_group: str | None = None,
+    calculate_next_run_from_previous: bool = False,
+    skip_if_older_than: Duration | None = None,
+) -> Layer[Never, Never, Sharding]: ...
+@overload
+def make[E, R](
+    *,
+    name: str,
+    cron: Cron,
+    execute: Effect[None, E, R | CurrentScheduledTime | Scope],
+    shard_group: str | None = None,
+    calculate_next_run_from_previous: bool = False,
+    skip_if_older_than: Duration | None = None,
+) -> Layer[Never, Never, Sharding | R]: ...
+def make(
+    *,
+    name: str,
+    cron: Cron,
+    execute: Effect[None, Any, Any],
+    shard_group: str | None = None,
+    calculate_next_run_from_previous: bool = False,
+    skip_if_older_than: Duration | None = None,
+) -> Layer[Never, Never, Any]:
+    """A layer running ``execute`` on ``cron`` through the cluster's
+    sharding (in ``shard_group``, ``"default"`` when ``None``).
+
+    The next run is due at the cron's next instant after now, or, with
+    ``calculate_next_run_from_previous``, after the previous run's instant:
+    then one executor entity runs every instant in order, and a failed run
+    is retried in place, holding back later runs. A run older than
+    ``skip_if_older_than`` (one day by default) is skipped. A failure of
+    ``execute`` is logged as a warning; the next run is scheduled anyway."""
+    require_effect(execute, "execute")
+    from_previous = calculate_next_run_from_previous
+    skip_after = to_seconds(timedelta(days=1) if skip_if_older_than is None else skip_if_older_than)
+    group = "default" if shard_group is None else shard_group
+
+    def next_run(after: datetime) -> datetime:
+        return cron_next(cron, after).astimezone(UTC)
+
+    def shard_group_of(_: EntityId) -> str:
+        return group
+
+    run_rpc = (
+        make_rpc("run", payload=CronPayload)
+        .annotate(Persisted, True)
+        .annotate(Uninterruptible, True)
+    )
+    start_rpc = make_rpc("start").annotate(Persisted, True)
+    cron_entity = cast(
+        "builtins.type[Entity]",
+        builtins.type(
+            f"ClusterCron[{name}]",
+            (Entity,),
+            {"run": run_rpc, "start": start_rpc},
+            type=f"ClusterCron/{name}",
+            annotations=Context.make(ShardGroup, shard_group_of).add(ClientTracingEnabled, False),
+        ),
+    )
+
+    def send(date_time: datetime) -> Effect[None, Any, Sharding]:
+        def call(make_client: Callable[[str], Any]) -> Effect[None, Any]:
+            if from_previous:
+                payload = CronPayload(date_time=date_time, executor=True)
+                return make_client(_EXECUTOR_ID).run.discard(payload)
+            return make_client(format_iso(date_time)).run.discard(CronPayload(date_time=date_time))
+
+        return cron_entity.client().flat_map(call)
+
+    def schedule(date_time: datetime) -> Effect[None, Never, Sharding]:
+        return _with_retry("Failed to schedule next run, retrying", send(date_time))
+
+    if from_previous:
+        # The executor schedules the next run if its chain has been lost.
+        # ``start`` has no primary key, so each retry is a new message.
+        initial: Effect[None, Never, Any] = _with_retry(
+            "Failed to start cron job, retrying",
+            cron_entity.client().flat_map(
+                lambda make_client: cast("Any", make_client(_EXECUTOR_ID)).start()
+            ),
+        )
+    else:
+        initial = now().flat_map(
+            lambda at: _with_retry("Failed to schedule initial run, retrying", send(next_run(at)))
+        )
+    initial_run = singleton.make(f"ClusterCron/{name}", initial, shard_group=shard_group)
+
+    def run_execute(date_time: datetime) -> Effect[None, Never, Any]:
+        def check(at: datetime) -> Effect[None, Any, Any]:
+            if date_time < at - timedelta(seconds=skip_after):
+                return void
+            return execute.pipe(provide_service(CurrentScheduledTime, date_time))
+
+        return now().flat_map(check).or_die()
+
+    def build() -> Generator[Effect[Any, Never, Any], Any, Any]:
+        address = yield from CurrentAddress()
+        # Runs are processed one at a time, so a run processed before
+        # ``start`` means the chain is alive.
+        processed_run = [False]
+
+        def run(request: Request[CronPayload]) -> Effect[None, Never, Any]:
+            date_time = request.payload.date_time
+            if from_previous and address.entity_id != _EXECUTOR_ID:
+                # A run scheduled by an earlier release joins the executor's chain.
+                return schedule(date_time)
+            processed_run[0] = True
+
+            def after(exit: Exit[None, Never]) -> Effect[None, Never, Any]:
+                warn = log_warning(exit.cause) if isinstance(exit, Failure) else void
+                following = (
+                    schedule(next_run(date_time))
+                    if from_previous
+                    else now().flat_map(lambda at: schedule(next_run(at)))
+                )
+                return warn.flat_map(lambda _: following)
+
+            return (
+                run_execute(date_time)
+                .match_cause_effect(
+                    on_failure=lambda cause: after(Failure(cause)).flat_map(
+                        lambda _: fail_cause(cause)
+                    ),
+                    on_success=lambda _: after(Success(None)),
+                )
+                .annotate_logs(
+                    {
+                        "module": "effect/cluster/ClusterCron",
+                        "name": name,
+                        "dateTime": date_time,
+                    }
+                )
+            )
+
+        def start(_: Request[None]) -> Effect[None, Never, Any]:
+            if processed_run[0]:
+                return void
+            # Deduplicated against the pending run if the chain is alive.
+            return now().flat_map(lambda at: schedule(next_run(at)))
+
+        return cron_entity.handlers().handle(run_rpc)(run).handle(start_rpc)(start)
+
+    entity_layer = cron_entity.to_layer_effect(gen(build))
+    return layer_.merge(initial_run, entity_layer)
+
+
+__all__ = ["CronPayload", "CurrentScheduledTime", "make"]

pyfect/cluster/cluster_error.py

@@ -0,0 +1,81 @@
+"""The errors of a cluster (Effect's ``ClusterError``). Each is a schema
+class, so it crosses the wire as an RPC failure."""
+
+from __future__ import annotations
+
+from pyfect import schema as S
+from pyfect._core import Effect
+from pyfect._effect_args import require_effect
+from pyfect.cluster.entity_address import EntityAddress
+from pyfect.cluster.runner_address import RunnerAddress
+from pyfect.cluster.snowflake import Snowflake, SnowflakeFromString
+from pyfect.effect import fail
+
+_ID = "~effect/cluster/ClusterError"
+
+
+class EntityNotAssignedToRunner(S.TaggedError, identifier=f"{_ID}/EntityNotAssignedToRunner"):
+    """The entity's shard is not assigned to the runner that got the message."""
+
+    address: EntityAddress
+
+
+class MalformedMessage(S.TaggedError, identifier=f"{_ID}/MalformedMessage"):
+    """A message could not be decoded or encoded."""
+
+    cause: object = S.field(S.Defect())
+
+    @staticmethod
+    def refail[A, R](effect: Effect[A, BaseException, R]) -> Effect[A, MalformedMessage, R]:
+        """``effect`` with its failure wrapped in ``MalformedMessage``."""
+        require_effect(effect, "effect")
+        return effect.map_error(lambda cause: MalformedMessage(cause=cause))
+
+
+class PersistenceError(S.TaggedError, identifier=f"{_ID}/PersistenceError"):
+    """Message or runner storage failed."""
+
+    cause: object = S.field(S.Defect())
+
+    @staticmethod
+    def refail[A, R](effect: Effect[A, object, R]) -> Effect[A, PersistenceError, R]:
+        """``effect`` with any failure or defect squashed into a
+        ``PersistenceError``."""
+        require_effect(effect, "effect")
+        return effect.catch_cause(lambda cause: fail(PersistenceError(cause=cause.raisable())))
+
+
+class RunnerNotRegistered(S.TaggedError, identifier=f"{_ID}/RunnerNotRegistered"):
+    """The runner is not registered with runner storage."""
+
+    address: RunnerAddress
+
+
+class RunnerUnavailable(S.TaggedError, identifier=f"{_ID}/RunnerUnavailable"):
+    """The runner could not be reached."""
+
+    address: RunnerAddress
+
+
+class MailboxFull(S.TaggedError, identifier=f"{_ID}/MailboxFull"):
+    """The entity's mailbox is at ``entity_mailbox_capacity``."""
+
+    address: EntityAddress
+
+
+class AlreadyProcessingMessage(S.TaggedError, identifier=f"{_ID}/AlreadyProcessingMessage"):
+    """The entity is already processing the message with this id."""
+
+    envelope_id: Snowflake = S.field(SnowflakeFromString, key="envelopeId")
+    address: EntityAddress
+
+
+__all__ = [
+    "AlreadyProcessingMessage",
+    "EntityNotAssignedToRunner",
+    "MailboxFull",
+    "MalformedMessage",
+    "PersistenceError",
+    "RunnerNotRegistered",
+    "RunnerUnavailable",
+]