pyfect-dev 0.1.0__py3-none-any.whl

pyfect/cli/terminal_capabilities.py

@@ -0,0 +1,118 @@
+"""What the terminal running the CLI can do (Effect's
+``TerminalCapabilities``): color output, animation and interactive prompts.
+
+The ``TerminalCapabilities`` reference defaults to the process's own
+streams and environment; ``layer`` reads them through ``Stdio`` and the
+current ``ConfigProvider`` instead, so tests can pick them::
+
+    caps = yield from TerminalCapabilities()
+    if caps.can_prompt: ...
+
+``TERM=dumb`` turns everything off; ``NO_COLOR`` (non-empty) turns color off;
+``FORCE_COLOR`` (non-empty, not ``0`` or ``false``) turns it on without a
+terminal.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from dataclasses import dataclass
+from typing import IO, Any, Never
+
+from pyfect import config as config_
+from pyfect.context import Reference
+from pyfect.effect import Effect, gen
+from pyfect.layer import Layer
+from pyfect.layer import effect as layer_effect
+from pyfect.option import Option, Some
+from pyfect.stdio import Stdio
+
+
+@dataclass(frozen=True, slots=True)
+class Capabilities:
+    can_color: bool
+    can_animate: bool
+    can_prompt: bool
+
+
+def _present(value: str | None) -> bool:
+    return value is not None and len(value) > 0
+
+
+def _forcing(value: str | None) -> bool:
+    return _present(value) and value != "0" and (value or "").lower() != "false"
+
+
+def detect(
+    *,
+    no_color: str | None,
+    force_color: str | None,
+    term: str | None,
+    stdin_is_terminal: bool,
+    stdout_is_terminal: bool,
+) -> Capabilities:
+    if term == "dumb":
+        return Capabilities(can_color=False, can_animate=False, can_prompt=False)
+    return Capabilities(
+        can_color=not _present(no_color) and (_forcing(force_color) or stdout_is_terminal),
+        can_animate=stdout_is_terminal,
+        can_prompt=stdin_is_terminal and stdout_is_terminal,
+    )
+
+
+def _isatty(stream: IO[Any] | None) -> bool:
+    try:
+        return stream is not None and stream.isatty()
+    except ValueError:
+        return False
+
+
+def from_process() -> Capabilities:
+    """The capabilities of this process's standard streams and environment."""
+    return detect(
+        no_color=os.environ.get("NO_COLOR"),
+        force_color=os.environ.get("FORCE_COLOR"),
+        term=os.environ.get("TERM"),
+        stdin_is_terminal=_isatty(sys.stdin),
+        stdout_is_terminal=_isatty(sys.stdout),
+    )
+
+
+class TerminalCapabilities(Reference[Capabilities]):
+    """The capabilities in use; defaults to ``from_process()``."""
+
+    @classmethod
+    def default(cls) -> Capabilities:
+        return from_process()
+
+
+def _value(option: Option[str]) -> str | None:
+    return option.value if isinstance(option, Some) else None
+
+
+def _make() -> Effect[Capabilities, Never, Stdio]:
+    def body():
+        stdio = yield from Stdio()
+        env = yield from config_.all(
+            config_.option(config_.string("NO_COLOR")),
+            config_.option(config_.string("FORCE_COLOR")),
+            config_.option(config_.string("TERM")),
+        ).or_die()
+        no_color, force_color, term = env
+        return detect(
+            no_color=_value(no_color),
+            force_color=_value(force_color),
+            term=_value(term),
+            stdin_is_terminal=(yield from stdio.stdin_is_terminal),
+            stdout_is_terminal=(yield from stdio.stdout_is_terminal),
+        )
+
+    return gen(body)
+
+
+layer: Layer[Never, Never, Stdio] = layer_effect(TerminalCapabilities, _make())
+"""Capabilities read from ``Stdio`` and the ``NO_COLOR``, ``FORCE_COLOR`` and
+``TERM`` config keys."""
+
+__all__ = ["Capabilities", "TerminalCapabilities", "detect", "from_process", "layer"]

pyfect/clock.py

@@ -0,0 +1,150 @@
+"""Time. ``Clock`` is a reference, so effects can sleep without declaring
+it, and tests can swap in ``pyfect.testing.TestClock``.
+
+Wall time (``now``) can jump, backwards too; monotonic time never does,
+so measure elapsed time with it. Each comes in float seconds and in whole
+nanoseconds (Effect's ``currentTimeNanos`` / ``monotonicTimeNanos``)."""
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+import math
+import time
+from collections.abc import Callable
+from typing import Any, Never, Protocol
+
+from pyfect._core import Effect, _Callback, _Gen, _Succeed, _Sync, _WithFiber, _YieldNow
+from pyfect._effect_args import require_effect
+from pyfect._fiber import Fiber
+from pyfect.context import Reference
+from pyfect.duration import Duration, Seconds, to_seconds
+from pyfect.effect import never
+
+
+class ClockService(Protocol):
+    def now_unsafe(self) -> float:
+        """Wall-clock time, seconds since the epoch."""
+        ...
+
+    def monotonic_unsafe(self) -> float:
+        """A clock that never goes backwards, in seconds."""
+        ...
+
+    def now_nanos_unsafe(self) -> int:
+        """Wall-clock time, whole nanoseconds since the epoch."""
+        ...
+
+    def monotonic_nanos_unsafe(self) -> int:
+        """Monotonic time in whole nanoseconds."""
+        ...
+
+    def sleep(self, seconds: Seconds) -> Effect[None]: ...
+
+
+_NANOS = 1_000_000_000
+
+
+class LiveClock:
+    """The system clock. A sleep of zero or less yields once; an infinite
+    sleep never wakes and holds no timer (as Effect's)."""
+
+    def now_unsafe(self) -> float:
+        return time.time_ns() / _NANOS
+
+    def monotonic_unsafe(self) -> float:
+        return time.monotonic_ns() / _NANOS
+
+    def now_nanos_unsafe(self) -> int:
+        return time.time_ns()
+
+    def monotonic_nanos_unsafe(self) -> int:
+        return time.monotonic_ns()
+
+    def sleep(self, seconds: Seconds) -> Effect[None]:
+        if seconds <= 0:
+            return _YieldNow()
+        if math.isinf(seconds):
+            return never
+
+        def body(fiber: Fiber[Any, Any]) -> Effect[None]:
+            def register(resume: Callable[[Effect[None]], None]) -> Effect[None]:
+                cancel = fiber._scheduler.call_later(seconds, lambda: resume(_Succeed(None)))
+                return _Sync(cancel)
+
+            return _Callback(register)
+
+        return _WithFiber(body)
+
+
+class Clock(Reference[ClockService]):
+    @classmethod
+    def default(cls) -> ClockService:
+        return LiveClock()
+
+
+def sleep(duration: Duration) -> Effect[None]:
+    seconds = to_seconds(duration)
+    return Clock().flat_map(lambda c: c.sleep(seconds))
+
+
+def now() -> Effect[float]:
+    """Wall-clock time, seconds since the epoch."""
+    return Clock().map(lambda c: c.now_unsafe())
+
+
+def monotonic() -> Effect[float]:
+    """Monotonic time in seconds."""
+    return Clock().map(lambda c: c.monotonic_unsafe())
+
+
+def now_millis_unsafe(clock: ClockService) -> int:
+    """``clock``'s wall-clock time in whole milliseconds since the epoch
+    (Effect's ``currentTimeMillisUnsafe``), floored from its nanoseconds:
+    ``now_unsafe() * 1000`` floors 1.001 s to 1000 ms."""
+    return clock.now_nanos_unsafe() // 1_000_000
+
+
+def now_nanos() -> Effect[int]:
+    """Wall-clock time, whole nanoseconds since the epoch."""
+    return Clock().map(lambda c: c.now_nanos_unsafe())
+
+
+def monotonic_nanos() -> Effect[int]:
+    """Monotonic time in whole nanoseconds."""
+    return Clock().map(lambda c: c.monotonic_nanos_unsafe())
+
+
+def clock_with[A, E = Never, R = Never](
+    f: Callable[[ClockService], Effect[A, E, R]],
+) -> Effect[A, E, R]:
+    """Run ``f`` with the current clock."""
+    return Clock().flat_map(f)
+
+
+def timed[A, E, R](eff: Effect[A, E, R]) -> Effect[tuple[float, A], E, R]:
+    """Pair ``eff``'s value with how long it took in seconds, measured in
+    whole monotonic nanoseconds."""
+    require_effect(eff, "eff")
+
+    def body():
+        start = yield from monotonic_nanos()
+        value = yield from eff
+        end = yield from monotonic_nanos()
+        return ((end - start) / _NANOS, value)
+
+    return _Gen(body)
+
+
+__all__ = [
+    "Clock",
+    "ClockService",
+    "LiveClock",
+    "clock_with",
+    "monotonic",
+    "monotonic_nanos",
+    "now",
+    "now_millis_unsafe",
+    "now_nanos",
+    "sleep",
+    "timed",
+]

pyfect/cluster/__init__.py

@@ -0,0 +1,155 @@
+"""Cluster sharding (Effect's ``effect/cluster``): shard, entity and
+runner addresses, snowflake ids, the sharding configuration, cluster
+errors and RPC annotations, envelopes, messages and replies, message and
+runner storage (in memory and in SQL), the Kubernetes API client,
+entities and their resources, singletons, the runner protocol
+(``Runners``, ``RunnerServer``, ``RunnerHealth``), ``Sharding`` itself,
+one-runner setups (``test_runner``, ``single_runner``), entity proxies
+(``entity_proxy``, ``entity_proxy_server``), cron jobs run once across the
+cluster (``cluster_cron``), durable workflows on cluster entities
+(``cluster_workflow_engine``) and runners served over sockets or HTTP
+(``socket_runner``, ``http_runner``). The ready-made runner layers of
+Effect's platform-node, ``cluster_socket`` and ``cluster_http``, are
+imported on their own (``from pyfect.cluster import cluster_socket``)."""
+
+from pyfect.cluster import (
+    cluster_cron,
+    cluster_error,
+    cluster_metrics,
+    cluster_schema,
+    cluster_workflow_engine,
+    deliver_at,
+    entity,
+    entity_address,
+    entity_id,
+    entity_proxy,
+    entity_proxy_server,
+    entity_resource,
+    entity_type,
+    envelope,
+    http_runner,
+    k8s_http_client,
+    k8s_types,
+    machine_id,
+    message,
+    message_storage,
+    reply,
+    runner,
+    runner_address,
+    runner_health,
+    runner_server,
+    runner_storage,
+    runners,
+    shard_id,
+    sharding,
+    sharding_config,
+    sharding_registration_event,
+    single_runner,
+    singleton,
+    singleton_address,
+    snowflake,
+    socket_runner,
+    sql_message_storage,
+    sql_runner_storage,
+    test_runner,
+)
+from pyfect.cluster.cluster_error import (
+    AlreadyProcessingMessage,
+    EntityNotAssignedToRunner,
+    MailboxFull,
+    MalformedMessage,
+    PersistenceError,
+    RunnerNotRegistered,
+    RunnerUnavailable,
+)
+from pyfect.cluster.deliver_at import DeliverAt
+from pyfect.cluster.entity import Entity
+from pyfect.cluster.entity_address import EntityAddress
+from pyfect.cluster.entity_id import EntityId
+from pyfect.cluster.entity_type import EntityType
+from pyfect.cluster.k8s_http_client import K8sHttpClient
+from pyfect.cluster.machine_id import MachineId
+from pyfect.cluster.message_storage import MessageStorage, MessageStorageService
+from pyfect.cluster.runner import Runner
+from pyfect.cluster.runner_address import RunnerAddress
+from pyfect.cluster.runner_health import RunnerHealth, RunnerHealthService
+from pyfect.cluster.runner_storage import RunnerStorage, RunnerStorageService
+from pyfect.cluster.runners import Runners, RunnersService
+from pyfect.cluster.shard_id import ShardId
+from pyfect.cluster.sharding import Sharding, ShardingService
+from pyfect.cluster.sharding_config import ShardingConfig, ShardingConfigService
+from pyfect.cluster.singleton_address import SingletonAddress
+from pyfect.cluster.snowflake import Snowflake
+
+__all__ = [
+    "AlreadyProcessingMessage",
+    "DeliverAt",
+    "Entity",
+    "EntityAddress",
+    "EntityId",
+    "EntityNotAssignedToRunner",
+    "EntityType",
+    "K8sHttpClient",
+    "MachineId",
+    "MailboxFull",
+    "MalformedMessage",
+    "MessageStorage",
+    "MessageStorageService",
+    "PersistenceError",
+    "Runner",
+    "RunnerAddress",
+    "RunnerHealth",
+    "RunnerHealthService",
+    "RunnerNotRegistered",
+    "RunnerStorage",
+    "RunnerStorageService",
+    "RunnerUnavailable",
+    "Runners",
+    "RunnersService",
+    "ShardId",
+    "Sharding",
+    "ShardingConfig",
+    "ShardingConfigService",
+    "ShardingService",
+    "SingletonAddress",
+    "Snowflake",
+    "cluster_cron",
+    "cluster_error",
+    "cluster_metrics",
+    "cluster_schema",
+    "cluster_workflow_engine",
+    "deliver_at",
+    "entity",
+    "entity_address",
+    "entity_id",
+    "entity_proxy",
+    "entity_proxy_server",
+    "entity_resource",
+    "entity_type",
+    "envelope",
+    "http_runner",
+    "k8s_http_client",
+    "k8s_types",
+    "machine_id",
+    "message",
+    "message_storage",
+    "reply",
+    "runner",
+    "runner_address",
+    "runner_health",
+    "runner_server",
+    "runner_storage",
+    "runners",
+    "shard_id",
+    "sharding",
+    "sharding_config",
+    "sharding_registration_event",
+    "single_runner",
+    "singleton",
+    "singleton_address",
+    "snowflake",
+    "socket_runner",
+    "sql_message_storage",
+    "sql_runner_storage",
+    "test_runner",
+]

pyfect/cluster/_cluster_abandon.py

@@ -0,0 +1,36 @@
+"""Abandoning a request (Effect's ``internal/clusterAbandon``): the caller
+is interrupted with the ``Abandon`` annotation, so the request is left for
+the next owner instead of being answered."""
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any, Never
+
+from pyfect._core import Effect
+from pyfect._fiber import Fiber
+from pyfect.cause import Cause, Interrupt
+from pyfect.cluster.cluster_schema import ABANDON_ANNOTATION, Abandon
+from pyfect.effect import callback, current_fiber
+
+
+def _abandon(fiber: Fiber[Any, Any]) -> Effect[Never]:
+    def register(_resume: Callable[[Effect[Never]], None]) -> None:
+        fiber.interrupt_as_unsafe(fiber.id, ABANDON_ANNOTATION)
+
+    return callback(register).interruptible()
+
+
+interrupt: Effect[Never] = current_fiber().flat_map(_abandon)
+"""Interrupt the current fiber as abandoned."""
+
+
+def is_cause(cause: Cause[Any]) -> bool:
+    """Whether ``cause`` holds an abandoning interruption."""
+    return any(
+        isinstance(reason, Interrupt) and Abandon in reason.annotations for reason in cause.reasons
+    )
+
+
+__all__ = ["interrupt", "is_cause"]

pyfect/cluster/_cluster_layer.py

@@ -0,0 +1,122 @@
+"""What ``cluster_socket.layer`` and ``cluster_http.layer`` share (the
+second half of Effect's ``NodeClusterSocket.layer`` /
+``NodeClusterHttp.layer``): runner health, storage, settings from the
+environment and the serialization around a transport layer."""
+
+from __future__ import annotations
+
+import ssl
+from collections.abc import Callable
+from typing import Any, Literal
+
+from pyfect import file_system
+from pyfect import layer as layer_
+from pyfect.cluster import (
+    k8s_http_client,
+    runner_health,
+    runner_storage,
+    runners,
+    sql_message_storage,
+    sql_runner_storage,
+)
+from pyfect.cluster import message_storage as message_storage_
+from pyfect.cluster import sharding_config as sharding_config_
+from pyfect.cluster.k8s_http_client import K8sHttpClient
+from pyfect.cluster.runners import RpcClientProtocol
+from pyfect.cluster.sharding_config import ShardingConfigService
+from pyfect.effect import gen, succeed
+from pyfect.file_system import FileSystem
+from pyfect.http import httpx_client
+from pyfect.layer import Layer
+from pyfect.rpc import serialization
+from pyfect.rpc.serialization import MaxBufferSize, RpcSerialization
+
+type Storage = Literal["local", "sql", "byo"]
+type Health = Literal["ping", "k8s"]
+type SerializationKind = Literal["binary", "ndjson"]
+type ShardingConfigUpdate = Callable[[ShardingConfigService], ShardingConfigService]
+
+_K8S_CA = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+
+
+@gen
+def _k8s_http_client():
+    fs = yield from FileSystem()
+    no_ca: str | None = None
+    ca = yield from fs.read_file_string(_K8S_CA).catch_all(lambda _: succeed(no_ca))
+    if ca is None:
+        return httpx_client.layer
+    return httpx_client.layer_with(verify=ssl.create_default_context(cadata=ca))
+
+
+layer_k8s_http_client: Layer[K8sHttpClient] = k8s_http_client.layer.pipe(
+    layer_.provide(layer_.unwrap(_k8s_http_client))
+).pipe(layer_.provide(file_system.layer))
+"""The ``K8sHttpClient`` of a pod: httpx trusting the service account's CA
+when it is mounted (Effect's ``layerK8sHttpClient``)."""
+
+
+def runner_health_layer(
+    *,
+    client_protocol: Layer[RpcClientProtocol, Any, Any],
+    health: Health,
+    namespace: str | None,
+    label_selector: str | None,
+) -> Layer[Any, Any, Any]:
+    if health == "k8s":
+        return runner_health.layer_k8s(namespace=namespace, label_selector=label_selector).pipe(
+            layer_.provide(layer_k8s_http_client)
+        )
+    return runner_health.layer_ping.pipe(layer_.provide(runners.layer_rpc)).pipe(
+        layer_.provide(client_protocol)
+    )
+
+
+def assemble(
+    transport: Layer[Any, Any, Any],
+    *,
+    health: Layer[Any, Any, Any] | None,
+    storage: Storage,
+    sharding_config: ShardingConfigUpdate | None,
+    serialization_kind: SerializationKind,
+    serialization_max_buffer_size: MaxBufferSize,
+) -> Layer[Any, Any, Any]:
+    """``transport`` (a ``Sharding | Runners`` layer) with its runner
+    health (``None`` for a client), storage, settings and serialization."""
+    with_health = transport if health is None else transport.pipe(layer_.provide(health))
+    messages: Layer[Any, Any, Any] = (
+        message_storage_.layer_noop
+        if storage == "local"
+        else layer_.empty
+        if storage == "byo"
+        else sql_message_storage.layer
+    )
+    locks: Layer[Any, Any, Any] = (
+        runner_storage.layer_memory
+        if storage == "local"
+        else layer_.empty
+        if storage == "byo"
+        else sql_runner_storage.layer.or_die()
+    )
+    wire: Layer[RpcSerialization] = (
+        serialization.layer_ndjson_with(max_buffer_size=serialization_max_buffer_size)
+        if serialization_kind == "ndjson"
+        else serialization.layer_schema_binary(max_frame_size=serialization_max_buffer_size)
+    )
+    return (
+        with_health.pipe(layer_.provide_merge(messages))
+        .pipe(layer_.provide(locks))
+        .pipe(layer_.provide(sharding_config_.layer_from_env(sharding_config)))
+        .pipe(layer_.provide(wire))
+    )
+
+
+__all__ = [
+    "Health",
+    "SerializationKind",
+    "ShardingConfigUpdate",
+    "Storage",
+    "assemble",
+    "layer_k8s_http_client",
+    "runner_health_layer",
+]

pyfect/cluster/_entity_activation.py

@@ -0,0 +1,16 @@
+"""The scope of an entity's activation (Effect's
+``internal/entityActivation``): closed when the entity is torn down."""
+
+from __future__ import annotations
+
+from pyfect.context import Tag
+from pyfect.scope import CloseableScope
+
+
+class CurrentActivationScope(
+    Tag[CloseableScope], key="effect/cluster/internal/CurrentActivationScope"
+):
+    """The scope of the running entity's activation."""
+
+
+__all__ = ["CurrentActivationScope"]

pyfect/cluster/_entity_manager/__init__.py

@@ -0,0 +1,22 @@
+"""An entity type's entities on this runner (Effect's
+``internal/entityManager``)."""
+
+from pyfect.cluster._entity_manager._manager import DEFAULT_RETRY_POLICY, EntityManager, make
+from pyfect.cluster._entity_manager._state import (
+    AbandonUndelivered,
+    EntityState,
+    ManagerOptions,
+    RequestEntry,
+    Residency,
+)
+
+__all__ = [
+    "DEFAULT_RETRY_POLICY",
+    "AbandonUndelivered",
+    "EntityManager",
+    "EntityState",
+    "ManagerOptions",
+    "RequestEntry",
+    "Residency",
+    "make",
+]