pyfect-dev 0.1.0__py3-none-any.whl

pyfect/persistence/persistence/_kvs.py

@@ -0,0 +1,135 @@
+"""``layer_backing_kvs``: stores in a ``KeyValueStore``."""
+
+from __future__ import annotations
+
+from collections.abc import Generator
+from typing import Any, Never, cast
+
+from pyfect import _json_text
+from pyfect._concurrency import for_each
+from pyfect._core import Effect
+from pyfect.clock import Clock, ClockService
+from pyfect.duration import Seconds
+from pyfect.effect import fail, gen, succeed, suspend
+from pyfect.layer import Layer
+from pyfect.layer import effect as layer_effect
+from pyfect.persistence._error import PersistenceError
+from pyfect.persistence.key_value_store import (
+    KeyValueStore,
+    KeyValueStoreError,
+    KeyValueStoreService,
+)
+from pyfect.persistence.persistence._service import (
+    BackingEntry,
+    BackingPersistence,
+    BackingPersistenceService,
+    BackingPersistenceStore,
+    now_millis,
+    to_json,
+    unsafe_ttl_to_expires,
+)
+
+
+def _store(
+    backing: KeyValueStoreService, clock: ClockService, store_id: str
+) -> BackingPersistenceStore:
+    store = backing.prefix(store_id)
+
+    def get(key: str) -> Effect[object | None, PersistenceError]:
+        def parse(text: str | None) -> Effect[object | None, PersistenceError]:
+            if text is None:
+                return succeed(None)
+            try:
+                parsed: object = _json_text.parse(text)
+            except ValueError as cause:
+                return fail(
+                    PersistenceError(
+                        message=f"Failed to parse value for key {key} from backing store",
+                        cause=cause,
+                    )
+                )
+            if not isinstance(parsed, list):
+                return succeed(None)
+            items = cast("list[object]", parsed)
+            value = items[0] if items else None
+            expires = items[1] if len(items) > 1 else None
+            if isinstance(expires, int | float) and expires <= now_millis(clock):
+                return store.remove(key).ignore().as_(None)
+            return succeed(value)
+
+        return (
+            store.get(key)
+            .map_error(
+                lambda error: PersistenceError(
+                    message=f"Failed to get key {key} from backing store", cause=error
+                )
+            )
+            .flat_map(parse)
+        )
+
+    def write(key: str, value: object, ttl: Seconds | None) -> Effect[None, KeyValueStoreError]:
+        return store.set(key, to_json([value, unsafe_ttl_to_expires(clock, ttl)]))
+
+    def set_value(key: str, value: object, ttl: Seconds | None) -> Effect[None, PersistenceError]:
+        def attempt() -> Effect[None, PersistenceError]:
+            try:
+                written = write(key, value, ttl)
+            except (TypeError, ValueError) as cause:
+                return fail(
+                    PersistenceError(
+                        message=f"Failed to serialize value for key {key} to backing store",
+                        cause=cause,
+                    )
+                )
+            return written.map_error(
+                lambda error: PersistenceError(
+                    message=f"Failed to set key {key} in backing store", cause=error
+                )
+            )
+
+        return suspend(attempt)
+
+    def set_many(entries: list[BackingEntry]) -> Effect[None, PersistenceError]:
+        def write_entry(entry: BackingEntry) -> Effect[None, KeyValueStoreError]:
+            key, value, ttl = entry
+            return suspend(lambda: write(key, value, ttl))
+
+        return (
+            for_each(entries, write_entry, concurrency="unbounded")
+            .as_void()
+            .map_error(
+                lambda error: PersistenceError(
+                    message="Failed to setMany in backing store", cause=error
+                )
+            )
+        )
+
+    return BackingPersistenceStore(
+        get=get,
+        get_many=lambda keys: for_each(keys, get, concurrency="unbounded"),
+        set=set_value,
+        set_many=lambda entries: set_many(list(entries)),
+        remove=lambda key: store.remove(key).map_error(
+            lambda error: PersistenceError(
+                message=f"Failed to remove key {key} from backing store", cause=error
+            )
+        ),
+        clear=store.clear.map_error(
+            lambda error: PersistenceError(message="Failed to clear backing store", cause=error)
+        ),
+    )
+
+
+def _build() -> Generator[Effect[Any, Never, KeyValueStore], Any, BackingPersistenceService]:
+    backing = yield from KeyValueStore()
+    clock = yield from Clock()
+    return BackingPersistenceService(
+        lambda store_id: suspend(lambda: succeed(_store(backing, clock, store_id)))
+    )
+
+
+layer_backing_kvs: Layer[BackingPersistence, Never, KeyValueStore] = layer_effect(
+    BackingPersistence, gen(_build)
+)
+"""Stores in the ``KeyValueStore``, each under its id as a key prefix;
+an entry is written as the JSON ``[value, expires]``."""

pyfect/persistence/persistence/_memory.py

@@ -0,0 +1,62 @@
+"""``layer_backing_memory``: stores in process memory."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from pyfect.clock import ClockService, clock_with
+from pyfect.effect import succeed, sync
+from pyfect.layer import Layer
+from pyfect.layer import sync as layer_sync
+from pyfect.persistence.persistence._service import (
+    BackingEntry,
+    BackingPersistence,
+    BackingPersistenceService,
+    BackingPersistenceStore,
+    now_millis,
+    unsafe_ttl_to_expires,
+)
+
+type _Map = dict[str, tuple[object, int | None]]
+
+
+def make_backing_memory() -> BackingPersistenceService:
+    """A backend whose stores live in dicts; stores with the same id share
+    one. An expired entry is dropped when next read."""
+    stores: dict[str, _Map] = {}
+
+    def make_store(store_id: str, clock: ClockService) -> BackingPersistenceStore:
+        entries = stores.setdefault(store_id, {})
+
+        def read(key: str) -> object | None:
+            found = entries.get(key)
+            if found is None:
+                return None
+            value, expires = found
+            if expires is not None and expires <= now_millis(clock):
+                del entries[key]
+                return None
+            return value
+
+        def write_many(written: Sequence[BackingEntry]) -> None:
+            for key, value, ttl in written:
+                entries[key] = (value, unsafe_ttl_to_expires(clock, ttl))
+
+        return BackingPersistenceStore(
+            get=lambda key: sync(lambda: read(key)),
+            get_many=lambda keys: sync(lambda: [read(key) for key in keys]),
+            set=lambda key, value, ttl: sync(lambda: write_many(((key, value, ttl),))),
+            set_many=lambda written: sync(lambda: write_many(written)),
+            remove=lambda key: sync(lambda: entries.pop(key, None)).as_void(),
+            clear=sync(entries.clear),
+        )
+
+    return BackingPersistenceService(
+        lambda store_id: clock_with(lambda clock: succeed(make_store(store_id, clock)))
+    )
+
+
+layer_backing_memory: Layer[BackingPersistence] = layer_sync(
+    BackingPersistence, make_backing_memory
+)
+"""Stores in memory, fresh each time the layer is built."""

pyfect/persistence/persistence/_redis.py

@@ -0,0 +1,162 @@
+"""``layer_backing_redis``: stores in Redis."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Callable, Generator, Mapping, Sequence
+from typing import Any, Never
+
+from pyfect import _json_text
+from pyfect._core import Effect
+from pyfect.duration import Seconds, to_millis
+from pyfect.effect import fail, gen, succeed, suspend, void
+from pyfect.layer import Layer
+from pyfect.layer import effect as layer_effect
+from pyfect.persistence._error import PersistenceError
+from pyfect.persistence.persistence._service import (
+    BackingEntry,
+    BackingPersistence,
+    BackingPersistenceService,
+    BackingPersistenceStore,
+    to_json,
+)
+from pyfect.persistence.redis import Redis, RedisError, RedisReply, RedisService, script
+
+
+def _set_many_params(sets: Mapping[str, str], expires: Mapping[str, int]) -> list[str | float]:
+    return [*sets, *expires, len(sets), len(expires), *sets.values(), *expires.values()]
+
+
+def _set_many_keys(sets: Mapping[str, str], expires: Mapping[str, int]) -> int:
+    return len(sets) + len(expires)
+
+
+SET_MANY = script(
+    _set_many_params,
+    number_of_keys=_set_many_keys,
+    lua="""
+local num_sets = tonumber(ARGV[1])
+local num_expires = tonumber(ARGV[2])
+local index = 3
+
+for i = 1, num_sets do
+  local key = KEYS[i]
+  local value = ARGV[index]
+  redis.call("SET", key, value)
+  index = index + 1
+end
+
+for i = 1, num_expires do
+  local key = KEYS[num_sets + i]
+  local expire = tonumber(ARGV[index])
+  redis.call("PEXPIRE", key, expire)
+  index = index + 1
+end
+""",
+)
+"""Sets ``sets`` and gives each key of ``expires`` its time to live in
+milliseconds, in one step."""
+
+
+def _ttl_millis(ttl: Seconds) -> int:
+    return math.ceil(to_millis(ttl))
+
+
+def _store(redis: RedisService, prefix: str) -> BackingPersistenceStore:
+    set_many_script = redis.eval(SET_MANY)
+
+    def prefixed(key: str) -> str:
+        return f"{prefix}:{key}"
+
+    def failed(message: str) -> Callable[[RedisError], PersistenceError]:
+        return lambda error: PersistenceError(message=message, cause=error.cause)
+
+    def parse(reply: RedisReply) -> Effect[object | None, PersistenceError]:
+        if reply is None:
+            return succeed(None)
+        try:
+            return succeed(_json_text.parse(str(reply)))
+        except ValueError as cause:
+            return fail(PersistenceError(message="Failed to parse value from Redis", cause=cause))
+
+    def parse_many(keys: Sequence[str], reply: RedisReply) -> list[object | None]:
+        values = reply if isinstance(reply, list) else []
+        out: list[object | None] = []
+        for index in range(len(keys)):
+            value = values[index] if index < len(values) else None
+            try:
+                out.append(None if value is None else _json_text.parse(str(value)))
+            except ValueError:
+                out.append(None)
+        return out
+
+    def get(key: str) -> Effect[object | None, PersistenceError]:
+        return (
+            redis.send("GET", prefixed(key))
+            .map_error(failed(f"Failed to get key {key} from Redis"))
+            .flat_map(parse)
+        )
+
+    def get_many(keys: Sequence[str]) -> Effect[list[object | None], PersistenceError]:
+        return (
+            redis.send("mget", *map(prefixed, keys))
+            .map_error(failed("Failed to getMany from Redis"))
+            .map(lambda reply: parse_many(keys, reply))
+        )
+
+    def set_value(key: str, value: object, ttl: Seconds | None) -> Effect[None, PersistenceError]:
+        def run() -> Effect[RedisReply, RedisError]:
+            text = to_json(value)
+            if ttl is None:
+                return redis.send("SET", prefixed(key), text)
+            return redis.send("SET", prefixed(key), text, "PX", str(_ttl_millis(ttl)))
+
+        return suspend(run).as_void().map_error(failed(f"Failed to set key {key} in Redis"))
+
+    def set_many(entries: Sequence[BackingEntry]) -> Effect[None, PersistenceError]:
+        def run() -> Effect[None, RedisError]:
+            sets: dict[str, str] = {}
+            expires: dict[str, int] = {}
+            for key, value, ttl in entries:
+                prefixed_key = prefixed(key)
+                sets[prefixed_key] = to_json(value)
+                if ttl is not None:
+                    expires[prefixed_key] = _ttl_millis(ttl)
+            return set_many_script(sets, expires).as_void()
+
+        return suspend(run).map_error(failed("Failed to setMany in Redis"))
+
+    def delete_all(keys: RedisReply) -> Effect[object, RedisError]:
+        found = [str(key) for key in keys] if isinstance(keys, list) else []
+        return void if not found else redis.send("DEL", *found)
+
+    return BackingPersistenceStore(
+        get=get,
+        get_many=get_many,
+        set=set_value,
+        set_many=set_many,
+        remove=lambda key: (
+            redis.send("DEL", prefixed(key))
+            .as_void()
+            .map_error(failed(f"Failed to remove key {key} from Redis"))
+        ),
+        clear=redis.send("KEYS", f"{prefix}:*")
+        .flat_map(delete_all)
+        .as_void()
+        .map_error(failed("Failed to clear keys from Redis")),
+    )
+
+
+def _build() -> Generator[Effect[Any, Never, Redis], Any, BackingPersistenceService]:
+    redis = yield from Redis()
+    return BackingPersistenceService(
+        lambda store_id: suspend(lambda: succeed(_store(redis, store_id)))
+    )
+
+
+layer_backing_redis: Layer[BackingPersistence, Never, Redis] = layer_effect(
+    BackingPersistence, gen(_build)
+)
+"""Stores in Redis, each under its id as a key prefix (``id:key``); values
+are JSON, and a time to live is a Redis expiry in whole milliseconds,
+rounded up."""

pyfect/persistence/persistence/_service.py

@@ -0,0 +1,276 @@
+"""The ``Persistence`` service over a ``BackingPersistence`` (Effect's
+``Persistence.layer``)."""
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+import math
+from collections.abc import Callable, Generator, Iterable, Sequence
+from dataclasses import dataclass
+from typing import Any, Final, Never, Protocol
+
+from pyfect import _json_text
+from pyfect._core import Effect
+from pyfect._effect_args import require_effect
+from pyfect.clock import ClockService
+from pyfect.context import Tag
+from pyfect.duration import Seconds, to_millis, to_seconds
+from pyfect.effect import fail, gen, succeed, suspend, void
+from pyfect.exit import Exit, Failure
+from pyfect.layer import Layer
+from pyfect.layer import effect as layer_effect
+from pyfect.persistence._error import PersistenceError
+from pyfect.persistence.persistable import (
+    AnyPersistable,
+    Persistable,
+    TimeToLive,
+    deserialize_exit,
+    serialize_exit,
+)
+from pyfect.schema import SchemaError
+from pyfect.scope import CloseableScope, Scope
+
+# --- the backing store: JSON values by string key -----------------------------------
+
+
+type BackingEntry = tuple[str, object, Seconds | None]
+"""A key, its JSON value and how long to keep it (``None``: forever)."""
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class BackingPersistenceStore:
+    """One store of a backend: JSON values by key, each with an optional
+    time to live. ``get_many`` gets at least one key and answers each, in
+    order."""
+
+    get: Callable[[str], Effect[object | None, PersistenceError]]
+    get_many: Callable[[Sequence[str]], Effect[list[object | None], PersistenceError]]
+    set: Callable[[str, object, Seconds | None], Effect[None, PersistenceError]]
+    set_many: Callable[[Sequence[BackingEntry]], Effect[None, PersistenceError]]
+    remove: Callable[[str], Effect[None, PersistenceError]]
+    clear: Effect[None, PersistenceError]
+
+    def __post_init__(self) -> None:
+        require_effect(self.clear, "clear")
+
+
+@dataclass(frozen=True, slots=True)
+class BackingPersistenceService:
+    """A backend: ``make(store_id)`` opens one of its stores, kept apart
+    from the others."""
+
+    make: Callable[[str], Effect[BackingPersistenceStore, Never, Scope]]
+
+
+class BackingPersistence(
+    Tag[BackingPersistenceService], key="effect/persistence/BackingPersistence"
+):
+    """Where ``Persistence`` keeps what it stores."""
+
+
+# --- the typed store: request outcomes by primary key --------------------------------
+
+
+class PersistenceStore(Protocol):
+    """Request outcomes by the request's primary key, written by the
+    request's schemas. ``get`` gives ``None`` for an outcome never stored
+    (or expired) and fails with ``SchemaError`` for one the schemas no
+    longer read; ``get_many`` gives ``None`` for those and removes them."""
+
+    def get[A, E, R](
+        self, key: Persistable[A, E, R]
+    ) -> Effect[Exit[A, E] | None, PersistenceError | SchemaError, R]: ...
+
+    def get_many[A, E, R](
+        self, keys: Iterable[Persistable[A, E, R]]
+    ) -> Effect[list[Exit[A, E] | None], PersistenceError | SchemaError, R]: ...
+
+    def set[A, E, R](
+        self, key: Persistable[A, E, R], value: Exit[A, E]
+    ) -> Effect[None, PersistenceError | SchemaError, R]: ...
+
+    def set_many[A, E, R](
+        self, entries: Iterable[tuple[Persistable[A, E, R], Exit[A, E]]]
+    ) -> Effect[None, PersistenceError | SchemaError, R]: ...
+
+    def remove(self, key: AnyPersistable) -> Effect[None, PersistenceError]: ...
+
+    @property
+    def clear(self) -> Effect[None, PersistenceError]: ...
+
+
+class PersistenceService(Protocol):
+    def make(
+        self,
+        *,
+        store_id: str,
+        time_to_live: TimeToLive[Any, Any, Any] | None = None,
+    ) -> Effect[PersistenceStore, Never, Scope]:
+        """Open the store ``store_id``. ``time_to_live`` says how long each
+        outcome is kept (default: forever); one of zero or less is not
+        written."""
+        ...
+
+
+class Persistence(Tag[PersistenceService], key="effect/persistence/Persistence"):
+    """Stores request outcomes outside the process."""
+
+
+class _Store:
+    __slots__ = ("_scope", "_storage", "_time_to_live")
+
+    def __init__(
+        self,
+        storage: BackingPersistenceStore,
+        time_to_live: TimeToLive[Any, Any, Any] | None,
+        scope: CloseableScope,
+    ) -> None:
+        self._storage = storage
+        self._time_to_live = time_to_live
+        self._scope = scope
+
+    def _ttl(self, value: Exit[Any, Any], key: AnyPersistable) -> Seconds:
+        if self._time_to_live is None:
+            return _FOREVER
+        return to_seconds(self._time_to_live(value, key))
+
+    def get[A, E, R](
+        self, key: Persistable[A, E, R]
+    ) -> Effect[Exit[A, E] | None, PersistenceError | SchemaError, R]:
+        def decoded(found: object | None) -> Effect[Exit[A, E] | None, SchemaError, R]:
+            return succeed(None) if found is None else deserialize_exit(key, found)
+
+        return self._storage.get(key.primary_key()).flat_map(decoded)
+
+    def get_many[A, E, R](
+        self, keys: Iterable[Persistable[A, E, R]]
+    ) -> Effect[list[Exit[A, E] | None], PersistenceError | SchemaError, R]:
+        requests = list(keys)
+        primary_keys = [key.primary_key() for key in requests]
+        storage = self._storage
+        scope = self._scope
+
+        def body() -> Generator[Effect[Any, Any, R], Any, list[Exit[A, E] | None]]:
+            if not primary_keys:
+                return []
+            results = yield from storage.get_many(primary_keys)
+            if len(results) != len(primary_keys):
+                return (
+                    yield from fail(
+                        PersistenceError(
+                            message=f"Expected {len(primary_keys)} results but got "
+                            f"{len(results)} from backing store"
+                        )
+                    )
+                )
+            out: list[Exit[A, E] | None] = []
+            for request, primary_key, result in zip(requests, primary_keys, results, strict=True):
+                if result is None:
+                    out.append(None)
+                    continue
+                decoded = yield from deserialize_exit(request, result).exit()
+                if isinstance(decoded, Failure):
+                    out.append(None)
+                    yield from storage.remove(primary_key).fork_in(scope)
+                    continue
+                out.append(decoded.value)
+            return out
+
+        return gen(body)
+
+    def set[A, E, R](
+        self, key: Persistable[A, E, R], value: Exit[A, E]
+    ) -> Effect[None, PersistenceError | SchemaError, R]:
+        storage = self._storage
+
+        def write() -> Effect[None, PersistenceError | SchemaError, R]:
+            ttl = self._ttl(value, key)
+            if ttl <= 0:
+                return void
+            return serialize_exit(key, value).flat_map(
+                lambda encoded: storage.set(key.primary_key(), encoded, _finite(ttl))
+            )
+
+        return suspend(write)
+
+    def set_many[A, E, R](
+        self, entries: Iterable[tuple[Persistable[A, E, R], Exit[A, E]]]
+    ) -> Effect[None, PersistenceError | SchemaError, R]:
+        pairs = list(entries)
+        storage = self._storage
+
+        def body() -> Generator[Effect[Any, Any, R], Any]:
+            encoded: list[BackingEntry] = []
+            for key, value in pairs:
+                ttl = self._ttl(value, key)
+                if ttl <= 0:
+                    continue
+                json_value = yield from serialize_exit(key, value)
+                encoded.append((key.primary_key(), json_value, _finite(ttl)))
+            if encoded:
+                yield from storage.set_many(encoded)
+
+        return gen(body)
+
+    def remove(self, key: AnyPersistable) -> Effect[None, PersistenceError]:
+        return self._storage.remove(key.primary_key())
+
+    @property
+    def clear(self) -> Effect[None, PersistenceError]:
+        return self._storage.clear
+
+
+_FOREVER: Final = Seconds(math.inf)
+
+
+def _finite(ttl: Seconds) -> Seconds | None:
+    return None if math.isinf(ttl) else ttl
+
+
+class _Persistence:
+    __slots__ = ("_backing", "_scope")
+
+    def __init__(self, backing: BackingPersistenceService, scope: CloseableScope) -> None:
+        self._backing = backing
+        self._scope = scope
+
+    def make(
+        self,
+        *,
+        store_id: str,
+        time_to_live: TimeToLive[Any, Any, Any] | None = None,
+    ) -> Effect[PersistenceStore, Never, Scope]:
+        scope = self._scope
+        return self._backing.make(store_id).map(
+            lambda storage: _Store(storage, time_to_live, scope)
+        )
+
+
+def _build() -> Generator[Effect[Any, Never, Any], Any, PersistenceService]:
+    backing = yield from BackingPersistence()
+    scope = yield from Scope()
+    return _Persistence(backing, scope)
+
+
+layer: Layer[Persistence, Never, BackingPersistence] = layer_effect(Persistence, gen(_build))
+"""``Persistence`` writing through the ``BackingPersistence``."""
+
+
+# --- helpers for backends -----------------------------------------------------------
+
+
+def unsafe_ttl_to_expires(clock: ClockService, ttl: Seconds | None) -> int | None:
+    """When an entry written now with ``ttl`` expires, in epoch
+    milliseconds (``None``: never)."""
+    if ttl is None:
+        return None
+    return clock.now_nanos_unsafe() // 1_000_000 + math.ceil(to_millis(ttl))
+
+
+def now_millis(clock: ClockService) -> int:
+    return clock.now_nanos_unsafe() // 1_000_000
+
+
+def to_json(value: object) -> str:
+    """``JSON.stringify``: compact, non-ASCII kept."""
+    return _json_text.stringify(value)