pyfect-dev 0.1.0__py3-none-any.whl

pyfect/persistence/key_value_store/_sql.py

@@ -0,0 +1,197 @@
+"""``layer_sql``: a key-value store in a SQL table."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Generator
+from typing import Any, Final, Never
+
+from pyfect._core import Effect
+from pyfect._js import text_decoder, text_encode
+from pyfect.effect import fail, gen, succeed
+from pyfect.layer import Layer
+from pyfect.layer import effect as layer_effect
+from pyfect.persistence._columns import column_bytes, column_int
+from pyfect.persistence._sql_keys import MSSQL_KEY, MYSQL_KEY, key_equals, mssql_match
+from pyfect.persistence.key_value_store._store import (
+    KeyValueStore,
+    KeyValueStoreError,
+    KeyValueStoreService,
+    Value,
+    base64_encode,
+    make,
+)
+from pyfect.sql.client import SqlClient, SqlClientService
+from pyfect.sql.connection import Row
+from pyfect.sql.error import SqlError
+from pyfect.sql.statement import Identifier, Statement
+
+_STRING: Final = 0
+_BYTES: Final = 1
+
+
+def layer_sql(*, table: str = "effect_key_value_store") -> Layer[KeyValueStore, Never, SqlClient]:
+    """A store in ``table`` (made if missing) of the ``SqlClient``: one row
+    per key, the value as bytes and whether it was text."""
+    return layer_effect(KeyValueStore, gen(lambda: _build(table)))
+
+
+def _build(table_name: str) -> Generator[Effect[Any, Any, SqlClient], Any, KeyValueStoreService]:
+    sql = (yield from SqlClient()).without_transforms()
+    table = sql.identifier(table_name)
+    yield from sql.on_dialect_or_else(
+        mysql=lambda: sql(
+            f"CREATE TABLE IF NOT EXISTS {{}} (id {MYSQL_KEY} PRIMARY KEY, value BLOB NOT NULL, "
+            "value_type SMALLINT NOT NULL)",
+            table,
+        ),
+        pg=lambda: sql(
+            "CREATE TABLE IF NOT EXISTS {} (id TEXT PRIMARY KEY, value BYTEA NOT NULL, "
+            "value_type SMALLINT NOT NULL)",
+            table,
+        ),
+        mssql=lambda: sql(
+            "IF NOT EXISTS (SELECT * FROM sysobjects WHERE name={} AND xtype='U') "
+            f"CREATE TABLE {{}} (id {MSSQL_KEY} PRIMARY KEY, value VARBINARY(MAX) NOT NULL, "
+            "value_type SMALLINT NOT NULL)",
+            table_name,
+            table,
+        ),
+        or_else=lambda: sql(
+            "CREATE TABLE IF NOT EXISTS {} (id TEXT PRIMARY KEY, value BLOB NOT NULL, "
+            "value_type INTEGER NOT NULL)",
+            table,
+        ),
+    ).or_die()
+    upsert = _upsert(sql, table)
+
+    def failed(
+        method: str, key: str | None, message: str
+    ) -> Callable[[SqlError], KeyValueStoreError]:
+        return lambda cause: KeyValueStoreError(message, method, key, cause)
+
+    def select(method: str, key: str) -> Effect[Row | None, KeyValueStoreError]:
+        return (
+            sql("SELECT value, value_type FROM {} WHERE {}", table, key_equals(sql, "id", key))
+            .map(lambda rows: rows[0] if rows else None)
+            .map_error(failed(method, key, f"Unable to get item with key {key}"))
+        )
+
+    def invalid(method: str, key: str, row: Row) -> Effect[Any, KeyValueStoreError]:
+        return fail(
+            KeyValueStoreError(
+                f"Invalid stored value type for key {key}: {row['value_type']}", method, key
+            )
+        )
+
+    def get(key: str) -> Effect[str | None, KeyValueStoreError]:
+        def read(row: Row | None) -> Effect[str | None, KeyValueStoreError]:
+            if row is None:
+                return succeed(None)
+            data = column_bytes(row["value"])
+            match row["value_type"]:
+                case 0:
+                    return succeed(text_decoder().decode(data, final=True))
+                case 1:
+                    return succeed(base64_encode(data))
+                case _:
+                    return invalid("get", key, row)
+
+        return select("get", key).flat_map(read)
+
+    def get_bytes(key: str) -> Effect[bytes | None, KeyValueStoreError]:
+        def read(row: Row | None) -> Effect[bytes | None, KeyValueStoreError]:
+            if row is None:
+                return succeed(None)
+            if row["value_type"] in (_STRING, _BYTES):
+                return succeed(column_bytes(row["value"]))
+            return invalid("get_bytes", key, row)
+
+        return select("get_bytes", key).flat_map(read)
+
+    def set_value(key: str, value: Value) -> Effect[None, KeyValueStoreError]:
+        is_text = isinstance(value, str)
+        data = text_encode(value) if isinstance(value, str) else value
+        return (
+            upsert(key, data, _STRING if is_text else _BYTES)
+            .map_error(failed("set", key, f"Unable to set item with key {key}"))
+            .as_void()
+        )
+
+    def remove(key: str) -> Effect[None, KeyValueStoreError]:
+        return (
+            sql("DELETE FROM {} WHERE {}", table, key_equals(sql, "id", key))
+            .map_error(failed("remove", key, f"Unable to remove item with key {key}"))
+            .as_void()
+        )
+
+    clear = (
+        sql("DELETE FROM {}", table)
+        .map_error(failed("clear", None, "Unable to clear storage"))
+        .as_void()
+    )
+    size = (
+        sql("SELECT COUNT(*) AS count FROM {}", table)
+        .map_error(failed("size", None, "Unable to get size"))
+        .map(lambda rows: column_int(rows[0]["count"]) if rows else 0)
+    )
+    return make(get=get, get_bytes=get_bytes, set=set_value, remove=remove, clear=clear, size=size)
+
+
+type _Upsert = Callable[[str, bytes, int], Effect[object, SqlError]]
+
+
+def _upsert(sql: SqlClientService, table: Identifier) -> _Upsert:
+    def pg() -> _Upsert:
+        return lambda key, value, value_type: _unprepared(
+            sql(
+                "INSERT INTO {} (id, value, value_type) VALUES ({}, {}, {}) "
+                "ON CONFLICT (id) DO UPDATE SET value=EXCLUDED.value, "
+                "value_type=EXCLUDED.value_type",
+                table,
+                key,
+                value,
+                value_type,
+            )
+        )
+
+    def mysql() -> _Upsert:
+        return lambda key, value, value_type: sql(
+            "INSERT INTO {} (id, value, value_type) VALUES ({}, {}, {}) "
+            "ON DUPLICATE KEY UPDATE value=VALUES(value), value_type=VALUES(value_type)",
+            table,
+            key,
+            value,
+            value_type,
+        )
+
+    def mssql() -> _Upsert:
+        return lambda key, value, value_type: sql(
+            "MERGE {} AS target USING (SELECT {} AS id, {} AS value, {} AS value_type) AS source "
+            f"ON {mssql_match('id')} "
+            "WHEN MATCHED THEN UPDATE SET value = source.value, value_type = source.value_type "
+            "WHEN NOT MATCHED THEN INSERT (id, value, value_type) "
+            "VALUES (source.id, source.value, source.value_type);",
+            table,
+            key,
+            value,
+            value_type,
+        )
+
+    def sqlite() -> _Upsert:
+        return lambda key, value, value_type: _unprepared(
+            sql(
+                "INSERT INTO {} (id, value, value_type) VALUES ({}, {}, {}) "
+                "ON CONFLICT(id) DO UPDATE SET value=excluded.value, "
+                "value_type=excluded.value_type",
+                table,
+                key,
+                value,
+                value_type,
+            )
+        )
+
+    return sql.on_dialect_or_else(pg=pg, mysql=mysql, mssql=mssql, or_else=sqlite)
+
+
+def _unprepared(statement: Statement[Row]) -> Effect[list[Row], SqlError]:
+    return statement.unprepared

pyfect/persistence/key_value_store/_store.py

@@ -0,0 +1,212 @@
+"""The ``KeyValueStore`` service, its error, and ``make`` /
+``make_string_only`` for backends."""
+
+from __future__ import annotations
+
+import base64
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+from pyfect._core import Effect
+from pyfect._effect_args import require_effect
+from pyfect._js import text_encode
+from pyfect.context import Tag
+from pyfect.effect import succeed, suspend
+from pyfect.errors import TaggedError
+
+if TYPE_CHECKING:
+    from pyfect.persistence.key_value_store._schema_store import SchemaStore
+    from pyfect.schema import Codec
+
+
+@dataclass(eq=False)
+class KeyValueStoreError(TaggedError):
+    """A store operation failed: ``method`` is the operation, ``key`` the
+    key it was given (if any) and ``cause`` what the backend raised."""
+
+    message: str
+    method: str
+    key: str | None = None
+    cause: object = None
+
+
+type Value = str | bytes
+"""What a store holds: text, or bytes (Effect's ``Uint8Array``)."""
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class KeyValueStoreService:
+    """A string-keyed store of text and bytes. Build one with ``make`` (or
+    ``make_string_only`` for a backend that holds only text)."""
+
+    get: Callable[[str], Effect[str | None, KeyValueStoreError]]
+    """The value as text (bytes as Base64); ``None`` when missing."""
+    get_bytes: Callable[[str], Effect[bytes | None, KeyValueStoreError]]
+    """The value as bytes (text as UTF-8); ``None`` when missing."""
+    set: Callable[[str, Value], Effect[None, KeyValueStoreError]]
+    remove: Callable[[str], Effect[None, KeyValueStoreError]]
+    clear: Effect[None, KeyValueStoreError]
+    size: Effect[int, KeyValueStoreError]
+    modify: Callable[[str, Callable[[str], str]], Effect[str | None, KeyValueStoreError]]
+    """Replace a present value with ``f(value)`` and return it; ``None``
+    (and nothing written) when missing."""
+    modify_bytes: Callable[
+        [str, Callable[[bytes], bytes]], Effect[bytes | None, KeyValueStoreError]
+    ]
+    has: Callable[[str], Effect[bool, KeyValueStoreError]]
+    is_empty: Effect[bool, KeyValueStoreError]
+
+    def __post_init__(self) -> None:
+        for name in ("clear", "size", "is_empty"):
+            require_effect(getattr(self, name), name)
+
+    def prefix(self, prefix: str) -> KeyValueStoreService:
+        """This store with ``prefix`` put before every key (``clear``,
+        ``size`` and ``is_empty`` still see the whole store)."""
+        return _prefixed(self, prefix)
+
+    def to_schema_store[T, RD, RE](self, schema: Codec[T, Any, RD, RE]) -> SchemaStore[T, RD, RE]:
+        """This store holding ``schema``'s values, written as JSON text."""
+        from pyfect.persistence.key_value_store._schema_store import to_schema_store
+
+        return to_schema_store(self, schema)
+
+
+class KeyValueStore(Tag[KeyValueStoreService], key="effect/persistence/KeyValueStore"):
+    """The key-value store a program uses."""
+
+
+def make(
+    *,
+    get: Callable[[str], Effect[str | None, KeyValueStoreError]],
+    get_bytes: Callable[[str], Effect[bytes | None, KeyValueStoreError]],
+    set: Callable[[str, Value], Effect[None, KeyValueStoreError]],
+    remove: Callable[[str], Effect[None, KeyValueStoreError]],
+    clear: Effect[None, KeyValueStoreError],
+    size: Effect[int, KeyValueStoreError],
+    has: Callable[[str], Effect[bool, KeyValueStoreError]] | None = None,
+    is_empty: Effect[bool, KeyValueStoreError] | None = None,
+    modify: Callable[[str, Callable[[str], str]], Effect[str | None, KeyValueStoreError]]
+    | None = None,
+    modify_bytes: Callable[
+        [str, Callable[[bytes], bytes]], Effect[bytes | None, KeyValueStoreError]
+    ]
+    | None = None,
+) -> KeyValueStoreService:
+    """A store from its backend's operations; ``has``, ``is_empty``,
+    ``modify`` and ``modify_bytes`` are derived unless given."""
+    require_effect(clear, "clear")
+    require_effect(size, "size")
+    if is_empty is not None:
+        require_effect(is_empty, "is_empty")
+
+    def derived_has(key: str) -> Effect[bool, KeyValueStoreError]:
+        return get(key).map(lambda value: value is not None)
+
+    def derived_modify(key: str, f: Callable[[str], str]) -> Effect[str | None, KeyValueStoreError]:
+        def update(value: str | None) -> Effect[str | None, KeyValueStoreError]:
+            if value is None:
+                return succeed(None)
+            updated = f(value)
+            return set(key, updated).as_(updated)
+
+        return get(key).flat_map(update)
+
+    def derived_modify_bytes(
+        key: str, f: Callable[[bytes], bytes]
+    ) -> Effect[bytes | None, KeyValueStoreError]:
+        def update(value: bytes | None) -> Effect[bytes | None, KeyValueStoreError]:
+            if value is None:
+                return succeed(None)
+            updated = f(value)
+            return set(key, updated).as_(updated)
+
+        return get_bytes(key).flat_map(update)
+
+    return KeyValueStoreService(
+        get=get,
+        get_bytes=get_bytes,
+        set=set,
+        remove=remove,
+        clear=clear,
+        size=size,
+        has=has or derived_has,
+        is_empty=is_empty or size.map(lambda n: n == 0),
+        modify=modify or derived_modify,
+        modify_bytes=modify_bytes or derived_modify_bytes,
+    )
+
+
+def make_string_only(
+    *,
+    get: Callable[[str], Effect[str | None, KeyValueStoreError]],
+    set: Callable[[str, str], Effect[None, KeyValueStoreError]],
+    remove: Callable[[str], Effect[None, KeyValueStoreError]],
+    clear: Effect[None, KeyValueStoreError],
+    size: Effect[int, KeyValueStoreError],
+    has: Callable[[str], Effect[bool, KeyValueStoreError]] | None = None,
+    is_empty: Effect[bool, KeyValueStoreError] | None = None,
+) -> KeyValueStoreService:
+    """A store over a backend that holds only text: bytes are written as
+    Base64, and ``get_bytes`` reads a value that is valid Base64 as the
+    bytes it encodes, any other as its UTF-8."""
+    require_effect(clear, "clear")
+    require_effect(size, "size")
+    if is_empty is not None:
+        require_effect(is_empty, "is_empty")
+
+    def get_bytes(key: str) -> Effect[bytes | None, KeyValueStoreError]:
+        return get(key).map(lambda value: None if value is None else _text_to_bytes(value))
+
+    def set_value(key: str, value: Value) -> Effect[None, KeyValueStoreError]:
+        if isinstance(value, str):
+            return set(key, value)
+        return suspend(lambda: set(key, base64_encode(value)))
+
+    return make(
+        get=get,
+        get_bytes=get_bytes,
+        set=set_value,
+        remove=remove,
+        clear=clear,
+        size=size,
+        has=has,
+        is_empty=is_empty,
+    )
+
+
+def base64_encode(data: bytes) -> str:
+    return base64.b64encode(data).decode("ascii")
+
+
+def base64_decode(text: str) -> bytes | None:
+    """``text`` decoded as Effect's ``Base64.decode`` does (line breaks
+    ignored, padding required), or ``None`` when it is not Base64."""
+    stripped = text.replace("\r", "").replace("\n", "")
+    if len(stripped) % 4 != 0:
+        return None
+    try:
+        return base64.b64decode(stripped, validate=True)
+    except ValueError:
+        return None
+
+
+def _text_to_bytes(text: str) -> bytes:
+    decoded = base64_decode(text)
+    return text_encode(text) if decoded is None else decoded
+
+
+def _prefixed(store: KeyValueStoreService, prefix: str) -> KeyValueStoreService:
+    return KeyValueStoreService(
+        get=lambda key: store.get(prefix + key),
+        get_bytes=lambda key: store.get_bytes(prefix + key),
+        set=lambda key, value: store.set(prefix + key, value),
+        remove=lambda key: store.remove(prefix + key),
+        clear=store.clear,
+        size=store.size,
+        modify=lambda key, f: store.modify(prefix + key, f),
+        modify_bytes=lambda key, f: store.modify_bytes(prefix + key, f),
+        has=lambda key: store.has(prefix + key),
+        is_empty=store.is_empty,
+    )

pyfect/persistence/persistable.py

@@ -0,0 +1,164 @@
+"""Requests whose results can be stored (Effect's
+``effect/persistence/Persistable``).
+
+A persistable request names itself with ``primary_key()`` and carries the
+schemas of its success and error, so its ``Exit`` can be written as JSON
+and read back. ``Class`` makes the base class from the schemas, and the
+request's types come from them::
+
+    @dataclass(frozen=True)
+    class GetUser(persistable.Class(success=User, error=S.String)):
+        id: int
+
+        def primary_key(self) -> str:
+            return f"GetUser:{self.id}"
+
+``GetUser(1)`` is a ``Request[User, str | PersistenceError | SchemaError]``:
+a store may fail to read or write, or hold a value the schemas reject.
+"""
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from typing import Any, ClassVar, Never, Protocol, TypeVar, cast
+
+from pyfect import schema as S
+from pyfect._core import Effect
+from pyfect._json_value import JsonValue
+from pyfect.batching._request import Request
+from pyfect.duration import Duration
+from pyfect.exit import Exit
+from pyfect.persistence._error import PersistenceError
+
+A_co = TypeVar("A_co", covariant=True)
+E_co = TypeVar("E_co", covariant=True, default=Never)
+R_co = TypeVar("R_co", covariant=True, default=Never)
+
+type AnyCodec = S.Codec[Any, Any, Any, Any]
+type _ExitCodec = S.Bottom[Exit[Any, Any], JsonValue, Any, Any]
+
+
+class Persistable(Request[A_co, E_co | PersistenceError | S.SchemaError, R_co], ABC):
+    """A request succeeding with ``A`` and failing with ``E`` (the types of
+    its schemas, which need the services ``R``), stored under
+    ``primary_key()``. Subclass ``Class(success=..., error=...)``."""
+
+    __slots__ = ()
+
+    _success_schema: ClassVar[AnyCodec]
+    _error_schema: ClassVar[AnyCodec]
+    _exit_codec: ClassVar[_ExitCodec | None]
+
+    @abstractmethod
+    def primary_key(self) -> str:
+        """The key the request's result is stored under."""
+
+    def _persistable_types(self) -> Exactly[A_co, E_co]:
+        """Never called: lets ``persisted_cache`` read the request's success
+        and error exactly (``Persistable.Success<K>`` / ``Error<K>``)."""
+        raise NotImplementedError
+
+
+class Exactly[A, E]:
+    """Invariant in ``A`` and ``E`` (each is taken and given back), so
+    matching a request's ``_persistable_types()`` pins its success and
+    error instead of widening them to what else they meet."""
+
+    __slots__ = ()
+
+    def pin(self, success: A, error: E) -> tuple[A, E]:
+        raise NotImplementedError
+
+
+class HasPersistableTypes[A, E](Protocol):
+    """A persistable request succeeding with exactly ``A`` and failing with
+    exactly ``E`` (its schemas' types)."""
+
+    def _persistable_types(self) -> Exactly[A, E]: ...
+
+
+type AnyPersistable = Persistable[Any, Any, Any]
+
+type TimeToLive[K, A, E] = Callable[[Exit[A, E], K], Duration]
+"""How long to keep a request's outcome: a function of the ``Exit`` and
+the request. Zero or less keeps nothing; infinity keeps it forever."""
+
+
+def Class[A = None, E = Never, RR = Never](
+    *,
+    success: S.Codec[A, Any, RR, RR] | None = None,
+    error: S.Codec[E, Any, RR, RR] | None = None,
+) -> type[Persistable[A, E, RR]]:
+    """The base class of a persistable request succeeding with ``success``'s
+    values (default: ``None``) and failing with ``error``'s (default:
+    never). Subclass it as a frozen dataclass and define
+    ``primary_key()``."""
+    success_schema: AnyCodec = S.Null if success is None else success
+    error_schema: AnyCodec = S.Never if error is None else error
+
+    class Base(Persistable[Any, Any, Any]):
+        __slots__ = ()
+        _success_schema = success_schema
+        _error_schema = error_schema
+        _exit_codec = None
+
+    return cast("type[Persistable[A, E, RR]]", Base)
+
+
+def success_schema[A](request: Persistable[A, Any, Any]) -> S.Codec[A, Any, Any, Any]:
+    return type(request)._success_schema
+
+
+def error_schema[E](request: Persistable[Any, E, Any]) -> S.Codec[E, Any, Any, Any]:
+    return type(request)._error_schema
+
+
+def exit_schema[A, E](
+    request: Persistable[A, E, Any],
+) -> S.ExitSchema[A, E, Any, Any, Any, Any]:
+    """The schema of the request's ``Exit``: its success, its error, or a
+    defect."""
+    cls = type(request)
+    return S.Exit(cls._success_schema, cls._error_schema, S.Defect())
+
+
+def _exit_codec(request: AnyPersistable) -> _ExitCodec:
+    """The JSON codec of the request's ``Exit``, made once per class."""
+    cls = type(request)
+    codec = cls._exit_codec
+    if codec is None:
+        codec = S.to_codec_json(exit_schema(request))
+        cls._exit_codec = codec
+    return codec
+
+
+def serialize_exit[A, E, R](
+    request: Persistable[A, E, R], exit: Exit[A, E]
+) -> Effect[JsonValue, S.SchemaError, R]:
+    """``exit`` as JSON, by the request's schemas."""
+    return S.encode_effect(_exit_codec(request))(exit)
+
+
+def deserialize_exit[A, E, R](
+    request: Persistable[A, E, R], encoded: object
+) -> Effect[Exit[A, E], S.SchemaError, R]:
+    """The ``Exit`` that ``encoded`` holds, by the request's schemas."""
+    return S.decode_unknown_effect(_exit_codec(request))(encoded)
+
+
+__all__ = [
+    "AnyCodec",
+    "AnyPersistable",
+    "Class",
+    "Exactly",
+    "HasPersistableTypes",
+    "Persistable",
+    "TimeToLive",
+    "deserialize_exit",
+    "error_schema",
+    "exit_schema",
+    "serialize_exit",
+    "success_schema",
+]