semiont 0.6.12.dev2__py3-none-any.whl

semiont/__init__.py

@@ -0,0 +1,20 @@
+"""Semiont's Python SDK.
+
+What the protocol states, as Python types: the kinds of id
+(`semiont.identifiers`), the shapes it sends and answers with
+(`semiont.types`), the channels of its bus and the requests made over them
+(`semiont.channels`, `semiont.operations`), and the tables a client keeps to
+(`semiont.error_codes`, `semiont.timing`, `semiont.refresh`,
+`semiont.telemetry_table`). All of it is generated from `specs/`.
+
+The wire itself: what a client needs of whatever carries its bus, its content
+and a gateway's own answers (`semiont.transport`), the bus typed by channel
+(`semiont.bus`), a knowledge base's gateway as the carrier, and how a process
+signs in there (`semiont.http`), and what is told to OpenTelemetry of it
+(`semiont.telemetry`).
+
+And the client every Semiont SDK has (`semiont.client`): its namespaces
+(`semiont.namespaces`), each method a row of `specs/src/client/surface.json`,
+its own bus (`semiont.event_bus`), and a session with one knowledge base
+(`semiont.session`).
+"""

semiont/bus.py

@@ -0,0 +1,236 @@
+"""A client of a knowledge base's bus, over any `Transport`: emit, read, and request.
+
+**By type** (`Bus`), a channel is one of `semiont.channels`' constants and a
+payload is that channel's own, so a type checker refuses a wrong payload for a
+channel, gives a frame its payload's type, and gives a request its
+operation's reply:
+
+    bus = Bus(transport)
+    await bus.emit(BECKON_FOCUS, BeckonFocusEvent(annotation_id=annotation))
+    async for frame in bus.frames(MARK_ADDED): ...          # frame.payload is that channel's
+    created = await bus.request(MARK_CREATE_REQUEST, command)
+
+**By name** (`request`, and the transport's own `emit` and `frames`), a
+channel is the registry's name and a payload a JSON object: for what relays
+frames it does not read, or is told its channels at run time.
+
+A request (`docs/protocol/TRANSPORT-CONTRACT.md` § Requests) is one emit that
+carries a correlation id of this client's making
+beside its payload, answered on its operation's result channel or its failure
+channel. It is not sent before the stream that carries its reply is open. Its
+id is tracked until it settles, so a reply published while the stream was down
+is sent again. And it settles once: with its result, with a failure under this
+vocabulary's code, or as a timeout.
+
+A caller abandons a request by cancelling the task that awaits it. What was
+sent stays sent, the reply stops being tracked, and one that comes anyway
+reaches nobody. Cancelled while it waits for the stream, it sends nothing.
+"""
+
+import asyncio
+import logging
+import uuid
+from collections.abc import Mapping
+from dataclasses import dataclass
+from types import TracebackType
+from typing import Final, Self, final
+
+from pydantic import JsonValue, ValidationError
+
+from semiont.channel import AnyOperation, Channel, Operation
+from semiont.errors import BusRequestError, TransportError
+from semiont.events import Events
+from semiont.identifiers import ResourceId, UserId
+from semiont.model import WireModel
+from semiont.timing import BUS_REQUEST_TIMEOUT_MS
+from semiont.transport import ConnectionState, Frame, TraceContext, Transport
+from semiont.watched import NeverReached, reached
+
+__all__ = ["Bus", "Delivered", "Typed", "decoded", "reply_channels_for", "request"]
+
+_LOG: Final = logging.getLogger("semiont.bus")
+
+# What the gateway stamps on a payload to say who emitted it.
+_EMITTED_BY: Final = "_userId"
+
+
+def reply_channels_for(*operations: AnyOperation) -> tuple[str, ...]:
+    """The result and failure channels of `operations`, each once.
+
+    What a process that awaits only those operations names as its transport's
+    channels: it hears its replies and nothing else.
+    """
+    replies: dict[str, None] = {}
+    for operation in operations:
+        replies[operation.result.name] = None
+        replies[operation.failure.name] = None
+    return tuple(replies)
+
+
+def _settled(state: ConnectionState) -> bool:
+    return state in ("open", "closed")
+
+
+async def request(
+    transport: Transport,
+    operation: AnyOperation,
+    payload: Mapping[str, JsonValue],
+    *,
+    timeout_ms: int = BUS_REQUEST_TIMEOUT_MS,
+) -> Mapping[str, JsonValue]:
+    """Send `payload` as the request of `operation`, and wait up to `timeout_ms` for its reply.
+
+    Returns the payload of the reply on the operation's result channel.
+    Raises `BusRequestError` when it is answered with a failure, when no
+    reply comes in time, when the transport is closed, and when the
+    transport's stream does not carry the operation's replies; and
+    `TransportError` when the emit itself is refused.
+    """
+    asked, result, failure = operation.request.name, operation.result.name, operation.failure.name
+    for channel in (result, failure):
+        if not transport.is_subscribed(channel):
+            raise BusRequestError(
+                "bus.unsubscribed",
+                f"Transport is not subscribed to reply channel {channel}: a reply to {asked} can never arrive. "
+                "Add this operation's reply channels to the transport's channels.",
+            )
+
+    deadline = asyncio.get_running_loop().time() + timeout_ms / 1000
+
+    def timed_out() -> BusRequestError:
+        return BusRequestError("bus.timeout", f"Bus request timed out after {timeout_ms}ms on {result}")
+
+    # No request before its reply can arrive: wait, inside the request's own
+    # deadline, for the stream to be open. Only `open` delivers. A closed bus
+    # fails at once, and does not spend the deadline.
+    try:
+        async with asyncio.timeout_at(deadline):
+            state = await reached(transport.state, _settled)
+    except TimeoutError:
+        raise timed_out() from None
+    except NeverReached:
+        # A state that will change no more, and is not open, will not deliver.
+        state = "closed"
+    if state != "open":
+        raise BusRequestError("bus.closed", f"Bus closed before emit on {asked}")
+
+    # Tracked before the emit, so a stream opened while the emit is in flight
+    # already names the reply. The emit is not cut short by the deadline: a
+    # request half sent is one nobody can account for.
+    correlation_id = str(uuid.uuid4())
+    with transport.track_reply(correlation_id, (result, failure)) as pending:
+        await transport.emit(asked, payload, correlation_id=correlation_id)
+        try:
+            async with asyncio.timeout_at(deadline):
+                reply = await pending.frame()
+        except TimeoutError:
+            raise timed_out() from None
+
+    if reply is None:
+        raise BusRequestError("bus.closed", f"Bus closed before a reply on {result}")
+    if reply.channel == result:
+        return reply.payload
+    raise BusRequestError.answered(reply.payload)
+
+
+# ── By type ─────────────────────────────────────────────────────────────
+
+
+def decoded[P: WireModel](channel: Channel[P], payload: Mapping[str, JsonValue]) -> P:
+    """A payload as `channel` types it. Raises `ValidationError` for one that is not that channel's.
+
+    What the gateway stamps on a payload (a member whose name begins `_`) is
+    the bus's, not the payload's: one the channel's type does not declare is
+    left out before the payload is read.
+    """
+    declared = {field.alias or name for name, field in channel.payload.model_fields.items()}
+    return channel.decode({name: value for name, value in payload.items() if not name.startswith("_") or name in declared})
+
+
+@final
+@dataclass(frozen=True, slots=True, kw_only=True)
+class Delivered[P: WireModel]:
+    """A frame of one channel, its payload decoded."""
+
+    payload: P
+    correlation_id: str | None = None
+    scope: ResourceId | None = None
+    user_id: UserId | None = None
+    """Who emitted it, as the gateway stamped it."""
+    trace: TraceContext | None = None
+
+
+@final
+class Typed[P: WireModel]:
+    """One channel's frames from now on, each with its payload decoded.
+
+    Iterated (`async for`), and can be held with `async with`, which stops
+    listening on the way out. A frame whose payload is not the channel's is
+    nobody's to act on: it is not given, and is said so in the log.
+    """
+
+    def __init__(self, channel: Channel[P], frames: Events[Frame]) -> None:
+        self._channel: Final = channel
+        self._frames: Final = frames
+
+    def __aiter__(self) -> Self:
+        return self
+
+    async def __anext__(self) -> Delivered[P]:
+        while True:
+            frame = await anext(self._frames)
+            try:
+                payload = decoded(self._channel, frame.payload)
+            except ValidationError as error:
+                _LOG.warning("a payload on %s is not that channel's: %s", self._channel.name, error)
+                continue
+            stamped = frame.payload.get(_EMITTED_BY)
+            return Delivered(
+                payload=payload,
+                correlation_id=frame.correlation_id,
+                scope=frame.scope,
+                user_id=UserId.parse(stamped) if isinstance(stamped, str) else None,
+                trace=frame.trace,
+            )
+
+    async def aclose(self) -> None:
+        """Stop listening."""
+        await self._frames.aclose()
+
+    async def __aenter__(self) -> Self:
+        return self
+
+    async def __aexit__(self, kind: type[BaseException] | None, error: BaseException | None, trace: TracebackType | None) -> None:
+        await self.aclose()
+
+
+@final
+class Bus:
+    """The bus over `transport`, typed by channel. See the module's documentation."""
+
+    def __init__(self, transport: Transport) -> None:
+        self.transport: Final = transport
+
+    async def emit[P: WireModel](
+        self, channel: Channel[P], payload: P, *, scope: ResourceId | None = None, correlation_id: str | None = None
+    ) -> int | None:
+        """Send one frame on `channel`. How many subscribers the gateway reached, or nothing when it kept no count."""
+        return await self.transport.emit(channel.name, channel.encode(payload), scope=scope, correlation_id=correlation_id)
+
+    def frames[P: WireModel](self, channel: Channel[P]) -> Typed[P]:
+        """The frames delivered on `channel` from now on. Refused, as `bus.unsubscribed`, for a channel this transport can never deliver."""
+        return Typed(channel, self.transport.frames(channel.name))
+
+    async def request[Q: WireModel, R: WireModel, F: WireModel](
+        self, operation: Operation[Q, R, F], payload: Q, *, timeout_ms: int = BUS_REQUEST_TIMEOUT_MS
+    ) -> R:
+        """Send `payload` as the request of `operation`, and wait up to `timeout_ms` for its result.
+
+        Raises as `request` does, and `TransportError` when the result that
+        came is not the operation's.
+        """
+        reply = await request(self.transport, operation, operation.request.encode(payload), timeout_ms=timeout_ms)
+        try:
+            return decoded(operation.result, reply)
+        except ValidationError as error:
+            raise TransportError("error", f"a payload on {operation.result.name} is not that channel's: {error}") from error

semiont/cache.py

@@ -0,0 +1,345 @@
+"""A read-through cache: what a client's live queries answer from (`docs/protocol/CACHE-SEMANTICS.md`, B1 to B20).
+
+**A key has one state, and every observer of the key holds it**: `Pending`
+(no value yet; a fetch may be in flight), `Ready` (a value, which may be the
+one shown while a newer is fetched), or `Failed` (a key with no value whose
+fetch, and its one retry, failed). `Failed` is a state, not the end of
+anything: an observer lives through it, and the next act on the key tries
+again.
+
+Two ways to read:
+
+- `observe` is the live view. The first observer of a key with nothing starts
+  a fetch; later ones are given what is there. A failed fetch is tried once
+  more; a key with a value keeps showing it whatever its refetches do.
+- `fetch` is the one-shot read. It always asks, gives what it was answered or
+  raises the failure it met, and never retries: its caller sees the failure
+  and decides.
+
+`invalidate` asks again and keeps showing the value meanwhile; `set` writes a
+value that is known; `remove` ends a key whose entity is gone. `dispose` is
+terminal: every observer's states end, and nothing is fetched, retried or
+failed after it.
+
+A cache with a `CachePersister` begins with what was kept and saves its values
+as they change. A value that was kept is shown at once and asked for again the
+first time it is observed: nothing says it is still true.
+
+An observer that falls behind is given the latest state, not each one it
+missed: a state is what is true now.
+"""
+
+import asyncio
+from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Hashable, Mapping
+from dataclasses import dataclass, field
+from typing import Final, Literal, Protocol, final
+
+from semiont.errors import BusRequestError, SemiontError
+from semiont.watched import Variable
+
+__all__ = ["SAVE_DEBOUNCE_MS", "Cache", "CachePersister", "CacheState", "Failed", "Pending", "Ready", "Runs", "closed"]
+
+SAVE_DEBOUNCE_MS: Final = 50
+"""How long a cache waits, after its values change, before it saves them: a burst of changes is one save."""
+
+
+@final
+@dataclass(frozen=True, slots=True)
+class Pending:
+    """No value yet."""
+
+    status: Literal["pending"] = field(default="pending", init=False)
+
+
+@final
+@dataclass(frozen=True, slots=True)
+class Ready[T]:
+    """A value."""
+
+    value: T
+    status: Literal["ready"] = field(default="ready", init=False)
+
+
+@final
+@dataclass(frozen=True, slots=True)
+class Failed:
+    """No value, and the fetch for one failed, and failed again."""
+
+    error: SemiontError
+    status: Literal["failed"] = field(default="failed", init=False)
+
+
+type CacheState[T] = Pending | Ready[T] | Failed
+"""The state of a key. A `match` over one names all three."""
+
+
+class CachePersister[K, V](Protocol):
+    """Where a cache keeps its values so a later one begins with them."""
+
+    def load(self) -> Mapping[K, V] | None:
+        """What was kept, asked once, when the cache is built."""
+        ...
+
+    def save(self, entries: Mapping[K, V]) -> None:
+        """Keep these: the cache's values, after they changed."""
+        ...
+
+
+class Runs(Protocol):
+    """What runs a cache's fetches: tasks that have ended by the time whoever owns them has closed."""
+
+    def run[T](self, work: Coroutine[object, object, T]) -> asyncio.Task[T]:
+        """Run `work` as a task."""
+        ...
+
+
+def closed() -> BusRequestError:
+    """What a read of a closed client fails as: what a request of a closed bus does."""
+    return BusRequestError("bus.closed", "The client is closed")
+
+
+@final
+class _Key[V]:
+    def __init__(self, state: CacheState[V]) -> None:
+        self.state: Final = Variable[CacheState[V]](state)
+        self.in_flight: tuple[int, asyncio.Task[V]] | None = None
+        """The fetch in flight for the key: its number, and its task. An
+        `invalidate` lets go of it without waiting, so a fetch whose answer
+        will never come cannot hold the key."""
+
+    def known(self) -> bool:
+        """Whether anything asked for the key and has not let go of it."""
+        return self.in_flight is not None or not isinstance(self.state.value, Pending)
+
+
+async def _joined[V](fetch: asyncio.Task[V]) -> V:
+    """What a fetch gives, to one of those waiting on it: whoever stops waiting does not stop the fetch."""
+    try:
+        return await asyncio.shield(fetch)
+    except asyncio.CancelledError:
+        # The fetch was ended by its client's closing, and its caller was not: that is a failure to hear, not a cancellation.
+        task = asyncio.current_task()
+        if fetch.cancelled() and (task is None or task.cancelling() == 0):
+            raise closed() from None
+        raise
+
+
+@final
+class _Nothing[T]:
+    """The states of a key of a disposed cache: none."""
+
+    def __aiter__(self) -> "_Nothing[T]":
+        return self
+
+    async def __anext__(self) -> T:
+        raise StopAsyncIteration
+
+
+@final
+class Cache[K: Hashable, V]:
+    """See the module's documentation.
+
+    `fetch` asks the service for a key's value. `tasks` runs the fetches.
+    With a `persister`, the cache begins with what it kept, and saves its
+    values there once they have stopped changing for `SAVE_DEBOUNCE_MS`.
+    """
+
+    def __init__(
+        self,
+        fetch: Callable[[K], Awaitable[V]],
+        *,
+        tasks: Runs,
+        persister: CachePersister[K, V] | None = None,
+    ) -> None:
+        self._fetch: Final = fetch
+        self._tasks: Final = tasks
+        self._persister: Final = persister
+        self._disposed = False
+        self._fetches = 0
+        self._saving: asyncio.TimerHandle | None = None
+        """The wait before the next save, while one is owed."""
+        kept = None if persister is None else persister.load()
+        self._keys: Final[dict[K, _Key[V]]] = {key: _Key(Ready(value)) for key, value in (kept or {}).items()}
+        self._kept: Final[set[K]] = set(self._keys)
+        """Keys whose value was kept by an earlier cache and has not been asked for by this one."""
+
+    def _held(self, key: K) -> _Key[V]:
+        held = self._keys.get(key)
+        if held is None:
+            held = self._keys[key] = _Key(Pending())
+        return held
+
+    def _values(self) -> dict[K, V]:
+        return {key: held.state.value.value for key, held in self._keys.items() if isinstance(held.state.value, Ready)}
+
+    def _owe_a_save(self) -> None:
+        """The values changed: save them once they have stopped changing."""
+        if self._persister is None:
+            return
+        if self._saving is not None:
+            self._saving.cancel()
+        self._saving = asyncio.get_running_loop().call_later(SAVE_DEBOUNCE_MS / 1000, self._save)
+
+    def _save(self) -> None:
+        self._saving = None
+        if self._persister is not None:
+            self._persister.save(self._values())
+
+    def _fetching(self, key: K) -> asyncio.Task[V]:
+        """Start a fetch for `key`, or join the one in flight."""
+        # Asked for by this cache now, whichever path asked.
+        self._kept.discard(key)
+        held = self._held(key)
+        if held.in_flight is not None:
+            return held.in_flight[1]
+        self._fetches += 1
+        fetch = self._tasks.run(self._fetched(key, self._fetches))
+        held.in_flight = (self._fetches, fetch)
+        return fetch
+
+    async def _fetched(self, key: K, number: int) -> V:
+        try:
+            value = await self._fetch(key)
+        finally:
+            # Only its own, and on every way out: an `invalidate` may have put a newer fetch in its place.
+            held = self._keys.get(key)
+            if held is not None and held.in_flight is not None and held.in_flight[0] == number:
+                held.in_flight = None
+        # A value from any fetch is written, the newest last: each is at least
+        # as new as what was there. A failure writes nothing. One answered
+        # after the cache was disposed has no key to write to.
+        held = self._keys.get(key)
+        if held is not None:
+            held.state.set(Ready(value))
+            self._owe_a_save()
+        return value
+
+    def _revalidate(self, key: K) -> None:
+        """Fetch for the live view: a failure is tried once more, and a key with no value whose retry also fails becomes `Failed`."""
+        self._tasks.run(self._chain(key, self._fetching(key)))
+
+    async def _chain(self, key: K, first: asyncio.Task[V]) -> None:
+        try:
+            await _joined(first)
+        except SemiontError:
+            pass
+        else:
+            return
+        if self._disposed:
+            return
+        try:
+            await _joined(self._fetching(key))
+        except SemiontError as failure:
+            held = self._keys.get(key)
+            if held is not None and not isinstance(held.state.value, Ready):
+                held.state.set(Failed(failure))
+
+    def observe(self, key: K) -> AsyncIterator[CacheState[V]]:
+        """The key's state, now and as it changes, until the cache is disposed.
+
+        Observing is what asks: a key with nothing is fetched, a failed key
+        is tried again, and a value an earlier cache kept is asked for anew
+        while it is shown.
+        """
+        if self._disposed:
+            return _Nothing[CacheState[V]]()
+        was_kept = key in self._kept
+        held = self._held(key)
+        state = held.state.value
+        if isinstance(state, Failed):
+            # An observer arriving at a failed key starts over, for everyone: a fetch is in flight for all of them.
+            held.state.set(Pending())
+            asks = True
+        elif isinstance(state, Ready):
+            asks = was_kept
+        else:
+            asks = held.in_flight is None
+        observing = aiter(held.state)
+        if asks:
+            self._revalidate(key)
+        return observing
+
+    async def fetch(self, key: K) -> V:
+        """Ask for the key's value now.
+
+        What the service answers, which every observer of the key is given
+        too; or the failure, which only this caller is. Concurrent asks for
+        one key share one fetch.
+        """
+        if self._disposed:
+            raise closed()
+        return await _joined(self._fetching(key))
+
+    def get(self, key: K) -> V | None:
+        """The key's value now, asking for nothing."""
+        held = self._keys.get(key)
+        state = None if held is None else held.state.value
+        return state.value if isinstance(state, Ready) else None
+
+    def known(self, key: K) -> bool:
+        """Whether anything has asked for the key and not let go of it: it has a value or a failure, or a fetch for it is in flight."""
+        held = self._keys.get(key)
+        return held is not None and held.known()
+
+    def keys(self) -> list[K]:
+        """Every key the cache knows."""
+        return [key for key, held in self._keys.items() if held.known()]
+
+    @property
+    def persistence_pending(self) -> bool:
+        """Whether a cache that keeps its values has not yet kept what it knows: a fetch is in flight, or a save is owed."""
+        if self._persister is None:
+            return False
+        return self._saving is not None or any(held.in_flight is not None for held in self._keys.values())
+
+    def invalidate(self, key: K) -> None:
+        """The key's value is out of date: ask again, showing what there is meanwhile.
+
+        A fetch already in flight is not waited on: its answer may never come.
+        """
+        if self._disposed:
+            return
+        held = self._held(key)
+        held.in_flight = None
+        if isinstance(held.state.value, Failed):
+            held.state.set(Pending())
+        self._revalidate(key)
+
+    def invalidate_all(self) -> None:
+        """`invalidate`, of every key the cache knows."""
+        for key in self.keys():
+            self.invalidate(key)
+
+    def remove(self, key: K, gone: SemiontError) -> None:
+        """The key's entity is gone: its value is dropped and the key is failed with `gone`, at once, for every observer.
+
+        Nothing is asked for.
+        """
+        if self._disposed:
+            return
+        held = self._held(key)
+        held.in_flight = None
+        held.state.set(Failed(gone))
+        self._owe_a_save()
+
+    def set(self, key: K, value: V) -> None:
+        """The key's value is known: write it, asking for nothing."""
+        if self._disposed:
+            return
+        self._kept.discard(key)
+        self._held(key).state.set(Ready(value))
+        self._owe_a_save()
+
+    def dispose(self) -> None:
+        """End the cache: a save that was owed is made now, every observer's states end, and nothing is fetched or written after.
+
+        Disposing twice is disposing once.
+        """
+        if self._saving is not None:
+            self._saving.cancel()
+            self._save()
+        self._disposed = True
+        for held in self._keys.values():
+            held.state.end()
+        self._keys.clear()
+        self._kept.clear()