fleetmesh-node 0.2.0__py3-none-any.whl

node/compartment/stores.py

@@ -0,0 +1,662 @@
+"""What a compartment holds: its in-memory collections, declared and bounded.
+
+The sixth thing a compartment declares, beside PARAMS (what it can be told),
+CHANNELS (what it says), METRICS (what it measures), COMMANDS (what it can be
+asked to do) and LIFECYCLE (how it fails): STORES, what it keeps.
+
+Before this a compartment's memory was whatever attributes its __init__
+happened to assign. Nothing outside it knew they existed, so nobody could ask
+how much announce was holding, which collections grow with the number of
+peers, or what a command reads. Some were trimmed by hand (`del self.log
+[:-200]` in capacity) and some were not trimmed at all, and the difference was
+visible only to someone reading the code.
+
+A Store declares one collection:
+
+    key        the attribute it is reached under, and its name everywhere else
+    kind       RING      a log, newest last, oldest dropped at `bound`
+               TABLE     a mapping, keyed by `keyed_by`
+               SET       members, keyed by `keyed_by`
+               COUNTERS  a fixed set of named integers, the fields of `shape`
+    shape      the type of one entry (one row, one member, one record). A
+               TypedDict is checked on every write: required keys present, no
+               key it does not name. A class is checked with isinstance. A
+               write that does not fit raises TypeError - a programming error,
+               and the lifecycle degrades the compartment that made it.
+    bound      how many entries it may hold. Required: a collection that grows
+               with the outside world and says nothing about how far is the
+               failure this exists to prevent. COUNTERS are bounded by their
+               shape and take no bound.
+    evict      what happens at the bound. OLDEST drops the earliest written,
+               LRU the least recently read or written, REFUSE refuses the
+               write (StoreFull, or False from `put`) - for state that must
+               not be silently lost, where the caller has to decide.
+    keyed_by   what makes it grow: 'peer', 'event', 'url'. Required on a
+               TABLE and a SET, because that is the question an operator asks.
+    persist    MEMORY (gone with the process) or SNAPSHOT (carried across a
+               process restart by the lifecycle, node/lifecycle phase 4)
+    sensitive  its entries are never shown in a report, only its size
+
+and the compartment base class builds a container for each one - Ring, Table,
+Members, Counters - that enforces the bound and the shape, and counts itself:
+inserts, evictions, refusals and its high-water mark. So the declaration and
+the behaviour cannot drift apart, and the node meters every compartment's
+memory without the compartment writing a metric.
+
+What is NOT a store: a list of callbacks (wiring), a config cache, a small
+fixed lookup table, a per-packet buffer on a hot path. The rule is whether
+the collection grows with the outside world.
+
+Durable data (the event store, grants, the change log) lives in sqlite and is
+not declared here yet. A persist value for it arrives with the code that
+honours it.
+
+A package rather than a root module for the reason node/channels is one:
+node/*.py is held to a monotone baseline (fleetmesh.org/scripts/check-root.py).
+Standard library only.
+"""
+from __future__ import annotations
+
+import collections
+import dataclasses
+import re
+import threading
+from typing import (
+    Any,
+    Generic,
+    Hashable,
+    Iterable,
+    Iterator,
+    Mapping,
+    MutableMapping,
+    Optional,
+    Sequence,
+    TypedDict,
+    TypeVar,
+    Union,
+    is_typeddict,
+)
+
+# ---- the vocabulary -------------------------------------------------------------
+RING = 'ring'
+TABLE = 'table'
+SET = 'set'
+COUNTERS = 'counters'
+KINDS: tuple[str, ...] = (RING, TABLE, SET, COUNTERS)
+
+OLDEST = 'oldest'
+LRU = 'lru'
+REFUSE = 'refuse'
+EVICTIONS: tuple[str, ...] = (OLDEST, LRU, REFUSE)
+
+MEMORY = 'memory'
+SNAPSHOT = 'snapshot'
+PERSISTS: tuple[str, ...] = (MEMORY, SNAPSHOT)
+
+# A REFUSE store this full is trouble before it is refusing: the operator
+# hears about it while there is still room to act.
+NEARLY_FULL = 0.9
+
+_KEY = re.compile(r'[a-z_][a-z0-9_]*$')
+
+
+class StoreFull(Exception):
+    """A REFUSE store at its bound was asked to take a new entry."""
+
+
+def shape_problem(shape: Optional[type], value: object) -> str:
+    """Why `value` is not one `shape`, or '' when it is (or nothing is declared).
+
+    Shallow by design: a TypedDict's keys, not its values' types. The keys are
+    where drift happens - a field renamed in one writer and not another - and
+    checking them costs a set comparison per write.
+    """
+    if shape is None:
+        return ''
+    if is_typeddict(shape):
+        if not isinstance(value, dict):
+            return 'a %s is a dict, not %s' % (shape.__name__, type(value).__name__)
+        required: frozenset[str] = getattr(shape, '__required_keys__', frozenset())
+        known = required | getattr(shape, '__optional_keys__', frozenset())
+        missing = sorted(required - value.keys())
+        extra = sorted(k for k in value if k not in known)
+        if missing or extra:
+            return 'a %s %s' % (shape.__name__, '; '.join(
+                (['is missing %s' % ', '.join(missing)] if missing else [])
+                + (['has no field %s' % ', '.join(extra)] if extra else [])))
+        return ''
+    if isinstance(shape, type) and not isinstance(value, shape):
+        return 'expected %s, got %s' % (shape.__name__, type(value).__name__)
+    return ''
+
+
+class StoreRecord(TypedDict, total=False):
+    """A store as the census, `meshnode subsystems` and the docs read it."""
+    key: str
+    description: str
+    kind: str
+    shape: str
+    bound: int
+    evict: str
+    keyed_by: str
+    persist: str
+    sensitive: bool
+    tags: list[str]
+    compartment: str
+
+
+class StoreLive(TypedDict):
+    """What a store is holding now, for a compartment's report."""
+    kind: str
+    size: int
+    bound: int
+    high_water: int
+    inserts: int
+    evictions: int
+    refused: int
+
+
+@dataclasses.dataclass(frozen=True)
+class Store:
+    """One collection a compartment keeps. See the module docstring."""
+    key: str
+    description: str
+    kind: str
+    shape: Optional[type] = None
+    bound: int = 0
+    evict: str = OLDEST
+    keyed_by: str = ''
+    persist: str = MEMORY
+    sensitive: bool = False
+    tags: tuple[str, ...] = ()
+
+    @property
+    def limit(self) -> int:
+        """The bound in force: declared, or a COUNTERS store's field count."""
+        if self.kind == COUNTERS:
+            return len(_fields(self.shape))
+        return self.bound
+
+    def problems(self) -> list[str]:
+        """Everything wrong with this declaration, in sentences, collected."""
+        out: list[str] = []
+        if not _KEY.match(self.key or ''):
+            out.append('store %r: a key is lowercase letters, digits and underscores' % self.key)
+        if not self.description:
+            out.append('store %s: say what it holds (description)' % self.key)
+        if self.kind not in KINDS:
+            out.append('store %s: kind must be one of %s' % (self.key, ', '.join(KINDS)))
+        if self.persist not in PERSISTS:
+            out.append('store %s: persist must be one of %s' % (self.key, ', '.join(PERSISTS)))
+        if self.kind == COUNTERS:
+            if not is_typeddict(self.shape):
+                out.append('store %s: COUNTERS name their fields with a TypedDict shape' % self.key)
+            if self.bound:
+                out.append('store %s: COUNTERS are bounded by their shape; drop bound' % self.key)
+            return out
+        if self.bound <= 0:
+            out.append('store %s: a %s must declare its bound - how far it may grow' % (self.key, self.kind))
+        if self.evict not in EVICTIONS:
+            out.append('store %s: evict must be one of %s' % (self.key, ', '.join(EVICTIONS)))
+        if self.kind == RING and self.evict != OLDEST:
+            out.append('store %s: a ring drops its oldest entry; evict is OLDEST' % self.key)
+        if self.kind in (TABLE, SET) and not self.keyed_by:
+            out.append('store %s: a %s says what it grows with (keyed_by: peer, event, url ...)'
+                       % (self.key, self.kind))
+        return out
+
+    def describe(self) -> StoreRecord:
+        out: StoreRecord = {'key': self.key, 'description': self.description, 'kind': self.kind,
+                            'bound': self.limit, 'persist': self.persist, 'tags': list(self.tags)}
+        if self.shape is not None:
+            out['shape'] = getattr(self.shape, '__name__', str(self.shape))
+        if self.kind != COUNTERS:
+            out['evict'] = self.evict
+        if self.keyed_by:
+            out['keyed_by'] = self.keyed_by
+        if self.sensitive:
+            out['sensitive'] = True
+        return out
+
+    def build(self) -> 'Container':
+        """The container that keeps this declaration's promises."""
+        match self.kind:
+            case 'ring':
+                return Ring(self)
+            case 'table':
+                return Table(self)
+            case 'set':
+                return Members(self)
+            case _:
+                return Counters(self)
+
+
+def _fields(shape: Optional[type]) -> tuple[str, ...]:
+    if shape is None or not is_typeddict(shape):
+        return ()
+    return tuple(getattr(shape, '__annotations__', {}))
+
+
+def declared(obj: Any) -> tuple[Store, ...]:
+    """The stores a compartment declares, or () for one not yet converted."""
+    return tuple(getattr(obj, 'STORES', ()) or ())
+
+
+def every_store(compartments: Sequence[Any]) -> list[StoreRecord]:
+    """Every declared store: what the node keeps, beside what it says
+    (every_channel), measures (every_metric) and can be told (every_parameter)."""
+    out: list[StoreRecord] = []
+    for c in compartments:
+        for s in declared(c):
+            row = s.describe()
+            row['compartment'] = getattr(c, 'name', '') or getattr(c, '__name__', str(c))
+            out.append(row)
+    return out
+
+
+# ---- the containers ---------------------------------------------------------------
+class _Container:
+    """What every container shares: the declaration, a lock, and its counts.
+
+    Locked because the tick writes on one thread while the control socket
+    reads `state` on another; iterating a dict while it is resized raises.
+    Every read that iterates takes a copy under the lock.
+    """
+
+    def __init__(self, store: Store) -> None:
+        self.store: Store = store
+        self._lock: threading.RLock = threading.RLock()
+        self.inserts: int = 0
+        self.evictions: int = 0
+        self.refused: int = 0
+        self.high_water: int = 0
+
+    # Each container says how big it is; the counts are shared.
+    def __len__(self) -> int:
+        raise NotImplementedError
+
+    def _check(self, value: object) -> None:
+        why = shape_problem(self.store.shape, value)
+        if why:
+            raise TypeError('store %s: %s' % (self.store.key, why))
+
+    def _grew(self) -> None:
+        self.inserts += 1
+        n = len(self)
+        if n > self.high_water:
+            self.high_water = n
+
+    def live(self) -> StoreLive:
+        return {'kind': self.store.kind, 'size': len(self), 'bound': self.store.limit,
+                'high_water': self.high_water, 'inserts': self.inserts,
+                'evictions': self.evictions, 'refused': self.refused}
+
+    def nearly_full(self) -> bool:
+        lim = self.store.limit
+        return bool(lim) and len(self) >= NEARLY_FULL * lim
+
+    # ---- carried across a process restart (persist=SNAPSHOT) -----------------
+    def dump(self) -> object:
+        """The contents as JSON-able data."""
+        raise NotImplementedError
+
+    def check(self, data: object) -> None:
+        """Raise ValueError unless `data` is a dump this store can load whole.
+        Separate from load() so a compartment's restore is all or nothing."""
+        raise NotImplementedError
+
+    def load(self, data: object) -> None:
+        """Replace the contents with a checked dump. Not counted as inserts."""
+        raise NotImplementedError
+
+
+V = TypeVar('V')
+K = TypeVar('K', bound=Hashable)
+
+
+class Ring(_Container, Generic[V]):
+    """A log: newest last, the oldest dropped at the bound."""
+
+    def __init__(self, store: Store) -> None:
+        super().__init__(store)
+        self._items: collections.deque[V] = collections.deque()
+
+    def __len__(self) -> int:
+        return len(self._items)
+
+    def append(self, item: V) -> None:
+        self._check(item)
+        with self._lock:
+            self._items.append(item)
+            while len(self._items) > self.store.bound:
+                self._items.popleft()
+                self.evictions += 1
+            self._grew()
+
+    def extend(self, items: Iterable[V]) -> None:
+        for item in items:
+            self.append(item)
+
+    def last(self, n: int) -> list[V]:
+        with self._lock:
+            return list(self._items)[-n:] if n > 0 else []
+
+    def discard(self, item: V) -> bool:
+        """Take back the newest entry equal to `item`; False if none is kept.
+
+        For a log written before the thing it records, and undone when that
+        thing failed (sync records a handover, then sends it). The newest,
+        because the entry being undone is the one just written. Not an
+        eviction: nothing was dropped for room, so the counts stay as they are.
+        """
+        with self._lock:
+            for i in range(len(self._items) - 1, -1, -1):
+                if self._items[i] == item:
+                    del self._items[i]
+                    return True
+            return False
+
+    def __getitem__(self, index: Union[int, slice]) -> Any:
+        with self._lock:
+            if isinstance(index, slice):
+                return list(self._items)[index]
+            return self._items[index]
+
+    def __iter__(self) -> Iterator[V]:
+        with self._lock:
+            return iter(list(self._items))
+
+    def clear(self) -> None:
+        with self._lock:
+            self._items.clear()
+
+    def dump(self) -> object:
+        with self._lock:
+            return list(self._items)
+
+    def check(self, data: object) -> None:
+        if not isinstance(data, list):
+            raise ValueError('store %s: a ring dump is a list' % self.store.key)
+        for item in data:
+            why = shape_problem(self.store.shape, item)
+            if why:
+                raise ValueError('store %s: %s' % (self.store.key, why))
+
+    def load(self, data: object) -> None:
+        self.check(data)
+        assert isinstance(data, list)
+        with self._lock:
+            self._items = collections.deque(data[-self.store.bound:])
+            self.high_water = max(self.high_water, len(self._items))
+
+
+class Table(_Container, MutableMapping[K, V]):
+    """A bounded mapping. Reads refresh an entry's place when evict is LRU."""
+
+    def __init__(self, store: Store) -> None:
+        super().__init__(store)
+        self._rows: collections.OrderedDict[K, V] = collections.OrderedDict()
+
+    def __len__(self) -> int:
+        return len(self._rows)
+
+    def __getitem__(self, key: K) -> V:
+        with self._lock:
+            v = self._rows[key]
+            if self.store.evict == LRU:
+                self._rows.move_to_end(key)
+            return v
+
+    def __setitem__(self, key: K, value: V) -> None:
+        if not self.put(key, value):
+            raise StoreFull('store %s is full (%d %s); it refuses new entries rather than drop one'
+                            % (self.store.key, self.store.bound, self.store.keyed_by or 'entries'))
+
+    def put(self, key: K, value: V) -> bool:
+        """Write one entry. False - and counted - when a REFUSE store is full."""
+        self._check(value)
+        with self._lock:
+            if key in self._rows:
+                self._rows[key] = value
+                if self.store.evict == LRU:
+                    self._rows.move_to_end(key)
+                return True
+            if len(self._rows) >= self.store.bound:
+                if self.store.evict == REFUSE:
+                    self.refused += 1
+                    return False
+                self._rows.popitem(last=False)
+                self.evictions += 1
+            self._rows[key] = value
+            self._grew()
+            return True
+
+    def __delitem__(self, key: K) -> None:
+        with self._lock:
+            del self._rows[key]
+
+    def __iter__(self) -> Iterator[K]:
+        with self._lock:
+            return iter(list(self._rows))
+
+    def __contains__(self, key: object) -> bool:
+        with self._lock:
+            return key in self._rows
+
+    def __repr__(self) -> str:
+        return 'Table(%s, %d/%d)' % (self.store.key, len(self), self.store.bound)
+
+    def dump(self) -> object:
+        # Pairs, not an object: a JSON object's keys are strings, and order
+        # is what LRU and OLDEST evict by.
+        with self._lock:
+            return [[k, v] for k, v in self._rows.items()]
+
+    def check(self, data: object) -> None:
+        if not isinstance(data, list) or not all(isinstance(p, list) and len(p) == 2 for p in data):
+            raise ValueError('store %s: a table dump is a list of [key, value] pairs' % self.store.key)
+        if len(data) > self.store.bound:
+            raise ValueError('store %s: %d entries, more than its bound of %d'
+                             % (self.store.key, len(data), self.store.bound))
+        for k, v in data:
+            if not isinstance(k, (str, int)):
+                raise ValueError('store %s: key %r is not a string or an integer' % (self.store.key, k))
+            why = shape_problem(self.store.shape, v)
+            if why:
+                raise ValueError('store %s: %s' % (self.store.key, why))
+
+    def load(self, data: object) -> None:
+        self.check(data)
+        assert isinstance(data, list)
+        with self._lock:
+            self._rows = collections.OrderedDict((k, v) for k, v in data)
+            self.high_water = max(self.high_water, len(self._rows))
+
+
+class Members(_Container, Generic[K]):
+    """A bounded set. Evicts in the order members were added (or touched, LRU)."""
+
+    def __init__(self, store: Store) -> None:
+        super().__init__(store)
+        self._m: collections.OrderedDict[K, None] = collections.OrderedDict()
+
+    def __len__(self) -> int:
+        return len(self._m)
+
+    def add(self, member: K) -> bool:
+        """Add one. False - and counted - when a REFUSE set is full."""
+        self._check(member)
+        with self._lock:
+            if member in self._m:
+                if self.store.evict == LRU:
+                    self._m.move_to_end(member)
+                return True
+            if len(self._m) >= self.store.bound:
+                if self.store.evict == REFUSE:
+                    self.refused += 1
+                    return False
+                self._m.popitem(last=False)
+                self.evictions += 1
+            self._m[member] = None
+            self._grew()
+            return True
+
+    def discard(self, member: K) -> None:
+        with self._lock:
+            self._m.pop(member, None)
+
+    def remove(self, member: K) -> None:
+        with self._lock:
+            if member not in self._m:
+                raise KeyError(member)
+            del self._m[member]
+
+    def clear(self) -> None:
+        with self._lock:
+            self._m.clear()
+
+    def __contains__(self, member: object) -> bool:
+        with self._lock:
+            return member in self._m
+
+    def __iter__(self) -> Iterator[K]:
+        with self._lock:
+            return iter(list(self._m))
+
+    def dump(self) -> object:
+        with self._lock:
+            return list(self._m)
+
+    def check(self, data: object) -> None:
+        if not isinstance(data, list) or len(data) > self.store.bound:
+            raise ValueError('store %s: a set dump is a list within its bound' % self.store.key)
+        for m in data:
+            why = shape_problem(self.store.shape, m)
+            if why:
+                raise ValueError('store %s: %s' % (self.store.key, why))
+
+    def load(self, data: object) -> None:
+        self.check(data)
+        assert isinstance(data, list)
+        with self._lock:
+            self._m = collections.OrderedDict((m, None) for m in data)
+            self.high_water = max(self.high_water, len(self._m))
+
+
+class Counters(_Container):
+    """A fixed set of named integers - the fields of the store's TypedDict."""
+
+    def __init__(self, store: Store) -> None:
+        super().__init__(store)
+        self._n: dict[str, int] = {f: 0 for f in _fields(store.shape)}
+        self.high_water = len(self._n)
+
+    def __len__(self) -> int:
+        return len(self._n)
+
+    def __getitem__(self, field: str) -> int:
+        return self._n[field]
+
+    def __setitem__(self, field: str, value: int) -> None:
+        if field not in self._n:
+            raise KeyError('store %s has no counter %r; its fields are %s'
+                           % (self.store.key, field, ', '.join(self._n)))
+        if not isinstance(value, int) or isinstance(value, bool):
+            raise TypeError('store %s: %s is a count, not %s' % (self.store.key, field, type(value).__name__))
+        with self._lock:
+            self._n[field] = value
+
+    def incr(self, field: str, n: int = 1) -> int:
+        with self._lock:
+            self[field] = self._n.get(field, 0) + n
+            self.inserts += 1
+            return self._n[field]
+
+    def as_dict(self) -> dict[str, int]:
+        with self._lock:
+            return dict(self._n)
+
+    def __iter__(self) -> Iterator[str]:
+        with self._lock:
+            return iter(list(self._n))
+
+    def items(self) -> list[tuple[str, int]]:
+        with self._lock:
+            return list(self._n.items())
+
+    def keys(self) -> list[str]:
+        with self._lock:
+            return list(self._n.keys())
+
+    def values(self) -> list[int]:
+        with self._lock:
+            return list(self._n.values())
+
+    def get(self, field: str, default: Optional[int] = None) -> Optional[int]:
+        with self._lock:
+            return self._n.get(field, default)
+
+    def dump(self) -> object:
+        return self.as_dict()
+
+    def check(self, data: object) -> None:
+        if not isinstance(data, dict):
+            raise ValueError('store %s: a counters dump is an object' % self.store.key)
+        for f in self._n:
+            v = data.get(f)
+            if not isinstance(v, int) or isinstance(v, bool) or v < 0:
+                raise ValueError('store %s: %s is %r, not a count' % (self.store.key, f, v))
+
+    def load(self, data: object) -> None:
+        self.check(data)
+        assert isinstance(data, dict)
+        with self._lock:
+            self._n = {f: int(data[f]) for f in self._n}
+
+
+Container = Union[Ring[Any], Table[Any, Any], Members[Any], Counters]
+
+
+class Holdings:
+    """One compartment's containers, built from its STORES. Owned by the base."""
+
+    def __init__(self, stores: Sequence[Store]) -> None:
+        self._by_key: dict[str, Container] = {s.key: s.build() for s in stores}
+
+    def __getitem__(self, key: str) -> Container:
+        return self._by_key[key]
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._by_key)
+
+    def __len__(self) -> int:
+        return len(self._by_key)
+
+    def items(self) -> list[tuple[str, Container]]:
+        return list(self._by_key.items())
+
+    def live(self) -> dict[str, StoreLive]:
+        return {k: c.live() for k, c in self._by_key.items()}
+
+    def persisted(self) -> dict[str, Container]:
+        return {k: c for k, c in self._by_key.items() if c.store.persist == SNAPSHOT}
+
+    def snapshot(self) -> Mapping[str, Any]:
+        return {k: c.dump() for k, c in self.persisted().items()}
+
+    def restore(self, state: Mapping[str, Any]) -> None:
+        """All or nothing: every persisted store checks its dump, then all load."""
+        mine = self.persisted()
+        for k, c in mine.items():
+            if k not in state:
+                raise ValueError('the snapshot has no store %s' % k)
+            c.check(state[k])
+        for k, c in mine.items():
+            c.load(state[k])
+
+
+__all__ = [
+    'COUNTERS', 'Container', 'Counters', 'EVICTIONS', 'Holdings', 'KINDS', 'LRU', 'MEMORY', 'Members',
+    'NEARLY_FULL', 'OLDEST', 'PERSISTS', 'REFUSE', 'RING', 'Ring', 'SET', 'SNAPSHOT', 'Store', 'StoreFull',
+    'StoreLive', 'StoreRecord', 'TABLE', 'Table', 'declared', 'every_store', 'shape_problem',
+]