fleetmesh-node 0.2.0__py3-none-any.whl

node/planes/store/projected/compartment.py

@@ -0,0 +1,258 @@
+"""ProjectedDB compartment: operator visibility, audit verification, and
+on-demand rebuilds for co-transactional SQL projections over `ProjectedNodeStore`.
+
+The write path itself is synchronous and co-transactional inside
+`ProjectedNodeStore._put`, `_drop`, and `forget`. This compartment exposes:
+- `projected/show`: registered `TableProjector`s, row counts, causal logs, and
+  recent payload refusals
+- `projected/verify`: audits every projected SQL table against the live `events`
+  table (`verify_projection()`) and raises `trouble()` if any drift is found
+- `projected/rebuild`: rebuilds one or all projected SQL tables from `events`
+"""
+from __future__ import annotations
+
+import dataclasses
+from typing import Any, Mapping, Optional
+
+from node.channels import NOTE, Channel
+from node.compartment import Command, Compartment, Every, Need
+from node.compartment import stores as st
+from node.config.parameter import ParamBlock, Parameter
+from node.metrics import COUNTER, Metric
+from node.storage.projected import (
+    CausalProjector,
+    ProjectedNodeStore,
+    ProjectionVerifyReport,
+    ProjectorSpec,
+    TableProjector,
+)
+
+from .types import ProjectedRebuildReply, ProjectedStateReport, ProjectedTableSummary
+
+VERIFY_SECONDS: int = 3600
+
+
+@dataclasses.dataclass(frozen=True)
+class ProjectedDBConfig(ParamBlock):
+    key = 'projected'
+    name = 'projected'
+    DERIVED: bool = False
+    PARAMS = (
+        Parameter(
+            key='verify_seconds',
+            name='Verify Cadence',
+            description='Seconds between background projection consistency audits',
+            type=int,
+            default=VERIFY_SECONDS,
+            tags=('projected', 'store'),
+        ),
+        Parameter(
+            key='auto_heal',
+            name='Auto-Heal Drift',
+            description='Automatically rebuild projected SQL tables if verification detects drift',
+            type=bool,
+            default=True,
+            tags=('projected', 'store'),
+        ),
+    )
+    verify_seconds: int = VERIFY_SECONDS
+    auto_heal: bool = True
+
+
+class ProjectedDB(Compartment):
+    """Operator surface and consistency auditor for `ProjectedNodeStore`."""
+
+    name = 'projected'
+    reads = ('projected', 'coverage')
+    live = ('projected',)
+    config_cls = ProjectedDBConfig
+    CHANNELS = (
+        Channel(
+            'table_rebuilt',
+            'Projected SQL table rebuilt from events',
+            level=NOTE,
+            say='projected table {table} rebuilt ({rows} rows)',
+            tags=('projected', 'store'),
+        ),
+        Channel(
+            'projection_mismatch',
+            'Projected SQL table drifted from events store',
+            level=NOTE,
+            say='projection mismatch detected ({missing} missing, {orphan} orphan)',
+            tags=('projected', 'store'),
+        ),
+    )
+    METRICS = (
+        Metric(
+            name='projected_tables_rebuilt',
+            description='Times a projected SQL table was rebuilt from events.',
+            kind=COUNTER,
+            counts='table_rebuilt',
+            tags=('projected', 'store'),
+        ),
+        Metric(
+            name='projected_mismatches',
+            description='Times projection verification found drift.',
+            kind=COUNTER,
+            counts='projection_mismatch',
+            tags=('projected', 'store'),
+        ),
+    )
+    NEEDS = (
+        Need('store', 'stores.events'),
+        Need('cfg', 'cfg', getter=True),
+    )
+    EVERY = (Every(seconds=VERIFY_SECONDS, run='verify'),)
+    COMMANDS = (
+        Command(
+            key='show',
+            name='Projected tables',
+            description='registered SQL table projectors, row counts, and payload refusals',
+            tags=('projected', 'status'),
+            permission='projected/show',
+            run='state',
+            reply='{enabled, tables, causal_logs, refusals, verified_at, last_verify_ok}',
+        ),
+        Command(
+            key='verify',
+            name='Verify projections',
+            description='audit projected SQL tables against the live events table',
+            tags=('projected',),
+            permission='projected/verify',
+            run='verify',
+            reply='{ok, tables, missing_events, orphan_rows, mismatched_rows}',
+        ),
+        Command(
+            key='rebuild',
+            name='Rebuild projections',
+            description='rebuild one or all projected SQL tables from the live events table',
+            tags=('projected',),
+            parameters=(
+                Parameter(
+                    key='table_name',
+                    name='table_name',
+                    description='optional projected SQL table name to rebuild (omit to rebuild all)',
+                    type=str,
+                    default='',
+                    required=False,
+                ),
+            ),
+            permission='projected/rebuild',
+            run='rebuild',
+            reply='{rebuilt: {table: rows}}',
+        ),
+    )
+    STORES = (
+        st.Store(
+            key='rebuilds',
+            description='total rebuilds performed per projected table',
+            kind=st.TABLE,
+            shape=int,
+            bound=256,
+            keyed_by='table',
+            evict=st.LRU,
+        ),
+    )
+
+    def __init__(self, store: Any, cfg: Any, clock: Any, node: Any = None) -> None:
+        Compartment.__init__(self, node, clock)
+        self.store = store
+        self._cfg = cfg
+        self.rebuilds: st.Table[str, int]
+        self.verified_at: Optional[int] = None
+        self.last_verify_ok: Optional[bool] = None
+
+    def is_projected_store(self) -> bool:
+        return isinstance(self.store, ProjectedNodeStore)
+
+    def register_projector(self, projector: TableProjector | ProjectorSpec) -> Optional[TableProjector]:
+        if not isinstance(self.store, ProjectedNodeStore):
+            return None
+        return self.store.register_projector(projector)
+
+    def register_causal_projector(self, causal: CausalProjector) -> Optional[CausalProjector]:
+        if not isinstance(self.store, ProjectedNodeStore):
+            return None
+        return self.store.register_causal_projector(causal)
+
+    def verify(self, now: Optional[int] = None) -> ProjectionVerifyReport:
+        now_ts = now if now is not None else self.clock()
+        self.verified_at = now_ts
+        if not isinstance(self.store, ProjectedNodeStore):
+            self.last_verify_ok = True
+            return {
+                'ok': True,
+                'tables': {},
+                'missing_events': [],
+                'orphan_rows': [],
+                'mismatched_rows': [],
+            }
+        report = self.store.verify_projection()
+        self.last_verify_ok = report['ok']
+        if not report['ok']:
+            self.say(
+                'projection_mismatch',
+                missing=len(report['missing_events']),
+                orphan=len(report['orphan_rows']),
+            )
+            raw_cfg = (self._cfg() or {}).get('projected') or {}
+            auto_heal = raw_cfg.get('auto_heal', True) if isinstance(raw_cfg, Mapping) else True
+            if auto_heal:
+                self.rebuild(now=now_ts)
+                report = self.store.verify_projection()
+                self.last_verify_ok = report['ok']
+        return report
+
+    def rebuild(
+        self,
+        table_name: Optional[str | Mapping[str, Any]] = None,
+        now: Optional[int] = None,
+    ) -> ProjectedRebuildReply:
+        if not isinstance(self.store, ProjectedNodeStore):
+            return {'rebuilt': {}}
+        target_table: Optional[str]
+        if isinstance(table_name, Mapping):
+            raw_tbl = str(table_name.get('table_name') or '').strip()
+            target_table = raw_tbl or None
+        else:
+            target_table = (table_name or '').strip() or None if isinstance(table_name, str) else None
+        rebuilt = self.store.backfill(table_name=target_table)
+        for tbl, rows in rebuilt.items():
+            self.rebuilds[tbl] = self.rebuilds.get(tbl, 0) + 1
+            self.say('table_rebuilt', table=tbl, rows=rows)
+        return {'rebuilt': rebuilt}
+
+    def state(self) -> ProjectedStateReport:
+        if not isinstance(self.store, ProjectedNodeStore):
+            return {
+                'enabled': False,
+                'tables': {},
+                'causal_logs': [],
+                'refusals': [],
+                'verified_at': self.verified_at,
+                'last_verify_ok': self.last_verify_ok,
+            }
+        summaries: dict[str, ProjectedTableSummary] = {}
+        for tp in self.store.projectors():
+            summaries[tp.table_name] = {
+                'table': tp.table_name,
+                'kind': tp.kind,
+                'mode': tp.spec.mode(),
+                'primary_key': list(tp.spec.primary_key),
+                'rows': self.store.table_count(tp.table_name),
+                'projected_total': self.store.projected_counts.get(tp.table_name, 0),
+            }
+        return {
+            'enabled': True,
+            'tables': summaries,
+            'causal_logs': [cp.log_id for cp in self.store.causal_projectors()],
+            'refusals': list(self.store.projection_refusals[-20:]),
+            'verified_at': self.verified_at,
+            'last_verify_ok': self.last_verify_ok,
+        }
+
+    def trouble(self) -> list[str]:
+        out: list[str] = []
+        if self.last_verify_ok is False:
+            out.append('one or more projected SQL tables failed verification against the events store')
+        return out

node/planes/store/projected/types.py

@@ -0,0 +1,29 @@
+"""Types and status records for the ProjectedDB compartment."""
+from __future__ import annotations
+
+from typing import Optional, TypedDict
+
+from node.storage.projected import ProjectionRefusal
+
+
+class ProjectedTableSummary(TypedDict):
+    table: str
+    kind: int
+    mode: str
+    primary_key: list[str]
+    rows: int
+    projected_total: int
+
+
+class ProjectedStateReport(TypedDict):
+    enabled: bool
+    tables: dict[str, ProjectedTableSummary]
+    causal_logs: list[str]
+    refusals: list[ProjectionRefusal]
+    verified_at: Optional[int]
+    last_verify_ok: Optional[bool]
+
+
+class ProjectedRebuildReply(TypedDict):
+    rebuilt: dict[str, int]
+

node/planes/store/retention/__init__.py

@@ -0,0 +1,7 @@
+"""What this node stops keeping, and the one thing it must never stop keeping.
+
+One-module compartment: everything is in compartment.py.
+"""
+from .compartment import Retention
+
+__all__ = ['Retention']

node/planes/store/retention/compartment.py

@@ -0,0 +1,214 @@
+"""What this node stops keeping, and the one thing it must never stop keeping.
+
+An open relay face collects what an open face invites. On the alpha mesh one
+node was holding 6,025 events to every other node's 38, of which 5,625 were
+NIP-59 gift wraps between strangers using it as an ordinary DM relay - about
+5,200 arriving a day, 62 MB and climbing, on a box with 113 MB of memory free.
+Nothing bounded it: NIP-40 expiry is enforced when an event is read and never on
+disk, and only 68 of those 6,025 carried an expiration tag at all, so leaning on
+other people's compliance would have reclaimed nothing.
+
+So retention is the operator's, stated per kind, and **the default keeps
+everything**. A relay that quietly began deleting what it was handed would be a
+worse thing than one that fills up, and the specification prices retrieval
+rather than retention precisely because a standing promise to hold something is
+the one product this protocol declines to sell. Nothing here is protocol. It is
+a disk, and whose disk it is decides.
+
+The guard is the part that is not a preference. A kind 30803 coverage claim
+names a shard as a filter and carries a negentropy fingerprint over the set;
+evicting an event inside a claimed shard would leave this node advertising a
+fingerprint it can no longer reproduce, and a peer reconciling against it would
+find the difference and be right. So an event matching any shard this node
+claims is never evicted, whatever the operator asked for, and the refusal is
+recorded rather than performed silently.
+"""
+from __future__ import annotations
+
+import dataclasses
+from typing import Any, Iterator, Mapping, Optional
+from fleetmesh_core.storage import store as store_mod  # noqa: E402
+from node.compartment import stores as st
+from node.channels import Channel, NOTE
+from node.metrics import COUNTER, Metric
+from node.compartment import Command, Compartment, Every, Need  # noqa: E402
+from node.config.parameter import Parameter, ParamBlock
+from .types import (
+    RetentionStateReport,
+)
+
+# A sweep walks the store, so it does not run on every tick. An hour is far
+# finer than any age worth setting and coarse enough to cost nothing.
+SWEEP_SECONDS: int = 3600
+# How many events one page of a sweep holds. The sweep reads events to decide
+# whether a shard claim protects them, and the backlog it reads on the first
+# sweep after a policy is set is unbounded by nature - days of an open face.
+SWEEP_PAGE: int = 500
+
+
+@dataclasses.dataclass(frozen=True)
+class RetentionConfig(ParamBlock):
+    key = 'retention'
+    name = 'retention'
+    DERIVED: bool = False
+    PARAMS = (
+        Parameter(key='sweep_seconds', name='Sweep Cadence', description='Seconds between retention sweeps', type=int, default=SWEEP_SECONDS, tags=('retention', 'sweep')),
+        Parameter(key='sweep_page', name='Sweep Page Size', description='Number of events examined per sweep page', type=int, default=SWEEP_PAGE, tags=('retention', 'sweep')),
+    )
+    sweep_seconds: int = SWEEP_SECONDS
+    sweep_page: int = SWEEP_PAGE
+
+
+class Retention(Compartment):
+    """Per-kind ages, swept on a slow cadence, with the shards held back."""
+
+    name = 'retention'
+    reads = ('retention', 'coverage')
+    live = ('retention',)
+    config_cls = RetentionConfig
+    CHANNELS = (
+        Channel('events_evicted', 'Expired events evicted during sweep', level=NOTE, say='retention sweep evicted {count} expired events', tags=('retention', 'sweep')),
+        Channel('eviction_refused', 'Event eviction refused because it matches a claimed shard', level=NOTE, say='retention eviction refused for event {id}', tags=('retention', 'sweep')),
+    )
+    METRICS = (
+        Metric(name='retention_events_evicted', description='Times expired events were evicted.', kind=COUNTER, counts='events_evicted', tags=('retention', 'sweep')),
+        Metric(name='retention_evictions_refused', description='Times evictions were refused.', kind=COUNTER, counts='eviction_refused', tags=('retention', 'sweep')),
+    )
+    NEEDS = (
+        Need('store', 'stores.events'),
+        Need('cfg', 'cfg', getter=True),
+    )
+    EVERY = (Every(seconds=SWEEP_SECONDS, run='sweep'),)
+    COMMANDS = (
+        Command(
+            key='show',
+            name='Retention',
+            description='the ages this operator set, what they protect, and what has been evicted',
+            tags=('retention', 'status'),
+            permission='retention/show',
+            run='state',
+            reply='{ages, protected, swept_at, evicted, held_back}',
+        ),
+        Command(
+            key='sweep',
+            name='Retention sweep',
+            description='run the sweep now rather than waiting for the cadence',
+            tags=('retention',),
+            permission='retention/sweep',
+            run='sweep',
+            reply='{evicted: {kind: n}, held_back: n, scanned: n}',
+        ),
+    )
+    STORES = (
+        st.Store(key='evicted', description='total ever evicted, per event kind',
+                 kind=st.TABLE, shape=int, bound=1024, keyed_by='kind', evict=st.LRU),
+    )
+
+    def __init__(self, store: Any, cfg: Any, clock: Any, node: Any = None) -> None:
+        Compartment.__init__(self, node, clock)
+        self.store = store
+        self._cfg = cfg                      # a callable returning the live config
+        self.swept_at: Optional[int] = None
+        self.evicted: st.Table[int, int]     # kind -> total ever evicted here
+        self.held_back: int = 0              # events an age would have taken and a shard kept
+
+    # ---- what the operator asked for ----------------------------------------
+    def ages(self) -> dict[int, int]:
+        """{kind: seconds}. Absent, empty or unreadable all mean keep everything."""
+        raw = (self._cfg() or {}).get('retention') or {}
+        if not isinstance(raw, dict):
+            return {}
+        out: dict[int, int] = {}
+        for k, v in raw.items():
+            try:
+                kind, age = int(k), int(v)
+            except (TypeError, ValueError):
+                self.refusals.refuse('retention', 'not a kind and a number of seconds',
+                                     at=self.clock(), detail={'kind': str(k)[:16], 'age': str(v)[:16]})
+                continue
+            if age > 0:
+                out[kind] = age
+        return out
+
+    def protected(self) -> list[Mapping[str, Any]]:
+        """The shard filters a coverage claim or ProjectedNodeStore covers. Never evicted."""
+        cov = (self._cfg() or {}).get('coverage') or []
+        out: list[Mapping[str, Any]] = [
+            c['filter'] for c in cov
+            if isinstance(c, dict) and isinstance(c.get('filter'), dict)
+        ]
+        if hasattr(self.store, 'protected_filters'):
+            for f in self.store.protected_filters():
+                if f not in out:
+                    out.append(f)
+        return out
+
+    # ---- the sweep ----------------------------------------------------------
+    def sweep(self, now: Optional[int] = None) -> Mapping[str, Any]:
+        now = now if now is not None else self.clock()
+        ages, guard = self.ages(), self.protected()
+        evicted: dict[int, int] = {}
+        held_back = 0
+        scanned = 0
+        if not ages:
+            self.swept_at = now
+            return {'evicted': evicted, 'held_back': held_back, 'scanned': scanned}
+        for kind, age in sorted(ages.items()):
+            cutoff = now - age
+            for ev in self._older(kind, cutoff):
+                scanned += 1
+                if any(store_mod.matches(ev, f) for f in guard):
+                    # Not a preference and not overridable: the fingerprint this
+                    # node published over that shard has to stay reproducible.
+                    held_back += 1
+                    self.held_back += 1
+                    self.refusals.refuse('retention', 'inside a shard this node claims coverage for',
+                                         at=now, detail={'id': ev['id'][:16], 'kind': kind})
+                    continue
+                self.store.forget(ev['id'])
+                evicted[kind] = evicted.get(kind, 0) + 1
+                self.evicted[kind] = self.evicted.get(kind, 0) + 1
+        if sum(evicted.values()) > 0 and hasattr(self.store, 'vacuum'):
+            try:
+                self.store.vacuum(pages=50)
+            except Exception:
+                pass
+        self.swept_at = now
+        return {'evicted': evicted, 'held_back': held_back, 'scanned': scanned}
+
+    def _older(self, kind: int, cutoff: int) -> Iterator[Mapping[str, Any]]:
+        """One page at a time, oldest first, so the sweep costs a page and not
+        the backlog. The first sweep after a policy is set is the big one."""
+        after: Optional[tuple[int, str]] = None
+        while True:
+            page = self.store.by_kind_before(kind, cutoff, SWEEP_PAGE, after)
+            if not page:
+                return
+            for ev in page:
+                yield ev
+            if len(page) < SWEEP_PAGE:
+                return
+            after = (page[-1]['created_at'], page[-1]['id'])
+
+    # ---- the contract -------------------------------------------------------
+    def _report_sweep(self, r: Mapping[str, Any]) -> list[tuple[Any, ...]]:
+        did: list[tuple[Any, ...]] = []
+        if r.get('evicted'):
+            did.append(('evicted', sum(r['evicted'].values()), r['evicted']))
+        if r.get('held_back'):
+            did.append(('held back', r['held_back'], 'inside a claimed shard'))
+        return did
+
+    def state(self) -> RetentionStateReport:
+        return {'ages': {str(k): v for k, v in self.ages().items()},
+                'protected': len(self.protected()),
+                'swept_at': self.swept_at,
+                'evicted': {str(k): v for k, v in self.evicted.items()},
+                'held_back': self.held_back}
+
+    def trouble(self) -> list[str]:
+        out: list[str] = []
+        if self.held_back:
+            out.append('%d event(s) an age would have taken are inside a shard this node claims, and '
+                       'were kept: a coverage fingerprint has to stay reproducible' % self.held_back)
+        return out

node/planes/store/retention/types.py

@@ -0,0 +1,12 @@
+"""Types and records for the Retention compartment."""
+from __future__ import annotations
+
+from typing import Optional, TypedDict
+
+
+class RetentionStateReport(TypedDict):
+    ages: dict[str, int]
+    protected: int
+    swept_at: Optional[int]
+    evicted: dict[str, int]
+    held_back: int

node/planes/store/sync/__init__.py

@@ -0,0 +1,35 @@
+"""sync/1.0 over the wire.
+
+fleetmesh_core/sync.py runs a session as two objects handing each other messages in one
+process. Here the two objects are on two nodes and the messages travel as
+DIDComm envelopes through the transport, on one thread per session. The
+reference still decides everything: what to ask for, what to serve, what to
+count, what to find.
+
+Two things are added at the wire. Every delivered event travels inside a
+delivery envelope with a hop signed by the node that sent it, carried as a
+DIDComm attachment named delivery-envelope, and the receiver verifies every
+hop before the reference sees the event; an event whose envelope does not
+verify is not delivered and is a finding. And each side's session is bound
+to the grant that governs it: the initiator pulls under the grant it holds
+from the peer, the responder serves under the grant it issued.
+
+One thing follows a session. A responder that asserts the shard first-hand,
+over a range that has closed, and has just served a peer to equality over
+that whole range, may hand the shard over: a signed statement the peer
+carries in its own coverage claim so it can assert the same set. The offer
+is the responder's to make (config `handover`), the statement copies its
+terms from the claim this node has already published and nothing else, and
+the receiver keeps it only if it fingerprints to what the receiver now holds.
+
+A package by seam: envelope.py (the wire vocabulary), core.py (the
+compartment), initiator.py (`pull`), responder.py (`handle`), handover.py.
+Every name the module exported is re-exported here.
+"""
+from __future__ import annotations
+
+from .core import SyncPeer
+from .envelope import ATTACHMENT, MAX_SESSIONS_PER_PEER, PIURI, SESSION_IDLE, SyncError, envelope_attachment, verified_events
+
+__all__ = ['ATTACHMENT', 'MAX_SESSIONS_PER_PEER', 'PIURI', 'SESSION_IDLE', 'SyncError', 'envelope_attachment',
+           'verified_events', 'SyncPeer']