fleetmesh-node 0.2.0__py3-none-any.whl

node/storage/sharding.py

@@ -0,0 +1,448 @@
+"""ShardCoverageManager: reusable Kind 30803 shard coverage claims, HandoverLink
+provenance chains, hole auditing, and claimed-shard retention sweeps over
+`ProjectedNodeStore`.
+
+Wraps `fleetmesh_core.peering.coverage` and `node.planes.store.sync.rules` into
+a single stateful manager backed by SQLite so any service or compartment using
+`ProjectedNodeStore` gets:
+  - `coverage_claim(shard_id, ...)`: signed Kind 30803 claim with monotonic
+    `forward_clock` and automatic inline handover chain attachment when
+    `handover_rules.still_fits` holds.
+  - `issue_handover(shard_id, to_pubkey, ...)`: signed `HandoverLink` +
+    `handover_rules.extend` provenance chain + `sync/1.0/handover` envelope.
+  - `accept_handover(shard_id, chain, sender_pubkey, ...)`: validates inbound
+    chain via `handover_rules.acceptable`, persists it in
+    `projected_shard_handovers`, and publishes the replica's asserted claim.
+  - `audit_claim(claim, ...)`: checks `coverage.barred_reporter`,
+    `coverage.hole`, and `coverage.hole_in_handover`.
+  - `sweep_retention(ages_by_kind, ...)`: paginated age sweep that never evicts
+    events inside any claimed shard filter (`protected_filters()`).
+"""
+from __future__ import annotations
+
+import json
+from typing import Any, Callable, Mapping, Optional, Sequence, TypedDict, cast
+
+from fleetmesh_core.peering import coverage
+from fleetmesh_core.storage import replication, store
+from fleetmesh_core.wire.wire_types import HandoverLink
+from node.compartment.schema import Column, Table
+from node.interfaces import NostrEvent, NostrFilter
+from node.planes.store.sync import rules as handover_rules
+
+from .projected import ProjectedNodeStore
+
+__all__ = [
+    'ClaimAuditReply',
+    'HandoverAcceptReply',
+    'HandoverIssueReply',
+    'SHARD_HANDOVERS_TABLE',
+    'ShardCoverageManager',
+    'ShardSummaryRecord',
+]
+
+
+class HandoverIssueReply(TypedDict):
+    shard_id: str
+    origin_claim: NostrEvent
+    link: HandoverLink
+    chain: list[HandoverLink]
+    handover_message: Mapping[str, Any]
+
+
+class HandoverAcceptReply(TypedDict):
+    ok: bool
+    shard_id: str
+    from_pubkey: str
+    chain_length: int
+    fingerprint: str
+    count: int
+    replica_claim: NostrEvent
+    check_coverage_problems: list[str]
+
+
+class ClaimAuditReply(TypedDict):
+    ok: bool
+    barred: bool
+    barred_reason: Optional[str]
+    holes: list[list[str]]
+    handover_holes: list[list[str]]
+
+
+class ShardSummaryRecord(TypedDict):
+    shard_id: str
+    filter: NostrFilter
+    event_count: int
+    fingerprint: str
+    coverage_claim: NostrEvent
+    handover_chain: list[HandoverLink]
+
+
+SHARD_HANDOVERS_TABLE = Table(
+    'projected_shard_handovers',
+    (
+        Column('shard_id', 'TEXT', primary_key=True),
+        Column('from_pk', 'TEXT', not_null=True),
+        Column('since_ts', 'INTEGER', not_null=True),
+        Column('until_ts', 'INTEGER', not_null=True),
+        Column('fingerprint', 'TEXT', not_null=True),
+        Column('event_count', 'INTEGER', not_null=True),
+        Column('chain_json', 'TEXT', not_null=True),
+        Column('claim_json', 'TEXT', not_null=True),
+        Column('updated_at', 'INTEGER', not_null=True),
+    ),
+)
+
+
+class ShardCoverageManager:
+    """Manages Kind 30803 coverage claims, handover chains, hole audits, and
+    retention sweeps over a `ProjectedNodeStore`."""
+
+    def __init__(
+        self,
+        projected_store: ProjectedNodeStore,
+        *,
+        node_sk: str,
+        node_pk: Optional[str] = None,
+        extra_shards: Optional[Mapping[str, Mapping[str, Any]]] = None,
+        clock: Optional[Callable[[], int]] = None,
+    ) -> None:
+        self.store = projected_store
+        self.node_sk = node_sk.strip().lower()
+        from fleetmesh_core.crypto import crypto
+        self.node_pk = (node_pk or crypto.pubkey(self.node_sk)).strip().lower()
+        self._extra_shards: dict[str, NostrFilter] = {
+            k: cast(NostrFilter, dict(v)) for k, v in (extra_shards or {}).items()
+        }
+        self.clock: Callable[[], int] = clock or self.store.clock
+        self._handovers: dict[str, list[HandoverLink]] = {}
+        self._claims: dict[str, NostrEvent] = {}
+        self._ensure_schema()
+        self._load_persisted()
+
+    def _ensure_schema(self) -> None:
+        with self.store._lock:
+            if hasattr(self.store.dialect, 'learn_table'):
+                self.store.dialect.learn_table(SHARD_HANDOVERS_TABLE)
+            self.store.execute(SHARD_HANDOVERS_TABLE.to_sql(dialect=self.store.dialect.name))
+            for sec in SHARD_HANDOVERS_TABLE.secondary_sql(dialect=self.store.dialect.name):
+                self.store.execute(sec)
+            self.store.commit()
+
+    def _load_persisted(self) -> None:
+        with self.store._lock:
+            rows = self.store._all(
+                'SELECT shard_id, chain_json, claim_json FROM projected_shard_handovers'
+            )
+            for sid, chain_raw, claim_raw in rows:
+                try:
+                    self._handovers[str(sid)] = json.loads(str(chain_raw))
+                    self._claims[str(sid)] = json.loads(str(claim_raw))
+                except ValueError:
+                    continue
+
+    def shard_filters(self) -> dict[str, NostrFilter]:
+        """Return all registered shard filters from `ProjectedNodeStore` merged
+        with any `extra_shards`."""
+        merged = self.store.shard_filters()
+        for sid, filt in self._extra_shards.items():
+            merged[sid] = cast(NostrFilter, dict(filt))
+        return merged
+
+    def shard_filter(self, shard_id: str) -> NostrFilter:
+        filters = self.shard_filters()
+        if shard_id not in filters:
+            raise ValueError(f'Unknown shard_id: {shard_id!r}')
+        return filters[shard_id]
+
+    def handover_chain(self, shard_id: str) -> list[HandoverLink]:
+        with self.store._lock:
+            return [cast(HandoverLink, dict(x)) for x in self._handovers.get(shard_id, [])]
+
+    def coverage_claim(
+        self,
+        shard_id: str,
+        *,
+        since: int = 0,
+        until: Optional[int] = None,
+        completeness: str = 'asserted',
+        now: Optional[int] = None,
+    ) -> NostrEvent:
+        """Build and sign a Kind 30803 `coverage_claim` for `shard_id`."""
+        filt = self.shard_filter(shard_id)
+        now_ts = int(now if now is not None else self.clock())
+
+        with self.store._lock:
+            chain = self._handovers.get(shard_id)
+            if chain and until is None:
+                art = chain[-1]
+                h_since = int(art.get('since') or 0)
+                h_until = int(art['until'])
+                held_ids = self.store.enumerate_range(filt, h_since, h_until, max(now_ts, h_until))
+                if handover_rules.still_fits(chain, filt, held_ids) is None:
+                    claim_ts = self.store.forward_clock(
+                        f'{coverage.COVERAGE}:{self.node_pk}:{shard_id}',
+                        max(now_ts, h_until),
+                    )
+                    unsigned = coverage.coverage_claim(
+                        self.node_pk,
+                        shard_id,
+                        filt,
+                        'asserted',
+                        held_ids,
+                        claim_ts,
+                        since=h_since if art.get('since') is not None else None,
+                        until=h_until,
+                        handover=chain,
+                    )
+                    signed = cast(
+                        NostrEvent,
+                        dict(
+                            store.make_signed_event(
+                                self.node_sk,
+                                unsigned['created_at'],
+                                unsigned['kind'],
+                                unsigned['tags'],
+                                unsigned['content'],
+                            )
+                        ),
+                    )
+                    self._claims[shard_id] = signed
+                    return cast(NostrEvent, dict(signed))
+
+            until_ts = int(until) if until is not None else (now_ts + 1)
+            claim_ts = self.store.forward_clock(
+                f'{coverage.COVERAGE}:{self.node_pk}:{shard_id}',
+                max(now_ts, until_ts),
+            )
+            ids = self.store.enumerate_range(filt, since, until_ts, claim_ts)
+            unsigned = coverage.coverage_claim(
+                self.node_pk,
+                shard_id,
+                filt,
+                completeness,
+                ids,
+                claim_ts,
+                since=since,
+                until=until_ts,
+            )
+            signed = cast(
+                NostrEvent,
+                dict(
+                    store.make_signed_event(
+                        self.node_sk,
+                        unsigned['created_at'],
+                        unsigned['kind'],
+                        unsigned['tags'],
+                        unsigned['content'],
+                    )
+                ),
+            )
+            self._claims[shard_id] = signed
+            return cast(NostrEvent, dict(signed))
+
+    def issue_handover(
+        self,
+        shard_id: str,
+        to_pubkey: str,
+        *,
+        since: int = 0,
+        until: Optional[int] = None,
+        now: Optional[int] = None,
+    ) -> HandoverIssueReply:
+        """Sign a `HandoverLink` (`coverage.handover`) over `[since, until)` for
+        `shard_id` to `to_pubkey`, extending any held upstream handover chain via
+        `handover_rules.extend`."""
+        filt = self.shard_filter(shard_id)
+        clean_to = (to_pubkey or '').strip().lower()
+        if not store.is_hex64(clean_to):
+            raise ValueError('to_pubkey must be a 64-character lowercase hex Nostr pubkey')
+
+        now_ts = int(now if now is not None else self.clock()) + 2
+        until_ts = int(until) if until is not None else now_ts
+        if until_ts > now_ts:
+            now_ts = until_ts
+
+        with self.store._lock:
+            held_chain = self._handovers.get(shard_id)
+            if held_chain and until is None:
+                art = held_chain[-1]
+                h_since = int(art.get('since') or 0)
+                h_until = int(art['until'])
+                held_ids = self.store.enumerate_range(filt, h_since, h_until, max(now_ts, h_until))
+                if handover_rules.still_fits(held_chain, filt, held_ids) is None:
+                    origin_claim = self.coverage_claim(shard_id, now=max(now_ts, h_until))
+                    link = coverage.handover(self.node_sk, clean_to, origin_claim, max(now_ts, h_until))
+                    chain, why_not = handover_rules.extend(held_chain, link)
+                    if why_not is not None or chain is None:
+                        raise ValueError(f'handover chain extension refused: {why_not}')
+                    msg = coverage.handover_message(chain, thid=f'sync-{shard_id}')
+                    return {
+                        'shard_id': shard_id,
+                        'origin_claim': origin_claim,
+                        'link': cast(HandoverLink, dict(link)),
+                        'chain': [cast(HandoverLink, dict(c)) for c in chain],
+                        'handover_message': dict(msg),
+                    }
+
+            origin_claim = self.coverage_claim(
+                shard_id,
+                since=since,
+                until=until_ts,
+                completeness='asserted',
+                now=now_ts,
+            )
+            link = coverage.handover(self.node_sk, clean_to, origin_claim, now_ts)
+            chain_out = [cast(HandoverLink, dict(link))]
+            msg = coverage.handover_message(chain_out, thid=f'sync-{shard_id}')
+            return {
+                'shard_id': shard_id,
+                'origin_claim': origin_claim,
+                'link': cast(HandoverLink, dict(link)),
+                'chain': chain_out,
+                'handover_message': dict(msg),
+            }
+
+    def accept_handover(
+        self,
+        shard_id: str,
+        chain: Sequence[Mapping[str, Any]],
+        sender_pubkey: str,
+        *,
+        now: Optional[int] = None,
+    ) -> HandoverAcceptReply:
+        """Validate an inbound `HandoverLink` chain using `handover_rules.acceptable`,
+        persist it, and publish this replica's `asserted` Kind 30803 claim."""
+        filt = self.shard_filter(shard_id)
+        clean_sender = (sender_pubkey or '').strip().lower()
+        if not chain:
+            raise ValueError('handover chain is empty')
+
+        art = chain[-1]
+        h_since = int(art.get('since') or 0)
+        h_until = int(art.get('until') or 0)
+        now_ts = max(int(now if now is not None else self.clock()), h_until)
+
+        with self.store._lock:
+            held_ids = self.store.enumerate_range(filt, h_since, h_until, now_ts)
+            shard_cfg = {'shard': shard_id, 'filter': filt}
+            why_not = handover_rules.acceptable(
+                chain,
+                self.node_pk,
+                clean_sender,
+                shard_cfg,
+                held_ids,
+            )
+            if why_not is not None:
+                raise ValueError(f'handover_refused: {why_not}')
+
+            clean_chain = [cast(HandoverLink, dict(link)) for link in chain]
+            self._handovers[shard_id] = clean_chain
+            replica_claim = self.coverage_claim(shard_id, now=now_ts)
+            problems = coverage.check_coverage(replica_claim)
+            if problems:
+                del self._handovers[shard_id]
+                raise ValueError(f"replica coverage claim invalid: {'; '.join(problems)}")
+
+            self.store.execute(
+                """INSERT OR REPLACE INTO projected_shard_handovers
+                   (shard_id, from_pk, since_ts, until_ts, fingerprint, event_count, chain_json, claim_json, updated_at)
+                   VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+                (
+                    shard_id,
+                    clean_sender,
+                    h_since,
+                    h_until,
+                    str(art['fingerprint']),
+                    int(art['count']),
+                    json.dumps(clean_chain, sort_keys=True),
+                    json.dumps(replica_claim, sort_keys=True),
+                    now_ts,
+                ),
+            )
+            self.store.commit()
+            return {
+                'ok': True,
+                'shard_id': shard_id,
+                'from_pubkey': clean_sender,
+                'chain_length': len(clean_chain),
+                'fingerprint': str(art['fingerprint']),
+                'count': int(art['count']),
+                'replica_claim': replica_claim,
+                'check_coverage_problems': problems,
+            }
+
+    def audit_claim(
+        self,
+        claim: Mapping[str, Any],
+        *,
+        claimant_store: Optional[ProjectedNodeStore] = None,
+        reference_events: Optional[Sequence[Mapping[str, Any]]] = None,
+        reporter_pubkey: Optional[str] = None,
+    ) -> ClaimAuditReply:
+        """Audit an `asserted` Kind 30803 `claim` for missing events (`hole` and
+        `hole_in_handover`) and check `coverage.barred_reporter`."""
+        rep_pk = (reporter_pubkey or self.node_pk).strip().lower()
+        bar_reason = coverage.barred_reporter(claim, rep_pk)
+        if bar_reason is not None:
+            return {
+                'ok': False,
+                'barred': True,
+                'barred_reason': bar_reason,
+                'holes': [],
+                'handover_holes': [],
+            }
+
+        target_store = claimant_store if claimant_store is not None else self.store
+        held_fn = lambda eid: target_store.get_event(eid) is not None
+
+        if reference_events is None:
+            f_tag = store.first_tag(claim, 'filter')
+            filt = json.loads(f_tag[1]) if (f_tag and len(f_tag) > 1) else {}
+            s_tag = store.first_tag(claim, 'since')
+            u_tag = store.first_tag(claim, 'until')
+            s_ts = int(s_tag[1]) if (s_tag and len(s_tag) > 1) else 0
+            u_ts = int(u_tag[1]) if (u_tag and len(u_tag) > 1) else (self.clock() + 120)
+            eids = self.store.enumerate_range(filt, s_ts, u_ts, u_ts)
+            reference_events = [
+                ev for eid in eids
+                if (ev := self.store.get_event(eid)) is not None
+            ]
+
+        holes: list[list[str]] = []
+        handover_holes: list[list[str]] = []
+        inline_ho = coverage.inline_handover(claim)
+        for ev in reference_events:
+            h_ev = coverage.hole(claim, ev, held_fn)
+            if h_ev is not None:
+                holes.append(h_ev)
+            if inline_ho is not None:
+                ho_ev = coverage.hole_in_handover(inline_ho, ev, held_fn)
+                if ho_ev is not None:
+                    handover_holes.append(ho_ev)
+
+        return {
+            'ok': len(holes) == 0,
+            'barred': False,
+            'barred_reason': None,
+            'holes': holes,
+            'handover_holes': handover_holes,
+        }
+
+    def summary(self, *, now: Optional[int] = None) -> dict[str, ShardSummaryRecord]:
+        """Return a per-shard summary with event count, NIP-77 fingerprint,
+        coverage claim, and active handover chain."""
+        now_ts = int(now if now is not None else self.clock()) + 60
+        out: dict[str, ShardSummaryRecord] = {}
+        for shard_id, filt in self.shard_filters().items():
+            claim = self.coverage_claim(shard_id, now=now_ts)
+            ids = self.store.enumerate_range(filt, 0, now_ts, now_ts)
+            out[shard_id] = {
+                'shard_id': shard_id,
+                'filter': filt,
+                'event_count': len(ids),
+                'fingerprint': replication.fingerprint(ids),
+                'coverage_claim': claim,
+                'handover_chain': self.handover_chain(shard_id),
+            }
+        return out