fleetmesh-node 0.2.0__py3-none-any.whl

node/storage/standing.py

@@ -0,0 +1,568 @@
+"""PeerStandingStore: SQLite-backed peer standing, window AIMD budget evaluation,
+reporter weighting, and standing-gated NIP-77 Negentropy shard reconciliation.
+
+Standardizes the integration of:
+  - `fleetmesh_core.accounting.standing.Ledger` (window-ordered streak tracking,
+    `tell` / `answer`, `report_record`, `on_revocation`)
+  - `fleetmesh_core.accounting.window` (`opening_limits`, `evaluate_window`,
+    `view`, `adjust`, `reporter_weight`, fixed-point millionths consensus)
+  - `fleetmesh_core.storage.sync` (`Grant`, `Session`, `run`)
+"""
+from __future__ import annotations
+
+import contextlib
+import json
+import sqlite3
+import threading
+import time
+from typing import Any, Callable, Iterator, Mapping, Optional, Sequence, TypedDict
+
+from fleetmesh_core.accounting import standing as standing_mod
+from fleetmesh_core.accounting import window
+from fleetmesh_core.storage import store, sync
+from node.compartment.schema import Column, Table
+
+from .projected import ProjectedNodeStore
+from .sharding import HandoverAcceptReply, ShardCoverageManager
+
+__all__ = [
+    'PEER_STANDING_TABLE',
+    'PeerStandingRecord',
+    'PeerStandingStore',
+    'ReconcileShardsReply',
+    'ShardReconcileEntry',
+    'WeightedReportEntry',
+    'WindowEvalRecord',
+]
+
+VALID_TIERS: tuple[str, ...] = ('anchor', 'trusted', 'member', 'probation')
+
+
+class PeerStandingRecord(TypedDict):
+    peer_pk: str
+    peer_did: str
+    tier: str
+    limits: dict[str, int]
+    clean_windows: int
+    evaluated_windows: int
+    contradicted_reports: int
+    last_severity: str
+    last_note: str
+    zero_until: int
+    reporter_weight: float
+    can_sync: bool
+    updated_at: int
+
+
+class WeightedReportEntry(TypedDict):
+    severity: str
+    weight: float
+
+
+# Optional keys sit in total=False bases: typing.NotRequired is Python 3.11+
+# and this package supports 3.10.
+class _WindowEvalOptional(TypedDict, total=False):
+    weighted_reports: list[WeightedReportEntry]
+
+
+class WindowEvalRecord(PeerStandingRecord, _WindowEvalOptional):
+    window_severity: str
+    final_severity: str
+    per_limit: Mapping[str, Any]
+
+
+class ShardReconcileEntry(TypedDict):
+    pull_status: Optional[str]
+    push_status: Optional[str]
+    fingerprint_match: bool
+    findings: list[tuple[str, str]]
+
+
+class _ReconcileShardsOptional(TypedDict, total=False):
+    standing: PeerStandingRecord
+    handovers: dict[str, HandoverAcceptReply]
+
+
+class ReconcileShardsReply(_ReconcileShardsOptional):
+    ok: bool
+    status: str
+    standing_before: PeerStandingRecord
+    standing_after: PeerStandingRecord | WindowEvalRecord
+    shards: dict[str, ShardReconcileEntry]
+
+
+PEER_STANDING_TABLE = Table(
+    'peer_standing',
+    (
+        Column('peer_pk', 'TEXT', primary_key=True),
+        Column('peer_did', 'TEXT', not_null=True),
+        Column('tier', 'TEXT', not_null=True),
+        Column('limits_json', 'TEXT', not_null=True),
+        Column('clean_windows', 'INTEGER', not_null=True),
+        Column('evaluated_windows', 'INTEGER', not_null=True),
+        Column('contradicted_reports', 'INTEGER', not_null=True),
+        Column('last_severity', 'TEXT', not_null=True),
+        Column('last_note', 'TEXT', not_null=True),
+        Column('zero_until', 'INTEGER', not_null=True),
+        Column('updated_at', 'INTEGER', not_null=True),
+    ),
+)
+
+
+class PeerStandingStore:
+    """Persistent peer standing book backed by SQLite and hydrated into a
+    `fleetmesh_core.accounting.standing.Ledger`."""
+
+    def __init__(
+        self,
+        conn: Optional[sqlite3.Connection] = None,
+        *,
+        lock: Optional[threading.RLock] = None,
+        clock: Optional[Callable[[], int]] = None,
+        tier_fallback_fn: Optional[Callable[[str], Optional[str]]] = None,
+        on_tier_change: Optional[Callable[[str, str], None]] = None,
+    ) -> None:
+        self._conn = conn if conn is not None else sqlite3.connect(':memory:', check_same_thread=False)
+        self._lock = lock if lock is not None else threading.RLock()
+        self.clock: Callable[[], int] = clock or (lambda: int(time.time()))
+        self._tier_fallback_fn = tier_fallback_fn
+        self._on_tier_change = on_tier_change
+        self.ledger = standing_mod.Ledger()
+        self._epoch_seq: int = 0
+        self._tables_ready = False
+        self.ensure_tables()
+
+    def close(self) -> None:
+        """Close the underlying SQLite connection."""
+        with self._lock:
+            self._conn.close()
+
+    @contextlib.contextmanager
+    def _tx(self) -> Iterator[sqlite3.Connection]:
+        with self._lock:
+            with self._conn:
+                yield self._conn
+
+    def ensure_tables(self) -> None:
+        if self._tables_ready:
+            return
+        with self._tx() as conn:
+            with contextlib.closing(conn.execute(PEER_STANDING_TABLE.to_sql())):
+                pass
+            for sec in PEER_STANDING_TABLE.secondary_sql():
+                with contextlib.closing(conn.execute(sec)):
+                    pass
+            self._tables_ready = True
+            self._hydrate_ledger(conn)
+
+    @staticmethod
+    def _build_window_history(clean_w: int, eval_w: int, last_sev: str = 'S0') -> list[tuple[int, str]]:
+        c_n = max(0, int(clean_w))
+        e_n = max(c_n, int(eval_w))
+        win_list: list[tuple[int, str]] = []
+        non_clean = e_n - c_n
+        non_clean_sev = str(last_sev) if str(last_sev) != 'S0' else 'S1'
+        for i in range(non_clean):
+            win_list.append((i + 1, non_clean_sev))
+        for i in range(c_n):
+            win_list.append((non_clean + i + 1, 'S0'))
+        return win_list
+
+    def _hydrate_ledger(self, conn: sqlite3.Connection) -> None:
+        with contextlib.closing(
+            conn.execute(
+                'SELECT peer_pk, tier, clean_windows, evaluated_windows, last_severity FROM peer_standing'
+            )
+        ) as cur:
+            rows = cur.fetchall()
+        for pk, tier, clean_w, eval_w, last_sev in rows:
+            clean_pk = str(pk).lower()
+            self.ledger.tier[clean_pk] = str(tier)
+            c_n = int(clean_w or 0)
+            e_n = max(c_n, int(eval_w or 0))
+            self.ledger.windows[clean_pk] = self._build_window_history(c_n, e_n, str(last_sev or 'S0'))
+            self._epoch_seq = max(self._epoch_seq, e_n)
+
+    @staticmethod
+    def _row_to_record(row: Sequence[Any], now_ts: int) -> PeerStandingRecord:
+        clean_pk = str(row[0]).lower()
+        tier = str(row[2])
+        limits = json.loads(str(row[3]))
+        clean_win = int(row[4])
+        eval_win = int(row[5])
+        contra = int(row[6])
+        last_sev = str(row[7])
+        last_note = str(row[8])
+        zero_until = int(row[9])
+        updated_at = int(row[10])
+
+        rep_weight = (
+            window.reporter_weight(tier, clean_win, contra, eval_win)
+            if tier in VALID_TIERS
+            else 0.0
+        )
+        can_sync = (
+            tier != 'revoked'
+            and int(limits.get('events_in', 0)) > 0
+            and now_ts >= zero_until
+        )
+        return {
+            'peer_pk': clean_pk,
+            'peer_did': str(row[1]),
+            'tier': tier,
+            'limits': limits,
+            'clean_windows': clean_win,
+            'evaluated_windows': eval_win,
+            'contradicted_reports': contra,
+            'last_severity': last_sev,
+            'last_note': last_note,
+            'zero_until': zero_until,
+            'reporter_weight': round(rep_weight, 6),
+            'can_sync': can_sync,
+            'updated_at': updated_at,
+        }
+
+    def get(
+        self,
+        peer_pk: str,
+        *,
+        default_tier: Optional[str] = None,
+        create_if_missing: bool = False,
+    ) -> PeerStandingRecord:
+        """Return (and optionally initialize when `create_if_missing=True`) the
+        standing record for `peer_pk`."""
+        self.ensure_tables()
+        clean_pk = (peer_pk or '').strip().lower()
+        if not store.is_hex64(clean_pk):
+            raise ValueError('peer_pk must be a 64-character lowercase hex pubkey')
+        now_ts = self.clock()
+        with self._lock:
+            with contextlib.closing(
+                self._conn.execute(
+                    'SELECT peer_pk, peer_did, tier, limits_json, clean_windows, evaluated_windows, '
+                    'contradicted_reports, last_severity, last_note, zero_until, updated_at '
+                    'FROM peer_standing WHERE peer_pk = ?',
+                    (clean_pk,),
+                )
+            ) as cur:
+                row = cur.fetchone()
+            if row is None:
+                fallback = self._tier_fallback_fn(clean_pk) if self._tier_fallback_fn else None
+                tier = (
+                    str(default_tier)
+                    if default_tier in VALID_TIERS
+                    else str(fallback)
+                    if fallback in VALID_TIERS
+                    else 'probation'
+                )
+                limits = window.opening_limits(tier)
+                rep_weight = (
+                    window.reporter_weight(tier, 0, 0, 0)
+                    if tier in VALID_TIERS
+                    else 0.0
+                )
+                default_rec: PeerStandingRecord = {
+                    'peer_pk': clean_pk,
+                    'peer_did': f'did:nostr:{clean_pk}',
+                    'tier': tier,
+                    'limits': limits,
+                    'clean_windows': 0,
+                    'evaluated_windows': 0,
+                    'contradicted_reports': 0,
+                    'last_severity': 'S0',
+                    'last_note': 'opening',
+                    'zero_until': 0,
+                    'reporter_weight': round(rep_weight, 6),
+                    'can_sync': tier != 'revoked' and int(limits.get('events_in', 0)) > 0,
+                    'updated_at': now_ts,
+                }
+                if not create_if_missing:
+                    return default_rec
+                with self._conn:
+                    with contextlib.closing(
+                        self._conn.execute(
+                            """INSERT INTO peer_standing
+                               (peer_pk, peer_did, tier, limits_json, clean_windows, evaluated_windows,
+                                contradicted_reports, last_severity, last_note, zero_until, updated_at)
+                               VALUES (?, ?, ?, ?, 0, 0, 0, 'S0', 'opening', 0, ?)""",
+                            (clean_pk, f'did:nostr:{clean_pk}', tier, json.dumps(limits, sort_keys=True), now_ts),
+                        )
+                    ):
+                        pass
+                self.ledger.tier[clean_pk] = tier
+                self.ledger.windows.setdefault(clean_pk, [])
+                return default_rec
+
+            return self._row_to_record(row, now_ts)
+
+    def set_tier(
+        self,
+        peer_pk: str,
+        tier: str,
+        *,
+        clean_windows: Optional[int] = None,
+        evaluated_windows: Optional[int] = None,
+        now: Optional[int] = None,
+    ) -> PeerStandingRecord:
+        """Seed or update an operator-granted tier (`anchor`, `trusted`,
+        `member`, `probation`) for `peer_pk` and reset its opening limits."""
+        self.ensure_tables()
+        if tier not in VALID_TIERS:
+            raise ValueError(f'Invalid Fleetmesh tier: {tier!r}')
+        st_doc = self.get(peer_pk, default_tier=tier, create_if_missing=True)
+        clean_pk = st_doc['peer_pk']
+        limits = window.opening_limits(tier)
+        now_ts = int(now if now is not None else self.clock())
+        c_win = int(clean_windows if clean_windows is not None else st_doc['clean_windows'])
+        e_win = int(evaluated_windows if evaluated_windows is not None else max(c_win, st_doc['evaluated_windows']))
+        with self._tx() as conn:
+            with contextlib.closing(
+                conn.execute(
+                    """UPDATE peer_standing
+                       SET tier = ?, limits_json = ?, clean_windows = ?, evaluated_windows = ?,
+                           last_severity = 'S0', last_note = 'operator_seed', zero_until = 0, updated_at = ?
+                       WHERE peer_pk = ?""",
+                    (tier, json.dumps(limits, sort_keys=True), c_win, e_win, now_ts, clean_pk),
+                )
+            ):
+                pass
+            self.ledger.tier[clean_pk] = tier
+            self.ledger.windows[clean_pk] = self._build_window_history(c_win, e_win, 'S0')
+            self._epoch_seq = max(self._epoch_seq, e_win)
+            if self._on_tier_change is not None:
+                self._on_tier_change(clean_pk, tier)
+        return self.get(clean_pk)
+
+    def evaluate_window(
+        self,
+        peer_pk: str,
+        *,
+        measured: Mapping[str, int | float],
+        claimed: Optional[Mapping[str, int | float]] = None,
+        reports: Optional[Sequence[Mapping[str, Any]]] = None,
+        own_severity: str = 'S0',
+        now: Optional[int] = None,
+    ) -> WindowEvalRecord:
+        """Evaluate a closed window for `peer_pk` via `window.evaluate_window`,
+        `window.view`, and `window.adjust`, updating both SQLite and
+        `self.ledger`."""
+        self.ensure_tables()
+        now_ts = int(now if now is not None else self.clock())
+        st_doc = self.get(peer_pk, create_if_missing=True)
+        clean_pk = st_doc['peer_pk']
+        cur_tier = st_doc['tier']
+        cur_limits = dict(st_doc['limits'])
+
+        if cur_tier == 'revoked':
+            return {
+                **st_doc,
+                'window_severity': 'S4',
+                'final_severity': 'S4',
+                'per_limit': {},
+            }
+
+        actual_claimed = claimed if claimed is not None else measured
+        win_eval = window.evaluate_window(cur_limits, measured, actual_claimed)
+        combined_own = window.worse(str(win_eval['severity']), own_severity)
+
+        weighted_reports: list[WeightedReportEntry] = []
+        for r in (reports or []):
+            sev = str(r.get('severity') or 'S0')
+            if 'weight' in r and isinstance(r['weight'], (int, float)):
+                w = float(r['weight'])
+            elif r.get('reporter_pk'):
+                rep_st = self.get(str(r['reporter_pk']))
+                w = float(rep_st['reporter_weight'])
+            else:
+                w = 0.0
+            weighted_reports.append({'severity': sev, 'weight': w})
+
+        final_sev = window.view(combined_own, weighted_reports)
+        new_limits, new_tier, note = window.adjust(
+            cur_limits,
+            cur_tier,
+            final_sev,
+            used=measured,
+        )
+
+        with self._tx() as conn:
+            self._epoch_seq += 1
+            if new_tier == 'revoked':
+                self.ledger.on_revocation(clean_pk)
+            else:
+                self.ledger.record(clean_pk, self._epoch_seq, final_sev, new_tier)
+            if final_sev in standing_mod.ADVERSE:
+                self.ledger.reported_against(clean_pk)
+
+            clean_win = st_doc['clean_windows']
+            eval_win = st_doc['evaluated_windows'] + 1
+            zero_until = st_doc['zero_until']
+            if final_sev == 'S0':
+                clean_win += 1
+            elif final_sev in ('S2', 'S3', 'S4'):
+                clean_win = 0
+                if final_sev == 'S3':
+                    s3_hours = int(window.C['grant']['s3_zero_hours'])
+                    zero_until = now_ts + s3_hours * 3600
+
+            with contextlib.closing(
+                conn.execute(
+                    """UPDATE peer_standing
+                       SET tier = ?, limits_json = ?, clean_windows = ?, evaluated_windows = ?,
+                           last_severity = ?, last_note = ?, zero_until = ?, updated_at = ?
+                       WHERE peer_pk = ?""",
+                    (
+                        new_tier,
+                        json.dumps(new_limits, sort_keys=True),
+                        clean_win,
+                        eval_win,
+                        final_sev,
+                        note,
+                        zero_until,
+                        now_ts,
+                        clean_pk,
+                    ),
+                )
+            ):
+                pass
+            if self._on_tier_change is not None and new_tier in (*VALID_TIERS, 'revoked'):
+                self._on_tier_change(clean_pk, new_tier)
+
+        updated = self.get(clean_pk)
+        return {
+            **updated,
+            'window_severity': str(win_eval['severity']),
+            'final_severity': final_sev,
+            'per_limit': dict(win_eval['per_limit']),
+            'weighted_reports': weighted_reports,
+        }
+
+    def list_all(self) -> list[PeerStandingRecord]:
+        """Return all tracked peer standing records ordered by `updated_at DESC`."""
+        self.ensure_tables()
+        now_ts = self.clock()
+        with self._lock:
+            with contextlib.closing(
+                self._conn.execute(
+                    'SELECT peer_pk, peer_did, tier, limits_json, clean_windows, evaluated_windows, '
+                    'contradicted_reports, last_severity, last_note, zero_until, updated_at '
+                    'FROM peer_standing ORDER BY updated_at DESC'
+                )
+            ) as cur:
+                rows = cur.fetchall()
+        return [self._row_to_record(r, now_ts) for r in rows]
+
+    def reconcile_shards(
+        self,
+        local_store: ProjectedNodeStore,
+        peer_store: ProjectedNodeStore,
+        peer_pk: str,
+        *,
+        shard_filters: Mapping[str, Mapping[str, Any]],
+        local_sharding: Optional[ShardCoverageManager] = None,
+        peer_sharding: Optional[ShardCoverageManager] = None,
+        issue_handovers: bool = True,
+        now: Optional[int] = None,
+    ) -> ReconcileShardsReply:
+        """Run standing-gated bidirectional NIP-77 Negentropy (`sync/1.0`)
+        reconciliation over `shard_filters`, metering session counters through
+        `evaluate_window` and optionally issuing/accepting `Kind 30803`
+        handover chains."""
+        clean_pk = (peer_pk or '').strip().lower()
+        st_before = self.get(clean_pk)
+        if not st_before['can_sync']:
+            return {
+                'ok': False,
+                'status': 'refused_by_standing',
+                'standing': st_before,
+                'standing_before': st_before,
+                'standing_after': st_before,
+                'shards': {},
+                'handovers': {},
+            }
+
+        now_ts = int(now if now is not None else self.clock()) + 120
+        agg_counters: dict[str, int] = dict.fromkeys(sync.COUNTERS, 0)
+        worst_finding = 'S0'
+        shard_reports: dict[str, ShardReconcileEntry] = {}
+        handovers_completed: dict[str, HandoverAcceptReply] = {}
+
+        for shard_id, raw_filt in shard_filters.items():
+            filt = dict(raw_filt)
+            grant = sync.Grant(
+                f'grant-{shard_id}-{clean_pk[:8]}',
+                st_before['tier'],
+                st_before['limits'],
+                scope={'pull': filt, 'push': filt},
+            )
+            sess_init = sync.Session(
+                local_store, grant, shard_id, filt, since=0, until=now_ts, now=now_ts, is_initiator=True
+            )
+            sess_resp = sync.Session(
+                peer_store, grant, shard_id, filt, since=0, until=now_ts, now=now_ts, is_initiator=False
+            )
+            rep_pull = sync.run(sess_init, sess_resp)
+
+            sess_push_init = sync.Session(
+                peer_store, grant, shard_id, filt, since=0, until=now_ts, now=now_ts, is_initiator=True
+            )
+            sess_push_resp = sync.Session(
+                local_store, grant, shard_id, filt, since=0, until=now_ts, now=now_ts, is_initiator=False
+            )
+            rep_push = sync.run(sess_push_init, sess_push_resp)
+
+            for k in sync.COUNTERS:
+                agg_counters[k] = max(agg_counters[k], int(sess_init.counters.get(k, 0)))
+            for sev, _why in rep_pull['findings'] + rep_push['findings']:
+                if sev in window.SEV_ORDER:
+                    worst_finding = window.worse(worst_finding, sev)
+
+            fp_match = (
+                local_store.fingerprint_range(filt, 0, now_ts, now_ts)
+                == peer_store.fingerprint_range(filt, 0, now_ts, now_ts)
+            )
+            shard_reports[shard_id] = {
+                'pull_status': rep_pull['status'],
+                'push_status': rep_push['status'],
+                'fingerprint_match': fp_match,
+                'findings': rep_pull['findings'] + rep_push['findings'],
+            }
+
+            if (
+                issue_handovers
+                and local_sharding is not None
+                and peer_sharding is not None
+                and fp_match
+                and rep_pull['status'] in ('complete', 'in_sync')
+                and rep_push['status'] in ('complete', 'in_sync')
+            ):
+                ho_issued = peer_sharding.issue_handover(
+                    shard_id, local_sharding.node_pk, since=0, until=now_ts, now=now_ts
+                )
+                ho_accepted = local_sharding.accept_handover(
+                    shard_id, ho_issued['chain'], peer_sharding.node_pk, now=now_ts
+                )
+                handovers_completed[shard_id] = ho_accepted
+
+        st_after = self.evaluate_window(
+            clean_pk,
+            measured=agg_counters,
+            claimed=agg_counters,
+            own_severity=worst_finding,
+            now=now_ts,
+        )
+
+        all_ok = all(
+            r['pull_status'] in ('complete', 'in_sync')
+            and r['push_status'] in ('complete', 'in_sync')
+            and r['fingerprint_match']
+            for r in shard_reports.values()
+        )
+        return {
+            'ok': all_ok and st_after['final_severity'] == 'S0',
+            'status': 'complete' if all_ok else 'incomplete',
+            'standing_before': st_before,
+            'standing_after': st_after,
+            'shards': shard_reports,
+            'handovers': handovers_completed,
+        }

node/testnet/__init__.py

@@ -0,0 +1,9 @@
+"""Local testnet tooling: a multi-node cluster on one machine and the in-process
+FIPS daemon that stands in for fipsd. Development and operator rehearsal only -
+nothing in the node runtime imports this package (test_imports holds that line);
+the `meshnode cluster` verb is its one caller outside the tests.
+"""
+from .cluster import ChaosSimulator, LocalCluster, TOPOLOGIES, generate_topology_edges
+from .fips_daemon import ClusterFipsDaemon
+
+__all__ = ['ChaosSimulator', 'LocalCluster', 'TOPOLOGIES', 'generate_topology_edges', 'ClusterFipsDaemon']