fleetmesh-node 0.2.0__py3-none-any.whl

node/storage/accounts.py

@@ -0,0 +1,305 @@
+"""What a node must not forget when it restarts: its accounting.
+
+Grants and events were on disk from the start. The meters, the receipts a
+node has received, and the chain state of the receipts it sends were not: they
+lived in memory, and a restart lost them. That is not a lost convenience, it
+is a correctness bug, and a two-machine run is what showed its shape.
+
+A receipt chain is `seq` and `prev`, and the issuer checks both. A subject
+that restarts mid-window begins again at seq 0 with a zero `prev`, while the
+issuer is still expecting seq N chained onto what it last held. The issuer
+reads that as a broken chain and files an S2 misreport. Its counters are
+cumulative within a window, too, so the restarted subject also reports fewer
+than it did before, which is a counter going backwards: the second half of the
+same finding. An honest node that was restarted is priced exactly as a liar.
+
+So the accounting is persisted here, in the node's own directory, beside the
+grants it already kept.
+"""
+from __future__ import annotations
+
+import json
+import time
+from typing import Any, Mapping, Optional, Sequence
+
+from fleetmesh_core.wire.interfaces import ReceiptBody
+from fleetmesh_core.wire.wire_types import HandoverLink
+from ..planes.peering.budget.types import OutChainDict
+from ..planes.peering.reporter.types import FindingDict
+from ..compartment.schema import Column, Migration, Table
+from . import SqliteStore
+
+BY: str = 'by'
+OF_ME: str = 'of_me'
+
+
+class AccountStore(SqliteStore):
+    SCHEMA = (
+        Table('meters', (
+            Column('peer', 'TEXT', not_null=True),
+            Column('epoch', 'INTEGER', not_null=True),
+            Column('side', 'TEXT', not_null=True),
+            Column('counters', 'TEXT'),
+            Column('at', 'INTEGER'),
+        ), primary_key=('peer', 'epoch', 'side')),
+        Table('receipts', (
+            Column('peer', 'TEXT', not_null=True),
+            Column('epoch', 'INTEGER', not_null=True),
+            Column('seq', 'INTEGER', not_null=True),
+            Column('body', 'TEXT'),
+        ), primary_key=('peer', 'epoch', 'seq')),
+        Table('chain', (
+            Column('peer', 'TEXT', not_null=True),
+            Column('epoch', 'INTEGER', not_null=True),
+            Column('state', 'TEXT'),
+        ), primary_key=('peer', 'epoch')),
+        Table('evaluated', (
+            Column('peer', 'TEXT', not_null=True),
+            Column('epoch', 'INTEGER', not_null=True),
+            Column('severity', 'TEXT'),
+        ), primary_key=('peer', 'epoch')),
+        Table('findings', (
+            Column('fid', 'TEXT', primary_key=True),
+            Column('peer', 'TEXT'),
+            Column('epoch', 'INTEGER'),
+            Column('body', 'TEXT'),
+        )),
+        Table('seen_wraps', (
+            Column('wid', 'TEXT', primary_key=True),
+            Column('at', 'INTEGER'),
+        )),
+        Table('awaiting_acks', (
+            Column('mid', 'TEXT', primary_key=True),
+            Column('peer', 'TEXT'),
+            Column('type', 'TEXT'),
+            Column('at', 'INTEGER'),
+            Column('relays', 'TEXT'),
+        )),
+        Table('reports_seen', (
+            Column('reporter', 'TEXT', not_null=True),
+            Column('report_id', 'TEXT', not_null=True),
+            Column('contradicted', 'INTEGER'),
+        ), primary_key=('reporter', 'report_id')),
+        Table('strain', (
+            Column('peer', 'TEXT', not_null=True),
+            Column('shard', 'TEXT', not_null=True),
+            Column('transport', 'TEXT', not_null=True),
+            Column('kind', 'TEXT', not_null=True),
+            Column('n', 'INTEGER'),
+            Column('last_at', 'INTEGER'),
+        ), primary_key=('peer', 'shard', 'transport', 'kind')),
+        Table('handovers', (
+            Column('shard', 'TEXT', primary_key=True),
+            Column('frm', 'TEXT'),
+            Column('body', 'TEXT'),
+            Column('at', 'INTEGER'),
+        )),
+    )
+    MIGRATIONS = (
+        Migration(1, up='ALTER TABLE evaluated ADD COLUMN tier TEXT', name='evaluated_tier'),
+    )
+
+    # ---- writes ---------------------------------------------------------------
+    def put_meter(
+        self,
+        peer: str,
+        epoch: int,
+        side: str,
+        counters: Mapping[str, int],
+        now: Optional[int] = None,
+    ) -> None:
+        with self._lock:
+            self.execute('INSERT OR REPLACE INTO meters VALUES (?,?,?,?,?)',
+                            (peer, int(epoch), side, json.dumps(counters, sort_keys=True),
+                             int(now or time.time())))
+            self.db.commit()
+
+    def put_receipt(self, peer: str, epoch: int, body: ReceiptBody) -> None:
+        with self._lock:
+            self.execute('INSERT OR REPLACE INTO receipts VALUES (?,?,?,?)',
+                            (peer, int(epoch), int(body.get('seq', 0)),
+                             json.dumps(body, sort_keys=True)))
+            self.db.commit()
+
+    def put_chain(self, peer: str, epoch: int, state: OutChainDict | Mapping[str, Any]) -> None:
+        """An OutChain, as its to_dict renders it; chains() hands the mapping back for from_dict."""
+        with self._lock:
+            self.execute('INSERT OR REPLACE INTO chain VALUES (?,?,?)',
+                            (peer, int(epoch), json.dumps(state, sort_keys=True)))
+            self.db.commit()
+
+    def put_finding(self, fid: str, peer: str, epoch: int, body: FindingDict | Mapping[str, Any]) -> None:
+        """A reporter Finding, as its to_dict renders it; findings() hands the mapping back for from_dict."""
+        with self._lock:
+            self.execute('INSERT OR REPLACE INTO findings VALUES (?,?,?,?)',
+                            (fid, peer, int(epoch), json.dumps(body, sort_keys=True)))
+            self.db.commit()
+
+    def drop_finding(self, fid: str) -> None:
+        with self._lock:
+            self.execute('DELETE FROM findings WHERE fid = ?', (fid,))
+            self.db.commit()
+
+
+
+    def findings(self) -> list[FindingDict]:
+        """Every pending finding, oldest opened first."""
+        rows = self._all('SELECT body FROM findings')
+        out: list[FindingDict] = [json.loads(r[0]) for r in rows]
+        out.sort(key=lambda b: (b.get('opened_at', 0), b.get('fid', '')))
+        return out
+
+    def seen_wrap(self, wid: str) -> bool:
+        with self._lock:
+            return self._one('SELECT 1 FROM seen_wraps WHERE wid = ?', (wid,)) is not None
+
+    def put_seen_wrap(self, wid: str, at: int) -> None:
+        with self._lock:
+            self.execute('INSERT OR IGNORE INTO seen_wraps VALUES (?,?)', (wid, int(at)))
+            # three days: a day past the replay horizon, so nothing replayable is forgotten
+            self.execute('DELETE FROM seen_wraps WHERE at < ?', (int(at) - 3 * 86400,))
+            self.db.commit()
+
+    def put_awaiting(self, mid: str, peer: str, mtype: str, at: int, relays: list[str]) -> None:
+        with self._lock:
+            self.execute('INSERT OR REPLACE INTO awaiting_acks VALUES (?,?,?,?,?)',
+                            (mid, peer, mtype, int(at), json.dumps(list(relays))))
+            self.db.commit()
+
+    def drop_awaiting(self, mid: str) -> None:
+        with self._lock:
+            self.execute('DELETE FROM awaiting_acks WHERE mid = ?', (mid,))
+            self.db.commit()
+
+    def awaiting(self) -> list[tuple[str, str, str, int, list[str]]]:
+        with self._lock:
+            return [(m, p, t, a, json.loads(r)) for m, p, t, a, r in
+                    self._all('SELECT mid, peer, type, at, relays FROM awaiting_acks ORDER BY at')]
+
+    def put_report_seen(self, reporter: str, report_id: str, contradicted: bool) -> None:
+        with self._lock:
+            self.execute('INSERT OR REPLACE INTO reports_seen VALUES (?,?,?)',
+                            (reporter, report_id, 1 if contradicted else 0))
+            self.db.commit()
+
+    def reports_seen(self) -> list[tuple[str, str, bool]]:
+        with self._lock:
+            return [(r[0], r[1], bool(r[2])) for r in
+                    self.execute('SELECT reporter, report_id, contradicted FROM reports_seen')]
+
+    def put_evaluated(
+        self,
+        peer: str,
+        epoch: int,
+        severity: Optional[str],
+        tier: Optional[str] = None,
+    ) -> None:
+        with self._lock:
+            self.execute('INSERT OR REPLACE INTO evaluated (peer, epoch, severity, tier) VALUES (?,?,?,?)',
+                            (peer, int(epoch), severity or '', tier))
+            self.db.commit()
+
+    def put_strain(
+        self,
+        peer: Optional[str],
+        shard: Optional[str],
+        transport: Optional[str],
+        kind: str,
+        at: int,
+        n: int = 1,
+    ) -> None:
+        with self._lock:
+            self.execute(
+                'INSERT INTO strain (peer, shard, transport, kind, n, last_at) VALUES (?,?,?,?,?,?) '
+                'ON CONFLICT(peer, shard, transport, kind) DO UPDATE SET '
+                'n = n + excluded.n, last_at = excluded.last_at',
+                (peer or '', shard or '', transport or '', kind, int(n), int(at)))
+            self.db.commit()
+
+    def put_handover(self, shard: str, chain: Sequence[HandoverLink]) -> None:
+        """The whole chain, ordered from the first-hand root. `frm` is the last
+        link's signer: the node that handed it to this one."""
+        art = chain[-1]
+        with self._lock:
+            self.execute('INSERT OR REPLACE INTO handovers VALUES (?,?,?,?)',
+                            (shard, art['from'], json.dumps(chain, sort_keys=True), int(art['at'])))
+            self.db.commit()
+
+    def drop_handover(self, shard: str) -> None:
+        with self._lock:
+            self.execute('DELETE FROM handovers WHERE shard = ?', (shard,))
+            self.db.commit()
+
+    def handover(self, shard: str) -> Optional[list[HandoverLink]]:
+        """The chain this node holds for a shard, root first, or None."""
+        row = self._one('SELECT body FROM handovers WHERE shard = ?', (shard,))
+        if not row:
+            return None
+        chain = json.loads(row[0])
+        return chain if isinstance(chain, list) else [chain]
+
+    def handovers(self) -> dict[str, list[HandoverLink]]:
+        return {r[0]: (json.loads(r[1]) if isinstance(json.loads(r[1]), list) else [json.loads(r[1])])
+                for r in self._all('SELECT shard, body FROM handovers')}
+
+    def strain(self) -> list[tuple[str, str, str, str, int, int]]:
+        """[(peer, shard, transport, kind, n, last_at)] - everything survived."""
+        with self._lock:
+            return [(str(r[0]), str(r[1]), str(r[2]), str(r[3]), int(r[4]), int(r[5]))
+                    for r in self.execute('SELECT peer, shard, transport, kind, n, last_at FROM strain')]
+
+    # ---- reads ----------------------------------------------------------------
+    def meters(self) -> dict[str, dict[str, dict[int, dict[str, int]]]]:
+        # Every write here took the lock and every read did not, on a
+        # connection opened with check_same_thread=False and shared by the
+        # tick, sync, peering and transport threads. sqlite answers that with
+        # an intermittent "bad parameter or other API misuse", which is what a
+        # sweep caught in grants.get after the capacity ladder added a reader
+        # on the peering path. A read is as much a use of the connection as a
+        # write.
+        with self._lock:
+            out: dict[str, dict[str, dict[int, dict[str, int]]]] = {BY: {}, OF_ME: {}}
+            for peer, epoch, side, counters in self.execute(
+                    'SELECT peer, epoch, side, counters FROM meters'):
+                out.setdefault(side, {}).setdefault(peer, {})[epoch] = json.loads(counters)
+            return out
+
+    def receipts(self) -> dict[str, dict[int, list[ReceiptBody]]]:
+        with self._lock:
+            out: dict[str, dict[int, list[ReceiptBody]]] = {}
+            for peer, epoch, body in self.execute(
+                    'SELECT peer, epoch, body FROM receipts ORDER BY peer, epoch, seq'):
+                out.setdefault(peer, {}).setdefault(epoch, []).append(json.loads(body))
+            return out
+
+    def chains(self) -> dict[str, dict[int, OutChainDict]]:
+        with self._lock:
+            out: dict[str, dict[int, OutChainDict]] = {}
+            for peer, epoch, state in self.execute('SELECT peer, epoch, state FROM chain'):
+                out.setdefault(peer, {})[epoch] = json.loads(state)
+            return out
+
+    def evaluated(self) -> set[tuple[str, int]]:
+        with self._lock:
+            return {(peer, epoch) for peer, epoch in
+                    self.execute('SELECT peer, epoch FROM evaluated')}
+
+    def evaluations(self) -> list[tuple[str, int, Optional[str], Optional[str]]]:
+        """(peer, epoch, severity) in window order, severity None for a window
+        that was not evaluated. The standing ledger is rebuilt from this on a
+        restart: it lived in memory, so every streak began again at zero."""
+        with self._lock:
+            return [(p, e, s or None, t) for p, e, s, t in
+                    self.execute('SELECT peer, epoch, severity, tier FROM evaluated ORDER BY epoch, peer')]
+
+    def forget_before(self, epoch: int) -> None:
+        """Windows old enough that nothing can still be said about them.
+
+        A closed window that has been evaluated is history; keeping every one
+        forever would grow without bound on a node that never restarts.
+        """
+        with self._lock:
+            for table in ('meters', 'receipts', 'chain', 'evaluated', 'findings'):
+                self.execute('DELETE FROM %s WHERE epoch < ?' % table, (int(epoch),))
+            self.db.commit()
+

node/storage/adapter.py

@@ -0,0 +1,161 @@
+"""The database adapter module: provides typed schema binding, query execution,
+transactions, and leases across database backends (SQLite, PostgreSQL, etc.).
+
+Instantiated with the compartment into a database adapter module.
+"""
+from __future__ import annotations
+
+from typing import Any, Callable, Optional, Sequence
+
+from .base import LEASE_SECONDS, BaseStore
+from .dialect import Dialect, SqliteDialect
+
+
+class DatabaseAdapter(BaseStore):
+    """The database adapter module: binds a compartment's declared SCHEMA to a
+    database backend and provides query execution, transactions, and leases."""
+
+    def __init__(
+        self,
+        target: str = ':memory:',
+        schema: Sequence[Any] = (),
+        migrations: Sequence[Any] = (),
+        timeout: float = 5.0,
+        dialect: Optional[Dialect] = None,
+        owner: Optional[str] = None,
+        lease_seconds: int = LEASE_SECONDS,
+        clock: Optional[Callable[[], int]] = None,
+        readonly: bool = False,
+    ) -> None:
+        self.raw_schema: tuple[Any, ...] = tuple(schema)
+        # If schema is provided, override SCHEMA attribute before base _ensure_schema runs
+        if self.raw_schema:
+            self.SCHEMA = self.raw_schema
+        self.raw_migrations: tuple[Any, ...] = tuple(migrations)
+        if self.raw_migrations:
+            self.MIGRATIONS = tuple(
+                m.to_tuple() if hasattr(m, 'to_tuple') else m
+                for m in self.raw_migrations
+            )
+        super().__init__(
+            path=target,
+            timeout=timeout,
+            dialect=dialect,
+            owner=owner,
+            lease_seconds=lease_seconds,
+            clock=clock,
+            readonly=readonly,
+            schema=self.SCHEMA,
+        )
+
+    def one(self, q: str, args: Sequence[Any] = ()) -> Optional[tuple[Any, ...]]:
+        """Execute query and fetch one row, closing the cursor before returning."""
+        return self._one(q, args)
+
+    def all(self, q: str, args: Sequence[Any] = ()) -> list[tuple[Any, ...]]:
+        """Execute query and fetch all rows, closing the cursor before returning."""
+        return self._all(q, args)
+
+
+class SqliteAdapter(DatabaseAdapter):
+    """SQLite database adapter module."""
+
+    def __init__(
+        self,
+        path: str = ':memory:',
+        schema: Sequence[Any] = (),
+        migrations: Sequence[Any] = (),
+        timeout: float = 5.0,
+        dialect: Optional[Dialect] = None,
+        owner: Optional[str] = None,
+        lease_seconds: int = LEASE_SECONDS,
+        clock: Optional[Callable[[], int]] = None,
+        readonly: bool = False,
+    ) -> None:
+        super().__init__(
+            target=path,
+            schema=schema,
+            migrations=migrations,
+            timeout=timeout,
+            dialect=dialect or SqliteDialect(),
+            owner=owner,
+            lease_seconds=lease_seconds,
+            clock=clock,
+            readonly=readonly,
+        )
+
+
+class PsqlAdapter(DatabaseAdapter):
+    """PostgreSQL database adapter module."""
+
+    def __init__(
+        self,
+        url: str,
+        schema: Sequence[Any] = (),
+        migrations: Sequence[Any] = (),
+        timeout: float = 5.0,
+        dialect: Optional[Dialect] = None,
+        owner: Optional[str] = None,
+        lease_seconds: int = LEASE_SECONDS,
+        clock: Optional[Callable[[], int]] = None,
+        readonly: bool = False,
+    ) -> None:
+        if dialect is None:
+            from .postgres import PostgresDialect
+            dialect = PostgresDialect()
+        super().__init__(
+            target=url,
+            schema=schema,
+            migrations=migrations,
+            timeout=timeout,
+            dialect=dialect,
+            owner=owner,
+            lease_seconds=lease_seconds,
+            clock=clock,
+            readonly=readonly,
+        )
+
+
+PostgresAdapter = PsqlAdapter
+
+
+def create_adapter(
+    backend: Optional[str] = None,
+    target: Optional[str] = None,
+    schema: Sequence[Any] = (),
+    migrations: Sequence[Any] = (),
+    **kw: Any,
+) -> DatabaseAdapter:
+    """Create a DatabaseAdapter for the specified backend or target."""
+    tgt = target or ':memory:'
+    b = (backend or '').lower()
+    if b in ('psql', 'postgres', 'postgresql') or tgt.startswith(('postgresql://', 'postgres://')):
+        return PsqlAdapter(url=tgt, schema=schema, migrations=migrations, **kw)
+    return SqliteAdapter(path=tgt, schema=schema, migrations=migrations, **kw)
+
+
+def adapter_from_locator(
+    locator: Any,
+    name: str,
+    schema: Sequence[Any] = (),
+    migrations: Sequence[Any] = (),
+    **kw: Any,
+) -> DatabaseAdapter:
+    """Instantiate a DatabaseAdapter using the node's storage Locator."""
+    target = locator.target(name)
+    dialect = getattr(locator, 'dialect', None)
+    owner = kw.pop('owner', getattr(locator, 'owner', None))
+    if dialect is not None and dialect.name == 'postgres':
+        return PsqlAdapter(url=target, schema=schema, migrations=migrations, dialect=dialect, owner=owner, **kw)
+    return SqliteAdapter(path=target, schema=schema, migrations=migrations, dialect=dialect, owner=owner, **kw)
+
+
+
+__all__ = [
+    'DatabaseAdapter',
+    'SqliteAdapter',
+    'PsqlAdapter',
+    'PostgresAdapter',
+    'create_adapter',
+    'adapter_from_locator',
+]