fleetmesh-node 0.2.0__py3-none-any.whl

node/storage/projected.py

@@ -0,0 +1,953 @@
+"""ProjectedNodeStore and TableProjector: relational SQL views materialized
+co-transactionally from MeshStore NIP-01 events and P2Panda causal logs.
+
+Every write to `ProjectedNodeStore` is a signed NIP-01 event (or a Kind 9850
+envelope carrying a signed `P2PandaOperation`). `MeshStore` remains the single
+source of truth for:
+- BIP-340 signature verification and canonical event IDs
+- Replaceable (0, 3, 10000..19999) and addressable (30000..39999) supersession
+- NIP-09 (`kind == 5`) `e`-tag and `a`-tag coordinate deletions and tombstones
+- NIP-40 expiration filtering
+- NIP-77 Negentropy `enumerate_range` and `fingerprint_range` set reconciliation
+- Kind 30803 shard coverage claims, handover, and retention guards
+
+Whenever `MeshStore` commits `_put(event)`, `_drop(event_id)`, or `forget(event_id)`
+inside `with self._lock, self._atomic():`, `ProjectedNodeStore` updates all
+registered relational SQL tables through `self.dialect.sql(...)` in the exact
+same database transaction. Because `ProjectorSpec` enforces a 1-to-1 primary
+key correspondence with the NIP-01 storage class of each event kind, the
+projected SQL tables converge identically across all nodes regardless of event
+arrival order, supersession order, deletion order, or backfill order.
+"""
+from __future__ import annotations
+
+import json
+from typing import Any, Callable, Mapping, Optional, Sequence, TypedDict
+
+from fleetmesh_core.crypto import crypto
+from fleetmesh_core.storage import store
+from fleetmesh_core.storage.projection import (
+    CAUSAL_KIND,
+    SYSTEM_COLUMNS,
+    FieldSpec,
+    ProjectedRow,
+    ProjectedValue,
+    ProjectionError,
+    ProjectorSpec,
+    encode_deletion,
+    encode_event,
+    extract_row,
+    unwrap_p2panda_event,
+    wrap_p2panda_event,
+)
+from fleetmesh_core.wire.p2panda import P2PandaOperation
+from node.compartment.schema import Column, Index, Table
+from node.interfaces import NostrEvent, NostrFilter
+from node.protocols.nostr.eventstore import NodeStore
+
+__all__ = [
+    'CAUSAL_KIND',
+    'CausalProjector',
+    'FieldSpec',
+    'ProjectedNodeStore',
+    'ProjectedRow',
+    'ProjectedValue',
+    'ProjectionError',
+    'ProjectionRefusal',
+    'ProjectionVerifyReport',
+    'ProjectorSpec',
+    'TableProjector',
+]
+
+
+class ProjectionRefusal(TypedDict):
+    event_id: str
+    kind: int
+    table: str
+    reason: str
+    at: int
+
+
+class ProjectionVerifyReport(TypedDict):
+    ok: bool
+    tables: dict[str, int]
+    missing_events: list[str]
+    orphan_rows: list[str]
+    mismatched_rows: list[str]
+
+
+class TableProjector:
+    """Binds a pure `ProjectorSpec` to a declarative `node.compartment.schema.Table`
+    and generates dialect-neutral SQL for co-transactional upsert and drop."""
+
+    def __init__(
+        self,
+        spec: ProjectorSpec,
+        *,
+        indices: Sequence[Index] = (),
+        without_rowid: bool = False,
+    ) -> None:
+        if not isinstance(spec, ProjectorSpec):
+            raise TypeError(f'spec must be a ProjectorSpec, got {type(spec).__name__}')
+        self.spec: ProjectorSpec = spec
+        self.without_rowid: bool = bool(without_rowid)
+
+        single_pk = spec.primary_key[0] if len(spec.primary_key) == 1 else None
+        cols: list[Column] = []
+        for f in spec.fields:
+            is_pk = (single_pk is not None and f.name == single_pk)
+            cols.append(
+                Column(
+                    f.name,
+                    f.sql_type(),
+                    not_null=(f.required or is_pk or f.name in spec.primary_key),
+                    primary_key=is_pk,
+                )
+            )
+        # System provenance columns on every projected table
+        cols.append(Column('_event_id', 'TEXT', not_null=True))
+        cols.append(Column('_pubkey', 'TEXT', not_null=True))
+        cols.append(Column('_created_at', 'INTEGER', not_null=True))
+
+        event_idx_name = f'idx_{spec.table_name}__event_id'
+        all_indices: list[Index] = [
+            Index(event_idx_name, ('_event_id',), table=spec.table_name),
+        ]
+        for idx in indices:
+            bound = idx.bind_table(spec.table_name)
+            if bound.name != event_idx_name:
+                all_indices.append(bound)
+
+        table_pk: tuple[str, ...] = () if single_pk is not None else spec.primary_key
+        self.table: Table = Table(
+            spec.table_name,
+            cols,
+            primary_key=table_pk,
+            indices=all_indices,
+            without_rowid=self.without_rowid,
+        )
+        probs = self.table.problems()
+        if probs:
+            raise ProjectionError('; '.join(str(p) for p in probs))
+
+        self._col_names: tuple[str, ...] = tuple(f.name for f in spec.fields) + SYSTEM_COLUMNS
+        placeholders = ', '.join('?' for _ in self._col_names)
+        col_list = ', '.join(self._col_names)
+        self._upsert_sql: str = (
+            f'INSERT OR REPLACE INTO {spec.table_name} ({col_list}) VALUES ({placeholders})'
+        )
+        self._delete_by_event_sql: str = (
+            f'DELETE FROM {spec.table_name} WHERE _event_id = ?'
+        )
+
+    @property
+    def table_name(self) -> str:
+        return self.spec.table_name
+
+    @property
+    def kind(self) -> int:
+        return self.spec.kind
+
+    def column_names(self) -> tuple[str, ...]:
+        return self._col_names
+
+    def upsert_sql(self) -> str:
+        return self._upsert_sql
+
+    def delete_by_event_sql(self) -> str:
+        return self._delete_by_event_sql
+
+    def project(
+        self,
+        event: Mapping[str, Any],
+    ) -> tuple[Optional[tuple[ProjectedValue, ...]], Optional[str]]:
+        """Extract a positional value tuple matching `column_names()`, or `(None, why_refused)`."""
+        row, err = extract_row(self.spec, event)
+        if err is not None or row is None:
+            return None, err
+        return tuple(row[c] for c in self._col_names), None
+
+
+class CausalProjector:
+    """Projects verified `P2PandaOperation` chains carried in Kind 9850 events
+    in strict per-`(log_id, author)` sequence order `0, 1, 2, ...`.
+
+    Out-of-order operations arriving via Negentropy or relay gossip are buffered
+    in `p2panda_causal_ops` (`applied = 0`) and automatically drained as soon
+    as their missing predecessor operations arrive.
+    """
+
+    def __init__(
+        self,
+        log_id: str,
+        tables: Sequence[Table],
+        apply_fn: Callable[[Any, P2PandaOperation, Mapping[str, Any]], Optional[str]],
+        *,
+        shard_id: Optional[str] = None,
+    ) -> None:
+        if not log_id or not isinstance(log_id, str):
+            raise ProjectionError('CausalProjector log_id must be a non-empty string')
+        self.log_id: str = log_id
+        self.shard_id: str = shard_id or f'causal:{log_id}'
+        self.tables: tuple[Table, ...] = tuple(tables)
+        self.apply_fn = apply_fn
+        for tbl in self.tables:
+            probs = tbl.problems()
+            if probs:
+                raise ProjectionError('; '.join(str(p) for p in probs))
+
+    def shard_filter(self) -> NostrFilter:
+        """Return the NIP-01 filter covering all Kind 9850 operations for `log_id`."""
+        return {'kinds': [CAUSAL_KIND], '#L': [self.log_id]}  # type: ignore[typeddict-unknown-key]
+
+
+_CAUSAL_OPS_TABLE = Table(
+    'p2panda_causal_ops',
+    (
+        Column('event_id', 'TEXT', primary_key=True),
+        Column('log_id', 'TEXT', not_null=True),
+        Column('op_id', 'TEXT', not_null=True),
+        Column('author', 'TEXT', not_null=True),
+        Column('seq_num', 'INTEGER', not_null=True),
+        Column('backlink', 'TEXT'),
+        Column('timestamp', 'INTEGER', not_null=True),
+        Column('applied', 'INTEGER', not_null=True, default=0),
+        Column('raw_event', 'TEXT', not_null=True),
+    ),
+    indices=(
+        Index('idx_p2panda_causal_ops_log_author_seq', ('log_id', 'author', 'seq_num')),
+        Index('idx_p2panda_causal_ops_op_id', ('op_id',)),
+    ),
+)
+
+
+class ProjectedNodeStore(NodeStore):
+    """`NodeStore` extended with co-transactional `TableProjector` and
+    `CausalProjector` materialization."""
+
+    SCHEMA: tuple[Any, ...] = NodeStore.SCHEMA
+    MAX_REFUSALS: int = 512
+
+    def __init__(
+        self,
+        path: str = ':memory:',
+        *,
+        projectors: Sequence[TableProjector | ProjectorSpec] = (),
+        causal_projectors: Sequence[CausalProjector] = (),
+        **kw: Any,
+    ) -> None:
+        self._projectors_by_table: dict[str, TableProjector] = {}
+        self._projectors_by_kind: dict[int, list[TableProjector]] = {}
+        self._causal_by_log: dict[str, CausalProjector] = {}
+        self.projection_refusals: list[ProjectionRefusal] = []
+        self.projected_counts: dict[str, int] = {}
+
+        extra_tables: list[Table] = [_CAUSAL_OPS_TABLE]
+        for p in projectors:
+            tp = p if isinstance(p, TableProjector) else TableProjector(p)
+            if tp.table_name in self._projectors_by_table:
+                raise ProjectionError(f'duplicate TableProjector for table {tp.table_name!r}')
+            self._projectors_by_table[tp.table_name] = tp
+            self._projectors_by_kind.setdefault(tp.kind, []).append(tp)
+            extra_tables.append(tp.table)
+
+        for cp in causal_projectors:
+            if cp.log_id in self._causal_by_log:
+                raise ProjectionError(f'duplicate CausalProjector for log_id {cp.log_id!r}')
+            self._causal_by_log[cp.log_id] = cp
+            for tbl in cp.tables:
+                extra_tables.append(tbl)
+
+        combined_schema = list(NodeStore.SCHEMA) + extra_tables
+        NodeStore.__init__(self, path, schema=combined_schema, **kw)
+        if not self.readonly:
+            self._ensure_projector_columns(extra_tables)
+            if (self._projectors_by_table or self._causal_by_log) and self.count() > 0:
+                self.backfill()
+
+    def _ensure_projector_columns(self, tables: Sequence[Table]) -> None:
+        """Ensure any newly added `Column`s on existing projected tables are
+        migrated via `ALTER TABLE ... ADD COLUMN` prior to `backfill()`."""
+        with self._lock:
+            for tbl in tables:
+                if not self.dialect.has_table(self.db, tbl.name):
+                    continue
+                for col in tbl.columns:
+                    if not col.primary_key and not self.dialect.has_column(self.db, tbl.name, col.name):
+                        default_clause = (
+                            f' DEFAULT {col.default}'
+                            if col.default is not None
+                            else (" DEFAULT ''" if col.type.upper() == 'TEXT' else ' DEFAULT 0')
+                        )
+                        null_clause = '' if col.nullable else ' NOT NULL'
+                        ddl = f'ALTER TABLE {tbl.name} ADD COLUMN {col.name} {col.type}{null_clause}{default_clause}'
+                        try:
+                            self.execute(ddl)
+                        except Exception as exc:
+                            if not self.dialect.is_duplicate_column(exc):
+                                raise
+
+    # ---- Projector Registration ---------------------------------------------
+
+    def projectors(self) -> tuple[TableProjector, ...]:
+        return tuple(self._projectors_by_table.values())
+
+    def causal_projectors(self) -> tuple[CausalProjector, ...]:
+        return tuple(self._causal_by_log.values())
+
+    def projector(self, table_name: str) -> TableProjector:
+        tp = self._projectors_by_table.get(table_name)
+        if tp is None:
+            raise KeyError(f'unknown projected table {table_name!r}')
+        return tp
+
+    def register_projector(self, projector: TableProjector | ProjectorSpec) -> TableProjector:
+        """Register a `TableProjector` on an already-open store, create its
+        table/indices if needed, and backfill from `events`."""
+        tp = projector if isinstance(projector, TableProjector) else TableProjector(projector)
+        with self._lock:
+            if tp.table_name in self._projectors_by_table:
+                return self._projectors_by_table[tp.table_name]
+            self._projectors_by_table[tp.table_name] = tp
+            self._projectors_by_kind.setdefault(tp.kind, []).append(tp)
+            self.SCHEMA = tuple(self.SCHEMA) + (tp.table,)
+            if not self.readonly:
+                if hasattr(self.dialect, 'learn_table'):
+                    self.dialect.learn_table(tp.table)
+                self.execute(tp.table.to_sql(dialect=self.dialect.name))
+                for sec in tp.table.secondary_sql(dialect=self.dialect.name):
+                    self.execute(sec)
+                self._ensure_projector_columns((tp.table,))
+                self.commit()
+                self.backfill(table_name=tp.table_name)
+        return tp
+
+    def register_causal_projector(self, causal: CausalProjector) -> CausalProjector:
+        """Register a `CausalProjector` on an already-open store and backfill
+        any Kind 9850 operations for its `log_id`."""
+        with self._lock:
+            if causal.log_id in self._causal_by_log:
+                return self._causal_by_log[causal.log_id]
+            self._causal_by_log[causal.log_id] = causal
+            self.SCHEMA = tuple(self.SCHEMA) + causal.tables
+            if not self.readonly:
+                for tbl in causal.tables:
+                    if hasattr(self.dialect, 'learn_table'):
+                        self.dialect.learn_table(tbl)
+                    self.execute(tbl.to_sql(dialect=self.dialect.name))
+                    for sec in tbl.secondary_sql(dialect=self.dialect.name):
+                        self.execute(sec)
+                self._ensure_projector_columns(causal.tables)
+                self.commit()
+                self._backfill_causal(causal.log_id)
+        return causal
+
+    # ---- Co-Transactional Store Hooks ---------------------------------------
+
+    def _record_refusal(self, event_id: str, kind: int, table: str, reason: str) -> None:
+        entry: ProjectionRefusal = {
+            'event_id': str(event_id),
+            'kind': int(kind),
+            'table': str(table),
+            'reason': str(reason),
+            'at': self.clock(),
+        }
+        self.projection_refusals.append(entry)
+        if len(self.projection_refusals) > self.MAX_REFUSALS:
+            del self.projection_refusals[: len(self.projection_refusals) - self.MAX_REFUSALS]
+
+    def _project_put(self, event: Mapping[str, Any]) -> None:
+        kind = int(event['kind'])
+        eid = str(event['id'])
+
+        # 1. Declarative TableProjectors for this kind
+        for tp in self._projectors_by_kind.get(kind, ()):
+            vals, why = tp.project(event)
+            if why is not None or vals is None:
+                self._record_refusal(eid, kind, tp.table_name, why or 'invalid payload')
+                continue
+            self.execute('SAVEPOINT sp_proj_row')
+            try:
+                self.execute(tp.upsert_sql(), vals)
+            except Exception as exc:
+                self.execute('ROLLBACK TO SAVEPOINT sp_proj_row')
+                self.execute('RELEASE SAVEPOINT sp_proj_row')
+                self._record_refusal(eid, kind, tp.table_name, f'{type(exc).__name__}: {exc}')
+                continue
+            self.execute('RELEASE SAVEPOINT sp_proj_row')
+            self.projected_counts[tp.table_name] = self.projected_counts.get(tp.table_name, 0) + 1
+
+        # 2. Causal P2Panda operation envelopes (Kind 9850)
+        if kind == CAUSAL_KIND and self._causal_by_log:
+            self._project_causal_put(event)
+
+    def _rebuild_causal_projector_tables(self, cp: CausalProjector) -> None:
+        """Reset `cp.tables` and replay all buffered operations in `p2panda_causal_ops`
+        for `cp.log_id` from genesis in strict causal sequence order."""
+        for tbl in cp.tables:
+            self.execute(f'DELETE FROM {tbl.name}')
+        self.execute(
+            'UPDATE p2panda_causal_ops SET applied = 0 WHERE log_id = ?',
+            (cp.log_id,),
+        )
+        self.projected_counts[cp.log_id] = 0
+        authors = [
+            str(r[0])
+            for r in self._all(
+                'SELECT DISTINCT author FROM p2panda_causal_ops WHERE log_id = ? ORDER BY author ASC',
+                (cp.log_id,),
+            )
+        ]
+        for author in authors:
+            self._drain_causal_author(cp, author)
+
+    def _project_drop(self, event_id: str, kind: Optional[int] = None) -> None:
+        target_projectors = (
+            self._projectors_by_kind.get(kind, ())
+            if kind is not None
+            else tuple(self._projectors_by_table.values())
+        )
+        for tp in target_projectors:
+            self.execute(tp.delete_by_event_sql(), (event_id,))
+        if self._causal_by_log and (kind is None or kind == CAUSAL_KIND):
+            causal_row = self._one(
+                'SELECT log_id, applied FROM p2panda_causal_ops WHERE event_id = ?',
+                (event_id,),
+            )
+            self.execute('DELETE FROM p2panda_causal_ops WHERE event_id = ?', (event_id,))
+            if causal_row is not None:
+                log_id_str, was_applied = str(causal_row[0]), int(causal_row[1])
+                if was_applied and log_id_str in self._causal_by_log:
+                    self._rebuild_causal_projector_tables(self._causal_by_log[log_id_str])
+
+    def _put(self, event: Mapping[str, Any]) -> None:
+        with self._lock, self._atomic():
+            store.SqliteStore._put(self, dict(event))
+            self._project_put(event)
+
+    def _drop(self, event_id: str) -> None:
+        with self._lock, self._atomic():
+            row = self._one('SELECT kind FROM events WHERE id = ?', (event_id,))
+            kind = int(row[0]) if row is not None else None
+            NodeStore._drop(self, event_id)
+            self._project_drop(event_id, kind=kind)
+
+    def forget(self, event_id: str) -> bool:
+        with self._lock, self._atomic():
+            row = self._one('SELECT kind FROM events WHERE id = ?', (event_id,))
+            kind = int(row[0]) if row is not None else None
+            cur = self.execute('DELETE FROM events WHERE id = ?', (event_id,))
+            try:
+                deleted = cur.rowcount > 0
+            finally:
+                cur.close()
+            self.execute('DELETE FROM origin WHERE id = ?', (event_id,))
+            self.execute('DELETE FROM tags WHERE id = ?', (event_id,))
+            self._project_drop(event_id, kind=kind)
+            return deleted
+
+    # ---- Causal P2Panda Ordering Engine -------------------------------------
+
+    def _project_causal_put(self, event: Mapping[str, Any]) -> None:
+        eid = str(event['id'])
+        log_id, op, why = unwrap_p2panda_event(event)
+        if why is not None or log_id is None or op is None:
+            self._record_refusal(eid, CAUSAL_KIND, 'p2panda_causal_ops', why or 'invalid causal event')
+            return
+        cp = self._causal_by_log.get(log_id)
+        if cp is None:
+            return
+        author = op.author.lower()
+        op_id = op.id.lower()
+        backlink = op.backlink.lower() if op.backlink else None
+
+        # Check for conflicting operation at the same (log_id, author, seq_num)
+        existing = self._one(
+            'SELECT op_id, applied FROM p2panda_causal_ops WHERE log_id = ? AND author = ? AND seq_num = ?',
+            (log_id, author, op.seq_num),
+        )
+        if existing is not None:
+            if str(existing[0]).lower() == op_id:
+                return
+            self._record_refusal(
+                eid,
+                CAUSAL_KIND,
+                log_id,
+                f'fork conflict at seq_num={op.seq_num} for author {author}',
+            )
+            return
+
+        self.execute(
+            'INSERT OR REPLACE INTO p2panda_causal_ops '
+            '(event_id, log_id, op_id, author, seq_num, backlink, timestamp, applied, raw_event) '
+            'VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?)',
+            (
+                eid,
+                log_id,
+                op_id,
+                author,
+                op.seq_num,
+                backlink,
+                op.header.timestamp,
+                crypto.canonical(event).decode(),
+            ),
+        )
+        self._drain_causal_author(cp, author)
+
+    def _drain_causal_author(self, cp: CausalProjector, author: str) -> None:
+        """Advance `(cp.log_id, author)` through any contiguous buffered operations
+        starting from `highest_applied_seq + 1`."""
+        log_id = cp.log_id
+        latest = self._one(
+            'SELECT seq_num, op_id, timestamp FROM p2panda_causal_ops '
+            'WHERE log_id = ? AND author = ? AND applied = 1 '
+            'ORDER BY seq_num DESC LIMIT 1',
+            (log_id, author),
+        )
+        if latest is None:
+            expected_seq = 0
+            prev_op_id: Optional[str] = None
+            prev_ts: int = 0
+        else:
+            expected_seq = int(latest[0]) + 1
+            prev_op_id = str(latest[1]).lower()
+            prev_ts = int(latest[2])
+
+        while True:
+            cand = self._one(
+                'SELECT event_id, op_id, backlink, timestamp, raw_event FROM p2panda_causal_ops '
+                'WHERE log_id = ? AND author = ? AND seq_num = ? AND applied = 0 '
+                'ORDER BY event_id ASC LIMIT 1',
+                (log_id, author, expected_seq),
+            )
+            if cand is None:
+                break
+            cand_eid = str(cand[0])
+            cand_op_id = str(cand[1]).lower()
+            cand_backlink = str(cand[2]).lower() if cand[2] else None
+            cand_ts = int(cand[3])
+            cand_event = json.loads(str(cand[4]))
+
+            if expected_seq == 0:
+                if cand_backlink is not None and cand_backlink != '':
+                    self._record_refusal(
+                        cand_eid,
+                        CAUSAL_KIND,
+                        log_id,
+                        f'genesis operation for {author} must have null backlink',
+                    )
+                    break
+            else:
+                if cand_backlink != prev_op_id:
+                    self._record_refusal(
+                        cand_eid,
+                        CAUSAL_KIND,
+                        log_id,
+                        f'backlink mismatch at seq_num={expected_seq}: expected {prev_op_id}, got {cand_backlink}',
+                    )
+                    break
+                if cand_ts < prev_ts:
+                    self._record_refusal(
+                        cand_eid,
+                        CAUSAL_KIND,
+                        log_id,
+                        f'operation timestamp {cand_ts} precedes predecessor {prev_ts}',
+                    )
+                    break
+
+            _, op, why = unwrap_p2panda_event(cand_event)
+            if why is not None or op is None:
+                self._record_refusal(cand_eid, CAUSAL_KIND, log_id, why or 'unwrap failed')
+                break
+
+            self.execute('SAVEPOINT sp_proj_causal')
+            try:
+                err = cp.apply_fn(self, op, cand_event)
+            except Exception as exc:
+                self.execute('ROLLBACK TO SAVEPOINT sp_proj_causal')
+                self.execute('RELEASE SAVEPOINT sp_proj_causal')
+                self._record_refusal(cand_eid, CAUSAL_KIND, log_id, f'{type(exc).__name__}: {exc}')
+                break
+            if err is not None:
+                self.execute('ROLLBACK TO SAVEPOINT sp_proj_causal')
+                self.execute('RELEASE SAVEPOINT sp_proj_causal')
+                self._record_refusal(cand_eid, CAUSAL_KIND, log_id, err)
+                break
+            self.execute('RELEASE SAVEPOINT sp_proj_causal')
+
+            self.execute(
+                'UPDATE p2panda_causal_ops SET applied = 1 WHERE event_id = ?',
+                (cand_eid,),
+            )
+            self.projected_counts[log_id] = self.projected_counts.get(log_id, 0) + 1
+            prev_op_id = cand_op_id
+            prev_ts = cand_ts
+            expected_seq += 1
+
+    # ---- High-Level Write Helpers -------------------------------------------
+
+    def _monotonic_timestamp(
+        self,
+        spec: ProjectorSpec,
+        pubkey: str,
+        row: Mapping[str, Any],
+        now: int,
+    ) -> int:
+        """Ensure a locally authored replaceable/addressable update or post-delete
+        re-creation has a `created_at` strictly greater than any prior live
+        event or NIP-09 coordinate deletion at the same coordinate."""
+        ts = int(now)
+        if store.is_addressable(spec.kind):
+            d_val = ''
+            for f in spec.fields:
+                if f.source == 'tag' and f.tag_name == 'd':
+                    d_val = str(row.get(f.name, f.default or ''))
+                    break
+            coord = f'{spec.kind}:{pubkey}:{d_val}'
+            for ev in self._by_author_kind(pubkey, spec.kind):
+                if store.coordinate(ev) == coord and int(ev['created_at']) >= ts:
+                    ts = int(ev['created_at']) + 1
+            for del_ev in self._by_author_kind(pubkey, 5):
+                if any(t[1] == coord for t in store.tags(del_ev, 'a') if len(t) > 1):
+                    if int(del_ev['created_at']) >= ts:
+                        ts = int(del_ev['created_at']) + 1
+            ts = self.forward_clock(coord, ts)
+        elif store.is_replaceable(spec.kind):
+            coord = f'{spec.kind}:{pubkey}:'
+            for ev in self._by_author_kind(pubkey, spec.kind):
+                if int(ev['created_at']) >= ts:
+                    ts = int(ev['created_at']) + 1
+            for del_ev in self._by_author_kind(pubkey, 5):
+                if any(t[1] == coord for t in store.tags(del_ev, 'a') if len(t) > 1):
+                    if int(del_ev['created_at']) >= ts:
+                        ts = int(del_ev['created_at']) + 1
+            ts = self.forward_clock(coord, ts)
+        return ts
+
+    def forward_clock(self, coord: str, now: int) -> int:
+        """Return a strictly monotonic timestamp for `coord` (matching
+        `Discovery.forward_clock`)."""
+        with self._lock:
+            if not hasattr(self, '_forward_clocks'):
+                self._forward_clocks: dict[str, int] = {}
+            last = self._forward_clocks.get(coord)
+            ts = int(now)
+            if last is not None and ts <= last:
+                ts = last + 1
+            self._forward_clocks[coord] = ts
+            return ts
+
+    def put_record(
+        self,
+        table_name: str,
+        row: Mapping[str, Any],
+        sk_hex: str,
+        *,
+        created_at: Optional[int] = None,
+        now: Optional[int] = None,
+        extra_tags: Sequence[Sequence[str]] = (),
+        expiration_ts: Optional[int] = None,
+        origin: str = NodeStore.LOCAL,
+    ) -> tuple[NostrEvent, str]:
+        """Encode `row` into a signed NIP-01 event for `table_name` and insert it
+        co-transactionally into both `events` and the projected SQL table."""
+        tp = self.projector(table_name)
+        effective_now = int(now) if now is not None else self.clock()
+        pubkey = crypto.pubkey(sk_hex)
+        with self._lock:
+            effective_ts = (
+                int(created_at)
+                if created_at is not None
+                else self._monotonic_timestamp(tp.spec, pubkey, row, effective_now)
+            )
+            ev = encode_event(
+                tp.spec,
+                sk_hex,
+                row,
+                effective_ts,
+                extra_tags=extra_tags,
+                expiration_ts=expiration_ts,
+            )
+            outcome = self.insert_event(ev, now=max(effective_now, effective_ts))
+            if outcome == 'stored':
+                self.note_origin(ev['id'], origin)
+            return ev, outcome
+
+    def delete_record(
+        self,
+        table_name: str,
+        sk_hex: str,
+        *,
+        d_tag: Optional[str] = None,
+        event_id: Optional[str] = None,
+        reason: str = '',
+        created_at: Optional[int] = None,
+        now: Optional[int] = None,
+        origin: str = NodeStore.LOCAL,
+    ) -> tuple[NostrEvent, str]:
+        """Sign and insert a NIP-09 (`kind == 5`) deletion event removing a
+        record from `table_name` and `events` co-transactionally."""
+        tp = self.projector(table_name)
+        effective_now = int(now) if now is not None else self.clock()
+        pubkey = crypto.pubkey(sk_hex)
+        with self._lock:
+            effective_ts = int(created_at) if created_at is not None else effective_now
+            if created_at is None:
+                if store.is_addressable(tp.kind) and d_tag is not None:
+                    coord = f'{tp.kind}:{pubkey}:{d_tag}'
+                    for ev in self._by_author_kind(pubkey, tp.kind):
+                        if store.coordinate(ev) == coord and int(ev['created_at']) > effective_ts:
+                            effective_ts = int(ev['created_at'])
+                    effective_ts = self.forward_clock(coord, effective_ts)
+                elif store.is_replaceable(tp.kind):
+                    coord = f'{tp.kind}:{pubkey}:'
+                    for ev in self._by_author_kind(pubkey, tp.kind):
+                        if int(ev['created_at']) > effective_ts:
+                            effective_ts = int(ev['created_at'])
+                    effective_ts = self.forward_clock(coord, effective_ts)
+            del_ev = encode_deletion(
+                tp.spec,
+                sk_hex,
+                effective_ts,
+                d_tag=d_tag,
+                event_id=event_id,
+                reason=reason,
+            )
+            outcome = self.insert_event(del_ev, now=max(effective_now, effective_ts))
+            if outcome == 'stored':
+                self.note_origin(del_ev['id'], origin)
+            return del_ev, outcome
+
+    def put_causal_operation(
+        self,
+        log_id: str,
+        op: P2PandaOperation,
+        sk_hex: str,
+        *,
+        shard: Optional[str] = None,
+        created_at: Optional[int] = None,
+        now: Optional[int] = None,
+        origin: str = NodeStore.LOCAL,
+    ) -> tuple[NostrEvent, str]:
+        """Wrap a signed `P2PandaOperation` in a Kind 9850 event and insert it."""
+        effective_now = int(now) if now is not None else self.clock()
+        effective_ts = int(created_at) if created_at is not None else effective_now
+        ev = wrap_p2panda_event(sk_hex, log_id, op, effective_ts, shard=shard)
+        with self._lock:
+            outcome = self.insert_event(ev, now=max(effective_now, effective_ts))
+            if outcome == 'stored':
+                self.note_origin(ev['id'], origin)
+            return ev, outcome
+
+    # ---- Query, Backfill & Verification Helpers -----------------------------
+
+    def query_records(
+        self,
+        table_name: str,
+        where: str = '',
+        args: Sequence[Any] = (),
+        order_by: str = '',
+        limit: Optional[int] = None,
+    ) -> list[ProjectedRow]:
+        """Query rows from a projected SQL table as typed dictionaries."""
+        tp = self.projector(table_name)
+        cols = tp.column_names()
+        sql = f"SELECT {', '.join(cols)} FROM {tp.table_name}"
+        if where:
+            sql += f' WHERE {where}'
+        if order_by:
+            sql += f' ORDER BY {order_by}'
+        if limit is not None:
+            sql += f' LIMIT {int(limit)}'
+        with self._lock:
+            rows = self._all(sql, args)
+        return [dict(zip(cols, r)) for r in rows]
+
+    def get_record(
+        self,
+        table_name: str,
+        pk: ProjectedValue | Sequence[ProjectedValue] | Mapping[str, ProjectedValue],
+    ) -> Optional[ProjectedRow]:
+        """Fetch one row from `table_name` by its primary key."""
+        tp = self.projector(table_name)
+        pk_cols = tp.spec.primary_key
+        vals: tuple[ProjectedValue, ...]
+        if isinstance(pk, Mapping):
+            vals = tuple(pk[c] for c in pk_cols)
+        elif isinstance(pk, Sequence) and not isinstance(pk, str):
+            vals = tuple(pk)
+        else:
+            vals = (pk,)
+        if len(vals) != len(pk_cols):
+            raise ValueError(f'expected {len(pk_cols)} primary key values for {table_name}, got {len(vals)}')
+        where = ' AND '.join(f'{c} = ?' for c in pk_cols)
+        matches = self.query_records(table_name, where=where, args=vals, limit=1)
+        return matches[0] if matches else None
+
+    def table_count(self, table_name: str) -> int:
+        tp = self.projector(table_name)
+        with self._lock:
+            row = self._one(f'SELECT COUNT(*) FROM {tp.table_name}')
+            return int(row[0]) if row else 0
+
+    def _backfill_causal(self, log_id: Optional[str] = None) -> int:
+        targets = (
+            [self._causal_by_log[log_id]]
+            if log_id is not None and log_id in self._causal_by_log
+            else list(self._causal_by_log.values())
+        )
+        if not targets:
+            return 0
+        count = 0
+        with self._lock, self._atomic():
+            rows = self._all(
+                'SELECT json FROM events WHERE kind = ? ORDER BY created_at ASC, id ASC',
+                (CAUSAL_KIND,),
+            )
+            for (raw_json,) in rows:
+                ev = json.loads(raw_json)
+                l_tag = store.first_tag(ev, 'L')
+                ev_log = l_tag[1] if (l_tag and len(l_tag) > 1) else None
+                if log_id is not None and ev_log != log_id:
+                    continue
+                self._project_causal_put(ev)
+                count += 1
+        return count
+
+    def backfill(self, table_name: Optional[str] = None) -> dict[str, int]:
+        """Rebuild one or all projected SQL tables from the live `events` table."""
+        targets = (
+            [self.projector(table_name)]
+            if table_name is not None
+            else list(self._projectors_by_table.values())
+        )
+        rebuilt: dict[str, int] = {}
+        with self._lock, self._atomic():
+            for tp in targets:
+                self.execute(f'DELETE FROM {tp.table_name}')
+                rows = self._all(
+                    'SELECT json FROM events WHERE kind = ? ORDER BY created_at ASC, id ASC',
+                    (tp.kind,),
+                )
+                n = 0
+                for (raw_json,) in rows:
+                    ev = json.loads(raw_json)
+                    vals, why = tp.project(ev)
+                    if why is not None or vals is None:
+                        self._record_refusal(str(ev['id']), tp.kind, tp.table_name, why or 'invalid payload')
+                        continue
+                    self.execute(tp.upsert_sql(), vals)
+                    n += 1
+                rebuilt[tp.table_name] = n
+                self.projected_counts[tp.table_name] = n
+            if table_name is None and self._causal_by_log:
+                self.execute('DELETE FROM p2panda_causal_ops')
+                for cp in self._causal_by_log.values():
+                    for tbl in cp.tables:
+                        self.execute(f'DELETE FROM {tbl.name}')
+                self._backfill_causal()
+        return rebuilt
+
+    def verify_projection(self) -> ProjectionVerifyReport:
+        """Audit every registered `TableProjector` and `CausalProjector` against
+        the live `events` table, checking for missing events, orphan rows, or
+        column mismatches."""
+        tables_count: dict[str, int] = {}
+        missing_events: list[str] = []
+        orphan_rows: list[str] = []
+        mismatched_rows: list[str] = []
+
+        with self._lock:
+            for tp in self._projectors_by_table.values():
+                cols = tp.column_names()
+                sql_rows = self._all(
+                    f"SELECT {', '.join(cols)} FROM {tp.table_name}"
+                )
+                tables_count[tp.table_name] = len(sql_rows)
+                orphan_eids = {
+                    str(r[0])
+                    for r in self._all(
+                        f'SELECT _event_id FROM {tp.table_name} WHERE _event_id NOT IN (SELECT id FROM events)'
+                    )
+                }
+                by_eid: dict[str, tuple[object, ...]] = {}
+                for r in sql_rows:
+                    row_map = dict(zip(cols, r))
+                    eid = str(row_map['_event_id'])
+                    by_eid[eid] = tuple(r)
+                    if eid in orphan_eids:
+                        orphan_rows.append(f'{tp.table_name}:{eid}')
+
+                ev_rows = self._all(
+                    'SELECT json FROM events WHERE kind = ? ORDER BY created_at ASC, id ASC',
+                    (tp.kind,),
+                )
+                for (raw_json,) in ev_rows:
+                    ev = json.loads(raw_json)
+                    eid = str(ev['id'])
+                    expected_vals, why = tp.project(ev)
+                    if why is not None or expected_vals is None:
+                        continue
+                    actual_vals = by_eid.get(eid)
+                    if actual_vals is None:
+                        missing_events.append(f'{tp.table_name}:{eid}')
+                    elif tuple(actual_vals) != tuple(expected_vals):
+                        mismatched_rows.append(f'{tp.table_name}:{eid}')
+
+            if self._causal_by_log:
+                causal_eids = {
+                    str(r[0])
+                    for r in self._all('SELECT event_id FROM p2panda_causal_ops')
+                }
+                for (orphan_eid,) in self._all(
+                    'SELECT event_id FROM p2panda_causal_ops '
+                    'WHERE event_id NOT IN (SELECT id FROM events) ORDER BY event_id ASC'
+                ):
+                    orphan_rows.append(f'p2panda_causal_ops:{orphan_eid}')
+                causal_ev_rows = self._all(
+                    'SELECT json FROM events WHERE kind = ? ORDER BY created_at ASC, id ASC',
+                    (CAUSAL_KIND,),
+                )
+                for (raw_json,) in causal_ev_rows:
+                    ev = json.loads(raw_json)
+                    log_id, op, why = unwrap_p2panda_event(ev)
+                    if why is not None or op is None or not log_id or log_id not in self._causal_by_log:
+                        continue
+                    eid = str(ev['id'])
+                    if eid not in causal_eids:
+                        missing_events.append(f'p2panda_causal_ops:{eid}')
+
+        is_ok = not (missing_events or orphan_rows or mismatched_rows)
+        return {
+            'ok': is_ok,
+            'tables': tables_count,
+            'missing_events': missing_events,
+            'orphan_rows': orphan_rows,
+            'mismatched_rows': mismatched_rows,
+        }
+
+    def shard_filters(self) -> dict[str, NostrFilter]:
+        """Return `{shard_id: nostr_filter}` aggregated across all registered
+        `TableProjector`s and `CausalProjector`s that declare a `shard_id`."""
+        with self._lock:
+            by_shard_kinds: dict[str, list[int]] = {}
+            out: dict[str, NostrFilter] = {}
+            for tp in self._projectors_by_table.values():
+                sid = tp.spec.shard_id
+                if not sid:
+                    continue
+                kinds = by_shard_kinds.setdefault(sid, [])
+                if tp.kind not in kinds:
+                    kinds.append(tp.kind)
+            for sid, kinds in by_shard_kinds.items():
+                out[sid] = {'kinds': sorted(kinds)}
+            for cp in self._causal_by_log.values():
+                out[cp.shard_id] = cp.shard_filter()
+            return out
+
+    def protected_filters(self) -> list[NostrFilter]:
+        """Return the list of NIP-01 filters covering all projected tables with
+        a `shard_id` and all registered `CausalProjector` logs so `Retention`
+        never evicts events backing claimed projected tables."""
+        return list(self.shard_filters().values())
+