fleetmesh-node 0.2.0__py3-none-any.whl

node/testnet/cluster.py

@@ -0,0 +1,731 @@
+"""Local Multi-Node Cluster & Chaos Testnet Engine.
+
+Enables developers and operators to bootstrap, peer, inspect, and stress-test
+multi-node MeshNode topologies with simulated network chaos:
+1. Topologies: mesh, star (hub-spoke), ring, and line.
+2. Isolated local node directories with distinct identities, SQLite stores, and face ports.
+3. Automated peering proposal, mutual grant exchange, and sync orchestration.
+4. Chaos Simulator:
+   - Network partitions and healing.
+   - Configurable packet loss injection.
+   - Adversarial bad-peer injection (duplicate flood, forged hops, active silence).
+   - Automated S1..S4 severity verification and Kind 30802 dispute tracking.
+"""
+from __future__ import annotations
+
+import json
+import os
+import random
+import tempfile
+import threading
+import time
+from typing import Any, Callable, Mapping, Optional, Sequence
+
+from fleetmesh_core.peering import badpeer
+from fleetmesh_core.wire.interfaces import NostrEvent
+from .. import app as app_mod, keystore
+from ..protocols.nostr import relayserver
+from ..storage import grants
+from ..protocols.nostr import client as relay
+from .fips_daemon import ClusterFipsDaemon
+
+TOPOLOGIES: tuple[str, ...] = ('mesh', 'star', 'ring', 'line')
+
+
+def generate_topology_edges(node_count: int, topology: str = 'mesh') -> list[tuple[int, int]]:
+    """Compute pairwise peering edges (a, b) for N nodes given a topology."""
+    if node_count < 2:
+        return []
+    edges: list[tuple[int, int]] = []
+    if topology == 'mesh':
+        for i in range(node_count):
+            for j in range(i + 1, node_count):
+                edges.append((i, j))
+    elif topology == 'star':
+        # Node 0 is the central hub/anchor; nodes 1..N-1 peer with node 0
+        for i in range(1, node_count):
+            edges.append((0, i))
+    elif topology == 'ring':
+        for i in range(node_count):
+            edges.append((i, (i + 1) % node_count))
+    elif topology == 'line':
+        for i in range(node_count - 1):
+            edges.append((i, i + 1))
+    else:
+        raise ValueError(f"unknown topology {topology!r}; expected one of {TOPOLOGIES}")
+    return edges
+
+
+# ---------------------------------------------------------------------------
+# Chaos Simulator
+# ---------------------------------------------------------------------------
+
+class ChaosSimulator:
+    """Network chaos injection for a running LocalCluster."""
+
+    def __init__(self, cluster: LocalCluster) -> None:
+        self.cluster: LocalCluster = cluster
+        self._lock: threading.RLock = threading.RLock()
+        self.partitions: list[tuple[set[str], set[str], Optional[str]]] = []  # list of (pubkeys_a, pubkeys_b, rung)
+        self.directed_partitions: list[tuple[set[str], set[str], Optional[str]]] = []  # list of (from_pks, to_pks, rung)
+        self.loss_rates: dict[str, float] = {}  # pubkey -> drop probability [0.0, 1.0]
+        self.latencies: dict[tuple[str, Optional[str]], tuple[float, float]] = {}  # (src_pk, dst_pk) -> (lat, jit)
+        self.adversarial_nodes: dict[str, Mapping[str, Any]] = {}  # pubkey -> offence config
+        self._original_sends: dict[str, Callable[..., Any]] = {}
+
+    def partition(
+        self,
+        group_a: Sequence[int | str],
+        group_b: Sequence[int | str],
+        rung: Optional[str] = None,
+    ) -> None:
+        """Cut network connectivity between two groups of nodes (symmetric)."""
+        with self._lock:
+            pks_a = {self.cluster.resolve_node_pubkey(x) for x in group_a}
+            pks_b = {self.cluster.resolve_node_pubkey(x) for x in group_b}
+            self.partitions.append((pks_a, pks_b, rung))
+            self._ensure_interceptors()
+
+    def directed_partition(
+        self,
+        from_nodes: Sequence[int | str],
+        to_nodes: Sequence[int | str],
+        rung: Optional[str] = None,
+    ) -> None:
+        """Cut directed network connectivity from from_nodes to to_nodes (asymmetric)."""
+        with self._lock:
+            pks_from = {self.cluster.resolve_node_pubkey(x) for x in from_nodes}
+            pks_to = {self.cluster.resolve_node_pubkey(x) for x in to_nodes}
+            self.directed_partitions.append((pks_from, pks_to, rung))
+            self._ensure_interceptors()
+
+    def heal(self) -> None:
+        """Remove all network partitions and restore normal connectivity."""
+        with self._lock:
+            self.partitions.clear()
+            self.directed_partitions.clear()
+
+    def set_packet_loss(self, node: int | str, loss_rate: float) -> None:
+        """Simulate envelope drop probability for a specific node."""
+        with self._lock:
+            pk = self.cluster.resolve_node_pubkey(node)
+            if loss_rate <= 0.0:
+                self.loss_rates.pop(pk, None)
+            else:
+                self.loss_rates[pk] = min(1.0, float(loss_rate))
+            self._ensure_interceptors()
+
+    def set_latency(
+        self,
+        node: int | str,
+        target: Optional[int | str] = None,
+        latency_s: float = 0.0,
+        jitter_s: float = 0.0,
+    ) -> None:
+        """Inject latency and jitter (in seconds) for outgoing transmissions."""
+        with self._lock:
+            pk = self.cluster.resolve_node_pubkey(node)
+            target_pk = self.cluster.resolve_node_pubkey(target) if target is not None else None
+            key = (pk, target_pk)
+            if latency_s <= 0.0 and jitter_s <= 0.0:
+                self.latencies.pop(key, None)
+            else:
+                self.latencies[key] = (max(0.0, float(latency_s)), max(0.0, float(jitter_s)))
+            self._ensure_interceptors()
+
+    def clear_latency(
+        self,
+        node: Optional[int | str] = None,
+        target: Optional[int | str] = None,
+    ) -> None:
+        """Remove latency settings for a specific node, link, or all nodes."""
+        with self._lock:
+            if node is None and target is None:
+                self.latencies.clear()
+            else:
+                pk = self.cluster.resolve_node_pubkey(node) if node is not None else None
+                target_pk = self.cluster.resolve_node_pubkey(target) if target is not None else None
+                if pk is not None:
+                    self.latencies.pop((pk, target_pk), None)
+                    if target_pk is None:
+                        for k in list(self.latencies.keys()):
+                            if k[0] == pk:
+                                self.latencies.pop(k, None)
+
+    def get_delay(self, pk_from: str, pk_to: str) -> float:
+        with self._lock:
+            spec = self.latencies.get((pk_from, pk_to)) or self.latencies.get((pk_from, None))
+            if not spec:
+                return 0.0
+            lat, jit = spec
+            if jit > 0.0:
+                d = lat + random.uniform(-jit, jit)
+            else:
+                d = lat
+            return max(0.0, d)
+
+    def inject_bad_peer(self, node: int | str, offence: str = 'duplicate-flood', **kwargs: Any) -> None:
+        """Configure node as an adversarial peer committing protocol violations."""
+        with self._lock:
+            node_app = self.cluster.get_node(node)
+            pk = node_app.ks.node_pk
+            self.adversarial_nodes[pk] = {'offence': offence, **kwargs}
+
+            # If offence is a sync session offence, wrap node's sync session factory
+            if offence in badpeer.OFFENCES:
+                import functools
+                flood_mult = int(kwargs.get('flood', 8))
+                node_app.sync.session_cls = functools.partial(
+                    badpeer.BadPeer,
+                    offence=offence,
+                    flood=flood_mult,
+                )
+
+    def is_partitioned(self, pk_from: str, pk_to: str, rung: Optional[str] = None) -> bool:
+        with self._lock:
+            for item in self.partitions:
+                group_a, group_b = item[0], item[1]
+                part_rung = item[2] if len(item) > 2 else None
+                if (pk_from in group_a and pk_to in group_b) or (pk_from in group_b and pk_to in group_a):
+                    if part_rung is None or part_rung == rung or (rung is not None and part_rung == 'direct' and rung != 'nostr'):
+                        return True
+            for item in self.directed_partitions:
+                src_group, dst_group = item[0], item[1]
+                part_rung = item[2] if len(item) > 2 else None
+                if pk_from in src_group and pk_to in dst_group:
+                    if part_rung is None or part_rung == rung or (rung is not None and part_rung == 'direct' and rung != 'nostr'):
+                        return True
+            return False
+
+    def should_drop(self, pk: str) -> bool:
+        with self._lock:
+            rate = self.loss_rates.get(pk, 0.0)
+            if rate <= 0.0:
+                return False
+            return random.random() < rate
+
+    def _ensure_interceptors(self) -> None:
+        """Hook into each node's transport router to enforce partition and loss rules."""
+        for name, node in self.cluster.nodes.items():
+            pk = node.ks.node_pk
+            if pk in self._original_sends:
+                continue
+
+            orig_send = getattr(node.transport, '_send_ladder', node.transport.send)
+            self._original_sends[pk] = orig_send
+
+            def make_interceptor(sender_pk: str, original: Callable[..., Any]):
+                def intercepted_send(
+                    message: Mapping[str, Any],
+                    to_pubkey: str,
+                    anonymous: bool = False,
+                    stored: bool = False,
+                ) -> dict[str, tuple[bool, str]]:
+                    if self.is_partitioned(sender_pk, to_pubkey, rung=None):
+                        return {'chaos_partition': (False, 'network partition in force')}
+                    if self.should_drop(sender_pk) or self.should_drop(to_pubkey):
+                        return {'chaos_loss': (False, 'simulated packet loss')}
+                    delay = self.get_delay(sender_pk, to_pubkey)
+                    if delay > 0.0:
+                        time.sleep(delay)
+                    return original(message, to_pubkey, anonymous=anonymous, stored=stored)
+                return intercepted_send
+
+            def make_rung_interceptor(sender_pk: str):
+                def intercepted_rung(
+                    rung_name: str,
+                    message: Mapping[str, Any],
+                    to_pubkey: str,
+                    target: Any,
+                    anonymous: bool,
+                ) -> Optional[dict[str, tuple[bool, str]]]:
+                    if self.is_partitioned(sender_pk, to_pubkey, rung=rung_name):
+                        raise ConnectionResetError(f"chaos: rung '{rung_name}' partitioned")
+                    return None
+                return intercepted_rung
+
+            interceptor = make_interceptor(pk, orig_send)
+            if hasattr(node.transport, 'set_send_interceptor'):
+                node.transport.set_send_interceptor(interceptor)
+            else:
+                # Fallback for custom or mock transports
+                setattr(node.transport, 'send', interceptor)
+
+            if hasattr(node.transport, 'set_rung_interceptor'):
+                node.transport.set_rung_interceptor(make_rung_interceptor(pk))
+
+    def restore(self) -> None:
+        """Restore all original transport methods and reset chaos state."""
+        with self._lock:
+            self.partitions.clear()
+            self.directed_partitions.clear()
+            self.loss_rates.clear()
+            self.latencies.clear()
+            self.adversarial_nodes.clear()
+            for name, node in self.cluster.nodes.items():
+                pk = node.ks.node_pk
+                if hasattr(node.transport, 'clear_send_interceptor'):
+                    node.transport.clear_send_interceptor()
+                if hasattr(node.transport, 'clear_rung_interceptor'):
+                    node.transport.clear_rung_interceptor()
+                elif pk in self._original_sends:
+                    setattr(node.transport, 'send', self._original_sends[pk])
+                from fleetmesh_core.storage import sync
+                node.sync.session_cls = sync.Session
+            self._original_sends.clear()
+
+    def get_status(self) -> Mapping[str, Any]:
+        with self._lock:
+            return {
+                'partitions_active': len(self.partitions),
+                'partitions': [
+                    {'group_a': list(p[0]), 'group_b': list(p[1]), 'rung': p[2] if len(p) > 2 else None}
+                    for p in self.partitions
+                ],
+                'directed_partitions_active': len(self.directed_partitions),
+                'directed_partitions': [
+                    {'from': list(p[0]), 'to': list(p[1]), 'rung': p[2] if len(p) > 2 else None}
+                    for p in self.directed_partitions
+                ],
+                'packet_loss_nodes': self.loss_rates,
+                'latencies_active': len(self.latencies),
+                'adversarial_nodes': self.adversarial_nodes,
+            }
+
+
+# ---------------------------------------------------------------------------
+# LocalCluster Engine
+# ---------------------------------------------------------------------------
+
+class LocalCluster:
+    """Manages an isolated multi-node MeshNode testnet cluster."""
+
+    def __init__(
+        self,
+        base_dir: str,
+        node_count: int = 3,
+        topology: str = 'mesh',
+        shared_relays: bool = True,
+        relay_urls: Optional[Sequence[str]] = None,
+        clock: Optional[Callable[[], int]] = None,
+        pow_bits: int = 8,
+        default_shard: str = 'bristol',
+        node_classes: Optional[Sequence[str]] = None,
+        enable_fips: bool = False,
+        fips_sock_path: Optional[str] = None,
+    ) -> None:
+        self.base_dir: str = os.path.abspath(base_dir)
+        self.node_count: int = max(2, node_count)
+        self.topology: str = topology
+        self.shared_relays: bool = shared_relays
+        self.clock: Callable[[], int] = clock or (lambda: int(time.time()))
+        self.pow_bits: int = pow_bits
+        self.default_shard: str = default_shard
+        self.node_classes: list[str] = list(node_classes or ['edge'] * self.node_count)
+        while len(self.node_classes) < self.node_count:
+            self.node_classes.append('edge')
+
+        self.enable_fips: bool = enable_fips
+        self.fips_sock_path: Optional[str] = fips_sock_path
+        self.fips_daemon: Optional[ClusterFipsDaemon] = None
+
+        self.relay_server: Optional[relayserver.RelayServer] = None
+        self.relay_urls: list[str] = list(relay_urls or [])
+        self.nodes: dict[str, app_mod.NodeApp] = {}
+        self.node_dirs: dict[str, str] = {}
+        self.chaos: ChaosSimulator = ChaosSimulator(self)
+        self.running: bool = False
+        self._lock: threading.RLock = threading.RLock()
+
+    @property
+    def status_path(self) -> str:
+        return os.path.join(self.base_dir, 'cluster.json')
+
+    def resolve_node_pubkey(self, node: int | str) -> str:
+        """Resolve node index or name or pubkey to a canonical 64-hex node pubkey."""
+        if isinstance(node, int):
+            if not 0 <= node < self.node_count:
+                raise IndexError(f"node index {node} out of range (0..{self.node_count-1})")
+            name = f'node_{node}'
+            if name in self.nodes:
+                return self.nodes[name].ks.node_pk
+            ndir = self.node_dirs.get(name) or os.path.join(self.base_dir, name)
+            kpath = os.path.join(ndir, 'keys.json')
+            if os.path.exists(kpath):
+                k = json.load(open(kpath, encoding='utf-8'))
+                return k.get('node_pk', '')
+            raise KeyError(f"node index {node} not initialized on disk")
+        s = str(node).strip()
+        if s in self.nodes:
+            return self.nodes[s].ks.node_pk
+        if s.startswith('did:nostr:'):
+            s = s.split(':')[-1]
+        for name, n in self.nodes.items():
+            if n.ks.node_pk.startswith(s.lower()):
+                return n.ks.node_pk
+        # If not running yet, inspect directory configs
+        for i in range(self.node_count):
+            ndir = os.path.join(self.base_dir, f'node_{i}')
+            kpath = os.path.join(ndir, 'keys.json')
+            if os.path.exists(kpath):
+                try:
+                    k = json.load(open(kpath, encoding='utf-8'))
+                    npk = k.get('node_pk', '')
+                    if npk.startswith(s.lower()):
+                        return npk
+                except Exception:
+                    pass
+        raise KeyError(f"node identifier {node!r} not found in cluster")
+
+    def get_node(self, node: int | str) -> app_mod.NodeApp:
+        pk = self.resolve_node_pubkey(node)
+        for name, n in self.nodes.items():
+            if n.ks.node_pk == pk:
+                return n
+        raise KeyError(f"node {node!r} is not currently running")
+
+    def init(self) -> LocalCluster:
+        """Initialize cluster directory and node configurations on disk."""
+        os.makedirs(self.base_dir, exist_ok=True)
+        # Separate cluster operator key ensuring 3-Layer Operator Independence Model
+        operator_ks = keystore.Keystore.generate()
+        operator_pk = operator_ks.node_pk
+
+        if self.enable_fips and self.fips_daemon is None:
+            if self.fips_sock_path:
+                sock_p = self.fips_sock_path
+            else:
+                candidate = os.path.join(self.base_dir, 'fips.sock')
+                if len(candidate) > 90:
+                    # sun_path is 108 bytes; a deep base_dir does not fit.
+                    # The fallback used to be a predictable name in /tmp,
+                    # which any local user could create ahead of us and
+                    # then answer as the daemon. A fresh private directory
+                    # (mode 0700) under $XDG_RUNTIME_DIR, or mkdtemp's
+                    # default, is ours alone; the name is unguessable.
+                    run_dir = os.environ.get('XDG_RUNTIME_DIR')
+                    if run_dir and not os.path.isdir(run_dir):
+                        run_dir = None
+                    sock_p = os.path.join(tempfile.mkdtemp(prefix='ms_fips_', dir=run_dir), 'fips.sock')
+                else:
+                    sock_p = candidate
+            self.fips_sock_path = sock_p
+            self.fips_daemon = ClusterFipsDaemon(sock_p)
+
+        for i in range(self.node_count):
+            name = f'node_{i}'
+            ndir = os.path.join(self.base_dir, name)
+            self.node_dirs[name] = ndir
+            os.makedirs(ndir, exist_ok=True)
+
+            ks = keystore.Keystore.load_or_generate(ndir)
+
+            # Node config
+            cpath = os.path.join(ndir, 'config.json')
+            if not os.path.exists(cpath):
+                node_cls = self.node_classes[i]
+                eps = []
+                if self.enable_fips and self.fips_daemon is not None:
+                    eps.append({
+                        'transport': 'fips-native',
+                        'address': f"{ks.node_pk}:11801",
+                        'priority': 5,
+                        'sock_path': self.fips_daemon.sock_path,
+                    })
+                # `class` is a keyword, so it goes in as a key. (It went in as
+                # `class_` once, and every node ran as the default class; the
+                # config check now refuses a key no compartment reads.)
+                cfg = dict(
+                    app_mod.defaults(),
+                    name=name,
+                    operator=operator_pk,
+                    pow_bits=self.pow_bits,
+                    heartbeat=True,
+                    tick_seconds=2,
+                    face={'host': '127.0.0.1', 'port': 0},
+                    relays=self.relay_urls,
+                    endpoints=eps,
+                    coverage=[{'shard': self.default_shard, 'filter': {'kinds': [1], '#t': [self.default_shard]},
+                               'completeness': 'asserted'}],
+                )
+                cfg['class'] = node_cls
+                app_mod.save_config(ndir, cfg)
+
+        self._save_cluster_meta()
+        return self
+
+    def start(self, timeout: float = 10.0) -> LocalCluster:
+        """Start the cluster: relay server, all node instances, and establish peering mesh."""
+        with self._lock:
+            if self.running:
+                return self
+
+            self.init()
+
+            if self.enable_fips and self.fips_daemon is None:
+                sock_p = self.fips_sock_path or os.path.join(self.base_dir, 'fips_control.sock')
+                self.fips_daemon = ClusterFipsDaemon(sock_p)
+
+            # 1. Start local relay server if requested and none specified
+            if self.shared_relays and not self.relay_urls:
+                self.relay_server = relayserver.RelayServer(clock=self.clock).start()
+                self.relay_urls = [self.relay_server.url]
+
+            # 2. Start each node
+            for i in range(self.node_count):
+                name = f'node_{i}'
+                ndir = self.node_dirs[name]
+                cfg = dict(app_mod.load_config(ndir))
+                cfg['relays'] = list(self.relay_urls)
+                if self.enable_fips and self.fips_daemon is not None:
+                    ks = keystore.Keystore.load(ndir)
+                    cfg['endpoints'] = [
+                        {
+                            'transport': 'fips-native',
+                            'address': f"{ks.node_pk}:11801",
+                            'priority': 5,
+                            'sock_path': self.fips_daemon.sock_path,
+                        }
+                    ]
+                app_mod.save_config(ndir, cfg)
+
+                node = app_mod.NodeApp(ndir, cfg, clock=self.clock)
+                node.start()
+                self.nodes[name] = node
+
+            # 3. Wait for all inboxes to listen
+            time.sleep(0.05)
+
+            # 4. If FIPS native is enabled, exchange descriptors directly across cluster
+            if self.enable_fips:
+                for name_a, node_a in self.nodes.items():
+                    for name_b, node_b in self.nodes.items():
+                        if name_a != name_b and node_b.descriptor is not None:
+                            node_a.peers.learn(node_b.descriptor)
+
+            # 5. Automate peering proposals based on configured topology
+            edges = generate_topology_edges(self.node_count, self.topology)
+            filter_spec = {'kinds': [1], '#t': [self.default_shard]}
+
+            for idx_a, idx_b in edges:
+                node_a = self.nodes[f'node_{idx_a}']
+                node_b = self.nodes[f'node_{idx_b}']
+                try:
+                    node_a.peering.propose(
+                        node_b.ks.node_pk,
+                        [{'shard': self.default_shard, 'filter': filter_spec, 'mode': 'both'}],
+                        timeout=min(5.0, timeout),
+                    )
+                except Exception:
+                    pass
+
+            # 6. Wait for mutual grants to converge
+            deadline = time.time() + timeout
+            while time.time() < deadline:
+                all_peered = True
+                for idx_a, idx_b in edges:
+                    na = self.nodes[f'node_{idx_a}']
+                    nb = self.nodes[f'node_{idx_b}']
+                    g_a = na.grants.get(nb.ks.node_pk, grants.ISSUED)
+                    g_b = nb.grants.get(na.ks.node_pk, grants.ISSUED)
+                    if not (g_a and g_b):
+                        all_peered = False
+                        break
+                if all_peered:
+                    break
+                time.sleep(0.02)
+
+            self.running = True
+            self._save_cluster_meta()
+            return self
+
+    def stop_node(self, node: int | str) -> None:
+        """Stop an individual node instance in the cluster (simulating node crash / churn)."""
+        with self._lock:
+            pk = self.resolve_node_pubkey(node)
+            for name, n in list(self.nodes.items()):
+                if n.ks.node_pk == pk:
+                    try:
+                        n.stop()
+                    except Exception:
+                        pass
+                    del self.nodes[name]
+                    return
+            raise KeyError(f"node {node!r} is not running")
+
+    def start_node(self, node_idx: int) -> app_mod.NodeApp:
+        """Start or restart a single node instance from its persisted directory."""
+        with self._lock:
+            name = f'node_{node_idx}'
+            if name in self.nodes:
+                return self.nodes[name]
+            ndir = self.node_dirs[name]
+            cfg = app_mod.load_config(ndir)
+            node = app_mod.NodeApp(ndir, cfg, clock=self.clock)
+            node.start()
+            self.nodes[name] = node
+            return node
+
+    def stop(self) -> None:
+        """Stop all cluster nodes and release local relays."""
+        with self._lock:
+            self.chaos.restore()
+            for name, node in self.nodes.items():
+                try:
+                    node.stop()
+                except Exception:
+                    pass
+            self.nodes.clear()
+
+            if self.fips_daemon is not None:
+                try:
+                    self.fips_daemon.stop()
+                except Exception:
+                    pass
+                if self.fips_sock_path and os.path.exists(self.fips_sock_path):
+                    try:
+                        os.unlink(self.fips_sock_path)
+                    except OSError:
+                        pass
+                self.fips_daemon = None
+
+            if self.relay_server is not None:
+                try:
+                    self.relay_server.stop()
+                except Exception:
+                    pass
+                self.relay_server = None
+
+            self.running = False
+            self._save_cluster_meta()
+
+    def tick(self, ticks: int = 1) -> list[list[Any]]:
+        """Advance time or execute tick cycles across all cluster nodes."""
+        results = []
+        for _ in range(ticks):
+            cycle = []
+            for name, node in self.nodes.items():
+                cycle.append((name, node.tick()))
+            results.append(cycle)
+        return results
+
+    def sync_pair(self, idx_a: int | str, idx_b: int | str, shard: Optional[str] = None) -> Mapping[str, Any]:
+        """Trigger an immediate sync pull from Node A to Node B."""
+        try:
+            node_a = self.get_node(idx_a)
+            node_b = self.get_node(idx_b)
+        except KeyError as exc:
+            return {'status': 'offline', 'error': str(exc)}
+        s = shard or self.default_shard
+        filter_spec = {'kinds': [1], '#t': [s]}
+        return node_a.sync.pull(node_b.ks.node_pk, s, filter_spec, timeout=10)
+
+    def sync_all(self, shard: Optional[str] = None) -> list[Mapping[str, Any]]:
+        """Trigger a sync cycle across all topology edges."""
+        s = shard or self.default_shard
+        edges = generate_topology_edges(self.node_count, self.topology)
+        results: list[Mapping[str, Any]] = []
+        for idx_a, idx_b in edges:
+            res_ab = self.sync_pair(idx_a, idx_b, shard=s)
+            res_ba = self.sync_pair(idx_b, idx_a, shard=s)
+            results.append({'pair': (idx_a, idx_b), 'pull_ab': res_ab, 'pull_ba': res_ba})
+        return results
+
+    def publish_event(self, node: int | str, event: NostrEvent) -> str:
+        """Store and publish an event at the specified node."""
+        target_node = self.get_node(node)
+        target_node.store._put(event)
+        if self.relay_urls:
+            relay.publish_all(self.relay_urls, event, timeout=5)
+        return event.get('id', '')
+
+    def get_status(self) -> Mapping[str, Any]:
+        """Collect cluster topology, health, active peerings, and metrics."""
+        node_summaries = []
+        edges = generate_topology_edges(self.node_count, self.topology)
+        active_edge_count = 0
+
+        for i in range(self.node_count):
+            name = f'node_{i}'
+            ndir = self.node_dirs.get(name) or os.path.join(self.base_dir, name)
+            info = {
+                'index': i,
+                'name': name,
+                'dir': ndir,
+                'status': 'running' if name in self.nodes else 'offline',
+            }
+
+            if name in self.nodes:
+                node = self.nodes[name]
+                info['pubkey'] = node.ks.node_pk
+                info['did'] = node.ks.did
+                info['class'] = node.cfg.get('class', 'edge')
+                info['face'] = node.face.url if getattr(node, 'face', None) else None
+                info['event_count'] = node.store.count()
+                # peer_view() carries issued/received/standing; GrantStore.list()
+                # never did, so these read None for every peer until the row was
+                # typed.
+                peer_list = []
+                for p in node.peer_view():
+                    issued = p['issued']
+                    received = p['received']
+                    peer_list.append({
+                        'peer': p['peer'],
+                        'issued_tier': issued['tier'] if issued else None,
+                        'received_tier': received['tier'] if received else None,
+                        'standing': p.get('standing'),
+                    })
+                info['peers'] = peer_list
+                info['refusals'] = len(node.subsystems.refusals(100))
+            else:
+                # Inspect offline files
+                if os.path.exists(os.path.join(ndir, 'keys.json')):
+                    ks_data = json.load(open(os.path.join(ndir, 'keys.json'), encoding='utf-8'))
+                    info['pubkey'] = ks_data.get('node_pk')
+                    info['did'] = f"did:nostr:{ks_data.get('node_pk')}"
+                if os.path.exists(os.path.join(ndir, 'status.json')):
+                    st = json.load(open(os.path.join(ndir, 'status.json'), encoding='utf-8'))
+                    info['class'] = st.get('class')
+                    info['face'] = st.get('face')
+
+            node_summaries.append(info)
+
+        # Count mutual active edges
+        if self.running:
+            for idx_a, idx_b in edges:
+                na = self.nodes.get(f'node_{idx_a}')
+                nb = self.nodes.get(f'node_{idx_b}')
+                if na and nb:
+                    if na.grants.get(nb.ks.node_pk, grants.ISSUED) and nb.grants.get(na.ks.node_pk, grants.ISSUED):
+                        active_edge_count += 1
+
+        status = {
+            'cluster_dir': self.base_dir,
+            'node_count': self.node_count,
+            'topology': self.topology,
+            'running': self.running,
+            'relays': self.relay_urls,
+            'active_peering_edges': f"{active_edge_count}/{len(edges)}",
+            'nodes': node_summaries,
+            'chaos': self.chaos.get_status(),
+        }
+        if self.enable_fips:
+            status['fips'] = {
+                'active': self.fips_daemon is not None and self.fips_daemon.running,
+                'sock_path': self.fips_daemon.sock_path if self.fips_daemon else None,
+                'listeners': len(self.fips_daemon.peer_listeners) if self.fips_daemon else 0,
+            }
+        return status
+
+    def _save_cluster_meta(self) -> None:
+        data = {
+            'base_dir': self.base_dir,
+            'node_count': self.node_count,
+            'topology': self.topology,
+            'running': self.running,
+            'relays': self.relay_urls,
+            'enable_fips': self.enable_fips,
+            'updated_at': int(time.time()),
+        }
+        with open(self.status_path, 'w', encoding='utf-8') as f:
+            json.dump(data, f, indent=2)