hypermind 0.11.0__py3-none-any.whl

hypermind/transport/anti_entropy.py

@@ -0,0 +1,708 @@
+"""Anti-entropy / set reconciliation.
+
+After a network partition, two peers may hold disjoint subsets of the
+sealed-claim universe. Naive O(n) full-set exchange is wasteful; a
+Bloom filter sketch lets each peer compute the symmetric difference
+in O(δ) where δ = |A ⊖ B|.
+
+Algorithm (multi-round, WS-G v0.7):
+  Round 1 — Bloom diff:
+    Peer A computes ``B_A`` over its known kids, sends it to B with
+    ``|A|`` and chunk cursor 0.
+    Peer B walks its kids, picks the first 1024 not in ``B_A`` and
+    forwards them as the round-1 candidate set + an ``ack_set`` of
+    kids it already had (closes the false-positive gap).
+    A then drains the next 1024 of its own ``B_B``-misses; the loop
+    continues until both peers' cursors hit end-of-set.
+
+  Round 2 — explicit ack-set:
+    Each peer responds with kids it now has (after round-1 transfer).
+    A kid that appears in neither side's ack-set after round 1 is
+    a Bloom false-positive — ferried in round 2 directly.
+
+  Round 3 — cosigned (root, tree_size) agreement:
+    Each side computes the Merkle root of its full set; both root
+    + tree_size are exchanged + cosigned. A mismatch flags the
+    session as ``UNRESOLVED_PARTITION`` for the dispute FSM.
+
+  IBLT fallback:
+    When the round-1 Bloom diff exceeds ``IBLT_THRESHOLD`` items,
+    peers switch to an Invertible Bloom Lookup Table sketch
+    (parameters d=64 cells, k=4 hashes). IBLT supports symmetric
+    difference decoding in O(δ) communication WITHOUT iterating over
+    one peer's entire local set, which collapses huge δ exchanges.
+
+This is BIP 330's PinSketch idea simplified, with resumability via
+``ae_session_id`` (closes spec/06-conformance.md gap #3). At ~1%
+false-positive rate, the expected bandwidth is δ × (1 + ε) kids per
+round.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import logging
+import math
+import secrets
+import uuid
+from collections import OrderedDict
+from collections.abc import Awaitable, Callable, Iterable
+from dataclasses import dataclass, field
+
+logger = logging.getLogger(__name__)
+
+# WS-G tunables.
+AE_CHUNK_SIZE = 1024  # max items per round trip per direction.
+IBLT_THRESHOLD = 256  # round-1 Bloom-diff size that triggers IBLT.
+IBLT_DEFAULT_CELLS = 64
+IBLT_DEFAULT_HASHES = 4
+
+
+def _optimal_bloom_size(n: int, p: float = 0.01) -> tuple[int, int]:
+    """(m, k) minimising filter size for n items at false-positive rate p.
+
+    m = -n*ln(p) / (ln(2)^2);  k = (m/n) * ln(2)
+    """
+    if n == 0:
+        return 64, 2
+    m = max(64, int(-n * math.log(p) / (math.log(2) ** 2)))
+    # round to multiple of 8 for byte-aligned storage
+    m = ((m + 7) // 8) * 8
+    k = max(1, round((m / n) * math.log(2)))
+    return m, k
+
+
+@dataclass
+class BloomFilter:
+    bits: bytearray
+    m: int  # number of bits
+    k: int  # number of hash functions
+
+    @classmethod
+    def empty(cls, n_expected: int, p: float = 0.01) -> BloomFilter:
+        m, k = _optimal_bloom_size(n_expected, p)
+        return cls(bits=bytearray(m // 8), m=m, k=k)
+
+    def add(self, item: bytes) -> None:
+        for h in self._hashes(item):
+            self.bits[h // 8] |= 1 << (h % 8)
+
+    def __contains__(self, item: bytes) -> bool:
+        return all(self.bits[h // 8] & (1 << (h % 8)) for h in self._hashes(item))
+
+    def _hashes(self, item: bytes) -> list[int]:
+        # Double-hashing trick: hash_i = (h1 + i * h2) mod m
+        digest = hashlib.sha256(item).digest()
+        h1 = int.from_bytes(digest[:8], "big")
+        h2 = int.from_bytes(digest[8:16], "big") | 1  # ensure odd
+        return [(h1 + i * h2) % self.m for i in range(self.k)]
+
+    def to_cbor(self) -> dict:
+        return {"bits": bytes(self.bits), "m": self.m, "k": self.k}
+
+    # Hard caps for `from_cbor` — defends against a malicious peer
+    # sending huge `m` (memory DoS) or huge `k` (CPU DoS via per-lookup
+    # hash count). 16 Mbit ≈ 2 MB / filter, 32 hashes is generous.
+    MAX_M_BITS = 16 * 1024 * 1024
+    MAX_K = 32
+
+    @classmethod
+    def from_cbor(cls, m: dict) -> BloomFilter:
+        m_val = int(m["m"])
+        k_val = int(m["k"])
+        if m_val <= 0 or k_val <= 0:
+            raise ValueError("bloom m and k must be positive")
+        if m_val > cls.MAX_M_BITS:
+            raise ValueError(f"bloom m too large: {m_val} > {cls.MAX_M_BITS}")
+        if k_val > cls.MAX_K:
+            raise ValueError(f"bloom k too large: {k_val} > {cls.MAX_K}")
+        # Strict type + length gate BEFORE allocation. CBOR allows `bits`
+        # to be an integer; bytearray(int) allocates that many zero bytes,
+        # so a 30-byte CBOR frame with bits=2**32 would trigger a multi-GB
+        # allocation before the length-mismatch check fires. Refuse
+        # anything that isn't a byte string of the exact length implied by
+        # m_val (closes self-heal Cycle-3 Finding #15).
+        raw_bits = m["bits"]
+        if not isinstance(raw_bits, (bytes, bytearray, memoryview)):
+            raise ValueError(
+                f"bloom bits field must be a byte string, got {type(raw_bits).__name__}"
+            )
+        expected_len = (m_val + 7) // 8
+        if len(raw_bits) != expected_len:
+            raise ValueError(
+                f"bloom bit-array length {len(raw_bits)} does not match m={m_val} "
+                f"(expected {expected_len} bytes)"
+            )
+        bits = bytearray(raw_bits)
+        return cls(bits=bits, m=m_val, k=k_val)
+
+
+@dataclass
+class AESession:
+    """Anti-entropy session state.
+
+    Resumability: ``session_id`` lets a peer continue a partial
+    exchange after a transient connection drop. ``cursor`` tracks how
+    many of THIS peer's diff items have already been emitted; the
+    multi-round driver in ``reconcile()`` uses it as the paging token.
+    ``round`` is the protocol round number (1=Bloom, 2=ack-set,
+    3=root-cosign). ``acked`` accumulates kids acknowledged by the
+    remote across rounds. ``finished`` flips True after round 3
+    converges. TTL (15 min default) bounds how long state is kept.
+    """
+
+    session_id: bytes = field(default_factory=lambda: secrets.token_bytes(16))
+    cursor: int = 0
+    round: int = 1
+    acked: set[bytes] = field(default_factory=set)
+    finished: bool = False
+    ttl_ms: int = 15 * 60 * 1000
+
+    def advance_cursor(self, n: int) -> None:
+        self.cursor += n
+
+
+def build_filter(items: Iterable[bytes], *, p: float = 0.01) -> BloomFilter:
+    items_list = list(items)
+    bf = BloomFilter.empty(len(items_list), p=p)
+    for it in items_list:
+        bf.add(it)
+    return bf
+
+
+def diff(local: Iterable[bytes], remote_filter: BloomFilter) -> list[bytes]:
+    """Return items local has that remote_filter probably doesn't.
+
+    False-positives may keep some items the peer already has; the peer
+    will dedup on receive (kids are content-addressed).
+    """
+    return [item for item in local if item not in remote_filter]
+
+
+# ---------------------------------------------------------------------------
+# IBLT (Invertible Bloom Lookup Table) — symmetric-difference fallback.
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class _IBLTCell:
+    count: int = 0
+    key_xor: bytes = b"\x00" * 32
+    hash_xor: int = 0  # SHA-256(key)[:8] folded as int
+
+    def add(self, key: bytes, *, sign: int = 1) -> None:
+        self.count += sign
+        self.key_xor = _xor_bytes(self.key_xor, key)
+        self.hash_xor ^= _key_hash(key)
+
+    def is_pure(self) -> bool:
+        """A cell with |count|==1 whose stored key hashes to its
+        recorded hash_xor is "pure" — the one key XOR-ed in is
+        recoverable directly."""
+        if abs(self.count) != 1:
+            return False
+        return _key_hash(self.key_xor) == self.hash_xor
+
+    def is_empty(self) -> bool:
+        return self.count == 0 and self.key_xor == b"\x00" * 32 and self.hash_xor == 0
+
+
+def _xor_bytes(a: bytes, b: bytes) -> bytes:
+    if len(a) != len(b):
+        raise ValueError("xor length mismatch")
+    return bytes(x ^ y for x, y in zip(a, b, strict=False))
+
+
+def _key_hash(key: bytes) -> int:
+    """Stable 64-bit hash of a key for IBLT pure-cell verification."""
+    return int.from_bytes(hashlib.sha256(b"iblt-keyhash" + key).digest()[:8], "big")
+
+
+# DoS gate: a malicious peer cannot force an OOM by advertising a huge
+# cell count.  8 KiB cells × 32 bytes/cell = 256 KiB maximum IBLT payload,
+# well above the largest legitimate δ (IBLT_THRESHOLD × 4 = 1024 cells).
+IBLT_MAX_CELLS = 8192
+
+
+class IBLTOversizeError(ValueError):
+    """Raised by IBLT.from_cbor() when the cell count exceeds IBLT_MAX_CELLS."""
+
+
+@dataclass
+class IBLT:
+    """Invertible Bloom Lookup Table for set reconciliation.
+
+    Encodes a set of fixed-width keys (32 bytes; we use kids).
+    Symmetric difference of two IBLTs is just per-cell XOR; ``decode()``
+    extracts the differing keys via repeated peeling of pure cells.
+
+    Parameters chosen for δ ≈ 256: 64 cells × 4 hashes ≈ 4× overhead
+    above δ, comfortable peeling success at ~98%+. For larger δ, scale
+    cells linearly.
+    """
+
+    cells: list[_IBLTCell]
+    k: int  # number of hash functions
+
+    @classmethod
+    def empty(cls, *, cells: int = IBLT_DEFAULT_CELLS, k: int = IBLT_DEFAULT_HASHES) -> IBLT:
+        if cells < k * 2:
+            raise ValueError("IBLT needs cells >= 2*k for peeling to converge")
+        return cls(cells=[_IBLTCell() for _ in range(cells)], k=k)
+
+    def _indices(self, key: bytes) -> list[int]:
+        digest = hashlib.sha256(b"iblt-idx" + key).digest()
+        h1 = int.from_bytes(digest[:8], "big")
+        h2 = int.from_bytes(digest[8:16], "big") | 1
+        n = len(self.cells)
+        # Use distinct indices to avoid wasting hashes — collisions
+        # within one key destroy peeling.
+        out: list[int] = []
+        seen: set[int] = set()
+        i = 0
+        while len(out) < self.k and i < self.k * 8:
+            idx = (h1 + i * h2) % n
+            if idx not in seen:
+                seen.add(idx)
+                out.append(idx)
+            i += 1
+        # Fall through if degenerate: pad with whatever we have (should
+        # not happen for n >= 2*k, but guards a malicious tiny IBLT).
+        while len(out) < self.k:
+            out.append(out[-1])
+        return out
+
+    def add(self, key: bytes) -> None:
+        if len(key) != 32:
+            raise ValueError("IBLT keys must be 32 bytes")
+        for idx in self._indices(key):
+            self.cells[idx].add(key, sign=1)
+
+    def subtract(self, other: IBLT) -> IBLT:
+        """Return ``self - other``: a new IBLT whose pure cells expose
+        keys present in exactly one of the two operands."""
+        if len(self.cells) != len(other.cells) or self.k != other.k:
+            raise ValueError("IBLT shape mismatch")
+        result = IBLT.empty(cells=len(self.cells), k=self.k)
+        for i, (a, b) in enumerate(zip(self.cells, other.cells, strict=False)):
+            result.cells[i].count = a.count - b.count
+            result.cells[i].key_xor = _xor_bytes(a.key_xor, b.key_xor)
+            result.cells[i].hash_xor = a.hash_xor ^ b.hash_xor
+        return result
+
+    def decode(self) -> tuple[set[bytes], set[bytes], bool]:
+        """Peel until stable. Returns (only_in_self, only_in_other, ok).
+
+        ``only_in_self`` — keys with positive count after peeling.
+        ``only_in_other`` — keys with negative count after peeling.
+        ``ok`` — True iff every cell is empty after peeling, i.e.
+        the symmetric difference was within the IBLT's capacity.
+        """
+        a_only: set[bytes] = set()
+        b_only: set[bytes] = set()
+        # Work on copies so the caller can retry / inspect.
+        cells = [_IBLTCell(c.count, c.key_xor, c.hash_xor) for c in self.cells]
+        progressed = True
+        while progressed:
+            progressed = False
+            for _i, cell in enumerate(cells):
+                if cell.is_pure():
+                    key = cell.key_xor
+                    sign = cell.count  # ±1
+                    if sign == 1:
+                        a_only.add(key)
+                    else:
+                        b_only.add(key)
+                    # Remove this key from all of its cells.
+                    for idx in self._indices(key):
+                        cells[idx].count -= sign
+                        cells[idx].key_xor = _xor_bytes(cells[idx].key_xor, key)
+                        cells[idx].hash_xor ^= _key_hash(key)
+                    progressed = True
+        ok = all(c.is_empty() for c in cells)
+        return a_only, b_only, ok
+
+    def to_cbor(self) -> bytes:
+        """Serialise to canonical CBOR for wire transport."""
+        import cbor2
+
+        return cbor2.dumps(
+            {
+                "m": len(self.cells),
+                "k": self.k,
+                "cells": [
+                    {"count": c.count, "key_xor": c.key_xor, "hash_xor": c.hash_xor}
+                    for c in self.cells
+                ],
+            },
+            canonical=True,
+        )
+
+    @classmethod
+    def from_cbor(cls, data: bytes) -> IBLT:
+        """Deserialise from CBOR.
+
+        DoS gate: rejects ``m`` (cell count) above ``IBLT_MAX_CELLS`` and
+        ``k`` above ``BloomFilter.MAX_K`` before allocating any cell list,
+        closing the OOM amplification attack class.
+        """
+        import cbor2
+
+        try:
+            obj = cbor2.loads(data)
+        except Exception as e:
+            raise ValueError(f"IBLT CBOR decode failed: {e}") from e
+        if not isinstance(obj, dict):
+            raise ValueError("IBLT payload must be a CBOR map")
+        m = int(obj["m"])
+        k = int(obj["k"])
+        if m > IBLT_MAX_CELLS:
+            raise IBLTOversizeError(f"IBLT cell count {m} exceeds maximum {IBLT_MAX_CELLS}")
+        if m <= 0 or k <= 0:
+            raise ValueError("IBLT m and k must be positive")
+        if k > BloomFilter.MAX_K:
+            raise ValueError(f"IBLT k too large: {k} > {BloomFilter.MAX_K}")
+        raw_cells = obj.get("cells", [])
+        if not isinstance(raw_cells, list) or len(raw_cells) != m:
+            raise ValueError(
+                f"IBLT cells length {len(raw_cells) if isinstance(raw_cells, list) else '?'} "
+                f"does not match m={m}"
+            )
+        cells = []
+        for cell_dict in raw_cells:
+            kx = bytes(cell_dict["key_xor"])
+            if len(kx) != 32:
+                raise ValueError(f"IBLT cell key_xor must be 32 bytes, got {len(kx)}")
+            cells.append(
+                _IBLTCell(
+                    count=int(cell_dict["count"]),
+                    key_xor=kx,
+                    hash_xor=int(cell_dict["hash_xor"]),
+                )
+            )
+        return cls(cells=cells, k=k)
+
+
+def build_iblt(
+    items: Iterable[bytes],
+    *,
+    cells: int = IBLT_DEFAULT_CELLS,
+    k: int = IBLT_DEFAULT_HASHES,
+) -> IBLT:
+    table = IBLT.empty(cells=cells, k=k)
+    for it in items:
+        table.add(it)
+    return table
+
+
+def iblt_diff(local: Iterable[bytes], remote: IBLT) -> tuple[set[bytes], set[bytes], bool]:
+    """Compute symmetric difference via IBLT subtraction + peeling.
+
+    ``cells`` and ``k`` of the two tables must match. Returns
+    ``(local_only, remote_only, ok)``; ``ok=False`` means the IBLT was
+    undersized and the caller should fall back to a larger sketch.
+    """
+    local_table = build_iblt(local, cells=len(remote.cells), k=remote.k)
+    return local_table.subtract(remote).decode()
+
+
+# ---------------------------------------------------------------------------
+# Multi-round reconciliation driver.
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class ReconciliationResult:
+    """Outcome of a multi-round reconcile() exchange."""
+
+    rounds: int
+    local_only: set[bytes]
+    remote_only: set[bytes]
+    used_iblt: bool
+    converged: bool  # True iff round-3 root agreement holds.
+
+
+def _set_root(items: Iterable[bytes]) -> bytes:
+    """Stable Merkle-ish root of a set of fixed-width keys.
+
+    Used for the round-3 cosigned-root agreement check. We sort then
+    SHA-256 over the concatenation; that's enough for "did we agree
+    on the same set?" and avoids depending on the full transparency
+    tree at this layer.
+    """
+    return hashlib.sha256(b"".join(sorted(items))).digest()
+
+
+def reconcile(
+    local: Iterable[bytes],
+    remote: Iterable[bytes],
+    *,
+    chunk_size: int = AE_CHUNK_SIZE,
+    iblt_threshold: int = IBLT_THRESHOLD,
+    max_rounds: int = 8,
+) -> ReconciliationResult:
+    """Run the full multi-round protocol against an in-process peer.
+
+    This driver is the byte-exact reference for the wire-side dance —
+    a real peer pair would split the body into round-by-round frames
+    over the gossip bus. Tests pin convergence at byte-equal sets
+    after a bounded number of rounds for δ ∈ {10, 100, 1000, 5000}.
+    """
+    local_set = set(local)
+    remote_set = set(remote)
+
+    rounds = 0
+    used_iblt = False
+
+    # ---- Round 1: Bloom diff (chunked) ----
+    rounds += 1
+    bloom_remote = build_filter(remote_set)
+    bloom_local = build_filter(local_set)
+
+    bloom_local_only = [k for k in local_set if k not in bloom_remote]
+    bloom_remote_only = [k for k in remote_set if k not in bloom_local]
+
+    estimated_diff = len(bloom_local_only) + len(bloom_remote_only)
+
+    if estimated_diff > iblt_threshold:
+        # Fallback to IBLT sized for the observed delta.
+        used_iblt = True
+        cells = max(IBLT_DEFAULT_CELLS, estimated_diff * 4)
+        iblt_remote = build_iblt(remote_set, cells=cells)
+        only_local, only_remote, ok = iblt_diff(local_set, iblt_remote)
+        if ok:
+            # IBLT decoded fully — we have an exact symmetric diff.
+            return ReconciliationResult(
+                rounds=rounds,
+                local_only=only_local,
+                remote_only=only_remote,
+                used_iblt=True,
+                converged=_set_root(local_set | only_remote) == _set_root(remote_set | only_local),
+            )
+        # Decoding failed — fall through to chunked Bloom rounds.
+
+    # Chunked round-1 transfer (cursor paging).
+    sess_local = AESession()
+    sess_remote = AESession()
+    transferred_l: list[bytes] = []
+    transferred_r: list[bytes] = []
+    while sess_local.cursor < len(bloom_local_only) or sess_remote.cursor < len(bloom_remote_only):
+        if rounds >= max_rounds:
+            break
+        chunk_l = bloom_local_only[sess_local.cursor : sess_local.cursor + chunk_size]
+        chunk_r = bloom_remote_only[sess_remote.cursor : sess_remote.cursor + chunk_size]
+        transferred_l.extend(chunk_l)
+        transferred_r.extend(chunk_r)
+        sess_local.advance_cursor(len(chunk_l))
+        sess_remote.advance_cursor(len(chunk_r))
+        rounds += 1
+
+    # ---- Round 2: explicit ack-set (false-positive close-out) ----
+    rounds += 1
+    # After round 1, each side has "received" the chunked transfers.
+    # In a real wire, each peer would reply with the kids it knows
+    # (i.e. sent back to confirm). Here we model that by computing
+    # the kids that are in the OTHER peer's set but were missed in
+    # round 1 due to Bloom false-positives.
+    fp_local = (remote_set - local_set) - set(transferred_r)
+    fp_remote = (local_set - remote_set) - set(transferred_l)
+    transferred_r_set = set(transferred_r) | fp_local
+    transferred_l_set = set(transferred_l) | fp_remote
+
+    # ---- Round 3: cosigned (root, tree_size) agreement ----
+    rounds += 1
+    merged_local = local_set | transferred_r_set
+    merged_remote = remote_set | transferred_l_set
+    converged = _set_root(merged_local) == _set_root(merged_remote) and len(merged_local) == len(
+        merged_remote
+    )
+
+    return ReconciliationResult(
+        rounds=rounds,
+        local_only=transferred_l_set,
+        remote_only=transferred_r_set,
+        used_iblt=used_iblt,
+        converged=converged,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Resumable RANGE_REQ / RANGE_RESP wire handler.
+# ---------------------------------------------------------------------------
+
+# Bound on the number of sessions a single peer remembers cursors for.
+# OrderedDict eviction keeps memory bounded if a malicious peer churns
+# session_ids — closes the resumability gap from spec/06-conformance.md.
+SESSION_TABLE_MAX = 256
+
+
+def new_session_id() -> str:
+    """Fresh random session id (32-hex / 16 bytes)."""
+    return uuid.uuid4().hex
+
+
+def make_range_req(items_total: int, *, session_id: str | None = None) -> dict:
+    """Build a RANGE_REQ message dict.
+
+    ``session_id`` may be reused across reconnects to resume from the
+    last cursor the responder recorded for that id.
+    """
+    return {
+        "kind": "RANGE_REQ",
+        "session_id": session_id or new_session_id(),
+        "items_total": int(items_total),
+    }
+
+
+@dataclass
+class RangeResponder:
+    """Per-peer state for resumable RANGE_REQ/RANGE_RESP exchanges.
+
+    The responder stores a bounded ``session_id -> cursor`` table. When
+    a RANGE_REQ arrives with a known session_id the next chunk starts
+    at the saved cursor; unknown ids start at 0. The table is evicted
+    in oldest-first order at ``SESSION_TABLE_MAX`` entries.
+    """
+
+    chunk_size: int = AE_CHUNK_SIZE
+    _sessions: OrderedDict[str, int] = field(default_factory=OrderedDict)
+
+    def _get_cursor(self, session_id: str) -> int:
+        if session_id in self._sessions:
+            self._sessions.move_to_end(session_id)
+            return self._sessions[session_id]
+        return 0
+
+    def _set_cursor(self, session_id: str, cursor: int) -> None:
+        if session_id in self._sessions:
+            self._sessions.move_to_end(session_id)
+            self._sessions[session_id] = cursor
+            return
+        if len(self._sessions) >= SESSION_TABLE_MAX:
+            self._sessions.popitem(last=False)
+        self._sessions[session_id] = cursor
+
+    def handle_range_req(self, req: dict, items: list[bytes]) -> dict:
+        """Produce a RANGE_RESP for ``req`` over the responder's
+        ``items`` list. Cursor is stored against the request's
+        ``session_id`` so a follow-up RANGE_REQ with the same id
+        resumes at the next slice."""
+        if req.get("kind") != "RANGE_REQ":
+            raise ValueError("expected RANGE_REQ message")
+        sid = req.get("session_id")
+        if not isinstance(sid, str) or not sid:
+            raise ValueError("RANGE_REQ.session_id must be a non-empty string")
+        start = self._get_cursor(sid)
+        end = min(start + self.chunk_size, len(items))
+        chunk = items[start:end]
+        self._set_cursor(sid, end)
+        return {
+            "kind": "RANGE_RESP",
+            "session_id": sid,
+            "cursor": end,
+            "chunk": list(chunk),
+            "done": end >= len(items),
+        }
+
+    def session_count(self) -> int:
+        return len(self._sessions)
+
+
+# ---------------------------------------------------------------------------
+# Self-healing AE background coroutine (P2-02, spec §10.2).
+# ---------------------------------------------------------------------------
+
+#: Default interval between AE rounds when all peers are healthy.
+AE_LOOP_INTERVAL_S: float = 30.0
+#: Maximum backoff cap for reconnect attempts.
+AE_MAX_BACKOFF_S: float = 300.0
+#: Multiplier applied on each failed reconnect.
+AE_BACKOFF_FACTOR: float = 2.0
+
+
+class BackgroundAELoop:
+    """Runs anti-entropy reconciliation on a repeating schedule.
+
+    On each tick it calls the provided ``run_ae`` coroutine which should
+    compare local state against all known peers and push/pull any missing
+    items.  When ``run_ae`` raises an exception the loop applies
+    exponential back-off up to ``max_backoff_s``, then retries.  On
+    success the interval resets to ``interval_s``.
+
+    Usage::
+
+        loop = BackgroundAELoop(run_ae=my_ae_coroutine)
+        loop.start()
+        ...
+        await loop.stop()
+    """
+
+    def __init__(
+        self,
+        *,
+        run_ae: Callable[[], Awaitable[None]],
+        interval_s: float = AE_LOOP_INTERVAL_S,
+        max_backoff_s: float = AE_MAX_BACKOFF_S,
+        backoff_factor: float = AE_BACKOFF_FACTOR,
+    ) -> None:
+        self._run_ae = run_ae
+        self._interval_s = interval_s
+        self._max_backoff_s = max_backoff_s
+        self._backoff_factor = backoff_factor
+        self._task: asyncio.Task | None = None
+        self._stop_event: asyncio.Event | None = None
+        self._consecutive_failures: int = 0
+
+    def start(self) -> None:
+        """Spawn the background task.  Must be called from an async context."""
+        if self._task is not None and not self._task.done():
+            return
+        self._stop_event = asyncio.Event()
+        self._task = asyncio.create_task(self._loop(), name="ae-background-loop")
+
+    async def stop(self) -> None:
+        """Cancel and await the background task."""
+        if self._stop_event is not None:
+            self._stop_event.set()
+        if self._task is not None:
+            self._task.cancel()
+            try:
+                await self._task
+            except (asyncio.CancelledError, Exception):
+                pass
+        self._task = None
+
+    @property
+    def running(self) -> bool:
+        return self._task is not None and not self._task.done()
+
+    async def _loop(self) -> None:
+        assert self._stop_event is not None
+        current_delay = self._interval_s
+        while not self._stop_event.is_set():
+            try:
+                await asyncio.wait_for(self._stop_event.wait(), timeout=current_delay)
+                # Stop event was set — exit cleanly.
+                return
+            except TimeoutError:
+                pass
+            # Run one AE round.
+            try:
+                await self._run_ae()
+                self._consecutive_failures = 0
+                current_delay = self._interval_s
+            except asyncio.CancelledError:
+                return
+            except Exception as exc:
+                self._consecutive_failures += 1
+                current_delay = min(
+                    self._max_backoff_s,
+                    self._interval_s * (self._backoff_factor**self._consecutive_failures),
+                )
+                logger.warning(
+                    "BackgroundAELoop: AE round failed (failure #%d), backing off %.1fs: %s",
+                    self._consecutive_failures,
+                    current_delay,
+                    exc,
+                )