fleetmesh-node 0.2.0__py3-none-any.whl

node/planes/store/sync/core.py

@@ -0,0 +1,311 @@
+"""The SyncPeer compartment: construction, the metered send/receive every
+role shares, the session table, and the compartment contract. The roles are
+in initiator.py / responder.py; the handover in handover.py."""
+from __future__ import annotations
+
+import threading
+from typing import Any, Callable, Mapping, Optional
+
+from fleetmesh_core.storage import sync
+from node.channels import bus as eventbus_mod
+from fleetmesh_core.identity.did import did_nostr
+from node.compartment import stores as st
+from node.channels import Channel, INFO
+from node.metrics import COUNTER, Metric
+from node.compartment import Command, Handler, Need, Outbound, WireProtocol
+from .envelope import KEEP, RECENT, SESSIONS_CEILING, SESSION_IDLE, SyncError
+from .handover import HandoverMixin
+from .initiator import InitiatorMixin
+from .responder import ResponderMixin
+from .shape import SyncSessionResult
+from .types import SyncStateReport
+
+
+from .types import SyncTotals
+
+
+def _log(key: str, what: str, bound: int) -> st.Store:
+    return st.Store(key=key, description='%s, newest last' % what, kind=st.RING, shape=tuple, bound=bound)
+
+
+import dataclasses
+from node.config.parameter import Parameter, ParamBlock
+
+
+@dataclasses.dataclass(frozen=True)
+class SyncConfig(ParamBlock):
+    key = 'sync'
+    name = 'sync'
+    DERIVED: bool = False
+    PARAMS = (
+        Parameter(
+            key='sessions_ceiling', name='Sessions Ceiling',
+            description='Maximum concurrent open sync sessions answered',
+            type=int, default=SESSIONS_CEILING, minimum=1, tags=('sync', 'concurrency'),
+        ),
+        Parameter(
+            key='recent_kept', name='Recent Kept',
+            description='Number of recent completed or refused sessions kept',
+            type=int, default=RECENT, minimum=10, tags=('sync', 'history'),
+        ),
+    )
+    sessions_ceiling: int = SESSIONS_CEILING
+    recent_kept: int = RECENT
+
+
+class SyncPeer(InitiatorMixin, ResponderMixin, HandoverMixin, WireProtocol):
+    """Both roles of sync/1.0 for one node, as a compartment.
+
+    Its refusals are the sessions it would not open - a grant that does not
+    cover one, a peer with too many already, a node over its own ceiling - and
+    the deliveries it would not store. Those lived in three separate lists that
+    only the CLI knew how to read.
+    """
+
+    name: str = 'sync'
+    requires: tuple[str, ...] = ('budget', 'strain')
+    piuris: tuple[str, ...] = (sync.PIURI,)
+    reads: tuple[str, ...] = ('handover', 'coverage')
+    config_cls = SyncConfig
+    live: tuple[str, ...] = ('handover',)
+    CHANNELS = (
+        Channel('handed', 'Handover signed and offered to peer', level=INFO,
+                say='handed {shard} to {peer} ({events} events)'),
+        Channel('held', 'Handover accepted from peer', level=INFO,
+                say='held {shard} handed over by {peer} ({events} events, {chain_len} in the chain)'),
+    )
+    METRICS = (
+        Metric(name='sync_handovers_offered', description='Times shard coverage was handed over.',
+               kind=COUNTER, counts='handed', tags=('sync', 'handover')),
+        Metric(name='sync_handovers_accepted', description='Times shard coverage was accepted.',
+               kind=COUNTER, counts='held', tags=('sync', 'handover')),
+    )
+    HANDLERS = (
+        Handler('open', run='handle_open', doc='answer inbound sync/1.0 open session request'),
+        Handler('handover', run='handle_handover', doc='accept inbound sync/1.0 handover envelope'),
+        Handler('range', run='handle_range', doc='answer inbound sync/1.0 range request'),
+        Handler('need', run='handle_need', doc='answer inbound sync/1.0 need request'),
+        Handler('done', run='handle_done', doc='answer inbound sync/1.0 done session finish'),
+    )
+    OUTBOUND = (
+        Outbound('open', reply=('range', 'deliver', 'done'), doc='open sync session against peer'),
+        Outbound('range', reply=('range', 'have', 'need', 'deliver', 'done'), doc='query or answer negentropy range'),
+        Outbound('have', reply=('need', 'done'), doc='answer negentropy have with event ids'),
+        Outbound('need', reply=('deliver', 'done'), doc='request event delivery for missing ids'),
+        Outbound('deliver', reply=('range', 'need', 'done'), doc='deliver events in signed envelope attachment'),
+        Outbound('done', doc='conclude sync session with final status'),
+        Outbound('handover', doc='hand over verified shard coverage chain'),
+    )
+    COMMANDS = (
+        Command(
+            key='show',
+            name='Sync',
+            description='sessions open, refused and aborted, and what could not be sent',
+            tags=('sync', 'status'),
+            permission='sync/show',
+            run='state',
+            reply='{open, refused, aborted, completed, unsent, malformed, handed, held}',
+        ),
+    )
+
+    # What it keeps. aborted, refused and completed were plain lists appended
+    # once per session and never trimmed - memory that grew with every peer's
+    # sessions for the life of the process - while nothing read more than
+    # their length and the newest few. Now a ring of the recent ones, with the
+    # length kept in `totals` so `state` reports the same numbers. The four
+    # that were already cut to 200 by hand (`del x[:-200]`) keep that bound
+    # as a declaration instead, and `state` still reports their length.
+    STORES = (
+        st.Store(key='totals', description='sessions since the process started, by how they ended',
+                 kind=st.COUNTERS, shape=SyncTotals),
+        _log('aborted', 'sessions a peer stopped answering, as (peer, thid)', RECENT),
+        _log('refused', 'sessions this node would not open, as (peer, thid, why)', RECENT),
+        _log('completed', 'sessions that ended, as (peer, role, result)', RECENT),
+        _log('rejected', 'messages without the shape of their type, as (peer, kind, why)', KEEP),
+        _log('unsent', 'messages no relay would take, as (peer, type, why, at)', KEEP),
+        _log('handed', 'handovers this node signed and sent, as (peer, shard, fingerprint, at)', KEEP),
+        _log('held', 'handovers this node accepted, as (peer, shard, at)', KEEP),
+        # REFUSE: an open session dropped for room is a peer left waiting on a
+        # done that never comes. A full table refuses the open instead, with
+        # a done and a reason, like the per-peer cap in front of it.
+        st.Store(key='sessions', description='sessions this node is answering, with when each last moved',
+                 kind=st.TABLE, shape=tuple, bound=SESSIONS_CEILING, evict=st.REFUSE, keyed_by='thid'),
+        # OLDEST, which the hand trim this replaces meant to be and was not: it
+        # cut a set to an arbitrary 2048 of its members, not the newest. A thid
+        # forgotten here only charges a late message on a long-finished
+        # session to the wrong side.
+        st.Store(key='_initiated', description='sessions this node opened, by thid', kind=st.SET,
+                 shape=str, bound=4096, keyed_by='thid'),
+        # OLDEST is the harmless direction: a memo forgotten is a handover
+        # offered twice, and the receiver fingerprints the second copy as the
+        # first (see HandoverMixin._unoffer).
+        st.Store(key='_offered', description='the fingerprint last handed over, per peer and shard',
+                 kind=st.TABLE, shape=str, bound=4096, keyed_by='peer and shard'),
+    )
+
+    NEEDS = (
+        Need('keystore', 'ks'),
+        Need('my_store', 'stores.events'),
+        Need('transport'),
+        Need('grant_store', 'stores.grants'),
+        Need('budget'),
+        Need('session_cls', 'cfg', field='_session_cls'),
+        Need('strain'),
+        Need('accounts', 'stores.accounts'),
+        Need('events'),
+        Need('discovery', late=True),
+        Need('presence', late=True),
+        Need('reporter', late=True),
+    )
+
+    def bind(self, discovery: Any = None, presence: Any = None) -> None:
+        """The two soft dependencies that tick after sync: discovery holds
+        the coverage claims a handover rests on, presence says whether a
+        peer that went quiet had announced it was leaving. Both are built
+        after sync, so they are bound once all three exist; either may stay
+        None (a bare SyncPeer in a suite), and the handover and partition
+        paths check for that."""
+        self.discovery = discovery
+        self.presence = presence
+
+    def start(self) -> None:
+        if self.reporter is not None and self.reporter.on_session not in self.on_session:
+            self.on_session.append(self.reporter.on_session)
+        if self.events is not None:
+            events = self.events
+            self.on_session.append(lambda *a: events.emit(eventbus_mod.TOPIC_SESSION_COMPLETED, *a))
+
+    def __init__(
+        self,
+        keystore: Any,
+        my_store: Any,
+        transport: Any,
+        grant_store: Any,
+        clock: Optional[Callable[[], int]] = None,
+        budget: Optional[Any] = None,
+        session_cls: Optional[Any] = None,
+        strain: Optional[Any] = None,
+        node: Optional[Any] = None,
+        accounts: Optional[Any] = None,
+        events: Optional[Any] = None,
+        discovery: Optional[Any] = None,
+        presence: Optional[Any] = None,
+        reporter: Optional[Any] = None,
+    ) -> None:
+        super().__init__(node, clock)
+        self.strain: Optional[Any] = strain            # what this node has survived, or None
+        self.accounts: Optional[Any] = accounts        # where handover chains are kept, or None
+        self.discovery = discovery                     # bound after assembly (ticks after sync)
+        self.presence = presence                       # bound after assembly (ticks after sync)
+        self.reporter = reporter
+        self.events = events
+        self.session_cls: Any = session_cls or sync.Session   # the bad peer plugs in here
+        self.ks: Any = keystore
+        self.store: Any = my_store
+        self.transport: Any = transport
+        self.grants: Any = grant_store
+        self.budget: Optional[Any] = budget            # meters every message both ways, if present
+        self.on_session: list[Callable[[str, str, SyncSessionResult], Any]] = []            # callbacks (peer, role, result) when a session ends
+        # The stores were built and bound by the base (see STORES).
+        self.totals: st.Counters
+        self.aborted: st.Ring[tuple[str, str]]                 # (peer, thid) sessions a peer stopped answering
+        self.refused: st.Ring[tuple[str, Optional[str], str]]  # (peer, thid, why) sessions the grant did not cover
+        self.sessions: st.Table[str, tuple[str, Any, int]]     # thid -> (peer_pk, Session, last activity)
+        # The sessions this node opened, by thid. A message is charged to the
+        # session's initiator on both sides, and this is how the wire knows
+        # which side that was. Kept after the session ends: a late message on
+        # a finished thid is still that session's, and still the initiator's.
+        self._initiated: st.Members[str]
+        self.rejected: st.Ring[tuple[str, str, str]]           # (peer, kind, why) messages that did not have the shape of their type
+        # (peer, type, why, at). The time is what stops one message lost in a
+        # restart warning for the rest of the process: this peering completed
+        # thirty-four sessions while still reporting it.
+        self.unsent: st.Ring[tuple[str, str, str, int]]        # (peer, type, why, at)
+        self.completed: st.Ring[tuple[str, str, SyncSessionResult]]   # (peer_pk, role, counters, findings, status)
+        self.handed: st.Ring[tuple[str, str, str, int]]        # (peer_pk, shard, fingerprint, at) handovers this node signed and sent
+        self.held: st.Ring[tuple[str, str, int]]               # (peer_pk, shard, at) handovers this node accepted
+        self._offered: st.Table[tuple[str, str], str]          # (peer_pk, shard) -> fingerprint last handed, so equality twice is one offer
+        self._lock: threading.Lock = threading.Lock()
+
+    def _record(self, ring: 'st.Ring[Any]', entry: tuple[Any, ...]) -> None:
+        """One session's end: kept in its ring, and counted in `totals`."""
+        ring.append(entry)
+        self.totals.incr(ring.store.key)
+
+    def _grant(self, peer_pk: str, direction: str) -> Any:
+        row = self.grants.get(peer_pk, direction)
+        if row is None or row['status'] != 'live':
+            raise SyncError('no live %s grant with %s' % (direction, peer_pk[:16]))
+        return sync.Grant.from_payload(row['payload'], row['envelope_id'])
+
+    def _send(self, m: Mapping[str, Any], peer_pk: str) -> dict[str, tuple[bool, str]]:
+        msg_dict = dict(m)
+        msg_dict['from'] = self.ks.did
+        msg_dict['to'] = [did_nostr(peer_pk)]
+        try:
+            res: dict[str, tuple[bool, str]] = self.wire_client.send(peer_pk, msg_dict)
+        except Exception as e:
+            # Nothing left the node: not metered, and said where an operator
+            # can see it rather than left as the peer's timeout.
+            self.unsent.append((peer_pk, msg_dict.get('type', '').rsplit('/', 1)[-1],
+                                '%s: %s' % (type(e).__name__, e), self.clock()))
+            return {}
+        # Meter what a relay took, not what was attempted. A deliver every
+        # relay refused never reached the peer; counting it claims traffic the
+        # peer will measure as absent, and the window prices that gap as a
+        # misreport against the sender. Three machines showed it: 111 events
+        # claimed, 37 arrived, S2 against an honest node.
+        if res and not any(ok for ok, _ in res.values()):
+            why = '; '.join(n for _, n in res.values() if n)[:160] or 'refused'
+            self.unsent.append((peer_pk, msg_dict.get('type', '').rsplit('/', 1)[-1], why, self.clock()))
+            return res
+        if self.budget is not None:
+            self.budget.observe_out(peer_pk, msg_dict, i_initiated=msg_dict.get('thid') in self._initiated)
+        return res
+
+    def _seen(self, peer_pk: str, m: Mapping[str, Any]) -> None:
+        if self.budget is not None:
+            self.budget.observe_in(peer_pk, m, i_initiated=m.get('thid') in self._initiated)
+
+
+    # ---- the compartment contract -------------------------------------------------
+    def state(self) -> SyncStateReport:
+        return {
+            'open': len(self.sessions),
+            'refused': self.totals['refused'],
+            'aborted': self.totals['aborted'],
+            'completed': self.totals['completed'],
+            'unsent': len(self.unsent),
+            'malformed': len(self.rejected),
+            'handed': [{'peer': p, 'shard': sh, 'at': at} for p, sh, _, at in self.handed.last(20)],
+            'held': [{'peer': p, 'shard': sh, 'at': at} for p, sh, at in self.held.last(20)],
+        }
+
+    def trouble(self) -> list[str]:
+        out: list[str] = []
+        recent = self.refusals.within(self.TROUBLE_WINDOW, self.clock())
+        if recent:
+            out.append('%d sync session(s) refused: %s'
+                       % (len(recent), '; '.join(sorted({r.why for r in recent}))[:120]))
+        still = [u for u in self.unsent if self.clock() - u[-1] <= self.TROUBLE_WINDOW]
+        if still:
+            # A relay that refused a message says nothing to the peer, which
+            # waits on an envelope that never finishes. Only the sender knows.
+            out.append('%d message(s) no relay would take - the peer is waiting on an '
+                       'envelope that will never finish' % len(still))
+        return out
+
+
+    def _expire(self, now: int) -> list[str]:
+        """Forget sessions whose peer went quiet. No finding: quiet is not lying."""
+        with self._lock:
+            stale = [t for t, v in self.sessions.items() if now - v[2] > SESSION_IDLE]
+            for t in stale:
+                self.sessions.pop(t, None)
+        return stale
+
+    def _finish(self, thid: str, peer_pk: str, sess: Any, status: Any) -> None:
+        with self._lock:
+            self.sessions.pop(thid, None)
+            self._record(self.completed, (peer_pk, 'responder', {'status': status, 'counters': dict(sess.counters),
+                                                                  'findings': list(sess.findings), 'thid': thid}))

node/planes/store/sync/envelope.py

@@ -0,0 +1,106 @@
+"""The wire vocabulary of sync/1.0 as this node carries it: the PIURI, the
+session bounds, the delivery-envelope attachment it sends and the
+verification it demands of one it receives."""
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping, Sequence
+
+from fleetmesh_core.storage import replication, sync
+from node.interfaces import (
+    DIDCommAttachment,
+    DIDCommMessage,
+    DeliveryEnvelope,
+    NostrEvent,
+)
+
+
+PIURI: str = sync.PIURI
+ATTACHMENT: str = sync.ENVELOPE_ATTACHMENT
+# A peer could open sessions and never finish them, one per thread id, and
+# the responder kept every one. Four at once is the old conn_max advisory
+# ceiling for probation and member; an idle session is forgotten after the
+# receipt cadence has passed twice.
+MAX_SESSIONS_PER_PEER: int = 4
+SESSION_IDLE: int = 1200
+# The cap above is per peer; this one is the node's. Many peers each within
+# their four can still add up, and past this the open is refused with a done
+# rather than one of the sessions already answering dropped.
+SESSIONS_CEILING: int = 1024
+# The history SyncPeer keeps: RECENT sessions of each ending (the counts are
+# in `totals`), and KEEP of each message log, the bound they were trimmed to
+# by hand before they were declared.
+RECENT: int = 256
+KEEP: int = 200
+
+
+class SyncError(Exception):
+    pass
+
+
+def _thid(shard: str) -> str:
+    return 'sync-%s-%s' % (shard, os.urandom(4).hex())
+
+
+def envelope_attachment(
+    events: Sequence[NostrEvent],
+    node_did: str,
+    sk: str,
+    now: int,
+) -> DIDCommAttachment:
+    """One delivery envelope per event, each with this node's hop appended."""
+    return {
+        'id': ATTACHMENT,
+        'mime_type': 'application/json',
+        'data': {
+            'json': {
+                'envelopes': [
+                    replication.append_hop({'event': ev, 'path': []}, node_did, sk, now)
+                    for ev in events
+                ]
+            }
+        },
+    }
+
+
+def verified_events(
+    message: DIDCommMessage | Mapping[str, Any],
+    findings: list[Any],
+) -> tuple[list[NostrEvent], dict[str, DeliveryEnvelope]]:
+    """(events, {event id: envelope}) for the deliveries whose envelopes verify.
+
+    A deliver carries its events in the delivery-envelope attachment and
+    nowhere else. The envelope is what makes a delivery citable: the peer
+    signed the hop even for an event somebody else authored, so it is the one
+    artefact that proves this peer sent this, and the only evidence available
+    for an offence the sender did not author.
+
+    Which is why a deliver with no attachment is refused rather than read out
+    of the body. Events accepted without provenance are events whose delivery
+    can never be reported: the reporter can cite an envelope, or an event the
+    peer itself authored, and a third party's event with no envelope is
+    neither. Omitting the attachment was a free way to commit the interest
+    offence unpunishably. A hop that does not verify is refused the same way.
+    """
+    body = message.get('body', {})
+    envs = None
+    for att in message.get('attachments', []) or []:
+        if att.get('id') == ATTACHMENT:
+            envs = (att.get('data', {}).get('json') or {}).get('envelopes')
+    if envs is None:
+        findings.append(('S3', 'deliver carries no %s attachment: %s event(s) with no provenance'
+                         % (ATTACHMENT, body.get('count', len(body.get('events') or [])))))
+        return [], {}
+    kept: list[NostrEvent] = []
+    by_id: dict[str, DeliveryEnvelope] = {}
+    for env in envs:
+        problems = replication.verify_envelope(env)
+        if problems:
+            findings.append(('S3', 'delivery envelope refused: %s' % '; '.join(problems)))
+            continue
+        kept.append(env['event'])
+        by_id[env['event']['id']] = env
+    if body.get('count') is not None and body['count'] != len(envs):
+        findings.append(('S2', 'count says %d, %d envelope(s) attached' % (body['count'], len(envs))))
+    return kept, by_id
+

node/planes/store/sync/handover.py

@@ -0,0 +1,206 @@
+"""The handover that may follow a done: offering a shard this session
+earned, handing over everything on retirement, and accepting one from a
+peer only if it fingerprints to what this node now holds."""
+from __future__ import annotations
+
+from typing import Any, Optional, Sequence
+
+from fleetmesh_core.peering import coverage
+from fleetmesh_core.wire.interfaces import NostrEvent
+from fleetmesh_core.wire.wire_types import HandoverLink, SyncHandoverBody
+from fleetmesh_core.identity.did import did_nostr
+from . import rules as handover
+from .envelope import _thid
+from .shape import SyncShape
+
+
+class HandoverMixin:
+    # ---- the handover ----------------------------------------------------------
+    def _offer_handover(self: SyncShape, thid: str, peer_pk: str, sess: Any, status: Any) -> Optional[str]:
+        """Hand the shard over if this session earned one. Returns what was done, or None.
+
+        Whether it earned one is `handover.offered_set`: this does the signing
+        and the sending, which are the parts that need a key and a transport.
+        The same set to the same peer is offered once.
+        """
+        discovery = self.discovery
+        claim = discovery.published_claim(sess.shard) if discovery is not None else None
+        now = self.clock()
+        fingerprint, _why = handover.offered_set(claim, sess, now, self.config('handover', 'offer'), status)
+        # offered_set returns None for a missing claim, so a fingerprint here
+        # means the claim exists; the check is stated for the type checker and
+        # for the next reader rather than left to be inferred.
+        if fingerprint is None or claim is None or self._offered.get((peer_pk, sess.shard)) == fingerprint:
+            return None
+        chain = self._chain_for(sess.shard, claim, peer_pk, now)
+        if chain is None:
+            return None
+        # Recorded before it is sent, like the done above it and the relay
+        # face's counters, and taken back if nothing left the node. Both
+        # orders are wrong in one direction. Written after the send, a peer
+        # holding the artefact can ask this node whether it handed one over
+        # and be told no. Written before and kept, a send no relay took is a
+        # handover this node believes it made and will never make again, so a
+        # replica that should be able to assert the shard silently cannot.
+        # The record goes first and the failure undoes it, which leaves
+        # handing the same set over twice as the only residual - and that is
+        # noise rather than damage: the artefact is signed and idempotent, and
+        # `_accept_handover` fingerprints what it is given against what the
+        # receiver holds, so the second copy is the first one again.
+        entry = (peer_pk, sess.shard, fingerprint, now)
+        previous = self._offered.get((peer_pk, sess.shard))
+        self._offered[(peer_pk, sess.shard)] = fingerprint
+        with self._lock:
+            self.handed.append(entry)
+        # Inserts, not the length: at its bound the log drops one for each it
+        # takes, so a failure there left the length unchanged and the memo of
+        # a handover that never left kept.
+        unsent = self.unsent.inserts
+        self._send(coverage.handover_message(chain, thid), peer_pk)
+        if self.unsent.inserts != unsent:
+            # `_send` is the one place that decides nothing left the node -
+            # the transport raised, or every relay refused - and it has
+            # already said so where the operator reads it. A second thread
+            # recording its own failure here would cost this offer a needless
+            # retry, which is the harmless direction.
+            self._unoffer((peer_pk, sess.shard), previous, entry)
+            return None
+        self.say('handed', shard=sess.shard, peer=peer_pk[:16], events=chain[-1]['count'])
+        return 'handed over ' + sess.shard
+
+    def _unoffer(self: SyncShape, key: tuple[str, str], previous: Optional[str], entry: tuple[Any, ...]) -> None:
+        """Undo the memo of a handover that never left the node."""
+        if previous is None:
+            self._offered.pop(key, None)
+        else:
+            self._offered[key] = previous
+        with self._lock:
+            self.handed.discard(entry)
+
+    def hand_over_all(self: SyncShape, peers: Sequence[str], now: Optional[int] = None) -> list[tuple[Any, ...]]:
+        """Hand every closed asserted shard to every peer named, on the way out.
+
+        The opportunistic offer waits for a peer to pull, which is the right
+        shape while a node is running and useless while it is retiring: a
+        drain would have to outlast the slowest peer's pull interval, and the
+        peer that is offline right now would never be reached at all. So a
+        retiring node sends the statement rather than waiting to be asked.
+
+        It is sent and not reconciled. Opening a session would contradict the
+        rung a drain is entering, put ingest load on neighbours whose own
+        capacity this node cannot see, and buy only the equality evidence the
+        receiver already checks for itself: `_accept_handover` refuses a
+        handover that does not fingerprint to what that node holds. So a
+        statement to a peer that does not hold the set is harmless, and one to
+        a peer that does is exactly right. The signature is the same one this
+        node would have given had the peer pulled.
+
+        Stored, because the peer this matters most to is the one that is
+        offline while its only first-hand holder retires.
+
+        Nothing here waits on an answer and nothing here may raise.
+        """
+        now_val = now if now is not None else self.clock()
+        did: list[tuple[Any, ...]] = []
+        discovery = self.discovery
+        if discovery is None or self.config('handover', 'offer') != 'offer':
+            return did
+        for claim in discovery.asserted_claims():
+            shard = [t[1] for t in claim['tags'] if t[0] == 'd'][0]
+            for peer_pk in peers:
+                entry: Optional[tuple[Any, ...]] = None
+                previous: Optional[str] = None
+                try:
+                    chain = self._chain_for(shard, claim, peer_pk, now_val)
+                    if chain is None:
+                        continue
+                    art = chain[-1]
+                    msg = coverage.handover_message(chain, _thid(shard))
+                    msg = dict(msg, **{'from': self.ks.did, 'to': [did_nostr(peer_pk)]})
+                    # Same order and same reason as the opportunistic offer,
+                    # and it matters more here: this is the last thing the
+                    # node does for this shard, so a memo kept over a send
+                    # that failed is a handover nobody will ever make.
+                    entry = (peer_pk, shard, art['fingerprint'], now_val)
+                    previous = self._offered.get((peer_pk, shard))
+                    self._offered[(peer_pk, shard)] = art['fingerprint']
+                    with self._lock:
+                        self.handed.append(entry)
+                    res = self.wire_client.send(peer_pk, msg, stored=True)
+                    if res and not any(ok for ok, _ in res.values()):
+                        self._unoffer((peer_pk, shard), previous, entry)
+                        why = '; '.join(n for _, n in res.values() if n)[:160] or 'refused'
+                        did.append(('handover unsent', shard, peer_pk, why))
+                        continue
+                    did.append(('handed over', shard, peer_pk, '%d events' % art['count']))
+                except Exception as e:
+                    # One peer that cannot be told must not cost the others
+                    # theirs. The memo written just above is taken back so a
+                    # peer the transport could not address now is still owed
+                    # the statement - and `entry` is None when the throw came
+                    # before there was one to take back.
+                    if entry is not None:
+                        self._unoffer((peer_pk, shard), previous, entry)
+                    did.append(('handover unsent', shard, peer_pk, '%s: %s' % (type(e).__name__, e)))
+        return did
+
+    def _chain_for(
+        self: SyncShape,
+        shard: str,
+        claim: NostrEvent,
+        peer_pk: str,
+        now: int,
+    ) -> Optional[list[HandoverLink]]:
+        """The chain to send: what this node was handed, plus its own link.
+
+        The signing is here because the key is; `handover.extend` decides
+        whether the result is a chain this node may stand behind.
+        """
+        try:
+            link = coverage.handover(self.ks.node_sk, peer_pk, claim, now)
+        except ValueError as e:
+            self.refusals.refuse('handover', str(e), subject=peer_pk, detail={'shard': shard}, at=now)
+            return None
+        accounts = self.accounts
+        held = (accounts.handover(shard) if accounts is not None else None) or []
+        chain, why = handover.extend(held, link)
+        if chain is None:
+            self.refusals.refuse('handover', why or 'the chain could not be extended', subject=peer_pk,
+                                 detail={'shard': shard}, at=now)
+        return chain
+
+    def _accept_handover(self: SyncShape, peer_pk: str, body: SyncHandoverBody) -> tuple[Any, ...]:
+        """One inbound sync/1.0/handover. Kept only if it is ours to carry.
+
+        `handover.acceptable` says whether; this does the looking up and the
+        keeping. The whole chain is kept, because the whole chain is what this
+        node will carry in its own claim and pass on if it hands the shard
+        along.
+        """
+        now = self.clock()
+        accounts = self.accounts
+        chain = [h for h in (body.get('chain') or []) if isinstance(h, dict)]
+        art = chain[-1] if chain else {}
+        shard_name = art.get('shard')
+        shard_cfg = (handover.shard_config(self.config('coverage'), shard_name)
+                     if isinstance(shard_name, str) else None)
+        held_ids = None
+        why: Optional[str]
+        if accounts is None:
+            why = 'this node keeps no account store'
+        else:
+            if shard_cfg is not None and art:
+                held_ids = self.store.enumerate_range(shard_cfg['filter'], art.get('since') or 0,
+                                                      art.get('until') or 0, now)
+            why = handover.acceptable(chain, self.ks.node_pk, peer_pk, shard_cfg, held_ids)
+        if why is None:
+            assert accounts is not None    # why is only None on the branch that had one
+            accounts.put_handover(art['shard'], chain)
+            with self._lock:
+                self.held.append((peer_pk, art['shard'], now))
+            self.say('held', shard=art['shard'], peer=peer_pk[:16],
+                     events=art['count'], chain_len=len(chain))
+            return ('handover', peer_pk, art['shard'])
+        self.refusals.refuse('handover', why, subject=peer_pk, detail={'shard': art.get('shard')}, at=now)
+        return ('handover refused', peer_pk, why)
+