memd-engine 0.5.0__py3-none-any.whl

memd/engine/forward.py

@@ -0,0 +1,1175 @@
+"""Write forwarding: several processes on one data root.
+
+memd has one writer per namespace: the process holding its lock file (a
+local data root) or its lease (s3://) - see NamespaceStore. A second
+process opening such a namespace used to get NamespaceBusyError. With
+forwarding on (`forwarding="auto"`, the default), its Memory sends that
+namespace's writes - and its strongly consistent reads - to the process that
+holds it instead, and serves eventual reads from its own read replica:
+
+  - a Memory that may write runs a small endpoint (TCP; 127.0.0.1 and an
+    ephemeral port unless `forward_host` / `forward_port` say otherwise) and
+    puts its address in what it holds: the lock file's content on a local
+    root, the lease holder string on S3
+    ("host:pid;fwd=127.0.0.1:41234;id=<endpoint id>");
+  - a process that finds a namespace busy reads the holder from there,
+    connects and runs the call ON THE HOLDER - the same code a local call
+    runs, so quarantine, consolidation, session taint and the audit ledger
+    stay the writer's;
+  - a connection is authenticated both ways with a secret every process on
+    the root can read (`<data dir>/forward.secret`, mode 0600, created on
+    first use - or `forward_secret` / MEMD_FORWARD_SECRET), and every frame
+    after the handshake carries an HMAC-SHA256 under a key derived for that
+    connection (integrity and replay protection; NOT confidentiality);
+  - every forwarded call carries a request id, and a call that creates
+    records carries their ids, generated by the caller. A retry - after a
+    lost reply, a timeout, or a holder that died - is applied at most once:
+    a live holder answers a repeated request id from memory (or waits for
+    the first attempt still running), and a new holder finds the records in
+    the log it replayed when it took the namespace;
+  - when the holder goes away its lock frees at once (a lease after its
+    TTL), and the next forwarded call takes the namespace: the caller
+    becomes its writer and runs the call itself.
+
+A process never applies a write to a namespace it does not hold: the local
+path always goes through the namespace's lock or lease, and a holder that
+no longer holds it answers "not owner" - nothing applied - to a forwarded
+call.
+"""
+from __future__ import annotations
+
+import hashlib
+import hmac
+import ipaddress
+import json
+import logging
+import os
+import random
+import secrets
+import select
+import socket
+import stat
+import struct
+import threading
+import time
+import uuid
+from collections import OrderedDict
+from dataclasses import dataclass
+
+from memd.metrics import METRICS
+from memd.storage.engine import NamespaceBusyError, NamespaceClosedError, owner_lock_path, read_owner_lock
+
+_log = logging.getLogger(__name__)
+
+PROTOCOL = 1
+FORWARDING_MODES = ("auto", "off")
+SECRET_FILE = "forward.secret"
+# a frame: 4-byte length, a kind byte, the body, and - once the handshake
+# keyed the connection - an HMAC-SHA256 of (direction, sequence, kind, body)
+_LEN = struct.Struct(">I")
+_SEQ = struct.Struct(">Q")
+_MAC_LEN = 32
+MAX_FRAME = 256 << 20
+_MAX_HELLO = 4096          # handshake frames: before authentication, keep them small
+_IDLE_S = 600.0            # an idle authenticated connection is closed after this
+_DONE_MAX = 8192           # write results remembered for a repeated request id...
+_DONE_TTL_S = 900.0        # ...for this long
+_STREAM_CHUNK = 256 << 10  # an export streams in chunks of whole lines about this big
+
+# kinds of frame
+_HELLO, _REQUEST, _REPLY, _STREAM, _DATA, _END = b"H", b"Q", b"R", b"S", b"D", b"E"
+
+# calls that change the namespace: a repeated request id gets the first
+# answer back instead of running again
+WRITE_OPS = frozenset({"add", "add_events", "remember", "close_session", "delete", "delete_many",
+                       "forget_ids", "destroy_namespace", "compact", "reembed", "audit"})
+
+
+class ForwardingError(NamespaceBusyError):
+    """Another process holds the namespace and the call could not be
+    forwarded to it (no writer answered in time, or it does not accept
+    forwarded calls). Nothing was applied."""
+
+
+class ForwardAuthError(ForwardingError):
+    """The namespace's holder refused this process's credentials (or could
+    not prove its own): the processes do not share the forwarding secret."""
+
+
+class ForwardTimeoutError(TimeoutError):
+    """A forwarded write got no answer and no writer answered its retries in
+    time: it may or may not have been applied."""
+
+
+class ForwardedCallError(RuntimeError):
+    """The holder raised an error that has no counterpart here."""
+
+
+class ForwardNotReady(RuntimeError):
+    """(holder side) the Memory is still opening: the caller retries."""
+
+
+class _Unreachable(Exception):
+    """Nothing was sent: the holder cannot be reached (or another process
+    answers at its address)."""
+
+
+class _Uncertain(Exception):
+    """The request was sent and no answer came back."""
+
+
+class _Wait(Exception):
+    """Nobody can be reached right now (a lock or lease changing hands)."""
+
+
+class _AuthFailed(Exception):
+    pass
+
+
+LOCAL = object()   # Forwarder.call: run the call here - this process took the namespace
+
+
+# ---------------------------------------------------------------- config
+
+
+@dataclass
+class ForwardConfig:
+    mode: str = "auto"
+    secret: str | None = None        # forward_secret / MEMD_FORWARD_SECRET
+    host: str = "127.0.0.1"          # the endpoint's bind address
+    port: int = 0                    # 0: an ephemeral port
+    advertise: str | None = None     # host:port other processes connect to (default: bind address)
+    wait_s: float = 30.0             # how long a call keeps looking for a writer that answers
+    timeout_s: float = 300.0         # how long one attempt waits for the holder's answer
+    connect_timeout_s: float = 2.0
+    max_inflight: int = 64           # holder: forwarded calls executing at once
+    max_conns: int = 256             # holder: connections at once
+    queue_wait_s: float = 10.0       # holder: a call waits this long for a slot, then "overloaded"
+    check_s: float = 0.0             # S3: how often a call waiting for its answer re-checks the lease
+
+    @property
+    def enabled(self) -> bool:
+        return self.mode == "auto"
+
+    @classmethod
+    def from_config(cls, cfg: dict, forwarding: str | None = None, *,
+                    lease_ttl_s: float | None = None) -> "ForwardConfig":
+        """From Memory(config=...) keys, else the MEMD_FORWARD* environment.
+        `lease_ttl_s` (an s3:// root): a crashed holder's lease frees only
+        after it, so a call waits that long plus a margin by default."""
+        def pick(key: str, env: str, default):
+            v = cfg.get(key)
+            if v is None or v == "":
+                v = os.environ.get(env)
+            return default if v is None or v == "" else v
+
+        mode = str(forwarding or pick("forwarding", "MEMD_FORWARDING", "auto")).strip().lower()
+        if mode not in FORWARDING_MODES:
+            raise ValueError(f"forwarding must be one of {FORWARDING_MODES}, got {mode!r}")
+        wait_default = 30.0 if lease_ttl_s is None else max(30.0, float(lease_ttl_s) + 15.0)
+        out = cls(
+            mode=mode,
+            secret=pick("forward_secret", "MEMD_FORWARD_SECRET", None),
+            host=str(pick("forward_host", "MEMD_FORWARD_HOST", "127.0.0.1")),
+            port=int(pick("forward_port", "MEMD_FORWARD_PORT", 0)),
+            advertise=pick("forward_advertise", "MEMD_FORWARD_ADVERTISE", None),
+            wait_s=float(pick("forward_wait_s", "MEMD_FORWARD_WAIT_S", wait_default)),
+            timeout_s=float(pick("forward_timeout_s", "MEMD_FORWARD_TIMEOUT_S", 300.0)),
+            max_inflight=int(pick("forward_max_inflight", "MEMD_FORWARD_MAX_INFLIGHT", 64)),
+            queue_wait_s=float(pick("forward_queue_wait_s", "MEMD_FORWARD_QUEUE_WAIT_S", 10.0)),
+            check_s=0.0 if lease_ttl_s is None else max(0.5, float(lease_ttl_s) / 3.0),
+        )
+        if out.secret is not None and len(out.secret) < 16:
+            raise ValueError("forward_secret / MEMD_FORWARD_SECRET must be at least 16 characters")
+        return out
+
+
+class ForwardSecretError(PermissionError):
+    """The forwarding secret file is not one this process may trust: a
+    symbolic link, not a regular file, another user's, or open to others."""
+
+
+def _read_secret(path: str) -> bytes | None:
+    """The secret file's content - None when there is none - read without
+    following a symbolic link, and only from a regular file of this
+    process's user that nobody else may read or write (0600 or narrower)."""
+    try:
+        fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+    except FileNotFoundError:
+        return None
+    except OSError as ex:
+        if os.path.islink(path):
+            raise ForwardSecretError(
+                f"the forwarding secret {path} is a symbolic link: refused") from ex
+        raise
+    try:
+        st = os.fstat(fd)
+        if not stat.S_ISREG(st.st_mode):
+            raise ForwardSecretError(f"the forwarding secret {path} is not a regular file: refused")
+        uid = os.getuid() if hasattr(os, "getuid") else st.st_uid
+        if st.st_uid != uid:
+            raise ForwardSecretError(
+                f"the forwarding secret {path} belongs to uid {st.st_uid}, not to this process's "
+                f"user (uid {uid}): refused")
+        if st.st_mode & 0o077:
+            raise ForwardSecretError(
+                f"the forwarding secret {path} has mode {stat.S_IMODE(st.st_mode):04o}: other users "
+                "may read or write it - refused (it must be 0600)")
+        return os.read(fd, 4096).strip()
+    finally:
+        os.close(fd)
+
+
+def load_secret(cfg: ForwardConfig, data_dir: str) -> str:
+    """The forwarding secret: the configured one, else `<data_dir>/
+    forward.secret` - created on first use (mode 0600), so every process of
+    the user that can read the data directory shares it. ForwardSecretError
+    when the file is not one to trust (see _read_secret)."""
+    if cfg.secret:
+        return cfg.secret
+    path = os.path.join(data_dir, SECRET_FILE)
+    for _ in range(100):
+        got = _read_secret(path)
+        if got is not None:
+            if len(got) >= 32:
+                return got.decode("ascii", errors="replace")
+            time.sleep(0.02)      # being written by the process creating it
+            continue
+        # created whole or not at all: written to a temp file, then linked
+        # into place - which fails if another process got there first
+        tmp = f"{path}.{os.getpid()}-{secrets.token_hex(4)}.tmp"
+        fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+        try:
+            os.write(fd, secrets.token_hex(32).encode())
+            os.fsync(fd)
+        finally:
+            os.close(fd)
+        try:
+            os.link(tmp, path)
+        except FileExistsError:
+            pass
+        except OSError:
+            # no hard links here: an exclusive create (a reader may see it
+            # empty for a moment - the loop above waits for that)
+            try:
+                fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL
+                             | getattr(os, "O_NOFOLLOW", 0), 0o600)
+                try:
+                    os.write(fd, secrets.token_hex(32).encode())
+                    os.fsync(fd)
+                finally:
+                    os.close(fd)
+            except FileExistsError:
+                pass
+        finally:
+            try:
+                os.unlink(tmp)
+            except OSError:
+                pass
+    raise RuntimeError(f"could not read or create the forwarding secret {path}")
+
+
+# ---------------------------------------------------------------- holders
+
+
+@dataclass(frozen=True)
+class Endpoint:
+    host: str
+    port: int
+    endpoint_id: str
+    hostname: str        # the machine the holder runs on
+
+
+def holder_string(address: str, endpoint_id: str) -> str:
+    """The lock-file content / lease holder of a process that accepts
+    forwarded calls (no newline: a lease body is "<holder>\\n<stamp>")."""
+    return f"{socket.gethostname()}:{os.getpid()};fwd={address};id={endpoint_id}"
+
+
+def parse_holder(holder: str | None) -> Endpoint | None:
+    """The endpoint a holder string advertises; None when it advertises
+    none (forwarding off, a cluster node, an older memd)."""
+    if not holder or ";fwd=" not in holder:
+        return None
+    head, *fields = holder.strip().split(";")
+    kv = dict(f.split("=", 1) for f in fields if "=" in f)
+    addr, eid = kv.get("fwd"), kv.get("id")
+    if not addr or not eid:
+        return None
+    host, _, port = addr.rpartition(":")
+    try:
+        return Endpoint(host.strip("[]"), int(port), eid, head.rsplit(":", 1)[0])
+    except ValueError:
+        return None
+
+
+def _unspecified(host: str) -> bool:
+    """A bind address meaning every interface: not one others can connect to."""
+    if not host:
+        return True
+    try:
+        return ipaddress.ip_address(host).is_unspecified
+    except ValueError:
+        return False
+
+
+def _loopback(host: str) -> bool:
+    if host == "localhost":
+        return True
+    try:
+        return ipaddress.ip_address(host).is_loopback
+    except ValueError:
+        return False
+
+
+# ---------------------------------------------------------------- framing
+
+
+def _shutdown(sock: socket.socket) -> None:
+    try:
+        sock.shutdown(socket.SHUT_RDWR)
+    except OSError:
+        pass
+
+
+def _jdump(obj) -> bytes:
+    return json.dumps(obj, separators=(",", ":"), default=_jdefault).encode()
+
+
+def _jdefault(o):
+    name = getattr(o, "name", None)
+    if isinstance(name, str) and hasattr(type(o), "__members__"):
+        return name                                 # an enum (Source): by name
+    if isinstance(o, (set, frozenset, tuple)):
+        return list(o)
+    item = getattr(o, "item", None)                 # numpy scalars
+    if callable(item):
+        return item()
+    raise TypeError(f"{type(o).__name__} cannot be forwarded")
+
+
+def _json_object(body: bytes) -> dict | None:
+    """A frame's JSON object, or None (not JSON, or not an object)."""
+    try:
+        got = json.loads(body)
+    except ValueError:
+        return None
+    return got if isinstance(got, dict) else None
+
+
+def _hmac(key: bytes, *parts: bytes) -> bytes:
+    return hmac.new(key, b"".join(parts), hashlib.sha256).digest()
+
+
+class _Channel:
+    """Length-prefixed frames over one socket; once keyed (after the
+    handshake), every frame carries an HMAC over its direction, its
+    sequence number and its bytes - a frame altered, dropped, replayed or
+    reflected fails the check and closes the connection."""
+
+    def __init__(self, sock: socket.socket, side: bytes):
+        self.sock = sock
+        self.key: bytes | None = None
+        self.out_tag, self.in_tag = (b"c", b"s") if side == b"c" else (b"s", b"c")
+        self.seq_out = self.seq_in = 0
+
+    def send(self, kind: bytes, body: bytes) -> None:
+        mac = b""
+        if self.key is not None:
+            mac = _hmac(self.key, self.out_tag, _SEQ.pack(self.seq_out), kind, body)
+            self.seq_out += 1
+        head = _LEN.pack(1 + len(body) + len(mac)) + kind
+        if len(body) < 65536:
+            self.sock.sendall(head + body + mac)
+        else:
+            self.sock.sendall(head)
+            self.sock.sendall(body)
+            self.sock.sendall(mac)
+
+    def recv(self, limit: int = MAX_FRAME) -> tuple[bytes, bytes]:
+        n = _LEN.unpack(self._exact(_LEN.size))[0]
+        extra = 1 + (_MAC_LEN if self.key is not None else 0)
+        if n < extra or n > limit + extra:
+            raise _AuthFailed(f"bad frame length {n}")
+        frame = self._exact(n)
+        kind = frame[:1]
+        if self.key is None:
+            return kind, frame[1:]
+        body, mac = frame[1:-_MAC_LEN], frame[-_MAC_LEN:]
+        want = _hmac(self.key, self.in_tag, _SEQ.pack(self.seq_in), kind, body)
+        if not hmac.compare_digest(mac, want):
+            raise _AuthFailed("frame authentication failed")
+        self.seq_in += 1
+        return kind, body
+
+    def _exact(self, n: int) -> bytes:
+        buf = bytearray(n)
+        view = memoryview(buf)
+        got = 0
+        while got < n:
+            k = self.sock.recv_into(view[got:], min(n - got, 1 << 20))
+            if not k:
+                raise ConnectionError("connection closed")
+            got += k
+        return bytes(buf)
+
+    def close(self) -> None:
+        try:
+            self.sock.close()
+        except OSError:
+            pass
+
+
+def _keys(secret: str, endpoint_id: str, nonce: str, client: str, cnonce: str) -> tuple[str, str, bytes]:
+    """(client proof, server proof, connection key) for one handshake - the
+    cluster router's request signature (memd.server.cluster._sign) over
+    both nonces, so neither side's proof can be replayed to the other.
+    (Imported here: a single server never loads the cluster module.)"""
+    from memd.server.cluster import _sign
+
+    cproof = _sign(secret, endpoint_id, nonce, client, "forward-client", cnonce)
+    sproof = _sign(secret, endpoint_id, nonce, client, "forward-server", cnonce)
+    key = bytes.fromhex(_sign(secret, endpoint_id, nonce, client, "forward-key", cnonce))
+    return cproof, sproof, key
+
+
+# ---------------------------------------------------------------- holder side
+
+
+class _Stream:
+    """A handler result sent as a stream (an export): `meta` first, then
+    the chunks of `chunks`."""
+
+    def __init__(self, meta: dict, chunks):
+        self.meta = meta
+        self.chunks = chunks
+
+
+def _error_payload(ex: BaseException) -> dict:
+    from memd.storage.objectstore import LeaseLostError
+
+    if isinstance(ex, (NamespaceBusyError, LeaseLostError, NamespaceClosedError)):
+        # this process does not hold the namespace (any more): nothing was
+        # applied, the caller finds the writer again
+        return {"code": "not_owner", "retry": True, "message": str(ex)[:500]}
+    if isinstance(ex, ForwardNotReady):
+        return {"code": "not_ready", "retry": True, "message": str(ex)[:500]}
+    return {"code": "error", "retry": False, "type": type(ex).__name__, "message": str(ex)[:4000]}
+
+
+class ForwardServer:
+    """The endpoint of a process that may hold namespaces. `handler(op, ns,
+    args, ids)` runs one forwarded call (see Memory._serve_forwarded) and
+    returns a JSON-able result or a _Stream; NamespaceBusyError /
+    LeaseLostError from it mean "not the writer, nothing applied"."""
+
+    def __init__(self, cfg: ForwardConfig, secret: str, handler):
+        self.cfg = cfg
+        self.secret = secret
+        self.handler = handler
+        self.endpoint_id = secrets.token_hex(8)
+        self._sock = socket.create_server((cfg.host, int(cfg.port)), backlog=128)
+        port = self._sock.getsockname()[1]
+        if cfg.advertise:
+            address = str(cfg.advertise)
+        elif _unspecified(cfg.host):
+            self._sock.close()
+            raise ValueError("forward_host binds every interface: set forward_advertise / "
+                             "MEMD_FORWARD_ADVERTISE to the host:port other processes reach this one at")
+        else:
+            address = f"[{cfg.host}]:{port}" if ":" in cfg.host else f"{cfg.host}:{port}"
+        self.address = address
+        self.holder = holder_string(address, self.endpoint_id)
+        self._lock = threading.Lock()
+        self._idle = threading.Condition(self._lock)
+        self._conns: set[socket.socket] = set()
+        self._inflight = 0
+        self._slots = threading.BoundedSemaphore(max(1, int(cfg.max_inflight)))
+        # Calls run on these threads, never on a connection's own: a thread
+        # that reads an index keeps a SQLite connection of its own until the
+        # index closes, so a thread per connection kept two descriptors per
+        # connection for good. At most max_inflight, created as needed, reused.
+        from concurrent.futures import ThreadPoolExecutor
+
+        self._pool = ThreadPoolExecutor(max_workers=max(1, int(cfg.max_inflight)),
+                                        thread_name_prefix="memd-forward-call")
+        self._done: "OrderedDict[str, tuple[float, dict]]" = OrderedDict()
+        self._running: dict[str, threading.Event] = {}
+        self._closing = False
+        self._thread: threading.Thread | None = None
+
+    def start(self) -> None:
+        self._thread = threading.Thread(target=self._accept_loop, daemon=True, name="memd-forward")
+        self._thread.start()
+
+    def stop(self, drain_s: float = 30.0) -> None:
+        """Stop serving: new calls are answered "not owner" (the callers
+        find the next writer), calls already running finish - within
+        `drain_s` - and then every connection is closed."""
+        with self._lock:
+            if self._closing:
+                return
+            self._closing = True
+        try:
+            self._sock.shutdown(socket.SHUT_RDWR)
+        except OSError:
+            pass
+        self._sock.close()
+        deadline = time.monotonic() + drain_s
+        with self._lock:
+            while self._inflight and time.monotonic() < deadline:
+                self._idle.wait(max(0.01, deadline - time.monotonic()))
+            conns = list(self._conns)
+        for c in conns:
+            try:
+                c.shutdown(socket.SHUT_RDWR)
+            except OSError:
+                pass
+        if self._thread is not None and self._thread is not threading.current_thread():
+            self._thread.join(timeout=5)
+        self._pool.shutdown(wait=False)
+
+    # ------------------------------------------------------------ connections
+
+    def _accept_loop(self) -> None:
+        # (a timeout, so the loop also ends where shutting the listening
+        # socket down does not wake an accept() blocked on it)
+        self._sock.settimeout(1.0)
+        while True:
+            try:
+                conn, _addr = self._sock.accept()
+            except socket.timeout:
+                if self._closing:
+                    return
+                continue
+            except OSError:
+                return                      # closed (stop)
+            with self._lock:
+                refuse = self._closing or len(self._conns) >= self.cfg.max_conns
+                if not refuse:
+                    self._conns.add(conn)
+            if refuse:
+                METRICS.inc("memd_forward_refused_total", reason="connections",
+                            help="forwarded connections refused (too many at once, or closing)")
+                conn.close()
+                continue
+            threading.Thread(target=self._serve_conn, args=(conn,), daemon=True,
+                             name="memd-forward-conn").start()
+
+    def _serve_conn(self, conn: socket.socket) -> None:
+        ch = _Channel(conn, b"s")
+        # the whole handshake, however slowly its bytes come, within 5 s
+        cut = threading.Timer(max(5.0, self.cfg.connect_timeout_s * 2), _shutdown, args=(conn,))
+        cut.daemon = True
+        try:
+            conn.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+            conn.settimeout(max(5.0, self.cfg.connect_timeout_s * 2))
+            cut.start()
+            ok = self._handshake(ch)
+            cut.cancel()
+            if not ok:
+                return
+            conn.settimeout(_IDLE_S)
+            while True:
+                kind, body = ch.recv()
+                req = _json_object(body) if kind == _REQUEST else None
+                if req is None:
+                    return
+                self._handle(ch, req)
+        except _AuthFailed as ex:
+            self._refused(ch, str(ex))
+        except (OSError, ConnectionError, ValueError):
+            pass                            # the caller went away (or sent garbage)
+        except Exception:  # noqa: BLE001 - one connection never takes the endpoint down
+            _log.exception("memd forward: connection failed")
+        finally:
+            cut.cancel()
+            with self._lock:
+                self._conns.discard(conn)
+            ch.close()
+
+    def _handshake(self, ch: _Channel) -> bool:
+        nonce = secrets.token_hex(16)
+        ch.send(_HELLO, _jdump({"v": PROTOCOL, "id": self.endpoint_id, "nonce": nonce}))
+        kind, body = ch.recv(_MAX_HELLO)
+        msg = _json_object(body) if kind == _HELLO else None
+        if msg is None:
+            self._refused(ch, "a malformed handshake")
+            return False
+        client, cnonce, proof = str(msg.get("client", ""))[:128], str(msg.get("cnonce", ""))[:64], \
+            str(msg.get("proof", ""))
+        cproof, sproof, key = _keys(self.secret, self.endpoint_id, nonce, client, cnonce)
+        if len(cnonce) < 16 or not hmac.compare_digest(proof, cproof):
+            self._refused(ch, "credentials that do not prove the secret")
+            ch.send(_HELLO, _jdump({"ok": False, "error": "auth"}))
+            return False
+        ch.send(_HELLO, _jdump({"ok": True, "proof": sproof}))
+        ch.key = key
+        return True
+
+    @staticmethod
+    def _refused(ch: _Channel, why: str) -> None:
+        """One line and one count per refused connection - never a traceback."""
+        METRICS.inc("memd_forward_auth_failures_total",
+                    help="forwarded connections refused: bad credentials or a tampered frame")
+        try:
+            peer = "%s:%s" % ch.sock.getpeername()[:2]
+        except OSError:
+            peer = "?"
+        _log.warning("memd forward: refused a connection from %s: %s", peer, why)
+
+    # ------------------------------------------------------------ calls
+
+    def _handle(self, ch: _Channel, req: dict) -> None:
+        op = str(req.get("op") or "")
+        rid = str(req.get("rid") or "")
+        if op in WRITE_OPS and rid:
+            resp = self._write(req, rid)
+        else:
+            resp = self._execute(req)
+        if isinstance(resp, _Stream):
+            self._send_stream(ch, resp)
+            return
+        try:
+            body = _jdump(resp)
+        except (TypeError, ValueError) as ex:   # a result that cannot travel
+            body = _jdump({"ok": False, "error": _error_payload(ex)})
+        ch.send(_REPLY, body)
+
+    def _write(self, req: dict, rid: str) -> dict:
+        """A write runs once per request id: a repeat gets the first
+        answer, or waits for the first attempt still running."""
+        while True:
+            with self._lock:
+                hit = self._done.get(rid)
+                if hit is not None:
+                    METRICS.inc("memd_forward_repeats_total",
+                                help="forwarded writes answered from an earlier attempt (not run again)")
+                    return hit[1]
+                ev = self._running.get(rid)
+                mine = ev is None
+                if mine:
+                    ev = self._running[rid] = threading.Event()
+            if not mine:
+                if not ev.wait(self.cfg.timeout_s):
+                    return {"ok": False, "error": {"code": "overloaded", "retry": True,
+                                                   "message": "the first attempt is still running"}}
+                continue
+            resp: dict = {"ok": False, "error": {"code": "error", "retry": False, "type": "RuntimeError",
+                                                 "message": "not run"}}
+            try:
+                resp = self._execute(req)
+            finally:
+                with self._lock:
+                    if resp.get("ok") or not (resp.get("error") or {}).get("retry"):
+                        now = time.monotonic()
+                        self._done[rid] = (now, resp)
+                        while self._done and (len(self._done) > _DONE_MAX
+                                              or next(iter(self._done.values()))[0] < now - _DONE_TTL_S):
+                            self._done.popitem(last=False)
+                    self._running.pop(rid, None)
+                ev.set()
+            return resp
+
+    def _begin(self) -> dict | None:
+        if not self._slots.acquire(timeout=self.cfg.queue_wait_s):
+            METRICS.inc("memd_forward_refused_total", reason="overloaded")
+            return {"ok": False, "error": {"code": "overloaded", "retry": True,
+                                           "message": "too many forwarded calls running"}}
+        with self._lock:
+            if self._closing:
+                self._slots.release()
+                return {"ok": False, "error": {"code": "not_owner", "retry": True,
+                                               "message": "the holder is closing"}}
+            self._inflight += 1
+        return None
+
+    def _end(self) -> None:
+        with self._lock:
+            self._inflight -= 1
+            if not self._inflight:
+                self._idle.notify_all()
+        self._slots.release()
+
+    def _execute(self, req: dict):
+        refused = self._begin()
+        if refused is not None:
+            return refused
+        op = str(req.get("op") or "")
+        t0 = time.monotonic()
+        streaming = False
+        try:
+            try:
+                fut = self._pool.submit(self.handler, op, str(req.get("ns") or ""),
+                                        req.get("args") or {}, req.get("ids"))
+            except RuntimeError:            # shut down meanwhile: closing
+                return {"ok": False, "error": {"code": "not_owner", "retry": True,
+                                               "message": "the holder is closing"}}
+            out = fut.result()
+            if isinstance(out, _Stream):
+                streaming = True        # _send_stream ends it
+                return out
+            METRICS.inc("memd_forward_served_total", op=op, outcome="ok",
+                        help="forwarded calls this process ran for another")
+            return {"ok": True, "result": out}
+        except BaseException as ex:  # noqa: BLE001 - every failure is the caller's answer
+            err = _error_payload(ex)
+            METRICS.inc("memd_forward_served_total", op=op, outcome=err["code"])
+            return {"ok": False, "error": err}
+        finally:
+            if not streaming:
+                self._end()
+            METRICS.observe("memd_forward_serve_ms", (time.monotonic() - t0) * 1000,
+                            help="forwarded calls run for another process (ms)", op=op)
+
+    def _send_stream(self, ch: _Channel, st: _Stream) -> None:
+        try:
+            ch.send(_STREAM, _jdump(st.meta))
+            buf: list[bytes] = []
+            size = 0
+            trailer: dict = {"ok": True}
+            try:
+                for piece in st.chunks:
+                    buf.append(piece)
+                    size += len(piece)
+                    if size >= _STREAM_CHUNK:
+                        ch.send(_DATA, b"".join(buf))
+                        buf, size = [], 0
+                if buf:
+                    ch.send(_DATA, b"".join(buf))
+            except (OSError, ConnectionError):
+                raise
+            except BaseException as ex:  # noqa: BLE001 - reported to the caller
+                trailer = {"ok": False, "error": _error_payload(ex)}
+            ch.send(_END, _jdump(trailer))
+        finally:
+            close = getattr(st.chunks, "close", None)
+            if callable(close):
+                close()
+            self._end()
+
+
+# ---------------------------------------------------------------- caller side
+
+
+class _StreamReader:
+    """The chunks of a streamed answer (an export), as whole lines."""
+
+    def __init__(self, client: "ForwardClient", key, ch: _Channel, meta: dict, timeout_s: float):
+        self._client, self._key, self._ch = client, key, ch
+        self.meta = meta
+        self._timeout_s = timeout_s
+
+    def __iter__(self):
+        ch = self._ch
+        ok = False
+        try:
+            ch.sock.settimeout(self._timeout_s)
+            while True:
+                try:
+                    kind, body = ch.recv()
+                except (OSError, ConnectionError, _AuthFailed) as ex:
+                    raise ForwardingError(f"the holder's stream broke off ({ex})") from ex
+                if kind == _DATA:
+                    yield from body.splitlines(keepends=True)
+                    continue
+                trailer = (_json_object(body) if kind == _END else None) or {}
+                if not trailer.get("ok"):
+                    _raise_remote(trailer.get("error") or {"message": "stream ended abnormally"})
+                ok = True
+                return
+        finally:
+            if ok:
+                self._client._give(self._key, ch)
+            else:
+                ch.close()
+
+
+def _raise_remote(err: dict):
+    """Raise the holder's error as the same type here, when there is one."""
+    from memd.engine import memory as _m
+    from memd.storage import crypto, objectstore, replica
+
+    t, msg = str(err.get("type") or ""), str(err.get("message") or "")
+    known = {
+        "ValueError": ValueError, "TypeError": TypeError, "KeyError": KeyError,
+        "PermissionError": PermissionError,
+        "ForgetPreviewMismatch": _m.ForgetPreviewMismatch,
+        "KeyCustodyError": crypto.KeyCustodyError,
+        "KeyUnavailableError": crypto.KeyUnavailableError,
+        "ReadOnlyError": objectstore.ReadOnlyError,
+        "ReplicaUnavailableError": replica.ReplicaUnavailableError,
+        "ForwardTimeoutError": ForwardTimeoutError,
+    }
+    cls = known.get(t)
+    if cls is None:
+        raise ForwardedCallError(f"{t or 'error'} in the namespace's writer: {msg}")
+    raise cls(msg)
+
+
+class ForwardClient:
+    """Authenticated connections to holders, pooled per endpoint."""
+
+    POOL_IDLE = 16       # idle connections kept per endpoint...
+    POOL_IDLE_S = 60.0   # ...for this long (the holder closes them after _IDLE_S)
+
+    def __init__(self, cfg: ForwardConfig, secret: str, client_id: str):
+        self.cfg = cfg
+        self.secret = secret
+        self.client_id = client_id[:128]
+        self._lock = threading.Lock()
+        self._pool: dict[tuple, list[_Channel]] = {}
+        self._closed = False
+
+    def close(self) -> None:
+        with self._lock:
+            self._closed = True
+            pool, self._pool = self._pool, {}
+        for chans in pool.values():
+            for _t, ch in chans:
+                ch.close()
+
+    def drop(self, ep: Endpoint) -> None:
+        with self._lock:
+            chans = self._pool.pop((ep.host, ep.port, ep.endpoint_id), [])
+        for _t, ch in chans:
+            ch.close()
+
+    def _give(self, key, ch: _Channel) -> None:
+        with self._lock:
+            if not self._closed:
+                chans = self._pool.setdefault(key, [])
+                if len(chans) < self.POOL_IDLE:
+                    chans.append((time.monotonic(), ch))
+                    return
+        ch.close()
+
+    def _take(self, key) -> _Channel | None:
+        stale: list[_Channel] = []
+        got = None
+        with self._lock:
+            chans = self._pool.get(key) or []
+            while chans:
+                t, ch = chans.pop()
+                if time.monotonic() - t <= self.POOL_IDLE_S:
+                    got = ch
+                    break
+                stale.append(ch)
+        for ch in stale:
+            ch.close()
+        return got
+
+    def _connect(self, ep: Endpoint) -> _Channel:
+        try:
+            sock = socket.create_connection((ep.host, ep.port), timeout=self.cfg.connect_timeout_s)
+        except OSError as ex:
+            raise _Unreachable(f"cannot connect to {ep.host}:{ep.port} ({ex})") from None
+        ch = _Channel(sock, b"c")
+        try:
+            sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+            sock.settimeout(max(5.0, self.cfg.connect_timeout_s * 2))
+            kind, body = ch.recv(_MAX_HELLO)
+            hello = (_json_object(body) if kind == _HELLO else None) or {}
+            if hello.get("id") != ep.endpoint_id:
+                raise _Unreachable(f"another process answers at {ep.host}:{ep.port} "
+                                   "(the namespace's holder is gone)")
+            nonce, cnonce = str(hello.get("nonce", "")), secrets.token_hex(16)
+            cproof, sproof, key = _keys(self.secret, ep.endpoint_id, nonce, self.client_id, cnonce)
+            ch.send(_HELLO, _jdump({"v": PROTOCOL, "client": self.client_id, "cnonce": cnonce,
+                                    "proof": cproof}))
+            kind, body = ch.recv(_MAX_HELLO)
+            ack = (_json_object(body) if kind == _HELLO else None) or {}
+        except _Unreachable:
+            ch.close()
+            raise
+        except (OSError, ConnectionError, ValueError, _AuthFailed) as ex:
+            ch.close()
+            raise _Unreachable(f"handshake with {ep.host}:{ep.port} failed ({ex})") from None
+        if not ack.get("ok"):
+            ch.close()
+            raise ForwardAuthError(
+                f"the namespace's writer at {ep.host}:{ep.port} refused this process's forwarding "
+                "credentials: processes on one data root must share the forwarding secret "
+                "(<data dir>/forward.secret, or forward_secret / MEMD_FORWARD_SECRET)")
+        if not hmac.compare_digest(str(ack.get("proof", "")), sproof):
+            ch.close()
+            raise ForwardAuthError(f"the process at {ep.host}:{ep.port} could not prove it holds the "
+                                   "forwarding secret; nothing was sent to it")
+        ch.key = key
+        return ch
+
+    def call(self, ep: Endpoint, req: dict, *, timeout_s: float, alive=None, check_s: float = 0.0):
+        """Send one request and wait for its answer: the reply dict, or a
+        _StreamReader. _Unreachable: nothing was sent; _Uncertain: sent, no
+        answer (within `timeout_s`, or `alive()` said the holder lost the
+        namespace meanwhile)."""
+        key = (ep.host, ep.port, ep.endpoint_id)
+        body = _jdump(req)
+        if len(body) > MAX_FRAME:
+            raise ValueError(f"a forwarded call is limited to {MAX_FRAME >> 20} MiB")
+        ch = self._take(key)
+        pooled = ch is not None
+        while True:
+            if ch is None:
+                ch = self._connect(ep)
+            try:
+                ch.sock.settimeout(max(5.0, timeout_s))
+                ch.send(_REQUEST, body)
+                break
+            except OSError:
+                ch.close()
+                if not pooled:
+                    raise _Uncertain(f"sending to {ep.host}:{ep.port} failed") from None
+                ch, pooled = None, False    # a pooled connection the holder closed: once more
+        try:
+            self._await(ch, timeout_s, alive, check_s)
+            ch.sock.settimeout(max(5.0, timeout_s))
+            kind, data = ch.recv()
+        except _Uncertain:
+            ch.close()
+            raise
+        except (OSError, ConnectionError, ValueError, _AuthFailed) as ex:
+            ch.close()
+            raise _Uncertain(f"no answer from {ep.host}:{ep.port} ({ex})") from None
+        if kind == _STREAM:
+            return _StreamReader(self, key, ch, _json_object(data) or {}, max(5.0, timeout_s))
+        reply = _json_object(data)
+        if reply is None:
+            ch.close()
+            raise _Uncertain("unreadable answer") from None
+        self._give(key, ch)
+        return reply
+
+    @staticmethod
+    def _await(ch: _Channel, timeout_s: float, alive, check_s: float) -> None:
+        deadline = time.monotonic() + timeout_s
+        next_check = time.monotonic() + check_s if alive is not None and check_s > 0 else None
+        poller = select.poll()
+        poller.register(ch.sock, select.POLLIN | select.POLLERR | select.POLLHUP)
+        while True:
+            left = deadline - time.monotonic()
+            if left <= 0:
+                raise _Uncertain(f"no answer within {timeout_s:.0f} s")
+            if poller.poll(int(min(left, 1.0) * 1000)):
+                return
+            if next_check is not None and time.monotonic() >= next_check:
+                next_check = time.monotonic() + check_s
+                try:
+                    still = alive()
+                except Exception:  # noqa: BLE001 - cannot tell: keep waiting
+                    still = True
+                if not still:
+                    raise _Uncertain("the holder lost the namespace while the call ran")
+
+
+# ---------------------------------------------------------------- routing
+
+
+class Forwarder:
+    """Where this process's calls for namespaces it does not hold go: the
+    holder's endpoint (cached per namespace), found again whenever a call
+    fails - and taken over, when its lock or lease is free."""
+
+    def __init__(self, engine, cfg: ForwardConfig, secret: str, own_holder: str):
+        self.engine = engine
+        self.cfg = cfg
+        self.own_holder = own_holder
+        self.hostname = socket.gethostname()
+        self.client = ForwardClient(cfg, secret, own_holder)
+        self._lock = threading.Lock()
+        self._routes: dict[str, Endpoint] = {}
+        self._written: set[str] = set()   # namespaces written through forwarding since the last flush
+
+    def close(self) -> None:
+        self.client.close()
+
+    def routes(self) -> dict[str, str]:
+        with self._lock:
+            return {ns: f"{ep.host}:{ep.port}" for ns, ep in self._routes.items()}
+
+    def _take(self, ns: str) -> bool:
+        """Become `ns`'s writer if its lock or lease is free."""
+        try:
+            self.engine.namespace(ns)
+        except NamespaceBusyError:
+            return False
+        if self.engine.holds(ns):
+            return True
+        # (a pinned store that lost its lease is still in the table: drop it
+        # and open the namespace again - or find it held)
+        self.engine.drop_lost(ns)
+        try:
+            self.engine.namespace(ns)
+        except NamespaceBusyError:
+            return False
+        return self.engine.holds(ns)
+
+    def _resolve(self, ns: str) -> Endpoint | None:
+        if self._take(ns):
+            return None
+        store = self.engine.store
+        reader = getattr(store, "read_owner", None)
+        if callable(reader):
+            info = reader(ns)
+            if info is None or not info.get("fresh"):
+                raise _Wait("its lease is changing hands")
+            holder = info.get("holder")
+        else:
+            holder = read_owner_lock(owner_lock_path(store.root, ns))
+            if not holder:
+                raise _Wait("its new writer is starting")
+        ep = parse_holder(holder)
+        if ep is None:
+            raise ForwardingError(
+                f"namespace {ns!r} is held by another process ({holder!r}) that does not accept "
+                "forwarded calls (forwarding off, a cluster node, or an older memd). "
+                "memd is single-writer per namespace: run one process per root, or turn forwarding "
+                "on there")
+        if _loopback(ep.host) and ep.hostname != self.hostname:
+            raise ForwardingError(
+                f"namespace {ns!r} is held by a process on {ep.hostname!r}, whose forwarding "
+                "endpoint listens on loopback only: set forward_host / forward_advertise there")
+        return ep
+
+    def _route(self, ns: str) -> Endpoint | None:
+        with self._lock:
+            ep = self._routes.get(ns)
+        if ep is not None:
+            return ep
+        ep = self._resolve(ns)
+        if ep is not None:
+            with self._lock:
+                self._routes[ns] = ep
+        return ep
+
+    def _forget(self, ns: str, ep: Endpoint | None) -> None:
+        with self._lock:
+            if ep is None or self._routes.get(ns) == ep:
+                self._routes.pop(ns, None)
+        if ep is not None:
+            self.client.drop(ep)
+
+    def _alive(self, ns: str, ep: Endpoint):
+        """S3: is `ep` still the namespace's (fresh) leaseholder? - checked
+        while a call waits for its answer, so a holder frozen past its TTL
+        is given up on (it is fenced when it resumes)."""
+        reader = getattr(self.engine.store, "read_owner", None)
+        if not callable(reader):
+            return None
+
+        def alive() -> bool:
+            info = reader(ns)
+            if info is None or not info.get("fresh"):
+                return False
+            got = parse_holder(info.get("holder"))
+            return got is not None and got.endpoint_id == ep.endpoint_id
+        return alive
+
+    def call(self, ns: str, op: str, args: dict, ids: list[str] | None, *, write: bool,
+             timeout_s: float | None = None):
+        """Run `op` on `ns`'s writer: its result, or LOCAL when this process
+        took the namespace (the caller runs it here, with the same `ids`).
+        Retries - same request id - until `wait_s` passes without a writer
+        answering; ForwardingError then (nothing applied), or
+        ForwardTimeoutError if a write's attempt may have been applied."""
+        uncertain: list[str] = []      # (a write's) attempts sent that got no answer
+        try:
+            return self._call(ns, op, args, ids, write=write, timeout_s=timeout_s, uncertain=uncertain)
+        except ForwardingError as ex:
+            if not uncertain:
+                raise
+            # "nothing was applied" would be false: an attempt may have been
+            METRICS.inc("memd_forward_calls_total", op=op, outcome="unknown", ns=ns)
+            raise ForwardTimeoutError(
+                f"namespace {ns!r}: an attempt of this write got no answer ({uncertain[-1]}) and "
+                f"no writer can run it again ({ex}); the earlier attempt may have been applied") from ex
+
+    def _call(self, ns: str, op: str, args: dict, ids: list[str] | None, *, write: bool,
+              timeout_s: float | None, uncertain: list[str]):
+        rid = uuid.uuid4().hex
+        t0 = time.monotonic()
+        deadline = t0 + self.cfg.wait_s
+        why = ""
+        attempt = 0
+        while True:
+            try:
+                ep = self._route(ns)
+            except _Wait as w:
+                ep, why = False, str(w)
+            if ep is None:
+                if attempt:
+                    METRICS.inc("memd_forward_takeovers_total", ns=ns,
+                                help="namespaces this process took over while forwarding to them")
+                return LOCAL
+            if ep is not False:
+                req = {"rid": rid, "op": op, "ns": ns, "args": args, "ids": ids}
+                try:
+                    reply = self.client.call(ep, req, timeout_s=timeout_s or self.cfg.timeout_s,
+                                             alive=self._alive(ns, ep), check_s=self.cfg.check_s)
+                except _Unreachable as ex:
+                    self._forget(ns, ep)
+                    why = str(ex)
+                except _Uncertain as ex:
+                    self._forget(ns, ep)
+                    why = str(ex)
+                    if write:
+                        uncertain.append(why)
+                else:
+                    if isinstance(reply, _StreamReader):
+                        return reply
+                    if reply.get("ok"):
+                        METRICS.inc("memd_forward_calls_total", op=op, outcome="ok", ns=ns,
+                                    help="calls forwarded to the namespace's writer in another process")
+                        METRICS.observe("memd_forward_call_ms", (time.monotonic() - t0) * 1000,
+                                        help="forwarded call latency, retries included (ms)", op=op)
+                        if write:
+                            with self._lock:
+                                self._written.add(ns)
+                        return reply.get("result")
+                    err = reply.get("error") or {}
+                    if not err.get("retry"):
+                        METRICS.inc("memd_forward_calls_total", op=op, outcome="error", ns=ns)
+                        _raise_remote(err)
+                    why = f"{err.get('code')}: {err.get('message', '')}"
+                    if err.get("code") != "overloaded":
+                        self._forget(ns, ep)
+            METRICS.inc("memd_forward_retries_total", op=op,
+                        help="forwarded call attempts that found no writer answering")
+            if time.monotonic() >= deadline:
+                METRICS.inc("memd_forward_calls_total", op=op, outcome="unavailable", ns=ns)
+                if uncertain:
+                    raise ForwardTimeoutError(
+                        f"namespace {ns!r}: no writer answered within {self.cfg.wait_s:.0f} s ({why}); "
+                        "an earlier attempt of this write may have been applied")
+                raise ForwardingError(
+                    f"namespace {ns!r} is held by another process and no writer answered within "
+                    f"{self.cfg.wait_s:.0f} s ({why}); nothing was applied")
+            time.sleep(min(0.5, 0.01 * (2 ** min(attempt, 6))) * (0.5 + random.random()))
+            attempt += 1
+
+    def probe(self, ns: str) -> bool:
+        """At open, for the facade's own namespace held elsewhere: True when
+        this process took it after all. Raises when forwarding to its holder
+        cannot work (refused credentials, a holder that does not forward);
+        one that cannot be reached right now is waited for by the calls."""
+        try:
+            ep = self._route(ns)
+        except _Wait:
+            return False
+        if ep is None:
+            return True
+        try:
+            reply = self.client.call(ep, {"rid": uuid.uuid4().hex, "op": "ping", "ns": ns,
+                                          "args": {}, "ids": None},
+                                     timeout_s=min(self.cfg.timeout_s, 10.0))
+        except (_Unreachable, _Uncertain):
+            self._forget(ns, ep)
+            return False
+        if isinstance(reply, dict) and not reply.get("ok"):
+            self._forget(ns, ep)
+        return False
+
+    def take_written(self) -> list[str]:
+        with self._lock:
+            out, self._written = sorted(self._written), set()
+        return out