fleetmesh-node 0.2.0__py3-none-any.whl

sidecar/fleetmesh_announce.py

@@ -0,0 +1,338 @@
+#!/usr/bin/env python3
+"""The libp2p half of the announce bridge, in its own process.
+
+    fleetmesh-announce DIR [--bootstrap MULTIADDR]... [--port N] [--pow-bits N]
+
+Joins the gossipsub topic `/fleetmesh/announce/1`, republishes the descriptor
+the node hands it, and hands back the descriptors it hears. One JSON object
+per line over a unix socket in the node's directory, the protocol
+`fleetmesh-announce/1`, the same shape `node/control.py` uses.
+
+Why a separate process. `fleetmesh_core/` is standard library only and `node/` depends on
+`cryptography` and nothing else. Installing the one maintained Python libp2p
+pulls in sixty-three packages, eight of them compiled; it will not install at
+all on Python 3.14 today; and on macOS it installs and then fails to import
+without a system GMP for `fastecdsa`. A second implementer reading the
+specification should inherit none of that to build a node, so it is here, and
+a node without it is a node with one discovery bridge instead of two.
+
+**This process holds the transport key and nothing else.** It reads
+`transport_seed` out of the node's `keys.json` and never touches `node_sk`.
+The specification says the transport key MAY rotate freely and that a
+verifier binds it to a DID only through a live descriptor, so a compromise
+here costs one descriptor republish. The node key, which never rotates,
+stays in the node.
+
+**What this validates, and why here.** A gossipsub topic is a flood channel
+with no grant in front of it, so the only thing standing between the mesh and
+descriptor spam is the validator that runs before propagation. 
+
+`node.announce.compartment.admit` is that validator, and it lives there rather than here
+because it needs nothing libp2p has: a size, a parse, a kind, a signature,
+`fleetmesh_core.identity.validate_descriptor`, a NIP-13 floor, and last the tie between
+the descriptor's `key transport` Multikey and the originator's PeerId. The
+node runs the same function again on what this process hands over, so a check
+that exists in two places cannot come to disagree with itself.
+
+The originator is `msg.from_id` and never the peer who forwarded the message.
+A forwarder does not re-sign under StrictSign, and a descriptor relayed by a
+third node is still valid on a flood topic.
+"""
+from __future__ import annotations
+
+import argparse
+import collections
+import json
+import os
+import stat
+import sys
+import time
+from typing import Any, Deque, Mapping, Optional, TypedDict
+
+import multiaddr
+import trio
+from libp2p import new_host
+from libp2p.crypto.ed25519 import create_new_key_pair
+from libp2p.custom_types import TProtocol
+from libp2p.peer.id import ID
+from libp2p.peer.peerinfo import info_from_p2p_addr
+from libp2p.pubsub.gossipsub import GossipSub
+from libp2p.pubsub.pubsub import Pubsub
+from libp2p.tools.anyio_service import background_trio_service
+
+_ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
+sys.path.insert(0, _ROOT)
+from node.planes.identity.announce.compartment import (KIND, MAX_LINE, PAYLOAD_MAX, PROTOCOL,      # noqa: E402
+                           TOPIC, admit, socket_path)
+
+GOSSIPSUB: TProtocol = TProtocol('/meshsub/1.0.0')
+# Descriptors held for the node to drain. The node ticks every thirty seconds
+# and a flood must not outgrow the interval, so the oldest goes and the loss
+# is counted rather than hidden.
+HEARD_MAX: int = 256
+# The control socket: a client that connects and does not finish a line
+# within this many seconds is dropped, so it cannot hold a task open.
+CONTROL_TIMEOUT: float = 10.0
+
+
+def transport_seed(directory: str) -> bytes:
+    """The Ed25519 seed out of the node's key file, and nothing else from it."""
+    path = os.path.join(directory, 'keys.json')
+    mode = stat.S_IMODE(os.stat(path).st_mode)
+    if mode & 0o077:
+        raise PermissionError('%s is readable by others (mode %o); refusing to load' % (path, mode))
+    with open(path, encoding='utf-8') as f:
+        d = json.load(f)
+    if d.get('format') != 'meshnode-keys/1':
+        raise ValueError('not a meshnode key file')
+    seed = bytes.fromhex(d['transport_seed'])
+    if len(seed) != 32:
+        raise ValueError('the transport seed is 32 bytes')
+    return seed
+
+
+AnnounceHeardRecord = TypedDict('AnnounceHeardRecord', {
+    'descriptor': Any,
+    'from': str,
+    'at': int,
+}, total=False)
+
+
+class AnnounceStatusReport(TypedDict, total=False):
+    """The report Announce.status() gives of itself."""
+    topic: str
+    peer_id: Optional[str]
+    mesh_peers: int
+    pow_bits: int
+    accepted: int
+    published: int
+    waiting: int
+    overflowed: int
+
+
+class Announce:
+    """The topic, what it carried, and what it refused."""
+
+    def __init__(self, pow_bits: int = 0, now: Any = None) -> None:
+        self.pow_bits: int = max(0, int(pow_bits))
+        self.now = now or (lambda: int(time.time()))
+        self.heard: Deque[AnnounceHeardRecord] = collections.deque(maxlen=HEARD_MAX)
+        self.dropped: collections.Counter[str] = collections.Counter()
+        self.accepted: int = 0
+        self.published: int = 0
+        self.overflowed: int = 0
+        self.pubsub: Optional[Pubsub] = None
+        self.host: Any = None
+
+    # ---- the validator, run before anything is propagated -------------------
+    def validate(self, msg_forwarder: ID, msg: Any) -> bool:
+        """gossipsub's pre-propagation hook. False stops the message dead.
+
+        `msg_forwarder` is the peer who handed it over and is deliberately
+        unused: under StrictSign the signature is the *originator's*, carried
+        in `msg.from_id`, and a forwarder does not re-sign. A descriptor
+        relayed by a third node is still valid on a flood topic, so the tie
+        to check is between the descriptor and whoever signed it.
+        """
+        origin = self._ed25519_of(ID(msg.from_id))
+        if origin is None:
+            self.dropped['originator is not an Ed25519 peer'] += 1
+            return False
+        desc, why = admit(msg.data, origin, self.now(), pow_bits=self.pow_bits,
+                          payload_max=PAYLOAD_MAX)
+        if desc is None:
+            self.dropped[str(why)] += 1
+            return False
+        if len(self.heard) == self.heard.maxlen:
+            self.overflowed += 1
+        self.heard.append({'descriptor': desc, 'from': origin, 'at': self.now()})
+        self.accepted += 1
+        return True
+
+    @staticmethod
+    def _ed25519_of(peer: ID) -> Optional[str]:
+        """The raw Ed25519 public key a PeerId carries, as hex.
+
+        An Ed25519 key is small enough that libp2p puts it in the PeerId
+        inline, under an identity multihash, so this needs no lookup. Anything
+        else - an RSA peer, whose id is a SHA-256 - returns None and is
+        refused, because fleetmesh nodes are Ed25519 by the key table.
+        """
+        try:
+            pub = peer.extract_public_key()
+        except Exception:
+            return None
+        if pub is None:
+            return None
+        raw = pub.to_bytes()
+        return raw.hex() if len(raw) == 32 else None
+
+    # ---- the socket ---------------------------------------------------------
+    async def serve(self, path: str) -> None:
+        # Refuse to take a socket another instance is answering on; a stale
+        # file left by a crash is unlinked.
+        if os.path.exists(path):
+            probe = trio.socket.socket(trio.socket.AF_UNIX, trio.socket.SOCK_STREAM)
+            try:
+                await probe.connect(path)
+            except OSError:
+                os.unlink(path)
+            else:
+                raise RuntimeError('another fleetmesh-announce is already serving %s' % path)
+            finally:
+                probe.close()
+        s = trio.socket.socket(trio.socket.AF_UNIX, trio.socket.SOCK_STREAM)
+        old = os.umask(0o177)
+        try:
+            await s.bind(path)
+            os.chmod(path, 0o600)
+        finally:
+            os.umask(old)
+        s.listen(8)
+        await trio.serve_listeners(self._one, [trio.SocketListener(s)])
+
+    async def _one(self, stream: trio.SocketStream) -> None:
+        try:
+            buf = b''
+            with trio.move_on_after(CONTROL_TIMEOUT):
+                while not buf.endswith(b'\n') and len(buf) <= MAX_LINE:
+                    chunk = await stream.receive_some(65536)
+                    if not chunk:
+                        break
+                    buf += chunk
+            try:
+                if not buf.endswith(b'\n') or len(buf) > MAX_LINE:
+                    raise ValueError('a request is one JSON line under %d bytes' % MAX_LINE)
+                req = json.loads(buf.decode('utf-8'))
+                if not isinstance(req, dict) or not isinstance(req.get('cmd'), str):
+                    raise ValueError('a request is {"cmd": ..., "args": {...}}')
+                args = req.get('args') or {}
+                if not isinstance(args, dict):
+                    raise ValueError('args is an object')
+                result = await self.dispatch(req['cmd'], args)
+                reply = {'ok': True, 'protocol': PROTOCOL, 'result': result}
+            except Exception as e:                  # one bad request answers one request
+                reply = {'ok': False, 'protocol': PROTOCOL, 'error': '%s: %s' % (type(e).__name__, e)}
+            # Everything status()/drain() return is str/int/dict already;
+            # no `default=str`, which would let a stray object type leak
+            # through as its repr and be parsed by the node as a value.
+            await stream.send_all((json.dumps(reply) + '\n').encode('utf-8'))
+        except (trio.BrokenResourceError, trio.ClosedResourceError, OSError):
+            pass
+        finally:
+            await trio.aclose_forcefully(stream)
+
+    async def dispatch(self, cmd: str, args: Mapping[str, Any]) -> Any:
+        if cmd == 'publish':
+            desc = args.get('descriptor')
+            if not isinstance(desc, dict) or desc.get('kind') != KIND:
+                raise ValueError('publish takes one kind %d event' % KIND)
+            if self.pubsub is None:
+                raise RuntimeError('the topic is not joined yet')
+            body = json.dumps(desc, separators=(',', ':'), sort_keys=True).encode('utf-8')
+            if len(body) > PAYLOAD_MAX:
+                raise ValueError('a descriptor of %d bytes is over the %d the topic carries'
+                                 % (len(body), PAYLOAD_MAX))
+            await self.pubsub.publish(TOPIC, body)
+            self.published += 1
+            return {'published': self.published, 'bytes': len(body)}
+        if cmd == 'drain':
+            try:
+                want = min(int(args.get('max') or HEARD_MAX), HEARD_MAX)
+            except (TypeError, ValueError):
+                want = HEARD_MAX
+            out = [self.heard.popleft() for _ in range(min(want, len(self.heard)))]
+            dropped = dict(self.dropped)
+            self.dropped.clear()
+            return dict(self.status(), heard=out, dropped=dropped)
+        if cmd == 'status':
+            return self.status()
+        raise KeyError(cmd)
+
+    def status(self) -> AnnounceStatusReport:
+        peers: list[str] = []
+        if self.pubsub is not None:
+            peers = [str(p) for p in self.pubsub.peer_topics.get(TOPIC, [])]
+        return {
+            'topic': TOPIC,
+            'peer_id': str(self.host.get_id()) if self.host is not None else None,
+            'mesh_peers': len(peers),
+            'pow_bits': self.pow_bits,
+            'accepted': self.accepted,
+            'published': self.published,
+            'waiting': len(self.heard),
+            'overflowed': self.overflowed,
+        }
+
+
+async def run(directory: str, bootstrap: list[str], port: int, pow_bits: int,
+              path: Optional[str] = None) -> None:
+    key_pair = create_new_key_pair(transport_seed(directory))
+    host = new_host(key_pair=key_pair)
+    gossipsub = GossipSub(protocols=[GOSSIPSUB], degree=6, degree_low=4, degree_high=12)
+    pubsub = Pubsub(host, gossipsub)                 # strict_signing defaults to True
+    state = Announce(pow_bits=pow_bits)
+    state.host, state.pubsub = host, pubsub
+    sock = path or socket_path(directory)
+    listen = [multiaddr.Multiaddr('/ip4/0.0.0.0/tcp/%d' % port)]
+
+    async with host.run(listen_addrs=listen), trio.open_nursery() as nursery:
+        async with background_trio_service(pubsub), background_trio_service(gossipsub):
+            await pubsub.wait_until_ready()
+            pubsub.set_topic_validator(TOPIC, state.validate, False)
+            await pubsub.subscribe(TOPIC)
+            print('%s  peer %s' % (TOPIC, host.get_id()), flush=True)
+            for a in host.get_addrs():
+                print('  listening %s' % a, flush=True)
+            for addr in bootstrap:
+                try:
+                    await host.connect(info_from_p2p_addr(multiaddr.Multiaddr(addr)))
+                    print('  bootstrap %s' % addr, flush=True)
+                except Exception as e:
+                    # A bootstrap address that does not answer is ordinary. The
+                    # topic works with one peer and the relay bridge works with
+                    # none, so this is never fatal.
+                    print('  bootstrap %s did not answer (%s)' % (addr, e), flush=True)
+            nursery.start_soon(state.serve, sock)
+            print('  socket %s' % sock, flush=True)
+            await trio.sleep_forever()
+
+
+def _leaves(e: BaseException) -> list[BaseException]:
+    """The real exceptions under however many ExceptionGroups trio nested."""
+    subs = getattr(e, 'exceptions', None)
+    if not subs:
+        return [e]
+    out: list[BaseException] = []
+    for s in subs:
+        out.extend(_leaves(s))
+    return out
+
+
+def main(argv: Optional[list[str]] = None) -> int:
+    ap = argparse.ArgumentParser(prog='fleetmesh-announce', description=__doc__.splitlines()[0])
+    ap.add_argument('directory', help="the node directory, for keys.json and the socket")
+    ap.add_argument('--bootstrap', action='append', default=[],
+                    metavar='MULTIADDR', help='a libp2p peer to dial on start; repeatable')
+    ap.add_argument('--port', type=int, default=0, help='tcp port to listen on (0 asks the kernel)')
+    ap.add_argument('--pow-bits', type=int, default=0,
+                    help='refuse a descriptor below this many bits of NIP-13 work. Local policy: '
+                         'two nodes with different floors interoperate, the stricter propagates less')
+    ap.add_argument('--socket', default=None, help='where to listen (default: announce.sock in the directory)')
+    a = ap.parse_args(argv)
+    try:
+        trio.run(run, a.directory, a.bootstrap, a.port, a.pow_bits, a.socket)
+    except KeyboardInterrupt:
+        return 0
+    except Exception as e:
+        # A refusal at startup - a live socket, a loose keys.json, a missing
+        # key file - is one line, not a nursery traceback. trio wraps what a
+        # nursery task raised in an ExceptionGroup; unwrap for the message.
+        for inner in _leaves(e):
+            print('fleetmesh-announce: %s: %s' % (type(inner).__name__, inner), file=sys.stderr, flush=True)
+        return 1
+    return 0
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())