fleetmesh-node 0.2.0__py3-none-any.whl

node/transport/netfetch.py

@@ -0,0 +1,283 @@
+"""netfetch: the one HTTP fetch helper every outbound request in node/ goes through.
+
+Why this exists. The node used to call urllib.request.urlopen() directly from
+half a dozen places -- compute.resolve_blob (URL chosen by the job requester),
+the Blossom /mirror handler (URL chosen by whoever holds an auth event),
+BlossomClient (peer servers), the Cashu mint client and the watcher. Each site
+re-invented the same three mistakes:
+
+  1. no scheme check, so ``file:///etc/passwd`` or ``ftp://`` were accepted by
+     urllib's default opener;
+  2. no address check, so a requester could point the node at
+     ``http://127.0.0.1:<port>/`` (its own control face) or at the cloud
+     metadata endpoint ``169.254.169.254`` -- classic SSRF;
+  3. ``resp.read()`` with no cap, so a hostile server could hand back gigabytes
+     and exhaust memory.
+
+This module fixes them once. ``fetch()`` allows only http/https, resolves the
+host and refuses any address that is loopback, link-local, private (RFC 1918
+and ULA), multicast, reserved or unspecified -- including IPv4-mapped IPv6
+forms -- unless the caller explicitly opts in with ``allow_private=True``
+(appropriate only for operator-configured URLs such as a local mint or a LAN
+CI API, never for requester-supplied ones). Redirects are re-checked against
+the same policy, so a public host cannot bounce us into the LAN. Bodies are
+read to ``max_bytes + 1`` and anything beyond raises ``FetchTooLarge``; a
+Content-Length larger than the cap is refused before a single body byte is
+read.
+
+Note on naming: this module is deliberately NOT called ``http.py``. Scripts
+run from inside ``node/`` (``python3 node/demo.py``) put ``node/`` first on
+``sys.path``, and a ``node/http.py`` would shadow the stdlib ``http`` package
+that ``http.server``, ``urllib`` and ``ws`` depend on.
+
+Note on DNS rebinding: the address check resolves the hostname once and then
+lets urllib resolve it again for the actual connection. A malicious resolver
+that answers differently on the second lookup could still reach a private
+address. Closing that hole properly means connecting to the vetted IP and
+sending the hostname in Host/SNI ourselves; that is more surgery than a
+stdlib opener allows cleanly, so it is recorded here rather than pretended
+away. Requester-supplied URLs are additionally gated by SHA-256 verification
+at every call site, which limits what a rebinding attacker can learn.
+"""
+from __future__ import annotations
+
+import http.client
+import ipaddress
+import socket
+import urllib.error
+import urllib.parse
+import urllib.request
+from typing import Any, Mapping, Optional, Sequence
+
+DEFAULT_TIMEOUT: float = 10.0
+USER_AGENT: str = 'fleetmesh-node/1.0'
+MAX_REDIRECTS: int = 5
+
+
+class FetchError(Exception):
+    """Base class: anything that stops fetch() returning a body."""
+
+
+class FetchRefused(FetchError):
+    """The URL was refused by policy (scheme, private address, redirect target)."""
+
+
+class FetchTooLarge(FetchError):
+    """The body (declared or actual) exceeds max_bytes."""
+
+
+class FetchHTTPError(FetchError):
+    """The server answered with a non-2xx status. ``code`` and ``body`` are kept
+    so callers that care (404 means 'not there', mint error JSON) can inspect
+    them without the raw urllib exception leaking through."""
+
+    def __init__(self, code: int, reason: str, body: bytes = b'', headers: Optional[Mapping[str, str]] = None) -> None:
+        super().__init__('HTTP %d: %s' % (code, reason))
+        self.code = code
+        self.reason = reason
+        self.body = body
+        self.headers = dict(headers or {})
+
+
+class FetchNetworkError(FetchError):
+    """DNS, connect, TLS, timeout or protocol failure."""
+
+
+def _is_forbidden_address(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
+    """True for any address a requester must not be able to make the node talk to.
+
+    IPv4-mapped IPv6 (``::ffff:127.0.0.1``) is unwrapped first so the IPv4
+    policy applies; otherwise ``ipaddress`` classifies it only by the IPv6
+    rules and, e.g., ``::ffff:10.0.0.1`` slips through as 'global'.
+    """
+    if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
+        ip = ip.ipv4_mapped
+    return (
+        ip.is_loopback
+        or ip.is_link_local
+        or ip.is_private
+        or ip.is_multicast
+        or ip.is_unspecified
+        or ip.is_reserved
+        or not ip.is_global
+    )
+
+
+def _private_exempt(host: str, addrs: Sequence[str], allow_private: Any) -> bool:
+    """Is this destination exempt from the address policy?
+
+    ``allow_private`` is either a bool (True: anything goes -- operator URLs
+    only) or a collection of hostnames / IP literals. The collection form
+    exempts exactly those destinations and nothing else, so an operator who
+    runs a mint on 127.0.0.1 does not thereby open the whole LAN, and a
+    redirect from the exempt host to some other private address is still
+    refused.
+    """
+    if allow_private is True:
+        return True
+    if not allow_private:
+        return False
+    exempt = {str(h).lower() for h in allow_private}
+    if host.lower() in exempt:
+        return True
+    return bool(addrs) and all(a in exempt for a in addrs)
+
+
+def check_url(url: str, *, allow_private: Any = False) -> urllib.parse.SplitResult:
+    """Validate scheme and destination address; return the split URL.
+
+    Raises FetchRefused for anything that fails policy. Resolution failures
+    are FetchNetworkError: a name that does not resolve is not an attack,
+    just an unreachable host, and callers treat it like a connection error.
+    """
+    try:
+        parts = urllib.parse.urlsplit(str(url))
+    except ValueError as e:
+        raise FetchRefused('malformed URL: %s' % e)
+    if parts.scheme not in ('http', 'https'):
+        raise FetchRefused('scheme %r not allowed (http/https only)' % parts.scheme)
+    host = parts.hostname
+    if not host:
+        raise FetchRefused('URL has no host')
+    if allow_private is True:
+        return parts
+    try:
+        port = parts.port or (443 if parts.scheme == 'https' else 80)
+    except ValueError as e:
+        raise FetchRefused('bad port: %s' % e)
+    # A literal address needs no DNS; check it directly (also catches
+    # bracketed IPv6 literals, which urlsplit already unbrackets).
+    try:
+        literal = ipaddress.ip_address(host)
+    except ValueError:
+        literal = None
+    if literal is not None:
+        if _is_forbidden_address(literal) and not _private_exempt(host, [host], allow_private):
+            raise FetchRefused('address %s is not a public address' % host)
+        return parts
+    try:
+        infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
+    except socket.gaierror as e:
+        raise FetchNetworkError('cannot resolve %s: %s' % (host, e))
+    if not infos:
+        raise FetchNetworkError('cannot resolve %s' % host)
+    addrs = [str(info[4][0]).split('%', 1)[0] for info in infos]
+    if _private_exempt(host, addrs, allow_private):
+        return parts
+    # Every answer must pass: a name with one public and one private A record
+    # would otherwise be a coin-flip on which the socket layer picks.
+    for addr in addrs:
+        try:
+            ip = ipaddress.ip_address(addr)
+        except ValueError:
+            raise FetchRefused('unparseable resolved address %r for %s' % (addr, host))
+        if _is_forbidden_address(ip):
+            raise FetchRefused('%s resolves to non-public address %s' % (host, addr))
+    return parts
+
+
+class _CheckedRedirectHandler(urllib.request.HTTPRedirectHandler):
+    """Re-run check_url() on every redirect target.
+
+    urllib follows 3xx by default and would happily hop from a vetted public
+    host to ``http://127.0.0.1/``; the first request's check says nothing
+    about where the chain ends.
+    """
+
+    max_redirections = MAX_REDIRECTS
+
+    def __init__(self, allow_private: bool) -> None:
+        super().__init__()
+        self.allow_private = allow_private
+
+    def http_error_302(self, req: Any, fp: Any, code: int, msg: str, headers: Any) -> Any:
+        # Hook the status handler rather than redirect_request(): the stdlib
+        # runs its own scheme check (raising a plain HTTPError) *before*
+        # redirect_request is consulted, and we want our FetchRefused to be
+        # the answer for every disallowed target, not just private hosts.
+        loc = headers.get('location') or headers.get('uri')
+        if loc:
+            newurl = urllib.parse.urljoin(req.full_url, loc.split(' ', 1)[0].strip())
+            check_url(newurl, allow_private=self.allow_private)
+        return super().http_error_302(req, fp, code, msg, headers)
+
+    http_error_301 = http_error_303 = http_error_307 = http_error_308 = http_error_302
+
+
+def fetch(
+    url: str,
+    *,
+    max_bytes: int,
+    timeout: float = DEFAULT_TIMEOUT,
+    allow_private: Any = False,
+    method: str = 'GET',
+    data: Optional[bytes] = None,
+    headers: Optional[Mapping[str, str]] = None,
+) -> tuple[bytes, dict[str, str]]:
+    """Fetch ``url`` under the policy described in the module docstring.
+
+    Returns ``(body, headers)`` where headers is a plain dict with the
+    original header names. ``max_bytes`` is mandatory on purpose: every
+    caller must state how much it is prepared to hold in memory.
+
+    ``allow_private`` is False (full policy), True (no address policy at
+    all: operator-configured URLs only) or a collection of hostnames / IP
+    literals that are exempt while everything else -- including redirect
+    targets -- still has to be public.
+    """
+    if max_bytes is None or int(max_bytes) < 0:
+        raise ValueError('max_bytes must be a non-negative int')
+    max_bytes = int(max_bytes)
+    check_url(url, allow_private=allow_private)
+
+    hdrs = {'User-Agent': USER_AGENT}
+    if headers:
+        hdrs.update(headers)
+    req = urllib.request.Request(url, data=data, headers=hdrs, method=method.upper())
+    opener = urllib.request.build_opener(_CheckedRedirectHandler(allow_private))
+    try:
+        resp = opener.open(req, timeout=timeout)
+    except FetchError:
+        # Raised from inside the redirect handler; urllib does not wrap it.
+        raise
+    except urllib.error.HTTPError as e:
+        # Bound the error body too; an error page can be as hostile as a 200.
+        try:
+            body = e.read(max_bytes + 1)
+        except Exception:
+            body = b''
+        finally:
+            e.close()      # the error response holds the socket until it is closed
+        if len(body) > max_bytes:
+            raise FetchTooLarge('error body exceeds %d bytes' % max_bytes)
+        raise FetchHTTPError(e.code, str(e.reason), body, dict(e.headers.items()) if e.headers else {})
+    except urllib.error.URLError as e:
+        # urllib wraps the redirect handler's refusal in URLError in some
+        # code paths; unwrap so callers see the policy error, not 'network'.
+        if isinstance(e.reason, FetchError):
+            raise e.reason
+        raise FetchNetworkError(str(e.reason))
+    except (socket.timeout, TimeoutError) as e:
+        raise FetchNetworkError('timeout: %s' % e)
+    except (http.client.HTTPException, OSError, ValueError) as e:
+        raise FetchNetworkError('%s: %s' % (type(e).__name__, e))
+
+    with resp:
+        resp_headers = dict(resp.headers.items())
+        clen = resp.headers.get('Content-Length')
+        # A HEAD response's Content-Length describes the body a GET *would*
+        # carry, not bytes on this connection, so the pre-check only applies
+        # to methods that actually return a body.
+        if clen is not None and method.upper() != 'HEAD':
+            try:
+                if int(clen) > max_bytes:
+                    raise FetchTooLarge('Content-Length %s exceeds %d bytes' % (clen, max_bytes))
+            except ValueError:
+                pass  # garbage Content-Length: fall through to the read cap
+        try:
+            body = resp.read(max_bytes + 1)
+        except (http.client.HTTPException, OSError) as e:
+            raise FetchNetworkError('read failed: %s: %s' % (type(e).__name__, e))
+    if len(body) > max_bytes:
+        raise FetchTooLarge('body exceeds %d bytes' % max_bytes)
+    return body, resp_headers

node/transport/relay_rung/__init__.py

@@ -0,0 +1,23 @@
+"""The relay rung: DIDComm JWEs in NIP-59 gift wraps over nostr relays.
+
+One rung of the transport ladder (router.py decides the order; this package
+never does). It is the rung that always exists, so the router delegates its
+inbox, thread waiters and acknowledgment ledger to it. Everything that knows
+a relay, a wrap kind or a subscription filter lives here and nowhere else in
+the node:
+
+    core.py      RelayRung - the object, its state, and how it is built
+    wire.py      wrap/fragment policy: which kind, which created_at, how big
+    outbound.py  send, request/converse, thread waiters, where a stored wrap goes
+    listen.py    one subscription per relay, reconnects, readiness
+    inbound.py   unwrap, dedup, reassemble, open the envelope, deliver
+    ledger.py    please_ack / ack: what was sent and is still owed an answer
+    shape.py     the Protocol the mixins share
+
+`NostrTransport` is the historical name of RelayRung and stays importable.
+"""
+from .core import RelayRung
+
+NostrTransport = RelayRung
+
+__all__ = ['RelayRung', 'NostrTransport']

node/transport/relay_rung/cooldown.py

@@ -0,0 +1,87 @@
+"""Per-relay cooldown after a relay says it is being asked too much.
+
+A public relay answers an event it will not take with an OK false whose
+reason is prefixed as NIP-01 suggests: `rate-limited:` and `banned:` are the
+two that mean "stop sending for a while", and a handshake refused with HTTP
+429 means the same before a word is spoken - as does a 503, which is what a
+relay that has banned the address answers once it stops talking at all. Publishing on regardless turns a
+minute of rate-limiting into a ban, and three nodes behind one address did
+exactly that within a minute of starting. The rung therefore holds such a
+relay out of publishing for a window that doubles on each repeat, keeps
+listening on it, and records the one fault that matters - that the relay
+was cooled, until when, and why - instead of one fault per refused wrap.
+
+Pure bookkeeping over an injected clock; no I/O, so a test drives it with
+reasons alone.
+"""
+from __future__ import annotations
+
+import re
+import threading
+from typing import Callable, Optional, TypedDict
+
+THROTTLE_PREFIXES: tuple[str, ...] = ('rate-limited', 'banned')
+_HTTP_429 = re.compile(r'\bHTTP/\d(?:\.\d)? (?:429|503)\b')   # 429 and 503 at the handshake
+
+
+class Cooling(TypedDict):
+    """A relay held out of publishing: until when, why, and how many times running."""
+    until: int
+    why: str
+    strikes: int
+
+
+def is_throttle(reason: str) -> bool:
+    """Whether an OK reason or connection error means 'you, specifically, stop for now'."""
+    head = reason.split(':', 1)[0].strip().lower()
+    return head in THROTTLE_PREFIXES or bool(_HTTP_429.search(reason))
+
+
+class RelayCooldown:
+    """Which relays are held, for how long, and the doubling that decides it."""
+
+    def __init__(self, clock: Callable[[], int], first: int = 60, longest: int = 900) -> None:
+        self.clock: Callable[[], int] = clock
+        self.first: int = first              # the first window, seconds
+        self.longest: int = longest          # the cap the doubling stops at
+        self._held: dict[str, Cooling] = {}
+        self._lock: threading.Lock = threading.Lock()
+
+    def refused(self, url: str, reason: str) -> Optional[Cooling]:
+        """Note a refusal. Returns the Cooling it started when the reason was a
+        throttle and the relay was not already held; None otherwise."""
+        if not is_throttle(reason):
+            return None
+        now = self.clock()
+        with self._lock:
+            held = self._held.get(url)
+            if held is not None and held['until'] > now:
+                return None                       # already cooling; the refusal is the old one echoing
+            strikes = (held['strikes'] + 1) if held is not None else 1
+            window = min(self.longest, self.first * (2 ** (strikes - 1)))
+            entry = Cooling(until=now + window, why=reason, strikes=strikes)
+            self._held[url] = entry
+            return entry
+
+    def accepted(self, url: str) -> None:
+        """An accepted publish after a cooling ended: the next strike starts over."""
+        with self._lock:
+            held = self._held.get(url)
+            if held is not None and held['until'] <= self.clock():
+                del self._held[url]
+
+    def holding(self, url: str) -> Optional[Cooling]:
+        """The Cooling on a relay if it is still in force."""
+        with self._lock:
+            held = self._held.get(url)
+        return held if held is not None and held['until'] > self.clock() else None
+
+    def open(self, urls: list[str]) -> list[str]:
+        """The relays in `urls` not currently held, in order."""
+        return [u for u in urls if self.holding(u) is None]
+
+    def state(self) -> dict[str, Cooling]:
+        """Every relay held right now."""
+        now = self.clock()
+        with self._lock:
+            return {u: dict(c) for u, c in self._held.items() if c['until'] > now}  # type: ignore[misc]

node/transport/relay_rung/core.py

@@ -0,0 +1,138 @@
+"""RelayRung: the relay rung's state and construction. Behaviour is in the
+sibling mixins; this file is what an operator or a test instantiates.
+"""
+from __future__ import annotations
+
+import collections
+import queue
+import threading
+import time
+from typing import Any, Callable, Counter, Deque, Mapping, Optional, Sequence
+
+from ...interfaces import NostrEvent, RelayHealthRow
+from .. import const
+from ..ladder import Fault, FaultLog
+from .cooldown import RelayCooldown
+from ..const import Inbound
+from . import wire
+from .types import AckVerdict, AwaitingEntry, Partial
+from .inbound import InboundMixin
+from .ledger import LedgerMixin
+from .listen import ListenMixin
+from .outbound import OutboundMixin
+
+
+class RelayRung(OutboundMixin, ListenMixin, InboundMixin, LedgerMixin):
+    scheme: str = 'nostr'
+
+    def __init__(
+        self,
+        keystore: Any,
+        peers: Any,
+        relay_urls: Sequence[str],
+        clock: Optional[Callable[[], int]] = None,
+        timeout: float = 15.0,
+        seen_store: Optional[tuple[Callable[[str], bool], Callable[[str, int], None]]] = None,
+        ack_store: Optional[Any] = None,
+        seen_max: int = const.SEEN_MAX,
+    ) -> None:
+        self.ks: Any = keystore
+        self.seen_max: int = seen_max           # wrap ids remembered for dedup before the oldest is forgotten
+        self.peers: Any = peers
+        # Stored messages sent and not yet acknowledged, by message id. On
+        # disk when the node backs it with its accounts (put_awaiting /
+        # drop_awaiting / awaiting), because the peer may be offline longer
+        # than this process lives and the question survives a restart.
+        self.ack_store: Optional[Any] = ack_store
+        self.awaiting: dict[str, AwaitingEntry] = {}
+        if ack_store is not None:
+            for mid, peer, mtype, at, relays in ack_store.awaiting():
+                self.awaiting[mid] = AwaitingEntry(peer=peer, type=mtype, at=at, relays=relays, heard_at=None)
+        self.heard: dict[str, int] = {}                  # peer -> when it was last heard from
+        self.acked: Counter[str] = collections.Counter()   # type -> stored messages acknowledged
+        self.acks_in: int = 0
+        self.acks_out: int = 0
+        self.unacked: Deque[AckVerdict] = collections.deque(maxlen=const.DROPPED_KEPT)   # verdicts, newest last
+        self._unacked_new: list[AckVerdict] = []      # verdicts not yet taken by the compartment
+        self._acks: queue.Queue[tuple[str, str]] = queue.Queue()   # (message id, peer) to acknowledge
+        # Stored wraps (kind 1059) are replayed by relays on every reconnect,
+        # for two days; a node that forgets what it opened across a restart
+        # would act on an old terminate after re-peering. The store is a
+        # pair (has(wid), add(wid, at)) the node backs with its accounts.
+        self.seen_store: Optional[tuple[Callable[[str], bool], Callable[[str, int], None]]] = seen_store
+        self.wraps_in: Counter[Optional[int]] = collections.Counter()          # inbound wraps by kind
+        self.wraps_seen_again: Counter[Optional[int]] = collections.Counter()  # and those a dedup discarded
+        self.relays: list[str] = list(relay_urls)
+        self.clock: Callable[[], int] = clock or (lambda: int(time.time()))
+        # peer -> (its kind 10050 relays, when read). Read once an hour, not
+        # per message: a list that changed under us costs one hour of wraps
+        # going to the old inbox, which the relay replay horizon covers.
+        self._dm_relays: dict[str, tuple[list[str], int]] = {}
+        self.timeout: float = timeout
+        self.inbox: queue.Queue[Inbound] = queue.Queue(maxsize=const.INBOX_MAX)
+        self._waiting: dict[str, queue.Queue[Inbound]] = {}          # thid -> Queue
+        self._seen: set[str] = set()                                 # wrap ids, so two relays deliver once
+        self._seen_order: Deque[str] = collections.deque()           # the same ids, oldest first, for the cap
+        self._partials: dict[tuple[str, str], Partial] = {}   # (sender, fragment id) -> pieces of a large envelope
+        self._lock: threading.Lock = threading.Lock()
+        self._stop: threading.Event = threading.Event()
+        self._threads: list[threading.Thread] = []
+        # What this rung refused or lost, with reasons; the ladder reads it
+        # through faults, Link exports it, and `meshctl watch` reds on it.
+        self.faults: FaultLog = FaultLog('nostr', clock=self.clock, keep=const.DROPPED_KEPT)
+        # Relays that asked to be left alone for a while; see cooldown.py.
+        self.cooldown: RelayCooldown = RelayCooldown(self.clock)
+        # One entry per relay, for the operator: whether the subscription is up,
+        # when it last delivered, how often it has had to reconnect and why.
+        self.health: dict[str, RelayHealthRow] = {
+            url: {
+                'connected': False,
+                # Whether the relay ANSWERED the subscription, which is not the
+                # same as having sent it. A kind 21059 is ephemeral: a wrap a
+                # peer sends before the REQ is registered is fanned out to
+                # nobody and no relay kept a copy, so a node that believes it
+                # is listening one round trip early loses exactly the live
+                # control traffic it cannot ask for again.
+                'confirmed': False,
+                'since': None,
+                'last_event_at': None,
+                'reconnects': 0,
+                'last_error': None,
+            }
+            for url in self.relays
+        }
+
+
+    def unfinished(self) -> int:
+        """Envelopes half-arrived: fragments held with pieces still missing.
+
+        A number the operator reads, so it is asked for rather than reached in
+        for. The compartment that reports it has no business in `_partials`.
+        """
+        return len(self._partials)
+
+    def _sign_auth(self, relay_url: str, challenge: str) -> NostrEvent:
+        """Sign a NIP-42 kind 22242 authentication event using the node's key."""
+        return wire.auth_event(self.ks.node_sk, int(self.clock()), relay_url, challenge)
+
+    def _drop(self, reason: str, wrap_id: Optional[str]) -> None:
+        self.faults.record(reason, wrap_id)
+
+    def recent_faults(self, since: Optional[int] = None) -> list[Fault]:
+        """What this rung refused or lost, newest last, with reasons."""
+        return self.faults.recent(since)
+
+    @property
+    def drop_counts(self) -> Counter[str]:
+        """Fault family -> how many, ever."""
+        return self.faults.counts
+
+    @property
+    def dropped(self) -> Deque[tuple[str, Optional[str]]]:
+        """(reason, wrap_id) of the most recent faults, oldest first."""
+        return collections.deque(((f['reason'], f['ident']) for f in self.faults.recent()), maxlen=self.faults.maxlen)
+
+
+    @staticmethod
+    def _fragments(jwe: Mapping[str, Any]) -> list[Any]:
+        return wire.fragments(jwe)