aics-cli 0.1.0__py3-none-any.whl

aics_cli/consistency.py

@@ -0,0 +1,280 @@
+"""Proving that one signed checkpoint is a prefix of another
+(docs/spec/AICS-1.md §14.2, §14.6; RFC 9162 §2.1.4).
+
+A proof may come from a proof file, the log's consistency endpoint
+(``GET /v2/tlog/consistency?from=<m>&to=<n>`` -> ``{from,to,hashes}``), or
+tiles.  Any source will do because the proof is checked against both
+log-signed roots; a source that serves bad data is reported and the next one
+is tried.
+
+Tiles are read through ``aics_tlog.VerifiedTileReader``, which authenticates
+every tile up to the signed root of a *reference* checkpoint R published by
+that source (normally the newer of the two notes).  Each note X is then
+tested as a prefix of R: the consistency proof and the root at X's size come
+from authenticated tiles, so if X's signed root differs from that
+authenticated prefix root, X and R are two log-signed, mutually inconsistent
+heads - constructive split-view evidence (§14.6 step 4) that anyone can
+re-check from the proof.
+
+Outcomes (:func:`relate`):
+
+* ``consistent`` - a valid proof (or equal size and root, or the empty
+  tree with the empty root);
+* ``split-proven`` - equal sizes with different roots, an authenticated
+  prefix root that differs from a signed root, or a size-0 checkpoint whose
+  root is not SHA-256("") (``bad-empty-root``, §14.2 step 1);
+* ``unproven`` - no valid proof was obtained.  A timeout, a missing tile or a
+  bad proof does not show that no valid proof exists (§14.6 step 3): this is
+  an availability alert, not evidence of a split.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from aics_tlog import EMPTY_HASH, VerifiedTileReader, verify_consistency
+from aics_tlog.tiles import TileVerificationError, consistency_proof_from_tiles, tree_root_from_tiles
+
+from .checkpoints import Cp
+from .encoding import b64d_strict, b64e
+from .errors import FetchError
+from .net import Fetcher
+from .tiles_http import TileSourceError
+
+PROOF_MAX_BYTES = 256 * 1024
+MAX_PROOF_HASHES = 128
+
+
+# ---------------------------------------------------------------------------
+# sources
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class TileSource:
+    label: str
+    reader: Any  # an aics_tlog TileReader (raw, unauthenticated)
+    ref: Cp | None = None  # checkpoint the source publishes tiles for (default: the newer note)
+
+
+@dataclass
+class EndpointSource:
+    label: str
+    template: str  # URL with {old} and {new}
+    fetcher: Fetcher
+
+
+@dataclass
+class GivenProof:
+    label: str
+    hashes: list[bytes]
+
+
+Source = TileSource | EndpointSource | GivenProof
+
+
+@dataclass
+class Attempt:
+    source: str
+    status: str  # consistent | split | invalid | incomplete
+    detail: str = ""
+    old: Cp | None = None
+    new: Cp | None = None
+    proof: list[bytes] | None = None
+    prefix_root: bytes | None = None  # authenticated root of `new` at old's size
+
+    def report(self) -> dict[str, Any]:
+        d: dict[str, Any] = {"source": self.source, "status": self.status}
+        if self.detail:
+            d["detail"] = self.detail
+        if self.old is not None and self.new is not None:
+            d["old_size"] = self.old.size
+            d["new_size"] = self.new.size
+        if self.proof is not None:
+            d["proof"] = [b64e(h) for h in self.proof]
+        if self.prefix_root is not None:
+            d["authenticated_prefix_root"] = b64e(self.prefix_root)
+        return d
+
+
+@dataclass
+class Relation:
+    status: str  # consistent | split-proven | unproven
+    a: Cp
+    b: Cp
+    reason: str | None = None
+    attempts: list[Attempt] = field(default_factory=list)
+    winning: Attempt | None = None
+    split_pairs: list[tuple[Cp, Cp, Attempt | None]] = field(default_factory=list)
+
+    def report(self) -> dict[str, Any]:
+        d: dict[str, Any] = {
+            "status": self.status,
+            "a": self.a.brief(),
+            "b": self.b.brief(),
+            "attempts": [x.report() for x in self.attempts],
+        }
+        if self.reason:
+            d["reason"] = self.reason
+        if self.winning is not None:
+            d["proof_source"] = self.winning.source
+        return d
+
+
+# ---------------------------------------------------------------------------
+# endpoint
+# ---------------------------------------------------------------------------
+
+
+def parse_proof_body(body: bytes, old: int, new: int) -> list[bytes]:
+    """``{"from": m, "to": n, "hashes": [B64(hash), ...]}`` (AICS-1 §12.1),
+    with canonical padded base64 and exactly 32-byte hashes."""
+    from .card import loads_strict
+
+    obj = loads_strict(body)
+    if not isinstance(obj, dict) or set(obj) != {"from", "to", "hashes"}:
+        raise ValueError("proof must be exactly {from, to, hashes}")
+    if obj["from"] != old or obj["to"] != new or isinstance(obj["from"], bool) or isinstance(obj["to"], bool):
+        raise ValueError(f"proof is for {obj['from']!r}->{obj['to']!r}, not {old}->{new}")
+    hashes = obj["hashes"]
+    if not isinstance(hashes, list) or len(hashes) > MAX_PROOF_HASHES:
+        raise ValueError("hashes must be a list of at most 128 entries")
+    out = []
+    for h in hashes:
+        raw = b64d_strict(h) if isinstance(h, str) else b""
+        if len(raw) != 32:
+            raise ValueError("each proof hash is canonical base64 of 32 bytes")
+        out.append(raw)
+    return out
+
+
+def proof_url(template: str, old: int, new: int) -> str:
+    return template.replace("{old}", str(old)).replace("{new}", str(new))
+
+
+def fetch_proof(src: EndpointSource, old: int, new: int) -> list[bytes]:
+    url = proof_url(src.template, old, new)
+    resp = src.fetcher.get(url, max_bytes=PROOF_MAX_BYTES, accept="application/json")
+    if resp.status != 200:
+        raise FetchError(f"{url}: HTTP {resp.status}")
+    return parse_proof_body(resp.body, old, new)
+
+
+# ---------------------------------------------------------------------------
+# the checks
+# ---------------------------------------------------------------------------
+
+
+def prefix_check(x: Cp, ref: Cp, reader: Any, label: str) -> Attempt:
+    """Is ``x`` a prefix of ``ref``?  ``reader`` serves the tiles of ``ref``."""
+    if x.size > ref.size:
+        return Attempt(label, "incomplete", f"reference is smaller ({ref.size} < {x.size})", x, ref)
+    if x.size == ref.size:
+        if x.root == ref.root:
+            return Attempt(label, "consistent", "same size and root", x, ref)
+        return Attempt(label, "split", "same size, different roots", x, ref)
+    if x.size == 0:
+        if x.root == EMPTY_HASH:
+            return Attempt(label, "consistent", "empty tree", x, ref)
+        return Attempt(label, "split", "a size-0 checkpoint whose root is not SHA-256('')", x, ref)
+    try:
+        verified = VerifiedTileReader(reader, ref.size, ref.root)
+        proof = consistency_proof_from_tiles(verified, x.size, ref.size)
+        prefix = tree_root_from_tiles(verified, x.size)
+    except TileSourceError as exc:
+        return Attempt(label, "incomplete", f"tile source failed: {exc}", x, ref)
+    except TileVerificationError as exc:
+        return Attempt(label, "invalid", f"tiles do not authenticate against the signed root: {exc}", x, ref)
+    except FileNotFoundError as exc:
+        return Attempt(label, "incomplete", f"tiles missing: {exc}", x, ref)
+    except (ValueError, OSError) as exc:
+        return Attempt(label, "invalid", f"unusable tile data: {exc}", x, ref)
+    if verify_consistency(x.size, ref.size, x.root, ref.root, proof):
+        return Attempt(label, "consistent", "valid consistency proof from authenticated tiles", x, ref, proof)
+    if prefix != x.root and verify_consistency(x.size, ref.size, prefix, ref.root, proof):
+        return Attempt(
+            label, "split",
+            "the root at this size, authenticated by the newer signed root, differs from the signed root",
+            x, ref, proof, prefix,
+        )
+    return Attempt(label, "invalid", "authenticated tiles yield no valid proof (inconsistent source data)",
+                   x, ref, proof)
+
+
+def relate(a: Cp, b: Cp, sources: list[Source]) -> Relation:
+    """Is the smaller of ``a`` and ``b`` a prefix of the larger?"""
+    if a.size > b.size:
+        a, b = b, a
+    rel = Relation(status="unproven", a=a, b=b)
+    # AICS-1 §14.2 step 1: for size 0 require the standard empty root and an
+    # empty proof.  No proof source is consulted (an empty "proof" verifies
+    # any old root at size 0).
+    bad_empty = [x for x in (a, b) if x.size == 0 and x.root != EMPTY_HASH]
+    if bad_empty:
+        rel.status, rel.reason = "split-proven", "bad-empty-root"
+        for x in bad_empty:
+            rel.split_pairs.append((x, b if x is a else a, None))
+        return rel
+    if a.size == 0:
+        rel.status, rel.reason = "consistent", "the empty tree is a prefix of every tree"
+        return rel
+    if a.size == b.size:
+        if a.root == b.root:
+            rel.status, rel.reason = "consistent", "same size and root"
+        else:
+            rel.status, rel.reason = "split-proven", "equal-size-different-root"
+            rel.split_pairs.append((a, b, None))
+        return rel
+    for src in sources:
+        if isinstance(src, (GivenProof, EndpointSource)):
+            if isinstance(src, GivenProof):
+                hashes = src.hashes
+            else:
+                try:
+                    hashes = fetch_proof(src, a.size, b.size)
+                except FetchError as exc:
+                    rel.attempts.append(Attempt(src.label, "incomplete", exc.message, a, b))
+                    continue
+                except (ValueError, UnicodeDecodeError) as exc:
+                    rel.attempts.append(Attempt(src.label, "invalid", f"unusable proof: {exc}", a, b))
+                    continue
+            ok = verify_consistency(a.size, b.size, a.root, b.root, hashes)
+            att = Attempt(src.label, "consistent" if ok else "invalid",
+                          "valid consistency proof" if ok else "proof does not verify", a, b, hashes)
+            rel.attempts.append(att)
+            if ok:
+                rel.status, rel.winning = "consistent", att
+                return rel
+            continue
+        ref = src.ref if src.ref is not None else b
+        if ref.size < b.size:
+            rel.attempts.append(Attempt(src.label, "incomplete", f"source is behind ({ref.size} < {b.size})", b, ref))
+            continue
+        rb = prefix_check(b, ref, src.reader, src.label)
+        if rb.status in ("invalid", "incomplete"):
+            rel.attempts.append(rb)
+            continue
+        ra = prefix_check(a, ref, src.reader, src.label)
+        if ra.status in ("invalid", "incomplete"):
+            rel.attempts.append(ra)
+            if rb.status == "split":
+                rel.attempts.append(rb)
+                rel.status, rel.reason = "split-proven", "not-a-prefix"
+                rel.split_pairs.append((b, ref, rb))
+                return rel
+            continue
+        rel.attempts += [x for x in (rb, ra) if x.old is not x.new]
+        if ra.status == "consistent" and rb.status == "consistent":
+            rel.status, rel.winning = "consistent", ra
+            return rel
+        rel.status = "split-proven"
+        equal = [x for x in (ra, rb) if x.status == "split" and x.old is not None and x.new is not None
+                 and x.old.size == x.new.size]
+        rel.reason = "equal-size-different-root" if equal else "not-a-prefix"
+        for x, chk in ((a, ra), (b, rb)):
+            if chk.status == "split":
+                rel.split_pairs.append((x, ref, chk))
+        return rel
+    rel.reason = "invalid-proof-data" if any(x.status == "invalid" for x in rel.attempts) else "no-proof-available"
+    return rel

aics_cli/dnstxt.py

@@ -0,0 +1,319 @@
+"""DNS TXT lookup: dnspython when installed (``pip install 'aics-cli[dns]'``),
+otherwise a small stdlib resolver (UDP with EDNS0, TCP on truncation).
+
+Neither path validates DNSSEC itself.  The query sets the AD bit and the
+result reports whether the resolver claimed the answer was authenticated
+(``ad``); that claim is only as good as the path to the resolver.
+
+The stdlib resolver accepts a response only if it answers the question that
+was asked: the same ID, exactly one question with the queried name
+(compared case-insensitively), type TXT and class IN, and it takes TXT
+records only from the queried name or a CNAME target reached from it in the
+answer section.
+"""
+
+from __future__ import annotations
+
+import secrets
+import socket
+import struct
+from dataclasses import dataclass, field
+
+from .errors import DnsError
+
+TYPE_CNAME = 5
+TYPE_TXT = 16
+TYPE_OPT = 41
+CLASS_IN = 1
+FLAG_QR = 0x8000
+FLAG_TC = 0x0200
+FLAG_RD = 0x0100
+FLAG_AD = 0x0020
+RCODE_NXDOMAIN = 3
+
+
+@dataclass
+class TxtResult:
+    name: str
+    records: list[str] = field(default_factory=list)  # each record's strings joined
+    ad: bool = False  # resolver said "authenticated data" (DNSSEC); not checked by us
+    resolver: str = ""  # "dnspython" or "stdlib:<ip>:<port>"
+    nxdomain: bool = False
+
+
+# ---------------------------------------------------------------------------
+# entry point
+# ---------------------------------------------------------------------------
+
+
+def lookup_txt(name: str, *, nameserver: str | None = None, timeout: float = 5.0,
+               use_dnspython: bool | None = None) -> TxtResult:
+    """TXT records at ``name``.  ``nameserver`` is ``ip`` or ``ip:port``
+    (``[v6]:port``).  No record or NXDOMAIN returns an empty list; other
+    failures raise :class:`DnsError`."""
+    name = name.rstrip(".")
+    if use_dnspython is None:
+        try:
+            import dns.resolver
+
+            use_dnspython = True
+        except ImportError:
+            use_dnspython = False
+    if use_dnspython:
+        return _lookup_dnspython(name, nameserver, timeout)
+    return lookup_txt_stdlib(name, nameserver=nameserver, timeout=timeout)
+
+
+def _split_ns(nameserver: str) -> tuple[str, int]:
+    ns = nameserver.strip()
+    if ns.startswith("["):
+        host, _, rest = ns[1:].partition("]")
+        port = int(rest[1:]) if rest.startswith(":") else 53
+        return host, port
+    if ns.count(":") == 1:
+        host, port = ns.split(":")
+        return host, int(port)
+    return ns, 53
+
+
+def _lookup_dnspython(name: str, nameserver: str | None, timeout: float) -> TxtResult:
+    import dns.exception
+    import dns.flags
+    import dns.resolver
+
+    resolver = dns.resolver.Resolver(configure=nameserver is None)
+    if nameserver is not None:
+        host, port = _split_ns(nameserver)
+        resolver.nameservers = [host]
+        resolver.port = port
+    resolver.lifetime = timeout
+    resolver.flags = dns.flags.RD | dns.flags.AD
+    try:
+        answer = resolver.resolve(name, "TXT", raise_on_no_answer=False)
+    except dns.resolver.NXDOMAIN:
+        return TxtResult(name=name, resolver="dnspython", nxdomain=True)
+    except dns.exception.DNSException as exc:
+        raise DnsError(f"TXT lookup for {name} failed: {exc}") from None
+    records: list[str] = []
+    for rr in answer:
+        try:
+            records.append(b"".join(rr.strings).decode("utf-8"))
+        except UnicodeDecodeError:
+            continue
+    ad = bool(answer.response.flags & dns.flags.AD) if answer.response is not None else False
+    return TxtResult(name=name, records=records, ad=ad, resolver="dnspython")
+
+
+# ---------------------------------------------------------------------------
+# stdlib resolver
+# ---------------------------------------------------------------------------
+
+
+def system_nameservers(path: str = "/etc/resolv.conf") -> list[str]:
+    out: list[str] = []
+    try:
+        with open(path, encoding="utf-8", errors="replace") as fh:
+            for line in fh:
+                parts = line.split()
+                if len(parts) >= 2 and parts[0] == "nameserver":
+                    out.append(parts[1])
+    except OSError:
+        pass
+    return out
+
+
+def encode_name(name: str) -> bytes:
+    out = bytearray()
+    for label in name.rstrip(".").split("."):
+        raw = label.encode("idna") if any(ord(c) > 127 for c in label) else label.encode("ascii")
+        if not 0 < len(raw) < 64:
+            raise DnsError(f"bad DNS label in {name!r}")
+        out.append(len(raw))
+        out += raw
+    out.append(0)
+    if len(out) > 255:
+        raise DnsError(f"DNS name too long: {name!r}")
+    return bytes(out)
+
+
+def build_query(name: str, qid: int, *, qtype: int = TYPE_TXT, udp_size: int = 1232) -> bytes:
+    header = struct.pack(">HHHHHH", qid, FLAG_RD | FLAG_AD, 1, 0, 0, 1)
+    question = encode_name(name) + struct.pack(">HH", qtype, CLASS_IN)
+    opt = b"\x00" + struct.pack(">HHIH", TYPE_OPT, udp_size, 0, 0)  # EDNS0, no DO bit
+    return header + question + opt
+
+
+def _read_name(msg: bytes, off: int) -> tuple[str, int]:
+    labels: list[str] = []
+    jumps = 0
+    end: int | None = None
+    while True:
+        if off >= len(msg):
+            raise DnsError("truncated DNS name")
+        length = msg[off]
+        if length & 0xC0 == 0xC0:
+            if off + 1 >= len(msg):
+                raise DnsError("truncated DNS pointer")
+            if end is None:
+                end = off + 2
+            off = ((length & 0x3F) << 8) | msg[off + 1]
+            jumps += 1
+            if jumps > 32:
+                raise DnsError("DNS name compression loop")
+            continue
+        if length & 0xC0:
+            raise DnsError("unsupported DNS label type")
+        off += 1
+        if length == 0:
+            break
+        labels.append(msg[off:off + length].decode("ascii", errors="replace"))
+        off += length
+    return ".".join(labels), (end if end is not None else off)
+
+
+def _norm(name: str) -> str:
+    """A name as it appears on the wire (IDNA labels), lowercased, no final dot."""
+    labels = []
+    for label in name.rstrip(".").split("."):
+        if any(ord(c) > 127 for c in label):
+            try:
+                label = label.encode("idna").decode("ascii")
+            except UnicodeError:
+                pass
+        labels.append(label.lower())
+    return ".".join(labels)
+
+
+def parse_response(msg: bytes, qid: int, qname: str | None = None,
+                   qtype: int = TYPE_TXT) -> tuple[int, int, list[bytes]]:
+    """Returns (flags, rcode, [TXT rdata strings joined per record]).
+
+    The response must carry exactly one question; with ``qname`` it must be
+    that name (case-insensitive), ``qtype`` and class IN, and only TXT
+    records owned by ``qname`` or by a CNAME target in the answer chain
+    from it are returned."""
+    if len(msg) < 12:
+        raise DnsError("short DNS response")
+    rid, flags, qd, an, _ns, _ar = struct.unpack(">HHHHHH", msg[:12])
+    if rid != qid:
+        raise DnsError("DNS response ID mismatch")
+    if not flags & FLAG_QR:
+        raise DnsError("DNS message is not a response")
+    rcode = flags & 0x000F
+    if qd != 1:
+        raise DnsError(f"DNS response carries {qd} questions, not the one that was asked")
+    name, off = _read_name(msg, 12)
+    if off + 4 > len(msg):
+        raise DnsError("truncated DNS question")
+    rqtype, rqclass = struct.unpack(">HH", msg[off:off + 4])
+    off += 4
+    if qname is not None and (_norm(name) != _norm(qname) or rqtype != qtype or rqclass != CLASS_IN):
+        raise DnsError(f"DNS response answers another question ({name!r} type {rqtype} class {rqclass})")
+    answers: list[tuple[str, int, int, int, bytes]] = []  # owner, type, class, rdata offset, rdata
+    for _ in range(an):
+        owner, off = _read_name(msg, off)
+        if off + 10 > len(msg):
+            raise DnsError("truncated DNS answer")
+        rtype, rclass, _ttl, rdlen = struct.unpack(">HHIH", msg[off:off + 10])
+        off += 10
+        rdata = msg[off:off + rdlen]
+        if len(rdata) != rdlen:
+            raise DnsError("truncated DNS rdata")
+        answers.append((_norm(owner), rtype, rclass, off, rdata))
+        off += rdlen
+    allowed = {_norm(qname if qname is not None else name)}
+    changed = True
+    while changed:  # follow CNAMEs from the queried name, in any order
+        changed = False
+        for owner, rtype, rclass, roff, _rdata in answers:
+            if rtype == TYPE_CNAME and rclass == CLASS_IN and owner in allowed:
+                target = _norm(_read_name(msg, roff)[0])
+                if target not in allowed:
+                    allowed.add(target)
+                    changed = True
+    records: list[bytes] = []
+    for owner, rtype, rclass, _roff, rdata in answers:
+        if rtype != TYPE_TXT or rclass != CLASS_IN or owner not in allowed:
+            continue  # a CNAME on the way, or a record about some other name
+        parts: list[bytes] = []
+        i = 0
+        while i < len(rdata):
+            n = rdata[i]
+            i += 1
+            if i + n > len(rdata):
+                raise DnsError("malformed TXT character-string")
+            parts.append(rdata[i:i + n])
+            i += n
+        records.append(b"".join(parts))
+    return flags, rcode, records
+
+
+def _query_udp(query: bytes, addr: tuple, family: int, qid: int, timeout: float) -> bytes:
+    with socket.socket(family, socket.SOCK_DGRAM) as s:
+        s.settimeout(timeout)
+        s.connect(addr)  # only accept datagrams from the server we asked
+        s.send(query)
+        while True:
+            data = s.recv(65535)
+            if len(data) >= 2 and struct.unpack(">H", data[:2])[0] == qid:
+                return data
+
+
+def _query_tcp(query: bytes, addr: tuple, family: int, timeout: float) -> bytes:
+    with socket.socket(family, socket.SOCK_STREAM) as s:
+        s.settimeout(timeout)
+        s.connect(addr)
+        s.sendall(struct.pack(">H", len(query)) + query)
+        head = _recv_exact(s, 2)
+        (n,) = struct.unpack(">H", head)
+        return _recv_exact(s, n)
+
+
+def _recv_exact(s: socket.socket, n: int) -> bytes:
+    buf = bytearray()
+    while len(buf) < n:
+        chunk = s.recv(n - len(buf))
+        if not chunk:
+            raise DnsError("DNS TCP connection closed early")
+        buf += chunk
+    return bytes(buf)
+
+
+def lookup_txt_stdlib(name: str, *, nameserver: str | None = None, timeout: float = 5.0) -> TxtResult:
+    servers = [nameserver] if nameserver else system_nameservers()
+    if not servers:
+        raise DnsError("no nameserver configured (no /etc/resolv.conf); pass --nameserver")
+    last: Exception | None = None
+    for ns in servers:
+        host, port = _split_ns(ns)
+        try:
+            infos = socket.getaddrinfo(host, port, type=socket.SOCK_DGRAM)
+        except socket.gaierror as exc:
+            last = exc
+            continue
+        family, _, _, _, addr = infos[0]
+        qid = secrets.randbelow(65536)
+        query = build_query(name, qid)
+        try:
+            data = _query_udp(query, addr, family, qid, timeout)
+            flags, rcode, records = parse_response(data, qid, name)
+            if flags & FLAG_TC:
+                data = _query_tcp(query, addr, family, timeout)
+                flags, rcode, records = parse_response(data, qid, name)
+        except (OSError, DnsError) as exc:
+            last = exc
+            continue
+        label = f"stdlib:{host}:{port}"
+        if rcode == RCODE_NXDOMAIN:
+            return TxtResult(name=name, resolver=label, nxdomain=True, ad=bool(flags & FLAG_AD))
+        if rcode != 0:
+            last = DnsError(f"DNS server {host} answered rcode {rcode}")
+            continue
+        texts: list[str] = []
+        for r in records:
+            try:
+                texts.append(r.decode("utf-8"))
+            except UnicodeDecodeError:
+                continue
+        return TxtResult(name=name, records=texts, ad=bool(flags & FLAG_AD), resolver=label)
+    raise DnsError(f"TXT lookup for {name} failed: {last}")