umbermark-cli 1.0.0__py3-none-any.whl

umbermark/core/external/socket_probe.py

@@ -0,0 +1,400 @@
+"""SocketProbe — the real outside-in network probe.
+
+The concrete `ExternalProbe`: given a hostname/IP it answers the same questions
+`FakeProbe` does, but over real sockets. This is the only Tier-1 component that
+touches the network. Every method is best-effort and timeout-bounded — any
+failure (host down, port filtered, handshake refused) returns a falsy value so
+the checks SKIP/PASS rather than crashing the scan. It probes only the public
+surface and never authenticates.
+
+The socket/TLS I/O glue is intentionally thin and is not unit-tested against
+live servers (mirrors ParamikoRunner). The deterministic logic that could hide
+bugs — the SSH KEXINIT parser and the certificate-facts derivation — is factored
+out as pure functions and tested directly.
+"""
+from __future__ import annotations
+
+import http.client
+import socket
+import ssl
+import warnings
+from datetime import datetime, timezone
+from typing import Optional
+
+from umbermark.core.external.ownership import dial_address
+from umbermark.core.external.probe import SshAlgorithms, TlsInfo
+
+# ---------------------------------------------------------------------------
+# Pure logic (unit-tested)
+# ---------------------------------------------------------------------------
+
+_SSH_MSG_KEXINIT = 20
+
+
+def _read_name_list(buf: bytes, off: int) -> tuple[list[str], int]:
+    """Decode one SSH name-list (uint32 length + comma-separated ASCII)."""
+    length = int.from_bytes(buf[off : off + 4], "big")
+    off += 4
+    raw = buf[off : off + length].decode("ascii", errors="replace")
+    off += length
+    names = [n for n in raw.split(",") if n] if raw else []
+    return names, off
+
+
+def parse_ssh_kexinit(payload: bytes) -> Optional[SshAlgorithms]:
+    """Parse an SSH KEXINIT message payload (RFC 4253 §7.1) into the algorithm
+    lists the server offers. Returns None if ``payload`` isn't a KEXINIT.
+
+    We surface the server->client encryption/MAC lists (what the server will use
+    to send us data) as the canonical ``ciphers``/``macs`` offer.
+    """
+    if not payload or payload[0] != _SSH_MSG_KEXINIT:
+        return None
+    try:
+        off = 1 + 16  # message type byte + 16-byte cookie
+        kex, off = _read_name_list(payload, off)
+        host_keys, off = _read_name_list(payload, off)
+        _enc_cs, off = _read_name_list(payload, off)
+        enc_sc, off = _read_name_list(payload, off)
+        _mac_cs, off = _read_name_list(payload, off)
+        mac_sc, off = _read_name_list(payload, off)
+    except Exception:
+        return None
+    return SshAlgorithms(kex=kex, ciphers=enc_sc, macs=mac_sc, host_keys=host_keys)
+
+
+def _cert_facts(cert: dict, now: datetime) -> tuple[int, bool]:
+    """Derive (days_until_expiry, self_signed) from an ssl ``getpeercert``-style
+    dict. ``days_until_expiry`` is negative once the cert has expired; a cert
+    whose subject equals its issuer is treated as self-signed."""
+    not_after_raw = cert.get("notAfter", "")
+    not_after = datetime.strptime(not_after_raw, "%b %d %H:%M:%S %Y %Z").replace(
+        tzinfo=timezone.utc
+    )
+    days = (not_after - now).days
+    subject = cert.get("subject")
+    issuer = cert.get("issuer")
+    self_signed = subject is not None and subject == issuer
+    return days, self_signed
+
+
+# ---------------------------------------------------------------------------
+# I/O helper
+# ---------------------------------------------------------------------------
+
+
+class _SockReader:
+    """Buffered reader so banner bytes and the following binary packet can be
+    read from the same stream without losing pipelined data."""
+
+    def __init__(self, sock: socket.socket) -> None:
+        self._sock = sock
+        self._buf = b""
+
+    def read_exact(self, n: int) -> Optional[bytes]:
+        while len(self._buf) < n:
+            chunk = self._sock.recv(4096)
+            if not chunk:
+                return None
+            self._buf += chunk
+        out, self._buf = self._buf[:n], self._buf[n:]
+        return out
+
+    def read_line(self, limit: int = 8192) -> Optional[bytes]:
+        while b"\n" not in self._buf:
+            if len(self._buf) > limit:
+                return None
+            chunk = self._sock.recv(4096)
+            if not chunk:
+                return None
+            self._buf += chunk
+        line, self._buf = self._buf.split(b"\n", 1)
+        return line + b"\n"
+
+
+def _der_to_cert_dict(der: bytes) -> Optional[dict]:
+    """Build an ssl-style cert dict from DER using cryptography (a hard dep via
+    paramiko). Returns None if the cert can't be parsed."""
+    try:
+        from cryptography import x509
+
+        cert = x509.load_der_x509_certificate(der)
+        try:
+            not_after = cert.not_valid_after_utc
+        except AttributeError:  # cryptography < 42
+            not_after = cert.not_valid_after.replace(tzinfo=timezone.utc)
+        subject = cert.subject.rfc4514_string()
+        issuer = cert.issuer.rfc4514_string()
+        return {
+            "notAfter": not_after.strftime("%b %d %H:%M:%S %Y GMT"),
+            "subject": subject,
+            "issuer": issuer,
+        }
+    except Exception:
+        return None
+
+
+# Legacy/modern TLS versions to probe, newest first. Probing the deprecated
+# versions is the whole point of the check, so silence the stdlib deprecation
+# warnings their enum members emit (Python 3.12+).
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore", DeprecationWarning)
+    _TLS_VERSIONS = [
+        ("TLSv1.3", ssl.TLSVersion.TLSv1_3),
+        ("TLSv1.2", ssl.TLSVersion.TLSv1_2),
+        ("TLSv1.1", ssl.TLSVersion.TLSv1_1),
+        ("TLSv1", ssl.TLSVersion.TLSv1),
+    ]
+
+
+# TLS 1.0/1.1 handshakes need SHA-1/MD5 signatures, which OpenSSL 3 only allows
+# at security level 0 -- at the default level it refuses to send the ClientHello.
+_LEGACY_TLS = {"TLSv1.1", "TLSv1"}
+_LEGACY_CIPHERS = "ALL:@SECLEVEL=0"
+
+
+def _probe_context(name: str, version: ssl.TLSVersion) -> Optional[ssl.SSLContext]:
+    """A non-verifying client context pinned to exactly ``version``, or None if
+    this ssl build cannot pin it. Legacy versions get security level 0 so a
+    modern OpenSSL can still *ask* the server whether it accepts them."""
+    ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    try:
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", DeprecationWarning)
+            ctx.minimum_version = version
+            ctx.maximum_version = version
+    except (ValueError, OSError):
+        return None
+    if name in _LEGACY_TLS:
+        try:
+            ctx.set_ciphers(_LEGACY_CIPHERS)
+        except ssl.SSLError:
+            pass  # _client_can_offer() reports whether it still works
+    return ctx
+
+
+def _client_can_offer(ctx: ssl.SSLContext) -> bool:
+    """True iff the local TLS stack can emit a ClientHello for ``ctx``.
+
+    Runs the first handshake step in memory, before any connection: a stack
+    that will not offer the pinned version (build flags, security level, crypto
+    policy) raises NO_PROTOCOLS_AVAILABLE here, whereas a usable context writes
+    its ClientHello and waits for the server (SSLWantReadError).
+    """
+    try:
+        obj = ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO())
+        obj.do_handshake()
+    except ssl.SSLWantReadError:
+        return True
+    except Exception:
+        return False
+    return True
+
+
+def _trust_context() -> ssl.SSLContext:
+    """A verifying client context over the Mozilla CA bundle (certifi, pulled in
+    by httpx), falling back to the system store. Hostname matching is off: the
+    question here is "does the chain lead to a public CA" (UMBR-40); targets are
+    often bare IPs, which no public certificate names."""
+    try:
+        import certifi
+
+        ctx = ssl.create_default_context(cafile=certifi.where())
+    except Exception:
+        ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    # Python 3.13+ turns on VERIFY_X509_STRICT, which rejects some chains OpenSSL and browsers
+    # accept (non-critical CA basicConstraints, missing key identifiers). The question here is
+    # only "does this lead to a public CA", so judge it the way clients do.
+    ctx.verify_flags &= ~ssl.VERIFY_X509_STRICT
+    return ctx
+
+
+def _utcnow() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+# ---------------------------------------------------------------------------
+# SocketProbe
+# ---------------------------------------------------------------------------
+
+
+class SocketProbe:
+    """Real ExternalProbe over sockets. ``host`` is a hostname or IP."""
+
+    def __init__(self, host: str, default_timeout: float = 3.0) -> None:
+        self.host = host
+        self.default_timeout = default_timeout
+        self._ips: list[str] = []
+
+    def pin(self, ips: list[str]) -> None:
+        """Connect only to these pre-vetted IPs; ``host`` stays the SNI/Host name.
+
+        Once pinned the probe never re-resolves the hostname (UMBR-26: the
+        worker vets the addresses once, so a flipped A record cannot redirect
+        probes to loopback / private / metadata / third-party addresses).
+        """
+        # IPv4 first: a worker without IPv6 egress would otherwise fail on a
+        # dual-stack host's v6 address and read every port as closed.
+        self._ips = sorted(ips, key=lambda ip: (":" in ip, ip))
+
+    def _connect(self, port: int, timeout: float) -> socket.socket:
+        if not self._ips:
+            return socket.create_connection((self.host, port), timeout=timeout)
+        last: Exception = OSError("no pinned addresses")
+        for ip in self._ips:  # first successful connection wins
+            try:
+                # dial_address: identity, except the scanner's own host (UMBR-65).
+                return socket.create_connection(
+                    (dial_address(ip, port), port), timeout=timeout
+                )
+            except OSError as exc:
+                last = exc
+        raise last
+
+    def tcp_connect(self, port: int, timeout: float = 3.0) -> bool:
+        try:
+            with self._connect(port, timeout):
+                return True
+        except Exception:
+            return False
+
+    def ssh_algorithms(
+        self, port: int = 22, timeout: float = 3.0
+    ) -> Optional[SshAlgorithms]:
+        try:
+            with self._connect(port, timeout) as s:
+                s.settimeout(timeout)
+                reader = _SockReader(s)
+                # Read lines until the server's "SSH-..." identification string.
+                banner = None
+                for _ in range(10):
+                    line = reader.read_line()
+                    if line is None:
+                        return None
+                    if line.startswith(b"SSH-"):
+                        banner = line
+                        break
+                if banner is None:
+                    return None
+                s.sendall(b"SSH-2.0-Umbermark-Probe\r\n")
+                header = reader.read_exact(4)
+                if header is None:
+                    return None
+                packet_len = int.from_bytes(header, "big")
+                if packet_len < 2 or packet_len > 100_000:
+                    return None
+                body = reader.read_exact(packet_len)
+                if body is None:
+                    return None
+                padding_len = body[0]
+                payload = body[1 : 1 + (packet_len - padding_len - 1)]
+                return parse_ssh_kexinit(payload)
+        except Exception:
+            return None
+
+    def tls_info(self, port: int = 443, timeout: float = 3.0) -> Optional[TlsInfo]:
+        protocols: list[str] = []
+        unprobed: list[str] = []
+        cert_dict: Optional[dict] = None
+        for name, version in _TLS_VERSIONS:
+            ctx = _probe_context(name, version)
+            if ctx is None or not _client_can_offer(ctx):
+                # Our TLS stack cannot even emit this ClientHello: the server's
+                # answer is unknown, which is NOT the same as "not offered".
+                unprobed.append(name)
+                continue
+            try:
+                raw = self._connect(port, timeout)
+            except Exception:
+                # Could not even reach the port for this attempt (rate limit,
+                # fail2ban, transient loss): the server was never asked.
+                unprobed.append(name)
+                continue
+            try:
+                with raw, ctx.wrap_socket(raw, server_hostname=self.host) as ss:
+                    protocols.append(name)
+                    if cert_dict is None:
+                        der = ss.getpeercert(binary_form=True)
+                        if der:
+                            cert_dict = _der_to_cert_dict(der)
+            except Exception:
+                continue  # handshake refused / alert / reset: not offered
+        if not protocols:
+            return None
+        chain_trusted, trust_error = self._verify_chain(port, timeout)
+        # An unreadable leaf is reported as such (UMBR-35) -- never defaulted to
+        # "valid for years, CA-issued", which the hygiene check would PASS.
+        days, self_signed, cert_readable = 0, False, False
+        if cert_dict is not None:
+            try:
+                days, self_signed = _cert_facts(cert_dict, _utcnow())
+                cert_readable = True
+            except Exception:
+                days, self_signed = 0, False
+        return TlsInfo(
+            protocols=protocols, days_until_expiry=days, self_signed=self_signed,
+            unprobed=unprobed, cert_readable=cert_readable,
+            chain_trusted=chain_trusted, trust_error=trust_error,
+        )
+
+    def _verify_chain(self, port: int, timeout: float) -> tuple[Optional[bool], str]:
+        """One verifying handshake (UMBR-40). The protocol probes above use
+        CERT_NONE, so they cannot tell a public-CA chain from a self-signed or
+        private-CA one (e.g. mod_ssl's sscg localhost.crt, whose issuer is a
+        throwaway local CA, so issuer != subject). Returns (True, "") when the
+        chain verifies, (False, reason) when verification fails, and (None, "")
+        when the handshake could not complete for any other reason."""
+        try:
+            raw = self._connect(port, timeout)
+        except Exception:
+            return None, ""
+        try:
+            with raw, _trust_context().wrap_socket(raw, server_hostname=self.host):
+                return True, ""
+        except ssl.SSLCertVerificationError as exc:
+            return False, str(exc.verify_message or exc.reason or exc)
+        except Exception:
+            return None, ""
+
+    def http_headers(
+        self, port: int = 443, tls: bool = True, timeout: float = 3.0
+    ) -> Optional[dict]:
+        conn = None
+        try:
+            if tls:
+                conn = http.client.HTTPSConnection(
+                    self.host,
+                    port,
+                    timeout=timeout,
+                    context=ssl._create_unverified_context(),
+                )
+            else:
+                conn = http.client.HTTPConnection(self.host, port, timeout=timeout)
+            if self._ips:
+                # Pre-connect to the vetted IP so http.client never re-resolves.
+                raw = self._connect(port, timeout)
+                if tls:
+                    raw = ssl._create_unverified_context().wrap_socket(
+                        raw, server_hostname=self.host
+                    )
+                conn.sock = raw
+            conn.request("HEAD", "/", headers={"User-Agent": "Umbermark-Probe/1.0"})
+            resp = conn.getresponse()
+            # A repeated header (app + proxy) is folded into one comma-joined
+            # value, as RFC 9110 allows, so none of the copies are dropped.
+            out: dict = {}
+            for k, v in resp.getheaders():
+                k = k.lower()
+                out[k] = f"{out[k]}, {v}" if k in out else v
+            return out
+        except Exception:
+            return None
+        finally:
+            if conn is not None:
+                try:
+                    conn.close()
+                except Exception:
+                    pass