umbermark-cli 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- umbermark/__init__.py +18 -0
- umbermark/checks/__init__.py +88 -0
- umbermark/checks/auto_updates_configured.py +149 -0
- umbermark/checks/dns_resolver_sanity.py +47 -0
- umbermark/checks/fail2ban_enabled.py +67 -0
- umbermark/checks/failed_login_anomaly.py +48 -0
- umbermark/checks/failed_systemd_units.py +41 -0
- umbermark/checks/firewall_protective.py +247 -0
- umbermark/checks/journald_persistence.py +55 -0
- umbermark/checks/kernel_age.py +74 -0
- umbermark/checks/listen_vs_firewall_delta.py +136 -0
- umbermark/checks/mac_framework_enforcing.py +97 -0
- umbermark/checks/pending_security_updates.py +188 -0
- umbermark/checks/public_listen_anomalies.py +200 -0
- umbermark/checks/reboot_pending.py +71 -0
- umbermark/checks/repo_gpg_trust.py +97 -0
- umbermark/checks/root_disk_free.py +46 -0
- umbermark/checks/ssh_non_default_port.py +62 -0
- umbermark/checks/ssh_password_auth.py +71 -0
- umbermark/checks/ssh_permit_root.py +59 -0
- umbermark/checks/ssh_weak_ciphers.py +85 -0
- umbermark/checks/sudo_requires_authentication.py +41 -0
- umbermark/checks/sudoers_nopasswd.py +111 -0
- umbermark/checks/swap_pressure.py +51 -0
- umbermark/checks/time_sync.py +55 -0
- umbermark/checks/tmp_hardening.py +45 -0
- umbermark/checks/user_password_aging.py +95 -0
- umbermark/checks/world_writable_suid_audit.py +171 -0
- umbermark/cli/__init__.py +4 -0
- umbermark/cli/app.py +23 -0
- umbermark/cli/auth.py +44 -0
- umbermark/cli/commands/__init__.py +0 -0
- umbermark/cli/commands/doctor.py +16 -0
- umbermark/cli/commands/login.py +80 -0
- umbermark/cli/commands/push.py +54 -0
- umbermark/cli/commands/report.py +26 -0
- umbermark/cli/commands/scan.py +77 -0
- umbermark/cli/commands/scan_local.py +72 -0
- umbermark/cli/commands/scan_wsl.py +35 -0
- umbermark/cli/commands/version.py +11 -0
- umbermark/cli/config.py +14 -0
- umbermark/core/__init__.py +0 -0
- umbermark/core/accepted.py +61 -0
- umbermark/core/access.py +896 -0
- umbermark/core/branding.py +71 -0
- umbermark/core/capture.py +228 -0
- umbermark/core/check.py +82 -0
- umbermark/core/checkin.py +89 -0
- umbermark/core/crypto.py +15 -0
- umbermark/core/duration.py +45 -0
- umbermark/core/engine.py +74 -0
- umbermark/core/evidence.py +83 -0
- umbermark/core/external/__init__.py +6 -0
- umbermark/core/external/check.py +54 -0
- umbermark/core/external/checks/__init__.py +31 -0
- umbermark/core/external/checks/exposed_datastores.py +55 -0
- umbermark/core/external/checks/http_security_headers.py +249 -0
- umbermark/core/external/checks/legacy_services.py +52 -0
- umbermark/core/external/checks/open_management_ports.py +65 -0
- umbermark/core/external/checks/ssh_alt_ports.py +77 -0
- umbermark/core/external/checks/ssh_weak_algorithms.py +93 -0
- umbermark/core/external/checks/tls_hygiene.py +107 -0
- umbermark/core/external/engine.py +89 -0
- umbermark/core/external/ownership.py +607 -0
- umbermark/core/external/probe.py +103 -0
- umbermark/core/external/socket_probe.py +400 -0
- umbermark/core/firewall.py +713 -0
- umbermark/core/fleet.py +92 -0
- umbermark/core/history.py +235 -0
- umbermark/core/html_report.py +308 -0
- umbermark/core/keyenc.py +42 -0
- umbermark/core/labels.py +29 -0
- umbermark/core/lifecycle.py +100 -0
- umbermark/core/logredact.py +55 -0
- umbermark/core/partial.py +57 -0
- umbermark/core/pdf/__init__.py +0 -0
- umbermark/core/pdf/renderer.py +26 -0
- umbermark/core/pdf/weasyprint_renderer.py +18 -0
- umbermark/core/platform.py +73 -0
- umbermark/core/redact.py +75 -0
- umbermark/core/remediation.py +121 -0
- umbermark/core/report.py +25 -0
- umbermark/core/scoring.py +105 -0
- umbermark/core/settings.py +238 -0
- umbermark/core/short_id.py +27 -0
- umbermark/core/signing.py +24 -0
- umbermark/core/ssh.py +156 -0
- umbermark/core/ssh_fingerprint.py +16 -0
- umbermark/core/tiers.py +138 -0
- umbermark/core/webhooks.py +177 -0
- umbermark/core/wording.py +371 -0
- umbermark_cli-1.0.0.dist-info/METADATA +11 -0
- umbermark_cli-1.0.0.dist-info/RECORD +96 -0
- umbermark_cli-1.0.0.dist-info/WHEEL +5 -0
- umbermark_cli-1.0.0.dist-info/entry_points.txt +2 -0
- umbermark_cli-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,607 @@
|
|
|
1
|
+
"""Target-ownership verification for the external scanner (UMBR-26).
|
|
2
|
+
|
|
3
|
+
Active probing (port checks) may only run against a host the requester has
|
|
4
|
+
proven they control. Proof is a random token published at the target:
|
|
5
|
+
|
|
6
|
+
* HTTP file ``/.well-known/umbermark-verify.txt`` containing the token
|
|
7
|
+
(https first, then plain http as a fallback), fetched from EACH resolved IP
|
|
8
|
+
separately. Only IPs that themselves serve the file are "proven" and may be
|
|
9
|
+
port-probed (strict model).
|
|
10
|
+
* DNS TXT ``umbermark-verify=<token>`` on ``<host>`` or ``_umbermark.<host>``
|
|
11
|
+
proves control of the NAME only. It is recorded but unlocks nothing, since a
|
|
12
|
+
name can be pointed at a server the requester does not run.
|
|
13
|
+
|
|
14
|
+
IP-literal targets have no DNS name; the file on the IP itself is the only proof.
|
|
15
|
+
|
|
16
|
+
This module is pure network/parse logic with no DB access (``core`` ships in the
|
|
17
|
+
slim CLI package). Everything that touches the network is injectable so tests
|
|
18
|
+
never leave the box.
|
|
19
|
+
|
|
20
|
+
SSRF posture for the HTTP fetch: resolve the name ourselves, refuse the request
|
|
21
|
+
if ANY resolved address is non-public, then connect to the vetted IP (pinned, so
|
|
22
|
+
DNS rebinding cannot swap it afterwards). Redirects are never followed, the body
|
|
23
|
+
is size-capped, and timeouts are tight.
|
|
24
|
+
"""
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import hashlib
|
|
28
|
+
import hmac
|
|
29
|
+
import http.client
|
|
30
|
+
import ipaddress
|
|
31
|
+
import re
|
|
32
|
+
import secrets
|
|
33
|
+
import socket
|
|
34
|
+
import ssl
|
|
35
|
+
import struct
|
|
36
|
+
import threading
|
|
37
|
+
import time
|
|
38
|
+
from dataclasses import dataclass, field
|
|
39
|
+
from typing import Callable, Optional
|
|
40
|
+
|
|
41
|
+
from umbermark.core.crypto import hash_with_pepper
|
|
42
|
+
|
|
43
|
+
TOKEN_PREFIX = "umbermark-verify="
|
|
44
|
+
WELL_KNOWN_PATH = "/.well-known/umbermark-verify.txt"
|
|
45
|
+
MAX_BODY_BYTES = 2048
|
|
46
|
+
HTTP_TIMEOUT = 1.5 # per ATTEMPT cap (see fetch_well_known)
|
|
47
|
+
MAX_FETCH_IPS = 4
|
|
48
|
+
DNS_TIMEOUT = 3.0
|
|
49
|
+
DEFAULT_RESOLVERS = ("1.1.1.1", "8.8.8.8")
|
|
50
|
+
|
|
51
|
+
_TOKEN_RE = re.compile(r"umbermark-verify=([A-Za-z0-9_\-]{16,128})")
|
|
52
|
+
_BARE_TOKEN_RE = re.compile(r"^[A-Za-z0-9_\-]{16,128}$")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# ---------------------------------------------------------------------------
|
|
56
|
+
# Tokens
|
|
57
|
+
# ---------------------------------------------------------------------------
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def generate_token() -> str:
|
|
61
|
+
return secrets.token_urlsafe(24)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def hash_token(token: str) -> str:
|
|
65
|
+
return hashlib.sha256(token.encode("utf-8")).hexdigest()
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def normalize_target(host: str) -> str:
|
|
69
|
+
"""Canonical form used to bind a verification to exactly one target."""
|
|
70
|
+
h = host.strip().lower().rstrip(".")
|
|
71
|
+
if h.startswith("[") and h.endswith("]"):
|
|
72
|
+
h = h[1:-1]
|
|
73
|
+
return h
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def is_ip_literal(host: str) -> bool:
|
|
77
|
+
try:
|
|
78
|
+
ipaddress.ip_address(normalize_target(host))
|
|
79
|
+
return True
|
|
80
|
+
except ValueError:
|
|
81
|
+
return False
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def extract_candidates(text: str) -> list[str]:
|
|
85
|
+
"""Token candidates from a TXT value or a .well-known file body."""
|
|
86
|
+
found = _TOKEN_RE.findall(text)
|
|
87
|
+
stripped = text.strip()
|
|
88
|
+
if _BARE_TOKEN_RE.match(stripped):
|
|
89
|
+
found.append(stripped)
|
|
90
|
+
return found
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def token_matches(text: str, token_hash: str) -> bool:
|
|
94
|
+
return any(
|
|
95
|
+
hmac.compare_digest(hash_token(c), token_hash) for c in extract_candidates(text)
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
# ---------------------------------------------------------------------------
|
|
100
|
+
# SSRF guard
|
|
101
|
+
# ---------------------------------------------------------------------------
|
|
102
|
+
|
|
103
|
+
_BLOCKED_NETS = [
|
|
104
|
+
ipaddress.ip_network(n)
|
|
105
|
+
for n in (
|
|
106
|
+
"0.0.0.0/8",
|
|
107
|
+
"10.0.0.0/8",
|
|
108
|
+
"100.64.0.0/10",
|
|
109
|
+
"127.0.0.0/8",
|
|
110
|
+
"169.254.0.0/16", # link-local incl. cloud metadata 169.254.169.254
|
|
111
|
+
"172.16.0.0/12",
|
|
112
|
+
"192.0.0.0/24",
|
|
113
|
+
"192.168.0.0/16",
|
|
114
|
+
"198.18.0.0/15",
|
|
115
|
+
"224.0.0.0/4",
|
|
116
|
+
"240.0.0.0/4",
|
|
117
|
+
"::/128",
|
|
118
|
+
"::1/128",
|
|
119
|
+
"fc00::/7",
|
|
120
|
+
"fe80::/10",
|
|
121
|
+
"ff00::/8",
|
|
122
|
+
)
|
|
123
|
+
]
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def is_public_ip(ip: str) -> bool:
|
|
127
|
+
try:
|
|
128
|
+
addr = ipaddress.ip_address(ip)
|
|
129
|
+
except ValueError:
|
|
130
|
+
return False
|
|
131
|
+
if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped is not None:
|
|
132
|
+
addr = addr.ipv4_mapped
|
|
133
|
+
if any(addr in net for net in _BLOCKED_NETS):
|
|
134
|
+
return False
|
|
135
|
+
return addr.is_global
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class BlockedTarget(Exception):
|
|
139
|
+
"""Target resolves to (or is) a non-public address."""
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
# ---------------------------------------------------------------------------
|
|
143
|
+
# Network path (UMBR-65): see the internet the way an outside observer does
|
|
144
|
+
# ---------------------------------------------------------------------------
|
|
145
|
+
#
|
|
146
|
+
# Two facts of the production host make a naive scan of a target that lives on
|
|
147
|
+
# the SAME server come back empty:
|
|
148
|
+
#
|
|
149
|
+
# 1. The host's resolver answers its own hostname from /etc/hosts
|
|
150
|
+
# (``<hostname> -> 127.0.1.1``); the SSRF guard then blocks the target.
|
|
151
|
+
# Fix: resolve scan targets through public resolvers (``SCAN_RESOLVERS``),
|
|
152
|
+
# i.e. ask what the internet sees, not what this box believes.
|
|
153
|
+
# 2. Rootless podman (pasta) gives the container namespace the host's own
|
|
154
|
+
# public IP, so a connect to that IP never leaves the namespace. Fix: dial
|
|
155
|
+
# the host through pasta's host mapping (``SCANNER_HAIRPIN_ADDR``) instead.
|
|
156
|
+
#
|
|
157
|
+
# The hairpin path lands on the host via loopback, which the host firewall
|
|
158
|
+
# accepts for EVERY port -- so it would show listeners the internet can't reach
|
|
159
|
+
# (e.g. sshd on 22). Only the ports that really are public on this host
|
|
160
|
+
# (``SCANNER_SELF_PUBLIC_PORTS``) are dialled; any other port fails like a
|
|
161
|
+
# filtered one, which is exactly what an outside observer sees. All of this is
|
|
162
|
+
# off until configured, and none of it touches the ownership gate.
|
|
163
|
+
|
|
164
|
+
_scan_resolvers: tuple[str, ...] = ()
|
|
165
|
+
_self_ips: frozenset[str] = frozenset()
|
|
166
|
+
_hairpin_addr: str = ""
|
|
167
|
+
_self_public_ports: frozenset[int] = frozenset()
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class SelfPortFiltered(OSError):
|
|
171
|
+
"""A port on the scanner's own host that is not reachable from outside."""
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _canon_ip(ip: str) -> str:
|
|
175
|
+
try:
|
|
176
|
+
return str(ipaddress.ip_address(ip))
|
|
177
|
+
except ValueError:
|
|
178
|
+
return ip
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def configure_network(
|
|
182
|
+
*,
|
|
183
|
+
resolvers: tuple[str, ...] = (),
|
|
184
|
+
self_ips: tuple[str, ...] = (),
|
|
185
|
+
hairpin_addr: str = "",
|
|
186
|
+
self_public_ports: tuple[int, ...] = (),
|
|
187
|
+
) -> None:
|
|
188
|
+
"""Install this process's network path (web + worker call it at boot)."""
|
|
189
|
+
global _scan_resolvers, _self_ips, _hairpin_addr, _self_public_ports
|
|
190
|
+
_scan_resolvers = tuple(resolvers)
|
|
191
|
+
_self_ips = frozenset(_canon_ip(i) for i in self_ips)
|
|
192
|
+
_hairpin_addr = hairpin_addr
|
|
193
|
+
_self_public_ports = frozenset(self_public_ports)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def reset_network() -> None:
|
|
197
|
+
configure_network()
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def dial_address(ip: str, port: int) -> str:
|
|
201
|
+
"""The address to connect to in order to reach vetted public ``ip:port``.
|
|
202
|
+
|
|
203
|
+
Identity for every target except this host's own IP (when a hairpin address
|
|
204
|
+
is configured): its public ports go via the hairpin, the rest raise
|
|
205
|
+
SelfPortFiltered so they read as filtered, as they do from outside.
|
|
206
|
+
"""
|
|
207
|
+
if not _hairpin_addr or _canon_ip(ip) not in _self_ips:
|
|
208
|
+
return ip
|
|
209
|
+
if port in _self_public_ports:
|
|
210
|
+
return _hairpin_addr
|
|
211
|
+
raise SelfPortFiltered(f"port {port} on the scanner host is not public")
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def resolve_public_ips(host: str) -> list[str]:
|
|
215
|
+
"""Resolve ``host``; raise BlockedTarget unless EVERY address is public."""
|
|
216
|
+
h = normalize_target(host)
|
|
217
|
+
if is_ip_literal(h):
|
|
218
|
+
ips = [h]
|
|
219
|
+
elif _scan_resolvers:
|
|
220
|
+
ips = sorted(set(udp_addr_lookup(h, _scan_resolvers)))
|
|
221
|
+
else:
|
|
222
|
+
try:
|
|
223
|
+
infos = socket.getaddrinfo(h, None, proto=socket.IPPROTO_TCP)
|
|
224
|
+
except OSError as exc:
|
|
225
|
+
raise BlockedTarget(f"cannot resolve {h}") from exc
|
|
226
|
+
ips = sorted({i[4][0] for i in infos})
|
|
227
|
+
if not ips:
|
|
228
|
+
raise BlockedTarget(f"no addresses for {h}")
|
|
229
|
+
for ip in ips:
|
|
230
|
+
if not is_public_ip(ip):
|
|
231
|
+
raise BlockedTarget(f"{h} resolves to non-public address")
|
|
232
|
+
return ips
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
# ---------------------------------------------------------------------------
|
|
236
|
+
# HTTP .well-known fetch
|
|
237
|
+
# ---------------------------------------------------------------------------
|
|
238
|
+
|
|
239
|
+
Fetcher = Callable[[str], dict[str, str]]
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _fetch_one(
|
|
243
|
+
host: str,
|
|
244
|
+
ip: str,
|
|
245
|
+
port: int,
|
|
246
|
+
use_tls: bool,
|
|
247
|
+
timeout: float,
|
|
248
|
+
verify_tls: bool,
|
|
249
|
+
) -> Optional[str]:
|
|
250
|
+
"""One fetch attempt. ``timeout`` is a hard TOTAL budget for the whole
|
|
251
|
+
attempt (connect + TLS + headers + body), not a per-socket-op timeout: a
|
|
252
|
+
slow-drip server cannot hold the thread past it. A watchdog shuts the live
|
|
253
|
+
socket down at the deadline, which unblocks any pending recv."""
|
|
254
|
+
sock = socket.create_connection((dial_address(ip, port), port), timeout=timeout)
|
|
255
|
+
live = {"sock": sock}
|
|
256
|
+
|
|
257
|
+
def _kill() -> None:
|
|
258
|
+
try:
|
|
259
|
+
live["sock"].shutdown(socket.SHUT_RDWR)
|
|
260
|
+
except Exception:
|
|
261
|
+
pass
|
|
262
|
+
|
|
263
|
+
watchdog = threading.Timer(timeout, _kill)
|
|
264
|
+
watchdog.daemon = True
|
|
265
|
+
watchdog.start()
|
|
266
|
+
try:
|
|
267
|
+
if use_tls:
|
|
268
|
+
ctx = ssl.create_default_context()
|
|
269
|
+
if not verify_tls:
|
|
270
|
+
ctx.check_hostname = False
|
|
271
|
+
ctx.verify_mode = ssl.CERT_NONE
|
|
272
|
+
sock = ctx.wrap_socket(sock, server_hostname=host)
|
|
273
|
+
live["sock"] = sock
|
|
274
|
+
conn = http.client.HTTPConnection(host, port, timeout=timeout)
|
|
275
|
+
conn.sock = sock # pinned to the vetted IP; http.client will not re-resolve
|
|
276
|
+
conn.request(
|
|
277
|
+
"GET",
|
|
278
|
+
WELL_KNOWN_PATH,
|
|
279
|
+
headers={
|
|
280
|
+
"Host": host,
|
|
281
|
+
"User-Agent": "Umbermark-Verify/1.0",
|
|
282
|
+
"Accept": "text/plain",
|
|
283
|
+
"Connection": "close",
|
|
284
|
+
},
|
|
285
|
+
)
|
|
286
|
+
resp = conn.getresponse()
|
|
287
|
+
if resp.status != 200: # includes 3xx: redirects are never followed
|
|
288
|
+
return None
|
|
289
|
+
body = resp.read(MAX_BODY_BYTES + 1)
|
|
290
|
+
if len(body) > MAX_BODY_BYTES:
|
|
291
|
+
return None
|
|
292
|
+
return body.decode("utf-8", errors="replace")
|
|
293
|
+
finally:
|
|
294
|
+
watchdog.cancel()
|
|
295
|
+
try:
|
|
296
|
+
sock.close()
|
|
297
|
+
except Exception:
|
|
298
|
+
pass
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def fetch_well_known(
|
|
302
|
+
host: str,
|
|
303
|
+
*,
|
|
304
|
+
resolve: Callable[[str], list[str]] = resolve_public_ips,
|
|
305
|
+
attempts: tuple[tuple[int, bool], ...] = ((443, True), (80, False)),
|
|
306
|
+
timeout: float = HTTP_TIMEOUT,
|
|
307
|
+
) -> dict[str, str]:
|
|
308
|
+
"""``{ip: body}`` of ``/.well-known/umbermark-verify.txt`` per resolved IP.
|
|
309
|
+
|
|
310
|
+
The file is fetched from EACH resolved public IP separately (pinned), so the
|
|
311
|
+
caller learns exactly which addresses serve it. Failures (blocked address,
|
|
312
|
+
timeout, non-200, redirect, oversize) simply omit that IP.
|
|
313
|
+
|
|
314
|
+
``timeout`` caps ONE attempt, so a firewall that drops 443 cannot starve the
|
|
315
|
+
port-80 fallback or later IPs. The total deadline is sized for 2 attempts per
|
|
316
|
+
IP and at most MAX_FETCH_IPS IPs, so the worst case stays bounded.
|
|
317
|
+
"""
|
|
318
|
+
h = normalize_target(host)
|
|
319
|
+
try:
|
|
320
|
+
ips = resolve(h)
|
|
321
|
+
except Exception: # BlockedTarget or resolver failure
|
|
322
|
+
return {}
|
|
323
|
+
ips = ips[:MAX_FETCH_IPS]
|
|
324
|
+
out: dict[str, str] = {}
|
|
325
|
+
deadline = time.monotonic() + timeout * len(attempts) * len(ips)
|
|
326
|
+
for ip in ips:
|
|
327
|
+
for port, use_tls in attempts:
|
|
328
|
+
remaining = deadline - time.monotonic()
|
|
329
|
+
if remaining <= 0.05:
|
|
330
|
+
return out
|
|
331
|
+
try:
|
|
332
|
+
# Verify certs for names; an IP literal has no matching cert to check.
|
|
333
|
+
body = _fetch_one(
|
|
334
|
+
h, ip, port, use_tls, min(remaining, timeout),
|
|
335
|
+
verify_tls=not is_ip_literal(h),
|
|
336
|
+
)
|
|
337
|
+
except Exception:
|
|
338
|
+
continue
|
|
339
|
+
if body is not None:
|
|
340
|
+
out[ip] = body
|
|
341
|
+
break
|
|
342
|
+
return out
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
# ---------------------------------------------------------------------------
|
|
346
|
+
# DNS TXT lookup (raw UDP; no extra dependency)
|
|
347
|
+
# ---------------------------------------------------------------------------
|
|
348
|
+
|
|
349
|
+
TxtLookup = Callable[[str], list[str]]
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
_QTYPE_A, _QTYPE_TXT, _QTYPE_AAAA = 1, 16, 28
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def build_query(name: str, qid: int, qtype: int) -> bytes:
|
|
356
|
+
header = struct.pack(">HHHHHH", qid, 0x0100, 1, 0, 0, 0) # RD set, 1 question
|
|
357
|
+
qname = (
|
|
358
|
+
b"".join(
|
|
359
|
+
bytes([len(p)]) + p.encode("ascii") for p in name.rstrip(".").split(".")
|
|
360
|
+
)
|
|
361
|
+
+ b"\x00"
|
|
362
|
+
)
|
|
363
|
+
return header + qname + struct.pack(">HH", qtype, 1) # qtype, IN
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def build_txt_query(name: str, qid: int) -> bytes:
|
|
367
|
+
return build_query(name, qid, _QTYPE_TXT)
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def _skip_name(buf: bytes, off: int) -> int:
|
|
371
|
+
while True:
|
|
372
|
+
ln = buf[off]
|
|
373
|
+
if ln == 0:
|
|
374
|
+
return off + 1
|
|
375
|
+
if ln & 0xC0 == 0xC0:
|
|
376
|
+
return off + 2
|
|
377
|
+
off += 1 + ln
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def parse_txt_response(buf: bytes, qid: int) -> list[str]:
|
|
381
|
+
"""TXT strings (a record's character-strings joined) from a DNS reply."""
|
|
382
|
+
try:
|
|
383
|
+
if len(buf) < 12:
|
|
384
|
+
return []
|
|
385
|
+
rid, flags, qd, an, _, _ = struct.unpack(">HHHHHH", buf[:12])
|
|
386
|
+
if rid != qid or flags & 0x000F: # id mismatch or non-zero RCODE
|
|
387
|
+
return []
|
|
388
|
+
off = 12
|
|
389
|
+
for _ in range(qd):
|
|
390
|
+
off = _skip_name(buf, off) + 4
|
|
391
|
+
out: list[str] = []
|
|
392
|
+
for _ in range(an):
|
|
393
|
+
off = _skip_name(buf, off)
|
|
394
|
+
rtype, _cls, _ttl, rdlen = struct.unpack(">HHIH", buf[off : off + 10])
|
|
395
|
+
off += 10
|
|
396
|
+
rdata = buf[off : off + rdlen]
|
|
397
|
+
off += rdlen
|
|
398
|
+
if rtype != 16:
|
|
399
|
+
continue
|
|
400
|
+
parts, i = [], 0
|
|
401
|
+
while i < len(rdata):
|
|
402
|
+
ln = rdata[i]
|
|
403
|
+
parts.append(
|
|
404
|
+
rdata[i + 1 : i + 1 + ln].decode("utf-8", errors="replace")
|
|
405
|
+
)
|
|
406
|
+
i += 1 + ln
|
|
407
|
+
out.append("".join(parts))
|
|
408
|
+
return out
|
|
409
|
+
except (struct.error, IndexError):
|
|
410
|
+
return []
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def udp_txt_lookup(
|
|
414
|
+
name: str,
|
|
415
|
+
resolvers: Optional[tuple[str, ...]] = None,
|
|
416
|
+
timeout: float = DNS_TIMEOUT,
|
|
417
|
+
) -> list[str]:
|
|
418
|
+
"""Single-shot UDP TXT query with a hard timeout. Empty list on any failure.
|
|
419
|
+
|
|
420
|
+
Default resolvers are the configured ``SCAN_RESOLVERS`` (UMBR-65: on the
|
|
421
|
+
production network UDP replies from 1.1.1.1/8.8.8.8 never arrive, so the
|
|
422
|
+
hard-coded defaults silently failed every DNS-TXT verification), else
|
|
423
|
+
DEFAULT_RESOLVERS.
|
|
424
|
+
"""
|
|
425
|
+
if resolvers is None:
|
|
426
|
+
resolvers = _scan_resolvers or DEFAULT_RESOLVERS
|
|
427
|
+
try:
|
|
428
|
+
name.encode("ascii")
|
|
429
|
+
except UnicodeEncodeError:
|
|
430
|
+
return []
|
|
431
|
+
qid = secrets.randbelow(65536)
|
|
432
|
+
try:
|
|
433
|
+
query = build_txt_query(name, qid)
|
|
434
|
+
except Exception:
|
|
435
|
+
return []
|
|
436
|
+
for server in resolvers:
|
|
437
|
+
try:
|
|
438
|
+
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
|
|
439
|
+
s.settimeout(timeout)
|
|
440
|
+
s.sendto(query, (server, 53))
|
|
441
|
+
data, _ = s.recvfrom(4096)
|
|
442
|
+
return parse_txt_response(data, qid)
|
|
443
|
+
except Exception:
|
|
444
|
+
continue
|
|
445
|
+
return []
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def parse_addr_response(buf: bytes, qid: int) -> Optional[list[str]]:
|
|
449
|
+
"""A/AAAA addresses from a DNS reply (UMBR-65).
|
|
450
|
+
|
|
451
|
+
Returns the addresses for a usable answer (NOERROR, possibly empty, or
|
|
452
|
+
NXDOMAIN -> []), and None when the reply is unusable (wrong id, not a
|
|
453
|
+
response, truncated with no answers, SERVFAIL/REFUSED, malformed) so the
|
|
454
|
+
caller can ask the next resolver. CNAME chains are followed by the
|
|
455
|
+
recursive resolver; every A/AAAA in the answer section is collected.
|
|
456
|
+
"""
|
|
457
|
+
try:
|
|
458
|
+
if len(buf) < 12:
|
|
459
|
+
return None
|
|
460
|
+
rid, flags, qd, an, _, _ = struct.unpack(">HHHHHH", buf[:12])
|
|
461
|
+
if rid != qid or not flags & 0x8000:
|
|
462
|
+
return None
|
|
463
|
+
rcode = flags & 0x000F
|
|
464
|
+
if rcode == 3: # NXDOMAIN: the name does not exist
|
|
465
|
+
return []
|
|
466
|
+
if rcode != 0:
|
|
467
|
+
return None
|
|
468
|
+
if flags & 0x0200 and an == 0: # truncated, nothing usable
|
|
469
|
+
return None
|
|
470
|
+
off = 12
|
|
471
|
+
for _ in range(qd):
|
|
472
|
+
off = _skip_name(buf, off) + 4
|
|
473
|
+
out: list[str] = []
|
|
474
|
+
for _ in range(an):
|
|
475
|
+
off = _skip_name(buf, off)
|
|
476
|
+
rtype, _cls, _ttl, rdlen = struct.unpack(">HHIH", buf[off : off + 10])
|
|
477
|
+
off += 10
|
|
478
|
+
rdata = buf[off : off + rdlen]
|
|
479
|
+
if len(rdata) != rdlen:
|
|
480
|
+
return None
|
|
481
|
+
off += rdlen
|
|
482
|
+
if rtype == _QTYPE_A and rdlen == 4:
|
|
483
|
+
out.append(socket.inet_ntop(socket.AF_INET, rdata))
|
|
484
|
+
elif rtype == _QTYPE_AAAA and rdlen == 16:
|
|
485
|
+
out.append(socket.inet_ntop(socket.AF_INET6, rdata))
|
|
486
|
+
return out
|
|
487
|
+
except (struct.error, IndexError, ValueError, OSError):
|
|
488
|
+
return None
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _udp_query(server: str, query: bytes, timeout: float) -> bytes:
|
|
492
|
+
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
|
|
493
|
+
s.settimeout(timeout)
|
|
494
|
+
s.connect((server, 53)) # connected UDP: replies from anyone else are dropped
|
|
495
|
+
s.send(query)
|
|
496
|
+
return s.recv(4096)
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def udp_addr_lookup(
|
|
500
|
+
name: str,
|
|
501
|
+
resolvers: tuple[str, ...] = DEFAULT_RESOLVERS,
|
|
502
|
+
timeout: float = DNS_TIMEOUT,
|
|
503
|
+
query_fn: Callable[[str, bytes, float], bytes] = _udp_query,
|
|
504
|
+
) -> list[str]:
|
|
505
|
+
"""A + AAAA for ``name`` from the first resolver that gives a usable A answer.
|
|
506
|
+
|
|
507
|
+
Used instead of the host resolver for scan targets (UMBR-65), so the result
|
|
508
|
+
is what the public internet sees. A failed AAAA query just means no IPv6.
|
|
509
|
+
Empty list when nothing usable came back (the caller fails closed).
|
|
510
|
+
"""
|
|
511
|
+
try:
|
|
512
|
+
name.encode("ascii")
|
|
513
|
+
except UnicodeEncodeError:
|
|
514
|
+
return []
|
|
515
|
+
for server in resolvers:
|
|
516
|
+
qid = secrets.randbelow(65536)
|
|
517
|
+
try:
|
|
518
|
+
v4 = parse_addr_response(
|
|
519
|
+
query_fn(server, build_query(name, qid, _QTYPE_A), timeout), qid
|
|
520
|
+
)
|
|
521
|
+
except Exception:
|
|
522
|
+
continue
|
|
523
|
+
if v4 is None:
|
|
524
|
+
continue
|
|
525
|
+
qid6 = secrets.randbelow(65536)
|
|
526
|
+
try:
|
|
527
|
+
v6 = parse_addr_response(
|
|
528
|
+
query_fn(server, build_query(name, qid6, _QTYPE_AAAA), timeout), qid6
|
|
529
|
+
)
|
|
530
|
+
except Exception:
|
|
531
|
+
v6 = None
|
|
532
|
+
return v4 + (v6 or [])
|
|
533
|
+
return []
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
# ---------------------------------------------------------------------------
|
|
537
|
+
# Orchestration
|
|
538
|
+
# ---------------------------------------------------------------------------
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
@dataclass
|
|
542
|
+
class OwnershipResult:
|
|
543
|
+
verified: bool
|
|
544
|
+
method: str = "" # "dns" | "http" | ""
|
|
545
|
+
detail: str = ""
|
|
546
|
+
# Public IPs that themselves served the verification file. Active checks
|
|
547
|
+
# later run only while EVERY resolved IP is inside this set.
|
|
548
|
+
ips: list[str] = field(default_factory=list)
|
|
549
|
+
# Resolved IPs that did NOT serve the file (no active checks for these).
|
|
550
|
+
unproven_ips: list[str] = field(default_factory=list)
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
def verify_target_hash(target: str, pepper: str) -> str:
|
|
554
|
+
"""Lookup key binding a verification to exactly one (normalized) target."""
|
|
555
|
+
return hash_with_pepper(f"ext-verify:{normalize_target(target)}", pepper)
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
def check_ownership(
|
|
559
|
+
host: str,
|
|
560
|
+
token_hash: str,
|
|
561
|
+
*,
|
|
562
|
+
txt_lookup: TxtLookup = udp_txt_lookup,
|
|
563
|
+
fetch: Fetcher = fetch_well_known,
|
|
564
|
+
resolve: Callable[[str], list[str]] = resolve_public_ips,
|
|
565
|
+
) -> OwnershipResult:
|
|
566
|
+
"""Strict model: only a token fetched from an IP's own .well-known file puts
|
|
567
|
+
that IP in ``ips`` (and so unlocks active checks). DNS TXT proves control of
|
|
568
|
+
the NAME only; it is recorded (verified=True, method "dns") but ``ips`` stays
|
|
569
|
+
empty because a name can be pointed at someone else's server."""
|
|
570
|
+
h = normalize_target(host)
|
|
571
|
+
try:
|
|
572
|
+
resolved = resolve(h)
|
|
573
|
+
except Exception:
|
|
574
|
+
return OwnershipResult(
|
|
575
|
+
False, "", "The host does not resolve to a public address."
|
|
576
|
+
)
|
|
577
|
+
dns_ok = False
|
|
578
|
+
if not is_ip_literal(h):
|
|
579
|
+
for name in (h, f"_umbermark.{h}"):
|
|
580
|
+
try:
|
|
581
|
+
records = txt_lookup(name)
|
|
582
|
+
except Exception:
|
|
583
|
+
records = []
|
|
584
|
+
if any(token_matches(r, token_hash) for r in records):
|
|
585
|
+
dns_ok = True
|
|
586
|
+
break
|
|
587
|
+
bodies = fetch(h) or {}
|
|
588
|
+
file_ips = sorted(ip for ip, b in bodies.items() if token_matches(b, token_hash))
|
|
589
|
+
if file_ips:
|
|
590
|
+
return OwnershipResult(
|
|
591
|
+
True, "http", "verification file found", ips=file_ips,
|
|
592
|
+
unproven_ips=sorted(set(resolved) - set(file_ips)),
|
|
593
|
+
)
|
|
594
|
+
if dns_ok:
|
|
595
|
+
return OwnershipResult(
|
|
596
|
+
True, "dns", "TXT record found", unproven_ips=sorted(resolved)
|
|
597
|
+
)
|
|
598
|
+
if is_ip_literal(h):
|
|
599
|
+
return OwnershipResult(
|
|
600
|
+
False, "", f"No matching token at http(s)://{h}{WELL_KNOWN_PATH}."
|
|
601
|
+
)
|
|
602
|
+
return OwnershipResult(
|
|
603
|
+
False,
|
|
604
|
+
"",
|
|
605
|
+
"No matching DNS TXT record or .well-known file found yet. "
|
|
606
|
+
"DNS changes can take a few minutes to propagate.",
|
|
607
|
+
)
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""Network probe abstraction for the external scanner.
|
|
2
|
+
|
|
3
|
+
`ExternalProbe` is to Tier-1 what `CommandRunner` is to Tier-3: the swappable
|
|
4
|
+
seam the checks run against. `FakeProbe` is the test double; `SocketProbe` (a
|
|
5
|
+
real socket implementation) is added in a later slice.
|
|
6
|
+
|
|
7
|
+
Every capability is optional and best-effort: a probe that cannot answer (host
|
|
8
|
+
down, port filtered, handshake failed) returns a falsy value (`False`/`None`)
|
|
9
|
+
rather than raising, so a single unreachable service yields SKIP/PASS — never a
|
|
10
|
+
crashed scan.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
from typing import Optional, Protocol
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class SshAlgorithms:
|
|
20
|
+
"""Algorithms an SSH server advertises in its pre-auth KEXINIT.
|
|
21
|
+
|
|
22
|
+
Read before any authentication — purely the public handshake offer, the
|
|
23
|
+
same data `ssh -Q` / `nmap --script ssh2-enum-algos` would surface.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
kex: list[str] = field(default_factory=list)
|
|
27
|
+
ciphers: list[str] = field(default_factory=list)
|
|
28
|
+
macs: list[str] = field(default_factory=list)
|
|
29
|
+
host_keys: list[str] = field(default_factory=list)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class TlsInfo:
|
|
34
|
+
"""What the TLS handshake reveals — all visible without authenticating.
|
|
35
|
+
|
|
36
|
+
``protocols`` is every TLS/SSL version the endpoint agrees to negotiate;
|
|
37
|
+
``days_until_expiry`` is negative once the leaf cert has expired.
|
|
38
|
+
``unprobed`` lists versions the *scanner's own* TLS stack could not offer
|
|
39
|
+
(e.g. TLS 1.0/1.1 under an OpenSSL 3 build that refuses them), so whether
|
|
40
|
+
the server accepts them is unknown -- never read that as "not offered".
|
|
41
|
+
``cert_readable`` is False when the leaf certificate could not be obtained
|
|
42
|
+
or parsed; ``days_until_expiry`` / ``self_signed`` are then meaningless.
|
|
43
|
+
``chain_trusted`` is the result of a verifying handshake against a public
|
|
44
|
+
trust store (UMBR-40): True = chain verified, False = verification failed
|
|
45
|
+
(``trust_error`` says why), None = could not be checked -- unknown, which
|
|
46
|
+
is never read as trusted.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
protocols: list[str] = field(default_factory=list)
|
|
50
|
+
days_until_expiry: int = 0
|
|
51
|
+
self_signed: bool = False
|
|
52
|
+
unprobed: list[str] = field(default_factory=list)
|
|
53
|
+
cert_readable: bool = True
|
|
54
|
+
chain_trusted: Optional[bool] = None
|
|
55
|
+
trust_error: str = ""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class ExternalProbe(Protocol):
|
|
59
|
+
def tcp_connect(self, port: int, timeout: float = 3.0) -> bool:
|
|
60
|
+
"""True if a TCP connection to ``port`` on the target succeeds."""
|
|
61
|
+
...
|
|
62
|
+
|
|
63
|
+
def ssh_algorithms(
|
|
64
|
+
self, port: int = 22, timeout: float = 3.0
|
|
65
|
+
) -> Optional[SshAlgorithms]:
|
|
66
|
+
"""Algorithms offered in the SSH KEXINIT, or None if unreadable."""
|
|
67
|
+
...
|
|
68
|
+
|
|
69
|
+
def tls_info(self, port: int = 443, timeout: float = 3.0) -> Optional[TlsInfo]:
|
|
70
|
+
"""TLS protocols + leaf-cert facts, or None if no TLS service answers."""
|
|
71
|
+
...
|
|
72
|
+
|
|
73
|
+
def http_headers(
|
|
74
|
+
self, port: int = 443, tls: bool = True, timeout: float = 3.0
|
|
75
|
+
) -> Optional[dict]:
|
|
76
|
+
"""Response headers (lowercased keys), or None if no HTTP service answers."""
|
|
77
|
+
...
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass
|
|
81
|
+
class FakeProbe:
|
|
82
|
+
"""Test double — a programmable set of open ports and canned probe results."""
|
|
83
|
+
|
|
84
|
+
open_ports: set[int] = field(default_factory=set)
|
|
85
|
+
ssh_algos: Optional[SshAlgorithms] = None
|
|
86
|
+
tls: Optional[TlsInfo] = None
|
|
87
|
+
headers: Optional[dict] = None
|
|
88
|
+
|
|
89
|
+
def tcp_connect(self, port: int, timeout: float = 3.0) -> bool:
|
|
90
|
+
return port in self.open_ports
|
|
91
|
+
|
|
92
|
+
def ssh_algorithms(
|
|
93
|
+
self, port: int = 22, timeout: float = 3.0
|
|
94
|
+
) -> Optional[SshAlgorithms]:
|
|
95
|
+
return self.ssh_algos
|
|
96
|
+
|
|
97
|
+
def tls_info(self, port: int = 443, timeout: float = 3.0) -> Optional[TlsInfo]:
|
|
98
|
+
return self.tls
|
|
99
|
+
|
|
100
|
+
def http_headers(
|
|
101
|
+
self, port: int = 443, tls: bool = True, timeout: float = 3.0
|
|
102
|
+
) -> Optional[dict]:
|
|
103
|
+
return self.headers
|