pubhost 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
pubhost/__init__.py ADDED
@@ -0,0 +1,24 @@
1
+ """pubhost: what makes a repository's sites and zones exist and stay right
2
+ at the outside systems that serve them. Its Terraform modules (terraform/)
3
+ turn pubrules.json's hosting block into DNS, mail, DNSSEC, Cloud Run and
4
+ bucket resources; `pubhost dns check` compares a zone's nameservers live;
5
+ terraform.yml applies a stack from CI; the actions build an image
6
+ (docker-build) and deploy a static site (deploy-static), which first runs
7
+ `pubhost site check`.
8
+
9
+ It uses pubrules as a library, for the hosting block's shape (pubrules.hosting),
10
+ which `pubrules sync` checks; pubrules never imports it.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+
16
+ def _version() -> str:
17
+ from importlib.metadata import PackageNotFoundError, version # noqa: PLC0415
18
+ try:
19
+ return version("pubhost")
20
+ except PackageNotFoundError:
21
+ return "0+source"
22
+
23
+
24
+ __version__ = _version()
pubhost/cli.py ADDED
@@ -0,0 +1,65 @@
1
+ """pubhost: a repository's hosting, checked live.
2
+
3
+ pubhost dns check ZONE [--record NAME/TYPE...]
4
+ pubhost site check DIR [--url URL]
5
+
6
+ Each runs in the repository's root, where pubrules.json is. The Terraform
7
+ modules, terraform.yml and the deploy-static and docker-build actions are
8
+ used from the repository by commit (README).
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import argparse
14
+ import os
15
+ import sys
16
+ import tempfile
17
+ from pathlib import Path
18
+
19
+ from . import __version__
20
+
21
+
22
+ def parser() -> argparse.ArgumentParser:
23
+ parser = argparse.ArgumentParser(prog="pubhost", description=__doc__.split("\n")[0])
24
+ parser.add_argument("--version", action="version", version=f"pubhost {__version__}")
25
+ commands = parser.add_subparsers(dest="command", required=True)
26
+ p = commands.add_parser("dns", help="check a zone in pubrules.json's hosting block, live")
27
+ dns = p.add_subparsers(dest="dns_command", required=True)
28
+ p = dns.add_parser("check", help="every nameserver gives the same answers and, with dnssec, "
29
+ "validates alone against the pinned keys")
30
+ p.add_argument("zone", help="a zone in pubrules.json's hosting block, such as example.eu")
31
+ p.add_argument("--record", action="append", default=[], metavar="NAME/TYPE",
32
+ help="another rrset to compare, such as www/A; repeat it")
33
+ p = commands.add_parser("site", help="check a public site's built pages")
34
+ site = p.add_subparsers(dest="site_command", required=True)
35
+ p = site.add_parser("check", help="every page names itself, has an Open Graph card with a "
36
+ "large enough image, and robots.txt lets preview bots in")
37
+ p.add_argument("dir", type=Path, help="the built site, such as _site")
38
+ p.add_argument("--url", help="the site's https:// address, to find og:image files in DIR")
39
+ return parser
40
+
41
+
42
+ # A copy of pubrules' paths.confined (pubrules 0.34.0): a path given on the
43
+ # command line lies in the working directory or a temporary directory.
44
+ def confined(path: Path | str, what: str) -> Path:
45
+ bases = [Path.cwd(), Path(tempfile.gettempdir())]
46
+ if os.environ.get("RUNNER_TEMP"):
47
+ bases.append(Path(os.environ["RUNNER_TEMP"]))
48
+ resolved = Path(path).resolve()
49
+ if any(resolved.is_relative_to(base.resolve()) for base in bases):
50
+ return resolved
51
+ sys.exit(f"pubhost: {what} ({path}) must be in the working directory or a temporary directory")
52
+
53
+
54
+ def main(argv: list[str] | None = None) -> int:
55
+ args = parser().parse_args(argv)
56
+ root = Path.cwd()
57
+ if args.command == "dns":
58
+ from .dns import check # noqa: PLC0415
59
+ return check(root, args.zone, args.record)
60
+ from .site import run as site # noqa: PLC0415
61
+ return site(root, confined(args.dir, "the site directory"), args.url)
62
+
63
+
64
+ if __name__ == "__main__":
65
+ sys.exit(main())
pubhost/dns.py ADDED
@@ -0,0 +1,318 @@
1
+ """`pubhost dns check ZONE`: does every nameserver of a zone in pubrules.json's
2
+ hosting block give the same answers, and, for a signed zone, does each one
3
+ validate on its own?
4
+
5
+ The nameservers are the ones the zone's providers are given in pubrules.json
6
+ (dns.external_nameservers), the zone's NS rrset as resolvers find it, and
7
+ the apex NS rrset each of those serves, so a provider joining a zone is
8
+ asked before the registrar lists it. Each is asked, without recursion, for
9
+ every rrset the hosting block describes (the apex NS rrset, the mail
10
+ provider's records, each site's) and any given with --record; their
11
+ answers must be the same, TTLs aside (deSEC raises a TTL below its minimum).
12
+
13
+ With the zone's dnssec block, each nameserver must also serve exactly the
14
+ keys it pins, every provider's, and validate alone: delv asks it, and only
15
+ it, for each rrset and for a name that does not exist, with the pinned
16
+ key-signing keys as trust anchors, which is what a resolver does once the
17
+ registrar has a DS record for each. A provider that cannot sign, or one
18
+ that changed its keys, fails here before any DS record is at the registrar.
19
+
20
+ dig and delv are BIND 9's (Debian's bind9-dnsutils). Nothing is changed
21
+ anywhere: every query is a read.
22
+
23
+ The hosting block's shape is pubrules' (`pubrules.hosting`, which `pubrules sync`
24
+ checks); this reads the block through it.
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ import json
30
+ import re
31
+ import secrets
32
+ import shutil
33
+ import subprocess
34
+ import sys
35
+ import tempfile
36
+ from dataclasses import dataclass, field
37
+ from pathlib import Path
38
+ from typing import Any
39
+
40
+ from pubrules.hosting import SEP, dnskey, settings
41
+
42
+ TIMEOUT = 30
43
+ RDATA = 4 # the fields of a record dig prints before its value: owner, TTL, class, type
44
+ KEY_FIELDS = 3 # a DNSKEY's flags, protocol and algorithm, before its key
45
+ NAME_TYPES = {"NS", "CNAME", "MX"}
46
+ VALIDATED = ("; fully validated", "; negative response, fully validated")
47
+
48
+ # What may reach dig and delv: absolute names in lower case, whose labels
49
+ # are letters, digits, hyphens and underscores and start with neither a
50
+ # hyphen nor a plus, and types in capitals. Names and types also go after
51
+ # -q and -t, so neither is ever read as an option, and nothing else (a
52
+ # name from pubrules.json, from --record, or from a nameserver's answer) is
53
+ # passed or written into delv's trust anchors.
54
+ NAME = re.compile(r"(?:[a-z0-9_][a-z0-9_-]{0,62}\.)+")
55
+ TYPE = re.compile(r"[A-Z][A-Z0-9]{0,9}")
56
+ LONGEST = 254 # an absolute name's length, its final dot included
57
+ KEY = re.compile(r"(256|257) 3 [0-9]{1,3} [A-Za-z0-9+/]+={0,2}")
58
+
59
+
60
+ @dataclass
61
+ class Answer:
62
+ """One nameserver's answer: its status, whether it is authoritative, and
63
+ the rrset's values, normalised so two providers' compare."""
64
+
65
+ status: str = ""
66
+ authoritative: bool = False
67
+ values: frozenset[str] = field(default_factory=frozenset)
68
+
69
+
70
+ def tool(name: str) -> str:
71
+ found = shutil.which(name)
72
+ if not found:
73
+ sys.exit(f"pubhost: dns check needs {name}, from BIND 9 (such as Debian's bind9-dnsutils)")
74
+ return found
75
+
76
+
77
+ def checked(name: str) -> str:
78
+ """An absolute name, as NAME allows; anything else stops the check."""
79
+ if not NAME.fullmatch(name) or len(name) > LONGEST:
80
+ sys.exit(f"pubhost: {name!r} is not a DNS name pubhost dns check asks about")
81
+ return name
82
+
83
+
84
+ def checked_type(rtype: str) -> str:
85
+ if not TYPE.fullmatch(rtype):
86
+ sys.exit(f"pubhost: {rtype!r} is not a DNS record type, such as A or MX")
87
+ return rtype
88
+
89
+
90
+ def absolute(name: str, zone: str) -> str:
91
+ """A name relative to the zone ("@" for the apex), made absolute."""
92
+ return checked(f"{zone}." if name == "@" else f"{name}.{zone}.")
93
+
94
+
95
+ def normalise(rtype: str, rdata: list[str]) -> str:
96
+ """A record's value as dig prints it, in one form: host names in lower
97
+ case, and a DNSKEY's key in one piece (dig may split a long one)."""
98
+ if rtype in NAME_TYPES:
99
+ return " ".join(rdata).lower()
100
+ if rtype == "DNSKEY" and len(rdata) > KEY_FIELDS:
101
+ return " ".join([*rdata[:KEY_FIELDS], "".join(rdata[KEY_FIELDS:])])
102
+ return " ".join(rdata)
103
+
104
+
105
+ def dig(server: str, fqdn: str, rtype: str) -> Answer:
106
+ """Ask one nameserver, without recursion, for one rrset."""
107
+ out = subprocess.run(
108
+ [tool("dig"), "+norec", "+time=3", "+tries=2", "+noall", "+comments", "+answer",
109
+ f"@{checked(server)}", "-q", checked(fqdn), "-t", checked_type(rtype)],
110
+ capture_output=True, text=True, timeout=TIMEOUT, check=False).stdout
111
+ answer = Answer()
112
+ values = set()
113
+ for line in out.splitlines():
114
+ if "status:" in line:
115
+ answer.status = line.split("status:")[1].split(",")[0].strip()
116
+ elif line.startswith(";; flags:"):
117
+ answer.authoritative = "aa" in line.split(":")[1].split(";")[0].split()
118
+ elif line and not line.startswith(";"):
119
+ fields = line.split()
120
+ if len(fields) > RDATA and fields[0].lower() == fqdn.lower() and fields[RDATA - 1] == rtype:
121
+ values.add(normalise(rtype, fields[RDATA:]))
122
+ answer.values = frozenset(values)
123
+ return answer
124
+
125
+
126
+ def resolved_nameservers(zone: str) -> set[str]:
127
+ """The zone's NS rrset as this machine's resolver finds it."""
128
+ out = subprocess.run([tool("dig"), "+short", "-q", checked(f"{zone}."), "-t", "NS"],
129
+ capture_output=True, text=True, timeout=TIMEOUT, check=False).stdout
130
+ return {line.strip().lower() for line in out.splitlines() if NAME.fullmatch(line.strip().lower())}
131
+
132
+
133
+ def configured_nameservers(zone: dict[str, Any], providers: dict[str, Any]) -> set[str]:
134
+ return {
135
+ f"{name.lower().rstrip('.')}."
136
+ for provider in zone["dns"]
137
+ for name in providers[provider].get("dns", {}).get("external_nameservers", [])
138
+ }
139
+
140
+
141
+ def nameservers(name: str, zone: dict[str, Any], providers: dict[str, Any]) -> list[str]:
142
+ """Every nameserver of the zone: the configured and resolved ones, and
143
+ those any of them lists in its apex NS rrset."""
144
+ found = configured_nameservers(zone, providers) | resolved_nameservers(name)
145
+ for server in sorted(found):
146
+ found |= {ns for ns in dig(server, f"{name}.", "NS").values if NAME.fullmatch(ns)}
147
+ return sorted(found)
148
+
149
+
150
+ def rrsets(zone: dict[str, Any], providers: dict[str, Any], extra: list[str]) -> list[tuple[str, str]]:
151
+ """The (name, type) of every rrset the hosting block describes for the
152
+ zone, and those --record adds."""
153
+ wanted = [("@", "NS")]
154
+ if "dnssec" in zone:
155
+ wanted.append(("@", "DNSKEY"))
156
+ email = zone.get("email")
157
+ mail = providers[email["provider"]] if email else {}
158
+ if mail.get("mx"):
159
+ wanted.append(("@", "MX"))
160
+ if mail.get("spf") or zone.get("apex_txt"):
161
+ wanted.append(("@", "TXT"))
162
+ if email and "dmarc" in email:
163
+ wanted.append(("_dmarc", "TXT"))
164
+ wanted += [(f"{selector}._domainkey", "CNAME") for selector in mail.get("dkim", {})]
165
+ wanted += [(name, "TXT") for name in (email or {}).get("verification", {})]
166
+ for name, site in zone.get("sites", {}).items():
167
+ cname = site["kind"] == "cloud-run" or (
168
+ site["kind"] == "pages" and site.get("served_by", "pages") == "pages")
169
+ wanted += [(name, "CNAME")] if cname else [(name, "A"), (name, "AAAA")]
170
+ for record in extra:
171
+ name, _, rtype = record.rpartition("/")
172
+ if not name or not rtype:
173
+ sys.exit(f"pubhost: --record {record} must be NAME/TYPE, the name relative to the zone "
174
+ "(\"@\" for the apex), such as www/A")
175
+ wanted.append((name, checked_type(rtype.upper())))
176
+ return list(dict.fromkeys(wanted))
177
+
178
+
179
+ def compare(zone: str, servers: list[str], wanted: list[tuple[str, str]]) -> list[str]:
180
+ """Every rrset at every nameserver: each must answer for the zone, and
181
+ all with the same values."""
182
+ problems = []
183
+ refusing: set[str] = set()
184
+ for name, rtype in wanted:
185
+ fqdn = absolute(name, zone)
186
+ answers = {server: dig(server, fqdn, rtype) for server in servers}
187
+ for server, answer in answers.items():
188
+ serving = answer.authoritative and answer.status in ("NOERROR", "NXDOMAIN")
189
+ if server not in refusing and not serving:
190
+ refusing.add(server)
191
+ problems.append(f"{server} does not serve {zone} (status {answer.status or 'none'}, "
192
+ f"{'authoritative' if answer.authoritative else 'not authoritative'})")
193
+ given = {server: answer.values for server, answer in answers.items() if server not in refusing}
194
+ if len(set(given.values())) > 1:
195
+ lines = [f" {server}: {', '.join(sorted(values)) or '(nothing)'}"
196
+ for server, values in sorted(given.items())]
197
+ problems.append(f"{fqdn} {rtype} differs between nameservers:\n" + "\n".join(lines))
198
+ return problems
199
+
200
+
201
+ def pinned_keys(dnssec: dict[str, Any]) -> set[str]:
202
+ return {normalise("DNSKEY", key.split(" ")) for keys in dnssec["keys"].values() for key in keys}
203
+
204
+
205
+ def check_keys(zone: str, servers: list[str], pinned: set[str]) -> list[str]:
206
+ """Each nameserver's DNSKEY rrset is the keys pubrules.json pins, all of them."""
207
+ problems = []
208
+ for server in servers:
209
+ served = dig(server, f"{zone}.", "DNSKEY").values
210
+ for key in sorted(served - pinned):
211
+ problems.append(f"{server} serves a key pubrules.json does not pin: {key}")
212
+ for key in sorted(pinned - served):
213
+ problems.append(f"{server} does not serve the pinned key {key}")
214
+ return problems
215
+
216
+
217
+ def anchors(zone: str, pinned: set[str]) -> str:
218
+ """delv's trust anchors: the pinned key-signing keys, which the
219
+ registrar's DS records will name."""
220
+ lines = []
221
+ for key in sorted(pinned):
222
+ flags, protocol, algorithm, _ = dnskey(key)
223
+ if flags == SEP and KEY.fullmatch(key):
224
+ lines.append(f' "{checked(f"{zone}.")}" static-key {flags} {protocol} {algorithm} '
225
+ f'"{key.split(" ")[3]}";')
226
+ return "trust-anchors {\n" + "\n".join(lines) + "\n};\n"
227
+
228
+
229
+ def validate(zone: str, servers: list[str], wanted: list[tuple[str, str]], pinned: set[str]) -> list[str]:
230
+ """delv at each nameserver alone, with the pinned keys as anchors: every
231
+ rrset, and a name that does not exist, must validate."""
232
+ problems = []
233
+ missing = f"pubhost-check-{secrets.token_hex(6)}"
234
+ with tempfile.TemporaryDirectory() as tmp:
235
+ anchor_file = Path(tmp) / "anchors.conf"
236
+ anchor_file.write_text(anchors(zone, pinned), encoding="utf-8")
237
+ for server in servers:
238
+ for name, rtype in [*wanted, (missing, "A")]:
239
+ fqdn = absolute(name, zone)
240
+ result = subprocess.run(
241
+ [tool("delv"), "-a", str(anchor_file), f"+root={checked(f'{zone}.')}",
242
+ f"@{checked(server)}", "-q", fqdn, "-t", checked_type(rtype)],
243
+ capture_output=True, text=True, timeout=TIMEOUT, check=False)
244
+ lines = (result.stdout + result.stderr).splitlines()
245
+ if not any(line.strip() in VALIDATED for line in lines):
246
+ reason = next((line.lstrip("; ") for line in lines if line.startswith(";;")),
247
+ "no answer")
248
+ what = "a name that does not exist" if name == missing else f"{fqdn} {rtype}"
249
+ problems.append(f"{server}: {what} does not validate against the pinned keys "
250
+ f"({reason})")
251
+ return problems
252
+
253
+
254
+ SERIAL = 2 # the serial's place in an SOA value: mname rname serial ...
255
+
256
+
257
+ def lagging(zone: str, servers: list[str]) -> list[str]:
258
+ """Nameservers serving an older version of the zone than another with
259
+ the same primary (one provider's): written moments ago, they have not
260
+ caught up yet, so the check is worth running again in a minute."""
261
+ serials: dict[str, dict[str, int]] = {}
262
+ for server in servers:
263
+ for value in dig(server, f"{zone}.", "SOA").values:
264
+ fields = value.split()
265
+ if len(fields) > SERIAL and fields[SERIAL].isdigit():
266
+ serials.setdefault(fields[0].lower(), {})[server] = int(fields[SERIAL])
267
+ hints = []
268
+ for primary, by_server in sorted(serials.items()):
269
+ newest = max(by_server.values())
270
+ for server, serial in sorted(by_server.items()):
271
+ if serial < newest:
272
+ hints.append(f"{server} serves serial {serial} of the zone, older than {newest} at "
273
+ f"the other nameservers of {primary}: it has not caught up with the "
274
+ "last change yet, so run the check again in a minute")
275
+ return hints
276
+
277
+
278
+ def read_config(root: Path) -> dict[str, Any]:
279
+ """pubrules.json, as JSON; its hosting block is checked by pubrules.hosting."""
280
+ path = root / "pubrules.json"
281
+ if not path.is_file():
282
+ sys.exit(f"pubhost: no pubrules.json in {root}; dns check reads its hosting block")
283
+ try:
284
+ config = json.loads(path.read_text(encoding="utf-8"))
285
+ except json.JSONDecodeError as e:
286
+ sys.exit(f"pubhost: pubrules.json is not JSON: {e}")
287
+ if not isinstance(config, dict):
288
+ sys.exit("pubhost: pubrules.json must be a JSON object")
289
+ return config
290
+
291
+
292
+ def check(root: Path, zone: str, extra: list[str]) -> int:
293
+ hosting = settings(read_config(root))
294
+ zones = hosting.get("zones", {})
295
+ if zone not in zones:
296
+ sys.exit(f"pubhost: pubrules.json's hosting block has no zone {zone}; it has "
297
+ f"{', '.join(sorted(zones)) or 'none'}")
298
+ providers = hosting.get("providers", {})
299
+ config = zones[zone]
300
+ servers = nameservers(zone, config, providers)
301
+ wanted = rrsets(config, providers, extra)
302
+ # The DNSKEY rrset is checked against the pinned keys, which says more.
303
+ problems = compare(zone, servers, [rrset for rrset in wanted if rrset[1] != "DNSKEY"])
304
+ if "dnssec" in config:
305
+ pinned = pinned_keys(config["dnssec"])
306
+ problems += check_keys(zone, servers, pinned)
307
+ problems += validate(zone, servers, wanted, pinned)
308
+ for problem in problems:
309
+ print(f"pubhost: {problem}", file=sys.stderr)
310
+ if problems:
311
+ for hint in lagging(zone, servers):
312
+ print(f"pubhost: {hint}", file=sys.stderr)
313
+ return 1
314
+ print(f"{zone}: {len(wanted)} rrsets agree at {len(servers)} nameservers: {', '.join(servers)}")
315
+ if "dnssec" in config:
316
+ print(f"{zone}: every nameserver serves the {len(pinned)} pinned keys and validates alone, "
317
+ "a name that does not exist included")
318
+ return 0