targetzero 0.3.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. targetzero/__init__.py +4 -0
  2. targetzero/__main__.py +6 -0
  3. targetzero/admins.py +109 -0
  4. targetzero/cli.py +366 -0
  5. targetzero/common/__init__.py +2 -0
  6. targetzero/common/changes.py +46 -0
  7. targetzero/common/config.py +83 -0
  8. targetzero/common/ids.py +17 -0
  9. targetzero/common/output.py +91 -0
  10. targetzero/common/resources.py +32 -0
  11. targetzero/common/roles.py +24 -0
  12. targetzero/common/schema.py +45 -0
  13. targetzero/config/approved_exposure.example.csv +4 -0
  14. targetzero/config/privileged_roles.yaml +50 -0
  15. targetzero/config/targetzero.example.yaml +57 -0
  16. targetzero/external/__init__.py +2 -0
  17. targetzero/external/dangling.py +106 -0
  18. targetzero/external/discover.py +181 -0
  19. targetzero/external/dnsinfo.py +54 -0
  20. targetzero/external/enrich.py +137 -0
  21. targetzero/external/http.py +33 -0
  22. targetzero/external/kev.py +222 -0
  23. targetzero/external/ports.py +63 -0
  24. targetzero/external/scan.py +260 -0
  25. targetzero/external/scope.py +54 -0
  26. targetzero/external/waf.py +202 -0
  27. targetzero/external/web.py +124 -0
  28. targetzero/gws/__init__.py +2 -0
  29. targetzero/gws/admins.py +230 -0
  30. targetzero/gws/changes.py +190 -0
  31. targetzero/gws/client.py +94 -0
  32. targetzero/gws/domains.py +36 -0
  33. targetzero/registers/__init__.py +2 -0
  34. targetzero/registers/exceptions.py +85 -0
  35. targetzero/registers/findings.py +108 -0
  36. targetzero/registers/inventory.py +142 -0
  37. targetzero/registers/manual_admins.py +43 -0
  38. targetzero/schemas/admins.schema.json +27 -0
  39. targetzero/schemas/alert_tests.schema.json +19 -0
  40. targetzero/schemas/config.schema.json +71 -0
  41. targetzero/schemas/exceptions.schema.json +20 -0
  42. targetzero/schemas/findings.schema.json +23 -0
  43. targetzero/schemas/inventory.schema.json +26 -0
  44. targetzero/schemas/summary.schema.json +36 -0
  45. targetzero/scorecard/__init__.py +2 -0
  46. targetzero/scorecard/scorecard.py +201 -0
  47. targetzero/templates/alert_tests.csv +1 -0
  48. targetzero/templates/exceptions.csv +1 -0
  49. targetzero/templates/findings.csv +1 -0
  50. targetzero/templates/inventory.csv +1 -0
  51. targetzero/templates/manual_admins.csv +1 -0
  52. targetzero/templates/origins.csv +1 -0
  53. targetzero/templates/recovery_page.md +51 -0
  54. targetzero/vendors/__init__.py +2 -0
  55. targetzero/vendors/cisco_asa.py +151 -0
  56. targetzero/vendors/common.py +84 -0
  57. targetzero/vendors/fortigate.py +176 -0
  58. targetzero/vendors/panos.py +141 -0
  59. targetzero/vendors/redfish.py +124 -0
  60. targetzero/vendors/vsphere.py +108 -0
  61. targetzero-0.3.2.dist-info/METADATA +116 -0
  62. targetzero-0.3.2.dist-info/RECORD +66 -0
  63. targetzero-0.3.2.dist-info/WHEEL +4 -0
  64. targetzero-0.3.2.dist-info/entry_points.txt +2 -0
  65. targetzero-0.3.2.dist-info/licenses/LICENSE +202 -0
  66. targetzero-0.3.2.dist-info/licenses/NOTICE +7 -0
@@ -0,0 +1,260 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Exposure scan (Zero 4): open TCP ports on your public IPs, compared with the approved list.
3
+
4
+ Run it from outside your network (a cloud VM, a home line), never from inside:
5
+ from inside you see your own LAN, not what the internet sees.
6
+
7
+ Only addresses inside scope.ip_ranges are scanned. Hostnames are resolved and
8
+ their addresses scanned only when those addresses are inside the ranges too:
9
+ a website on a CDN is not yours to scan.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import asyncio
15
+ import json
16
+ import shutil
17
+ import socket
18
+ import subprocess
19
+ import urllib.error
20
+ from collections.abc import Iterable
21
+ from dataclasses import dataclass, field
22
+ from datetime import date
23
+ from pathlib import Path
24
+
25
+ from defusedxml import ElementTree
26
+
27
+ from targetzero.common.config import Config
28
+ from targetzero.common.ids import make_id
29
+ from targetzero.common.output import read_csv, run_dir, write_csv, write_summary
30
+ from targetzero.external.http import get_json
31
+ from targetzero.external.ports import PORTS, describe
32
+ from targetzero.external.scope import Scope
33
+ from targetzero.registers import inventory
34
+
35
+ FIELDS = (
36
+ "ip",
37
+ "hostnames",
38
+ "port",
39
+ "protocol",
40
+ "category",
41
+ "service",
42
+ "seen_by",
43
+ "approved",
44
+ "justification",
45
+ )
46
+ INTERNETDB_FILE = "external_internetdb.json"
47
+
48
+
49
+ @dataclass
50
+ class Target:
51
+ ip: str
52
+ hostnames: set[str] = field(default_factory=set)
53
+
54
+
55
+ def collect_targets(
56
+ config: Config, scope: Scope, max_hosts: int, resolve: bool = True
57
+ ) -> tuple[dict[str, Target], list[str]]:
58
+ """Addresses in scope, plus in-scope addresses of known hostnames."""
59
+ targets = {ip: Target(ip) for ip in scope.addresses(max_hosts)}
60
+ notes: list[str] = []
61
+ if not resolve:
62
+ return targets, notes
63
+ names = set(scope.domains)
64
+ names |= {
65
+ r["asset"] for r in inventory.load(config) if r.get("type") in ("domain", "subdomain")
66
+ }
67
+ for name in sorted(names):
68
+ try:
69
+ addresses = {
70
+ info[4][0] for info in socket.getaddrinfo(name, None, type=socket.SOCK_STREAM)
71
+ }
72
+ except socket.gaierror:
73
+ continue
74
+ for ip in addresses:
75
+ if scope.contains_ip(str(ip)):
76
+ targets.setdefault(str(ip), Target(str(ip))).hostnames.add(name)
77
+ else:
78
+ notes.append(f"{name} -> {ip}: outside scope.ip_ranges, not scanned")
79
+ return targets, notes
80
+
81
+
82
+ def scan_nmap(ips: Iterable[str], ports: Iterable[int], nmap: str) -> dict[str, set[int]]:
83
+ command = [
84
+ nmap,
85
+ "-Pn",
86
+ "-n",
87
+ "-sT",
88
+ "--open",
89
+ "-T4",
90
+ "-p",
91
+ ",".join(str(p) for p in sorted(ports)),
92
+ "-oX",
93
+ "-",
94
+ *sorted(ips),
95
+ ]
96
+ result = subprocess.run(command, capture_output=True, text=True, check=True, timeout=3600) # noqa: S603
97
+ return parse_nmap_xml(result.stdout)
98
+
99
+
100
+ def parse_nmap_xml(xml: str) -> dict[str, set[int]]:
101
+ found: dict[str, set[int]] = {}
102
+ root = ElementTree.fromstring(xml)
103
+ for host in root.iter("host"):
104
+ address = host.find("address")
105
+ if address is None:
106
+ continue
107
+ ip = address.get("addr", "")
108
+ for port in host.iter("port"):
109
+ state = port.find("state")
110
+ if state is not None and state.get("state") == "open" and port.get("protocol") == "tcp":
111
+ found.setdefault(ip, set()).add(int(port.get("portid", "0")))
112
+ return found
113
+
114
+
115
+ async def _probe(ip: str, port: int, timeout: float, limit: asyncio.Semaphore) -> bool:
116
+ async with limit:
117
+ try:
118
+ _, writer = await asyncio.wait_for(asyncio.open_connection(ip, port), timeout)
119
+ except (TimeoutError, OSError):
120
+ return False
121
+ writer.close()
122
+ try:
123
+ await writer.wait_closed()
124
+ except OSError:
125
+ pass
126
+ return True
127
+
128
+
129
+ async def _scan_connect(
130
+ ips: list[str], ports: list[int], timeout: float, concurrency: int
131
+ ) -> dict[str, set[int]]:
132
+ limit = asyncio.Semaphore(concurrency)
133
+ pairs = [(ip, port) for ip in ips for port in ports]
134
+ results = await asyncio.gather(*(_probe(ip, port, timeout, limit) for ip, port in pairs))
135
+ found: dict[str, set[int]] = {}
136
+ for (ip, port), is_open in zip(pairs, results, strict=True):
137
+ if is_open:
138
+ found.setdefault(ip, set()).add(port)
139
+ return found
140
+
141
+
142
+ def scan_connect(
143
+ ips: Iterable[str], ports: Iterable[int], timeout: float = 1.5, concurrency: int = 200
144
+ ) -> dict[str, set[int]]:
145
+ return asyncio.run(_scan_connect(sorted(ips), sorted(ports), timeout, concurrency))
146
+
147
+
148
+ def internetdb(ips: Iterable[str]) -> tuple[dict[str, dict[str, list[object]]], list[str]]:
149
+ """Shodan InternetDB: free, no key. Data can be up to a week old."""
150
+ data, notes = {}, []
151
+ for ip in sorted(ips):
152
+ try:
153
+ data[ip] = get_json(f"https://internetdb.shodan.io/{ip}", timeout=15)
154
+ except urllib.error.HTTPError as exc:
155
+ if exc.code != 404: # 404: InternetDB has nothing on this address
156
+ notes.append(f"InternetDB {ip}: HTTP {exc.code}")
157
+ except (urllib.error.URLError, TimeoutError, ValueError) as exc:
158
+ notes.append(f"InternetDB {ip}: {exc}")
159
+ return data, notes
160
+
161
+
162
+ def load_approved(config: Config) -> dict[tuple[str, int], str]:
163
+ path = config.registers_dir / "approved_exposure.csv"
164
+ if not path.exists():
165
+ return {}
166
+ approved = {}
167
+ for row in read_csv(path):
168
+ if row.get("protocol", "tcp").lower() == "tcp" and row.get("port", "").isdigit():
169
+ approved[(row["host"].lower(), int(row["port"]))] = row.get("justification", "")
170
+ return approved
171
+
172
+
173
+ def run(
174
+ config: Config,
175
+ run_date: date,
176
+ *,
177
+ use_nmap: bool = True,
178
+ use_internetdb: bool = False,
179
+ max_hosts: int = 1024,
180
+ timeout: float = 1.5,
181
+ ) -> list[Path]:
182
+ scope = Scope.from_config(config)
183
+ targets, notes = collect_targets(config, scope, max_hosts)
184
+ complete = True
185
+ if not targets:
186
+ notes.append("Nothing to scan: scope.ip_ranges is empty and no hostname resolves inside it")
187
+ complete = False
188
+
189
+ ports = list(PORTS)
190
+ nmap = shutil.which("nmap") if use_nmap else None
191
+ try:
192
+ if nmap:
193
+ open_ports = scan_nmap(targets, ports, nmap)
194
+ notes.append("Scanned with nmap (TCP connect)")
195
+ else:
196
+ open_ports = scan_connect(targets, ports, timeout=timeout)
197
+ notes.append("Scanned with the built-in TCP connect scanner")
198
+ except (subprocess.SubprocessError, OSError) as exc:
199
+ notes.append(f"Scan failed: {exc}")
200
+ open_ports, complete = {}, False
201
+
202
+ directory = run_dir(config.output_dir, run_date)
203
+ seen_by: dict[tuple[str, int], set[str]] = {}
204
+ for ip, found in open_ports.items():
205
+ for port in found:
206
+ seen_by.setdefault((ip, port), set()).add("scan")
207
+ if use_internetdb and targets:
208
+ idb, idb_notes = internetdb(targets)
209
+ notes += idb_notes
210
+ (directory / INTERNETDB_FILE).write_text(json.dumps(idb, indent=2), encoding="utf-8")
211
+ for ip, entry in idb.items():
212
+ for reported in entry.get("ports", []):
213
+ if isinstance(reported, int):
214
+ seen_by.setdefault((ip, reported), set()).add("internetdb")
215
+
216
+ approved = load_approved(config)
217
+ records, items = [], []
218
+ for (ip, port), sources in sorted(seen_by.items(), key=lambda kv: (kv[0][0], kv[0][1])):
219
+ target = targets.get(ip, Target(ip))
220
+ keys = [ip, *sorted(target.hostnames)]
221
+ justification = next((approved[(k, port)] for k in keys if (k, port) in approved), None)
222
+ category, service = describe(port)
223
+ records.append(
224
+ {
225
+ "ip": ip,
226
+ "hostnames": ";".join(sorted(target.hostnames)),
227
+ "port": port,
228
+ "protocol": "tcp",
229
+ "category": category,
230
+ "service": service,
231
+ "seen_by": ";".join(sorted(sources)),
232
+ "approved": "Y" if justification is not None else "N",
233
+ "justification": justification or "",
234
+ }
235
+ )
236
+ # Count only what our own scan confirmed; InternetDB alone may be stale.
237
+ if justification is None and "scan" in sources:
238
+ items.append(
239
+ {
240
+ "id": make_id("external", ip, f"tcp-{port}"),
241
+ "reason": f"{service or f'port {port}'} ({category}) open, "
242
+ "not on the approved list",
243
+ }
244
+ )
245
+ record_file = "4_external_exposure.csv"
246
+ return [
247
+ write_csv(directory / record_file, records, FIELDS),
248
+ write_summary(
249
+ directory,
250
+ zero=4,
251
+ source="external",
252
+ name="exposure",
253
+ items=items,
254
+ tool="targetzero external scan",
255
+ run_date=run_date,
256
+ complete=complete,
257
+ notes=notes,
258
+ record_file=record_file,
259
+ ),
260
+ ]
@@ -0,0 +1,54 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """The scan scope from targetzero.yaml. Nothing outside it is ever scanned."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import ipaddress
7
+ from dataclasses import dataclass
8
+
9
+ from targetzero.common.config import Config
10
+
11
+ IPNetwork = ipaddress.IPv4Network | ipaddress.IPv6Network
12
+ IPAddress = ipaddress.IPv4Address | ipaddress.IPv6Address
13
+
14
+
15
+ @dataclass(frozen=True)
16
+ class Scope:
17
+ domains: tuple[str, ...]
18
+ networks: tuple[IPNetwork, ...]
19
+
20
+ @classmethod
21
+ def from_config(cls, config: Config) -> Scope:
22
+ scope = config.data.get("scope", {})
23
+ return cls(
24
+ domains=tuple(d.lower().strip(".") for d in scope.get("domains", [])),
25
+ networks=tuple(
26
+ ipaddress.ip_network(r, strict=False) for r in scope.get("ip_ranges", [])
27
+ ),
28
+ )
29
+
30
+ def contains_ip(self, ip: str) -> bool:
31
+ try:
32
+ address = ipaddress.ip_address(ip)
33
+ except ValueError:
34
+ return False
35
+ return any(address in n for n in self.networks)
36
+
37
+ def contains_name(self, name: str) -> bool:
38
+ name = name.lower().strip(".")
39
+ return any(name == d or name.endswith("." + d) for d in self.domains)
40
+
41
+ def addresses(self, max_hosts: int) -> list[str]:
42
+ total = sum(n.num_addresses for n in self.networks)
43
+ if total > max_hosts:
44
+ raise ValueError(
45
+ f"Scope has {total} addresses, more than --max-hosts {max_hosts}. "
46
+ "Split the ranges or raise the limit."
47
+ )
48
+ found: list[str] = []
49
+ for network in self.networks:
50
+ hosts = (
51
+ [network.network_address] if network.num_addresses == 1 else list(network.hosts())
52
+ )
53
+ found += [str(h) for h in hosts]
54
+ return found
@@ -0,0 +1,202 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """WAF / CDN check (Zero 4): are your public websites behind a WAF or CDN, and does the origin
3
+ server refuse direct traffic?
4
+
5
+ A web name counts when it does not resolve into a known CDN/WAF (by IP range or CNAME), or
6
+ when its origin, listed in registers/origins.csv, answers directly. Names whose inventory row
7
+ has behind_waf = n/a (a VPN portal, a mail server) are skipped.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import ipaddress
13
+ import socket
14
+ import urllib.error
15
+ from collections.abc import Iterable, Mapping
16
+ from datetime import date
17
+ from pathlib import Path
18
+ from typing import Any
19
+
20
+ from targetzero.common.config import Config
21
+ from targetzero.common.ids import make_id
22
+ from targetzero.common.output import read_csv, run_dir, write_csv, write_summary
23
+ from targetzero.external import dnsinfo, web
24
+ from targetzero.external.http import cached_json
25
+ from targetzero.external.scope import Scope
26
+ from targetzero.registers import inventory
27
+
28
+ FIELDS = ("name", "ips", "cnames", "web", "provider", "origin_ip", "origin_reachable", "result")
29
+ CNAME_PROVIDERS = {
30
+ "Akamai": ("edgekey.net", "edgesuite.net", "akamaiedge.net", "akamai.net", "akamaized.net"),
31
+ "Azure Front Door / CDN": ("azurefd.net", "azureedge.net", "t-msedge.net"),
32
+ "Cloudflare": ("cdn.cloudflare.net",),
33
+ "CloudFront": ("cloudfront.net",),
34
+ "Fastly": ("fastly.net", "fastlylb.net"),
35
+ "Imperva": ("incapdns.net", "impervadns.net"),
36
+ "Sucuri": ("sucuri.net",),
37
+ }
38
+
39
+ Network = ipaddress.IPv4Network | ipaddress.IPv6Network
40
+
41
+
42
+ def provider_ranges(cache: Path) -> tuple[dict[str, list[Network]], list[str]]:
43
+ """Published IP ranges of the large CDNs, refreshed daily."""
44
+ sources: dict[str, tuple[str, Any]] = {
45
+ "Cloudflare": (
46
+ "https://api.cloudflare.com/client/v4/ips",
47
+ lambda d: d["result"]["ipv4_cidrs"] + d["result"]["ipv6_cidrs"],
48
+ ),
49
+ "CloudFront": (
50
+ "https://ip-ranges.amazonaws.com/ip-ranges.json",
51
+ lambda d: (
52
+ [p["ip_prefix"] for p in d["prefixes"] if p["service"] == "CLOUDFRONT"]
53
+ + [p["ipv6_prefix"] for p in d["ipv6_prefixes"] if p["service"] == "CLOUDFRONT"]
54
+ ),
55
+ ),
56
+ "Fastly": (
57
+ "https://api.fastly.com/public-ip-list",
58
+ lambda d: d["addresses"] + d.get("ipv6_addresses", []),
59
+ ),
60
+ }
61
+ ranges: dict[str, list[Network]] = {}
62
+ notes = []
63
+ for name, (url, extract) in sources.items():
64
+ try:
65
+ data = cached_json(url, cache / f"ranges-{name.lower()}.json", max_age_hours=24)
66
+ ranges[name] = [ipaddress.ip_network(c, strict=False) for c in extract(data)]
67
+ except (urllib.error.URLError, TimeoutError, ValueError, KeyError) as exc:
68
+ notes.append(f"Could not load {name} IP ranges: {exc}")
69
+ return ranges, notes
70
+
71
+
72
+ def provider_for(found: dnsinfo.Lookup, ranges: Mapping[str, list[Network]]) -> str:
73
+ for name, suffixes in CNAME_PROVIDERS.items():
74
+ if any(c == s or c.endswith("." + s) for c in found.cnames for s in suffixes):
75
+ return name
76
+ for ip in found.ips:
77
+ address = ipaddress.ip_address(ip)
78
+ for name, networks in ranges.items():
79
+ if any(address in n for n in networks):
80
+ return name
81
+ return ""
82
+
83
+
84
+ def is_web(name: str, timeout: float = 4) -> bool:
85
+ for port in (443, 80):
86
+ try:
87
+ with socket.create_connection((name, port), timeout=timeout):
88
+ return True
89
+ except OSError:
90
+ continue
91
+ return False
92
+
93
+
94
+ def web_names(config: Config, scope: Scope) -> list[str]:
95
+ skip = {r["asset"].lower() for r in inventory.load(config) if r.get("behind_waf") == "n/a"}
96
+ names = set(scope.domains) | {
97
+ r["asset"].lower()
98
+ for r in inventory.load(config)
99
+ if r.get("type") in ("domain", "subdomain") and r.get("status") != "retired"
100
+ }
101
+ return sorted(n for n in names if scope.contains_name(n) and n not in skip)
102
+
103
+
104
+ def origin_reachable(name: str, origin_ip: str) -> bool:
105
+ for use_tls, port in ((True, 443), (False, 80)):
106
+ if web.get(name, port=port, use_tls=use_tls, connect_to=origin_ip, timeout=6).status:
107
+ return True
108
+ return False
109
+
110
+
111
+ def check(
112
+ names: Iterable[str],
113
+ ranges: Mapping[str, list[Network]],
114
+ origins: Mapping[str, str],
115
+ scope: Scope,
116
+ *,
117
+ test_origins: bool,
118
+ ) -> tuple[list[dict[str, str]], list[dict[str, str]], list[str]]:
119
+ res = dnsinfo.resolver()
120
+ records, items, notes = [], [], []
121
+ for name in names:
122
+ found = dnsinfo.lookup(name, res)
123
+ record = {
124
+ "name": name,
125
+ "ips": ";".join(found.ips),
126
+ "cnames": ";".join(found.cnames),
127
+ "web": "",
128
+ "provider": "",
129
+ "origin_ip": origins.get(name, ""),
130
+ "origin_reachable": "",
131
+ "result": "",
132
+ }
133
+ if not found.ips:
134
+ records.append(record | {"result": "does not resolve"})
135
+ continue
136
+ if not is_web(name):
137
+ records.append(record | {"web": "N", "result": "no web service"})
138
+ continue
139
+ record["web"] = "Y"
140
+ record["provider"] = provider_for(found, ranges)
141
+ if not record["provider"]:
142
+ record["result"] = "not behind a known WAF/CDN"
143
+ items.append(
144
+ {
145
+ "id": make_id("external", name, "no-waf"),
146
+ "reason": f"{name} serves web traffic directly ({record['ips']})",
147
+ }
148
+ )
149
+ else:
150
+ record["result"] = f"behind {record['provider']}"
151
+ origin = record["origin_ip"]
152
+ if origin and record["provider"]:
153
+ if not test_origins:
154
+ record["origin_reachable"] = "not tested"
155
+ elif not scope.contains_ip(origin):
156
+ record["origin_reachable"] = "not tested"
157
+ notes.append(f"{name}: origin {origin} is outside scope.ip_ranges, not tested")
158
+ elif origin_reachable(name, origin):
159
+ record["origin_reachable"] = "Y"
160
+ items.append(
161
+ {
162
+ "id": make_id("external", name, "origin-reachable"),
163
+ "reason": f"origin {origin} of {name} answers without the CDN",
164
+ }
165
+ )
166
+ else:
167
+ record["origin_reachable"] = "N"
168
+ records.append(record)
169
+ return records, items, notes
170
+
171
+
172
+ def run(config: Config, run_date: date, *, test_origins: bool = False) -> list[Path]:
173
+ scope = Scope.from_config(config)
174
+ ranges, notes = provider_ranges(config.output_dir / "cache")
175
+ origins_file = config.registers_dir / "origins.csv"
176
+ origins = (
177
+ {r["host"].lower(): r["origin_ip"] for r in read_csv(origins_file)}
178
+ if origins_file.exists()
179
+ else {}
180
+ )
181
+ if origins and not test_origins:
182
+ notes.append("Origins not tested: pass --authorised to test registers/origins.csv")
183
+ records, items, check_notes = check(
184
+ web_names(config, scope), ranges, origins, scope, test_origins=test_origins
185
+ )
186
+ directory = run_dir(config.output_dir, run_date)
187
+ record_file = "4_external_waf.csv"
188
+ return [
189
+ write_csv(directory / record_file, records, FIELDS),
190
+ write_summary(
191
+ directory,
192
+ zero=4,
193
+ source="external",
194
+ name="waf",
195
+ items=items,
196
+ tool="targetzero external waf",
197
+ run_date=run_date,
198
+ complete=len(ranges) == 3 and not (origins and not test_origins),
199
+ notes=notes + check_notes,
200
+ record_file=record_file,
201
+ ),
202
+ ]
@@ -0,0 +1,124 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """A minimal HTTP(S) probe: one GET, no redirects, TLS details kept.
3
+
4
+ Used by the WAF, enrichment and dangling-DNS checks. It can connect to a given
5
+ address while sending a different Host / SNI name, which is how a website's
6
+ origin server is tested behind a CDN.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import socket
12
+ import ssl
13
+ from dataclasses import dataclass, field
14
+
15
+ from targetzero.external.http import USER_AGENT
16
+
17
+ MAX_BYTES = 65536
18
+
19
+
20
+ @dataclass
21
+ class Tls:
22
+ version: str = ""
23
+ verified: bool = False
24
+ issuer: str = ""
25
+ not_after: str = ""
26
+ error: str = ""
27
+
28
+
29
+ @dataclass
30
+ class Response:
31
+ status: int = 0
32
+ headers: dict[str, str] = field(default_factory=dict)
33
+ body: str = ""
34
+ tls: Tls | None = None
35
+ error: str = ""
36
+
37
+
38
+ def _issuer(cert: dict[str, object]) -> str:
39
+ parts = []
40
+ for rdn in cert.get("issuer", ()): # type: ignore[attr-defined]
41
+ for key, value in rdn:
42
+ if key in ("organizationName", "commonName"):
43
+ parts.append(str(value))
44
+ return ", ".join(parts)
45
+
46
+
47
+ def _wrap(sock: socket.socket, host: str, verify: bool) -> ssl.SSLSocket:
48
+ context = ssl.create_default_context()
49
+ if not verify:
50
+ # Second attempt only, to read headers of a site whose certificate is invalid.
51
+ context.check_hostname = False
52
+ context.verify_mode = ssl.CERT_NONE
53
+ return context.wrap_socket(sock, server_hostname=host)
54
+
55
+
56
+ def get(
57
+ host: str,
58
+ *,
59
+ port: int = 443,
60
+ use_tls: bool = True,
61
+ connect_to: str | None = None,
62
+ timeout: float = 8.0,
63
+ path: str = "/",
64
+ ) -> Response:
65
+ """GET http(s)://host/path, optionally connecting to another address (origin tests)."""
66
+ response = Response()
67
+ address = connect_to or host
68
+ tls: Tls | None = Tls() if use_tls else None
69
+ for verify in (True, False) if use_tls else (False,):
70
+ try:
71
+ raw = socket.create_connection((address, port), timeout=timeout)
72
+ except OSError as exc:
73
+ response.error = f"connect: {exc}"
74
+ return response
75
+ try:
76
+ sock: socket.socket = raw
77
+ if tls is not None:
78
+ try:
79
+ sock = _wrap(raw, host, verify)
80
+ except ssl.SSLCertVerificationError as exc:
81
+ raw.close()
82
+ tls.error = exc.verify_message or str(exc)
83
+ continue
84
+ tls.version = sock.version() or ""
85
+ tls.verified = verify
86
+ cert = sock.getpeercert() if verify else None
87
+ if cert:
88
+ tls.issuer = _issuer(cert) # type: ignore[arg-type]
89
+ tls.not_after = str(cert.get("notAfter", ""))
90
+ request = (
91
+ f"GET {path} HTTP/1.1\r\nHost: {host}\r\nUser-Agent: {USER_AGENT}\r\n"
92
+ "Accept: */*\r\nConnection: close\r\n\r\n"
93
+ )
94
+ sock.sendall(request.encode())
95
+ data = b""
96
+ while len(data) < MAX_BYTES:
97
+ chunk = sock.recv(8192)
98
+ if not chunk:
99
+ break
100
+ data += chunk
101
+ except (OSError, ssl.SSLError) as exc:
102
+ response.error = f"{type(exc).__name__}: {exc}"
103
+ return response
104
+ finally:
105
+ raw.close()
106
+ _parse(data, response)
107
+ response.tls = tls
108
+ return response
109
+ response.tls = tls
110
+ response.error = response.error or "TLS handshake failed"
111
+ return response
112
+
113
+
114
+ def _parse(data: bytes, response: Response) -> None:
115
+ head, _, body = data.partition(b"\r\n\r\n")
116
+ lines = head.decode("iso-8859-1").split("\r\n")
117
+ if lines and lines[0].startswith("HTTP/"):
118
+ parts = lines[0].split(" ", 2)
119
+ response.status = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else 0
120
+ for line in lines[1:]:
121
+ key, sep, value = line.partition(":")
122
+ if sep:
123
+ response.headers[key.strip().lower()] = value.strip()
124
+ response.body = body.decode("utf-8", errors="replace")
@@ -0,0 +1,2 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Google Workspace checks (Admin SDK Directory and Reports APIs). Install with the [gws] extra."""