targetzero 0.3.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- targetzero/__init__.py +4 -0
- targetzero/__main__.py +6 -0
- targetzero/admins.py +109 -0
- targetzero/cli.py +366 -0
- targetzero/common/__init__.py +2 -0
- targetzero/common/changes.py +46 -0
- targetzero/common/config.py +83 -0
- targetzero/common/ids.py +17 -0
- targetzero/common/output.py +91 -0
- targetzero/common/resources.py +32 -0
- targetzero/common/roles.py +24 -0
- targetzero/common/schema.py +45 -0
- targetzero/config/approved_exposure.example.csv +4 -0
- targetzero/config/privileged_roles.yaml +50 -0
- targetzero/config/targetzero.example.yaml +57 -0
- targetzero/external/__init__.py +2 -0
- targetzero/external/dangling.py +106 -0
- targetzero/external/discover.py +181 -0
- targetzero/external/dnsinfo.py +54 -0
- targetzero/external/enrich.py +137 -0
- targetzero/external/http.py +33 -0
- targetzero/external/kev.py +222 -0
- targetzero/external/ports.py +63 -0
- targetzero/external/scan.py +260 -0
- targetzero/external/scope.py +54 -0
- targetzero/external/waf.py +202 -0
- targetzero/external/web.py +124 -0
- targetzero/gws/__init__.py +2 -0
- targetzero/gws/admins.py +230 -0
- targetzero/gws/changes.py +190 -0
- targetzero/gws/client.py +94 -0
- targetzero/gws/domains.py +36 -0
- targetzero/registers/__init__.py +2 -0
- targetzero/registers/exceptions.py +85 -0
- targetzero/registers/findings.py +108 -0
- targetzero/registers/inventory.py +142 -0
- targetzero/registers/manual_admins.py +43 -0
- targetzero/schemas/admins.schema.json +27 -0
- targetzero/schemas/alert_tests.schema.json +19 -0
- targetzero/schemas/config.schema.json +71 -0
- targetzero/schemas/exceptions.schema.json +20 -0
- targetzero/schemas/findings.schema.json +23 -0
- targetzero/schemas/inventory.schema.json +26 -0
- targetzero/schemas/summary.schema.json +36 -0
- targetzero/scorecard/__init__.py +2 -0
- targetzero/scorecard/scorecard.py +201 -0
- targetzero/templates/alert_tests.csv +1 -0
- targetzero/templates/exceptions.csv +1 -0
- targetzero/templates/findings.csv +1 -0
- targetzero/templates/inventory.csv +1 -0
- targetzero/templates/manual_admins.csv +1 -0
- targetzero/templates/origins.csv +1 -0
- targetzero/templates/recovery_page.md +51 -0
- targetzero/vendors/__init__.py +2 -0
- targetzero/vendors/cisco_asa.py +151 -0
- targetzero/vendors/common.py +84 -0
- targetzero/vendors/fortigate.py +176 -0
- targetzero/vendors/panos.py +141 -0
- targetzero/vendors/redfish.py +124 -0
- targetzero/vendors/vsphere.py +108 -0
- targetzero-0.3.2.dist-info/METADATA +116 -0
- targetzero-0.3.2.dist-info/RECORD +66 -0
- targetzero-0.3.2.dist-info/WHEEL +4 -0
- targetzero-0.3.2.dist-info/entry_points.txt +2 -0
- targetzero-0.3.2.dist-info/licenses/LICENSE +202 -0
- targetzero-0.3.2.dist-info/licenses/NOTICE +7 -0
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Exposure scan (Zero 4): open TCP ports on your public IPs, compared with the approved list.
|
|
3
|
+
|
|
4
|
+
Run it from outside your network (a cloud VM, a home line), never from inside:
|
|
5
|
+
from inside you see your own LAN, not what the internet sees.
|
|
6
|
+
|
|
7
|
+
Only addresses inside scope.ip_ranges are scanned. Hostnames are resolved and
|
|
8
|
+
their addresses scanned only when those addresses are inside the ranges too:
|
|
9
|
+
a website on a CDN is not yours to scan.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import asyncio
|
|
15
|
+
import json
|
|
16
|
+
import shutil
|
|
17
|
+
import socket
|
|
18
|
+
import subprocess
|
|
19
|
+
import urllib.error
|
|
20
|
+
from collections.abc import Iterable
|
|
21
|
+
from dataclasses import dataclass, field
|
|
22
|
+
from datetime import date
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
from defusedxml import ElementTree
|
|
26
|
+
|
|
27
|
+
from targetzero.common.config import Config
|
|
28
|
+
from targetzero.common.ids import make_id
|
|
29
|
+
from targetzero.common.output import read_csv, run_dir, write_csv, write_summary
|
|
30
|
+
from targetzero.external.http import get_json
|
|
31
|
+
from targetzero.external.ports import PORTS, describe
|
|
32
|
+
from targetzero.external.scope import Scope
|
|
33
|
+
from targetzero.registers import inventory
|
|
34
|
+
|
|
35
|
+
FIELDS = (
|
|
36
|
+
"ip",
|
|
37
|
+
"hostnames",
|
|
38
|
+
"port",
|
|
39
|
+
"protocol",
|
|
40
|
+
"category",
|
|
41
|
+
"service",
|
|
42
|
+
"seen_by",
|
|
43
|
+
"approved",
|
|
44
|
+
"justification",
|
|
45
|
+
)
|
|
46
|
+
INTERNETDB_FILE = "external_internetdb.json"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass
|
|
50
|
+
class Target:
|
|
51
|
+
ip: str
|
|
52
|
+
hostnames: set[str] = field(default_factory=set)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def collect_targets(
|
|
56
|
+
config: Config, scope: Scope, max_hosts: int, resolve: bool = True
|
|
57
|
+
) -> tuple[dict[str, Target], list[str]]:
|
|
58
|
+
"""Addresses in scope, plus in-scope addresses of known hostnames."""
|
|
59
|
+
targets = {ip: Target(ip) for ip in scope.addresses(max_hosts)}
|
|
60
|
+
notes: list[str] = []
|
|
61
|
+
if not resolve:
|
|
62
|
+
return targets, notes
|
|
63
|
+
names = set(scope.domains)
|
|
64
|
+
names |= {
|
|
65
|
+
r["asset"] for r in inventory.load(config) if r.get("type") in ("domain", "subdomain")
|
|
66
|
+
}
|
|
67
|
+
for name in sorted(names):
|
|
68
|
+
try:
|
|
69
|
+
addresses = {
|
|
70
|
+
info[4][0] for info in socket.getaddrinfo(name, None, type=socket.SOCK_STREAM)
|
|
71
|
+
}
|
|
72
|
+
except socket.gaierror:
|
|
73
|
+
continue
|
|
74
|
+
for ip in addresses:
|
|
75
|
+
if scope.contains_ip(str(ip)):
|
|
76
|
+
targets.setdefault(str(ip), Target(str(ip))).hostnames.add(name)
|
|
77
|
+
else:
|
|
78
|
+
notes.append(f"{name} -> {ip}: outside scope.ip_ranges, not scanned")
|
|
79
|
+
return targets, notes
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def scan_nmap(ips: Iterable[str], ports: Iterable[int], nmap: str) -> dict[str, set[int]]:
|
|
83
|
+
command = [
|
|
84
|
+
nmap,
|
|
85
|
+
"-Pn",
|
|
86
|
+
"-n",
|
|
87
|
+
"-sT",
|
|
88
|
+
"--open",
|
|
89
|
+
"-T4",
|
|
90
|
+
"-p",
|
|
91
|
+
",".join(str(p) for p in sorted(ports)),
|
|
92
|
+
"-oX",
|
|
93
|
+
"-",
|
|
94
|
+
*sorted(ips),
|
|
95
|
+
]
|
|
96
|
+
result = subprocess.run(command, capture_output=True, text=True, check=True, timeout=3600) # noqa: S603
|
|
97
|
+
return parse_nmap_xml(result.stdout)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def parse_nmap_xml(xml: str) -> dict[str, set[int]]:
|
|
101
|
+
found: dict[str, set[int]] = {}
|
|
102
|
+
root = ElementTree.fromstring(xml)
|
|
103
|
+
for host in root.iter("host"):
|
|
104
|
+
address = host.find("address")
|
|
105
|
+
if address is None:
|
|
106
|
+
continue
|
|
107
|
+
ip = address.get("addr", "")
|
|
108
|
+
for port in host.iter("port"):
|
|
109
|
+
state = port.find("state")
|
|
110
|
+
if state is not None and state.get("state") == "open" and port.get("protocol") == "tcp":
|
|
111
|
+
found.setdefault(ip, set()).add(int(port.get("portid", "0")))
|
|
112
|
+
return found
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
async def _probe(ip: str, port: int, timeout: float, limit: asyncio.Semaphore) -> bool:
|
|
116
|
+
async with limit:
|
|
117
|
+
try:
|
|
118
|
+
_, writer = await asyncio.wait_for(asyncio.open_connection(ip, port), timeout)
|
|
119
|
+
except (TimeoutError, OSError):
|
|
120
|
+
return False
|
|
121
|
+
writer.close()
|
|
122
|
+
try:
|
|
123
|
+
await writer.wait_closed()
|
|
124
|
+
except OSError:
|
|
125
|
+
pass
|
|
126
|
+
return True
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
async def _scan_connect(
|
|
130
|
+
ips: list[str], ports: list[int], timeout: float, concurrency: int
|
|
131
|
+
) -> dict[str, set[int]]:
|
|
132
|
+
limit = asyncio.Semaphore(concurrency)
|
|
133
|
+
pairs = [(ip, port) for ip in ips for port in ports]
|
|
134
|
+
results = await asyncio.gather(*(_probe(ip, port, timeout, limit) for ip, port in pairs))
|
|
135
|
+
found: dict[str, set[int]] = {}
|
|
136
|
+
for (ip, port), is_open in zip(pairs, results, strict=True):
|
|
137
|
+
if is_open:
|
|
138
|
+
found.setdefault(ip, set()).add(port)
|
|
139
|
+
return found
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def scan_connect(
|
|
143
|
+
ips: Iterable[str], ports: Iterable[int], timeout: float = 1.5, concurrency: int = 200
|
|
144
|
+
) -> dict[str, set[int]]:
|
|
145
|
+
return asyncio.run(_scan_connect(sorted(ips), sorted(ports), timeout, concurrency))
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def internetdb(ips: Iterable[str]) -> tuple[dict[str, dict[str, list[object]]], list[str]]:
|
|
149
|
+
"""Shodan InternetDB: free, no key. Data can be up to a week old."""
|
|
150
|
+
data, notes = {}, []
|
|
151
|
+
for ip in sorted(ips):
|
|
152
|
+
try:
|
|
153
|
+
data[ip] = get_json(f"https://internetdb.shodan.io/{ip}", timeout=15)
|
|
154
|
+
except urllib.error.HTTPError as exc:
|
|
155
|
+
if exc.code != 404: # 404: InternetDB has nothing on this address
|
|
156
|
+
notes.append(f"InternetDB {ip}: HTTP {exc.code}")
|
|
157
|
+
except (urllib.error.URLError, TimeoutError, ValueError) as exc:
|
|
158
|
+
notes.append(f"InternetDB {ip}: {exc}")
|
|
159
|
+
return data, notes
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def load_approved(config: Config) -> dict[tuple[str, int], str]:
|
|
163
|
+
path = config.registers_dir / "approved_exposure.csv"
|
|
164
|
+
if not path.exists():
|
|
165
|
+
return {}
|
|
166
|
+
approved = {}
|
|
167
|
+
for row in read_csv(path):
|
|
168
|
+
if row.get("protocol", "tcp").lower() == "tcp" and row.get("port", "").isdigit():
|
|
169
|
+
approved[(row["host"].lower(), int(row["port"]))] = row.get("justification", "")
|
|
170
|
+
return approved
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def run(
|
|
174
|
+
config: Config,
|
|
175
|
+
run_date: date,
|
|
176
|
+
*,
|
|
177
|
+
use_nmap: bool = True,
|
|
178
|
+
use_internetdb: bool = False,
|
|
179
|
+
max_hosts: int = 1024,
|
|
180
|
+
timeout: float = 1.5,
|
|
181
|
+
) -> list[Path]:
|
|
182
|
+
scope = Scope.from_config(config)
|
|
183
|
+
targets, notes = collect_targets(config, scope, max_hosts)
|
|
184
|
+
complete = True
|
|
185
|
+
if not targets:
|
|
186
|
+
notes.append("Nothing to scan: scope.ip_ranges is empty and no hostname resolves inside it")
|
|
187
|
+
complete = False
|
|
188
|
+
|
|
189
|
+
ports = list(PORTS)
|
|
190
|
+
nmap = shutil.which("nmap") if use_nmap else None
|
|
191
|
+
try:
|
|
192
|
+
if nmap:
|
|
193
|
+
open_ports = scan_nmap(targets, ports, nmap)
|
|
194
|
+
notes.append("Scanned with nmap (TCP connect)")
|
|
195
|
+
else:
|
|
196
|
+
open_ports = scan_connect(targets, ports, timeout=timeout)
|
|
197
|
+
notes.append("Scanned with the built-in TCP connect scanner")
|
|
198
|
+
except (subprocess.SubprocessError, OSError) as exc:
|
|
199
|
+
notes.append(f"Scan failed: {exc}")
|
|
200
|
+
open_ports, complete = {}, False
|
|
201
|
+
|
|
202
|
+
directory = run_dir(config.output_dir, run_date)
|
|
203
|
+
seen_by: dict[tuple[str, int], set[str]] = {}
|
|
204
|
+
for ip, found in open_ports.items():
|
|
205
|
+
for port in found:
|
|
206
|
+
seen_by.setdefault((ip, port), set()).add("scan")
|
|
207
|
+
if use_internetdb and targets:
|
|
208
|
+
idb, idb_notes = internetdb(targets)
|
|
209
|
+
notes += idb_notes
|
|
210
|
+
(directory / INTERNETDB_FILE).write_text(json.dumps(idb, indent=2), encoding="utf-8")
|
|
211
|
+
for ip, entry in idb.items():
|
|
212
|
+
for reported in entry.get("ports", []):
|
|
213
|
+
if isinstance(reported, int):
|
|
214
|
+
seen_by.setdefault((ip, reported), set()).add("internetdb")
|
|
215
|
+
|
|
216
|
+
approved = load_approved(config)
|
|
217
|
+
records, items = [], []
|
|
218
|
+
for (ip, port), sources in sorted(seen_by.items(), key=lambda kv: (kv[0][0], kv[0][1])):
|
|
219
|
+
target = targets.get(ip, Target(ip))
|
|
220
|
+
keys = [ip, *sorted(target.hostnames)]
|
|
221
|
+
justification = next((approved[(k, port)] for k in keys if (k, port) in approved), None)
|
|
222
|
+
category, service = describe(port)
|
|
223
|
+
records.append(
|
|
224
|
+
{
|
|
225
|
+
"ip": ip,
|
|
226
|
+
"hostnames": ";".join(sorted(target.hostnames)),
|
|
227
|
+
"port": port,
|
|
228
|
+
"protocol": "tcp",
|
|
229
|
+
"category": category,
|
|
230
|
+
"service": service,
|
|
231
|
+
"seen_by": ";".join(sorted(sources)),
|
|
232
|
+
"approved": "Y" if justification is not None else "N",
|
|
233
|
+
"justification": justification or "",
|
|
234
|
+
}
|
|
235
|
+
)
|
|
236
|
+
# Count only what our own scan confirmed; InternetDB alone may be stale.
|
|
237
|
+
if justification is None and "scan" in sources:
|
|
238
|
+
items.append(
|
|
239
|
+
{
|
|
240
|
+
"id": make_id("external", ip, f"tcp-{port}"),
|
|
241
|
+
"reason": f"{service or f'port {port}'} ({category}) open, "
|
|
242
|
+
"not on the approved list",
|
|
243
|
+
}
|
|
244
|
+
)
|
|
245
|
+
record_file = "4_external_exposure.csv"
|
|
246
|
+
return [
|
|
247
|
+
write_csv(directory / record_file, records, FIELDS),
|
|
248
|
+
write_summary(
|
|
249
|
+
directory,
|
|
250
|
+
zero=4,
|
|
251
|
+
source="external",
|
|
252
|
+
name="exposure",
|
|
253
|
+
items=items,
|
|
254
|
+
tool="targetzero external scan",
|
|
255
|
+
run_date=run_date,
|
|
256
|
+
complete=complete,
|
|
257
|
+
notes=notes,
|
|
258
|
+
record_file=record_file,
|
|
259
|
+
),
|
|
260
|
+
]
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""The scan scope from targetzero.yaml. Nothing outside it is ever scanned."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import ipaddress
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
|
|
9
|
+
from targetzero.common.config import Config
|
|
10
|
+
|
|
11
|
+
IPNetwork = ipaddress.IPv4Network | ipaddress.IPv6Network
|
|
12
|
+
IPAddress = ipaddress.IPv4Address | ipaddress.IPv6Address
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class Scope:
|
|
17
|
+
domains: tuple[str, ...]
|
|
18
|
+
networks: tuple[IPNetwork, ...]
|
|
19
|
+
|
|
20
|
+
@classmethod
|
|
21
|
+
def from_config(cls, config: Config) -> Scope:
|
|
22
|
+
scope = config.data.get("scope", {})
|
|
23
|
+
return cls(
|
|
24
|
+
domains=tuple(d.lower().strip(".") for d in scope.get("domains", [])),
|
|
25
|
+
networks=tuple(
|
|
26
|
+
ipaddress.ip_network(r, strict=False) for r in scope.get("ip_ranges", [])
|
|
27
|
+
),
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
def contains_ip(self, ip: str) -> bool:
|
|
31
|
+
try:
|
|
32
|
+
address = ipaddress.ip_address(ip)
|
|
33
|
+
except ValueError:
|
|
34
|
+
return False
|
|
35
|
+
return any(address in n for n in self.networks)
|
|
36
|
+
|
|
37
|
+
def contains_name(self, name: str) -> bool:
|
|
38
|
+
name = name.lower().strip(".")
|
|
39
|
+
return any(name == d or name.endswith("." + d) for d in self.domains)
|
|
40
|
+
|
|
41
|
+
def addresses(self, max_hosts: int) -> list[str]:
|
|
42
|
+
total = sum(n.num_addresses for n in self.networks)
|
|
43
|
+
if total > max_hosts:
|
|
44
|
+
raise ValueError(
|
|
45
|
+
f"Scope has {total} addresses, more than --max-hosts {max_hosts}. "
|
|
46
|
+
"Split the ranges or raise the limit."
|
|
47
|
+
)
|
|
48
|
+
found: list[str] = []
|
|
49
|
+
for network in self.networks:
|
|
50
|
+
hosts = (
|
|
51
|
+
[network.network_address] if network.num_addresses == 1 else list(network.hosts())
|
|
52
|
+
)
|
|
53
|
+
found += [str(h) for h in hosts]
|
|
54
|
+
return found
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""WAF / CDN check (Zero 4): are your public websites behind a WAF or CDN, and does the origin
|
|
3
|
+
server refuse direct traffic?
|
|
4
|
+
|
|
5
|
+
A web name counts when it does not resolve into a known CDN/WAF (by IP range or CNAME), or
|
|
6
|
+
when its origin, listed in registers/origins.csv, answers directly. Names whose inventory row
|
|
7
|
+
has behind_waf = n/a (a VPN portal, a mail server) are skipped.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import ipaddress
|
|
13
|
+
import socket
|
|
14
|
+
import urllib.error
|
|
15
|
+
from collections.abc import Iterable, Mapping
|
|
16
|
+
from datetime import date
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from targetzero.common.config import Config
|
|
21
|
+
from targetzero.common.ids import make_id
|
|
22
|
+
from targetzero.common.output import read_csv, run_dir, write_csv, write_summary
|
|
23
|
+
from targetzero.external import dnsinfo, web
|
|
24
|
+
from targetzero.external.http import cached_json
|
|
25
|
+
from targetzero.external.scope import Scope
|
|
26
|
+
from targetzero.registers import inventory
|
|
27
|
+
|
|
28
|
+
FIELDS = ("name", "ips", "cnames", "web", "provider", "origin_ip", "origin_reachable", "result")
|
|
29
|
+
CNAME_PROVIDERS = {
|
|
30
|
+
"Akamai": ("edgekey.net", "edgesuite.net", "akamaiedge.net", "akamai.net", "akamaized.net"),
|
|
31
|
+
"Azure Front Door / CDN": ("azurefd.net", "azureedge.net", "t-msedge.net"),
|
|
32
|
+
"Cloudflare": ("cdn.cloudflare.net",),
|
|
33
|
+
"CloudFront": ("cloudfront.net",),
|
|
34
|
+
"Fastly": ("fastly.net", "fastlylb.net"),
|
|
35
|
+
"Imperva": ("incapdns.net", "impervadns.net"),
|
|
36
|
+
"Sucuri": ("sucuri.net",),
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
Network = ipaddress.IPv4Network | ipaddress.IPv6Network
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def provider_ranges(cache: Path) -> tuple[dict[str, list[Network]], list[str]]:
|
|
43
|
+
"""Published IP ranges of the large CDNs, refreshed daily."""
|
|
44
|
+
sources: dict[str, tuple[str, Any]] = {
|
|
45
|
+
"Cloudflare": (
|
|
46
|
+
"https://api.cloudflare.com/client/v4/ips",
|
|
47
|
+
lambda d: d["result"]["ipv4_cidrs"] + d["result"]["ipv6_cidrs"],
|
|
48
|
+
),
|
|
49
|
+
"CloudFront": (
|
|
50
|
+
"https://ip-ranges.amazonaws.com/ip-ranges.json",
|
|
51
|
+
lambda d: (
|
|
52
|
+
[p["ip_prefix"] for p in d["prefixes"] if p["service"] == "CLOUDFRONT"]
|
|
53
|
+
+ [p["ipv6_prefix"] for p in d["ipv6_prefixes"] if p["service"] == "CLOUDFRONT"]
|
|
54
|
+
),
|
|
55
|
+
),
|
|
56
|
+
"Fastly": (
|
|
57
|
+
"https://api.fastly.com/public-ip-list",
|
|
58
|
+
lambda d: d["addresses"] + d.get("ipv6_addresses", []),
|
|
59
|
+
),
|
|
60
|
+
}
|
|
61
|
+
ranges: dict[str, list[Network]] = {}
|
|
62
|
+
notes = []
|
|
63
|
+
for name, (url, extract) in sources.items():
|
|
64
|
+
try:
|
|
65
|
+
data = cached_json(url, cache / f"ranges-{name.lower()}.json", max_age_hours=24)
|
|
66
|
+
ranges[name] = [ipaddress.ip_network(c, strict=False) for c in extract(data)]
|
|
67
|
+
except (urllib.error.URLError, TimeoutError, ValueError, KeyError) as exc:
|
|
68
|
+
notes.append(f"Could not load {name} IP ranges: {exc}")
|
|
69
|
+
return ranges, notes
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def provider_for(found: dnsinfo.Lookup, ranges: Mapping[str, list[Network]]) -> str:
|
|
73
|
+
for name, suffixes in CNAME_PROVIDERS.items():
|
|
74
|
+
if any(c == s or c.endswith("." + s) for c in found.cnames for s in suffixes):
|
|
75
|
+
return name
|
|
76
|
+
for ip in found.ips:
|
|
77
|
+
address = ipaddress.ip_address(ip)
|
|
78
|
+
for name, networks in ranges.items():
|
|
79
|
+
if any(address in n for n in networks):
|
|
80
|
+
return name
|
|
81
|
+
return ""
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def is_web(name: str, timeout: float = 4) -> bool:
|
|
85
|
+
for port in (443, 80):
|
|
86
|
+
try:
|
|
87
|
+
with socket.create_connection((name, port), timeout=timeout):
|
|
88
|
+
return True
|
|
89
|
+
except OSError:
|
|
90
|
+
continue
|
|
91
|
+
return False
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def web_names(config: Config, scope: Scope) -> list[str]:
|
|
95
|
+
skip = {r["asset"].lower() for r in inventory.load(config) if r.get("behind_waf") == "n/a"}
|
|
96
|
+
names = set(scope.domains) | {
|
|
97
|
+
r["asset"].lower()
|
|
98
|
+
for r in inventory.load(config)
|
|
99
|
+
if r.get("type") in ("domain", "subdomain") and r.get("status") != "retired"
|
|
100
|
+
}
|
|
101
|
+
return sorted(n for n in names if scope.contains_name(n) and n not in skip)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def origin_reachable(name: str, origin_ip: str) -> bool:
|
|
105
|
+
for use_tls, port in ((True, 443), (False, 80)):
|
|
106
|
+
if web.get(name, port=port, use_tls=use_tls, connect_to=origin_ip, timeout=6).status:
|
|
107
|
+
return True
|
|
108
|
+
return False
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def check(
|
|
112
|
+
names: Iterable[str],
|
|
113
|
+
ranges: Mapping[str, list[Network]],
|
|
114
|
+
origins: Mapping[str, str],
|
|
115
|
+
scope: Scope,
|
|
116
|
+
*,
|
|
117
|
+
test_origins: bool,
|
|
118
|
+
) -> tuple[list[dict[str, str]], list[dict[str, str]], list[str]]:
|
|
119
|
+
res = dnsinfo.resolver()
|
|
120
|
+
records, items, notes = [], [], []
|
|
121
|
+
for name in names:
|
|
122
|
+
found = dnsinfo.lookup(name, res)
|
|
123
|
+
record = {
|
|
124
|
+
"name": name,
|
|
125
|
+
"ips": ";".join(found.ips),
|
|
126
|
+
"cnames": ";".join(found.cnames),
|
|
127
|
+
"web": "",
|
|
128
|
+
"provider": "",
|
|
129
|
+
"origin_ip": origins.get(name, ""),
|
|
130
|
+
"origin_reachable": "",
|
|
131
|
+
"result": "",
|
|
132
|
+
}
|
|
133
|
+
if not found.ips:
|
|
134
|
+
records.append(record | {"result": "does not resolve"})
|
|
135
|
+
continue
|
|
136
|
+
if not is_web(name):
|
|
137
|
+
records.append(record | {"web": "N", "result": "no web service"})
|
|
138
|
+
continue
|
|
139
|
+
record["web"] = "Y"
|
|
140
|
+
record["provider"] = provider_for(found, ranges)
|
|
141
|
+
if not record["provider"]:
|
|
142
|
+
record["result"] = "not behind a known WAF/CDN"
|
|
143
|
+
items.append(
|
|
144
|
+
{
|
|
145
|
+
"id": make_id("external", name, "no-waf"),
|
|
146
|
+
"reason": f"{name} serves web traffic directly ({record['ips']})",
|
|
147
|
+
}
|
|
148
|
+
)
|
|
149
|
+
else:
|
|
150
|
+
record["result"] = f"behind {record['provider']}"
|
|
151
|
+
origin = record["origin_ip"]
|
|
152
|
+
if origin and record["provider"]:
|
|
153
|
+
if not test_origins:
|
|
154
|
+
record["origin_reachable"] = "not tested"
|
|
155
|
+
elif not scope.contains_ip(origin):
|
|
156
|
+
record["origin_reachable"] = "not tested"
|
|
157
|
+
notes.append(f"{name}: origin {origin} is outside scope.ip_ranges, not tested")
|
|
158
|
+
elif origin_reachable(name, origin):
|
|
159
|
+
record["origin_reachable"] = "Y"
|
|
160
|
+
items.append(
|
|
161
|
+
{
|
|
162
|
+
"id": make_id("external", name, "origin-reachable"),
|
|
163
|
+
"reason": f"origin {origin} of {name} answers without the CDN",
|
|
164
|
+
}
|
|
165
|
+
)
|
|
166
|
+
else:
|
|
167
|
+
record["origin_reachable"] = "N"
|
|
168
|
+
records.append(record)
|
|
169
|
+
return records, items, notes
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def run(config: Config, run_date: date, *, test_origins: bool = False) -> list[Path]:
|
|
173
|
+
scope = Scope.from_config(config)
|
|
174
|
+
ranges, notes = provider_ranges(config.output_dir / "cache")
|
|
175
|
+
origins_file = config.registers_dir / "origins.csv"
|
|
176
|
+
origins = (
|
|
177
|
+
{r["host"].lower(): r["origin_ip"] for r in read_csv(origins_file)}
|
|
178
|
+
if origins_file.exists()
|
|
179
|
+
else {}
|
|
180
|
+
)
|
|
181
|
+
if origins and not test_origins:
|
|
182
|
+
notes.append("Origins not tested: pass --authorised to test registers/origins.csv")
|
|
183
|
+
records, items, check_notes = check(
|
|
184
|
+
web_names(config, scope), ranges, origins, scope, test_origins=test_origins
|
|
185
|
+
)
|
|
186
|
+
directory = run_dir(config.output_dir, run_date)
|
|
187
|
+
record_file = "4_external_waf.csv"
|
|
188
|
+
return [
|
|
189
|
+
write_csv(directory / record_file, records, FIELDS),
|
|
190
|
+
write_summary(
|
|
191
|
+
directory,
|
|
192
|
+
zero=4,
|
|
193
|
+
source="external",
|
|
194
|
+
name="waf",
|
|
195
|
+
items=items,
|
|
196
|
+
tool="targetzero external waf",
|
|
197
|
+
run_date=run_date,
|
|
198
|
+
complete=len(ranges) == 3 and not (origins and not test_origins),
|
|
199
|
+
notes=notes + check_notes,
|
|
200
|
+
record_file=record_file,
|
|
201
|
+
),
|
|
202
|
+
]
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""A minimal HTTP(S) probe: one GET, no redirects, TLS details kept.
|
|
3
|
+
|
|
4
|
+
Used by the WAF, enrichment and dangling-DNS checks. It can connect to a given
|
|
5
|
+
address while sending a different Host / SNI name, which is how a website's
|
|
6
|
+
origin server is tested behind a CDN.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import socket
|
|
12
|
+
import ssl
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
|
|
15
|
+
from targetzero.external.http import USER_AGENT
|
|
16
|
+
|
|
17
|
+
MAX_BYTES = 65536
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class Tls:
|
|
22
|
+
version: str = ""
|
|
23
|
+
verified: bool = False
|
|
24
|
+
issuer: str = ""
|
|
25
|
+
not_after: str = ""
|
|
26
|
+
error: str = ""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class Response:
|
|
31
|
+
status: int = 0
|
|
32
|
+
headers: dict[str, str] = field(default_factory=dict)
|
|
33
|
+
body: str = ""
|
|
34
|
+
tls: Tls | None = None
|
|
35
|
+
error: str = ""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _issuer(cert: dict[str, object]) -> str:
|
|
39
|
+
parts = []
|
|
40
|
+
for rdn in cert.get("issuer", ()): # type: ignore[attr-defined]
|
|
41
|
+
for key, value in rdn:
|
|
42
|
+
if key in ("organizationName", "commonName"):
|
|
43
|
+
parts.append(str(value))
|
|
44
|
+
return ", ".join(parts)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _wrap(sock: socket.socket, host: str, verify: bool) -> ssl.SSLSocket:
|
|
48
|
+
context = ssl.create_default_context()
|
|
49
|
+
if not verify:
|
|
50
|
+
# Second attempt only, to read headers of a site whose certificate is invalid.
|
|
51
|
+
context.check_hostname = False
|
|
52
|
+
context.verify_mode = ssl.CERT_NONE
|
|
53
|
+
return context.wrap_socket(sock, server_hostname=host)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def get(
|
|
57
|
+
host: str,
|
|
58
|
+
*,
|
|
59
|
+
port: int = 443,
|
|
60
|
+
use_tls: bool = True,
|
|
61
|
+
connect_to: str | None = None,
|
|
62
|
+
timeout: float = 8.0,
|
|
63
|
+
path: str = "/",
|
|
64
|
+
) -> Response:
|
|
65
|
+
"""GET http(s)://host/path, optionally connecting to another address (origin tests)."""
|
|
66
|
+
response = Response()
|
|
67
|
+
address = connect_to or host
|
|
68
|
+
tls: Tls | None = Tls() if use_tls else None
|
|
69
|
+
for verify in (True, False) if use_tls else (False,):
|
|
70
|
+
try:
|
|
71
|
+
raw = socket.create_connection((address, port), timeout=timeout)
|
|
72
|
+
except OSError as exc:
|
|
73
|
+
response.error = f"connect: {exc}"
|
|
74
|
+
return response
|
|
75
|
+
try:
|
|
76
|
+
sock: socket.socket = raw
|
|
77
|
+
if tls is not None:
|
|
78
|
+
try:
|
|
79
|
+
sock = _wrap(raw, host, verify)
|
|
80
|
+
except ssl.SSLCertVerificationError as exc:
|
|
81
|
+
raw.close()
|
|
82
|
+
tls.error = exc.verify_message or str(exc)
|
|
83
|
+
continue
|
|
84
|
+
tls.version = sock.version() or ""
|
|
85
|
+
tls.verified = verify
|
|
86
|
+
cert = sock.getpeercert() if verify else None
|
|
87
|
+
if cert:
|
|
88
|
+
tls.issuer = _issuer(cert) # type: ignore[arg-type]
|
|
89
|
+
tls.not_after = str(cert.get("notAfter", ""))
|
|
90
|
+
request = (
|
|
91
|
+
f"GET {path} HTTP/1.1\r\nHost: {host}\r\nUser-Agent: {USER_AGENT}\r\n"
|
|
92
|
+
"Accept: */*\r\nConnection: close\r\n\r\n"
|
|
93
|
+
)
|
|
94
|
+
sock.sendall(request.encode())
|
|
95
|
+
data = b""
|
|
96
|
+
while len(data) < MAX_BYTES:
|
|
97
|
+
chunk = sock.recv(8192)
|
|
98
|
+
if not chunk:
|
|
99
|
+
break
|
|
100
|
+
data += chunk
|
|
101
|
+
except (OSError, ssl.SSLError) as exc:
|
|
102
|
+
response.error = f"{type(exc).__name__}: {exc}"
|
|
103
|
+
return response
|
|
104
|
+
finally:
|
|
105
|
+
raw.close()
|
|
106
|
+
_parse(data, response)
|
|
107
|
+
response.tls = tls
|
|
108
|
+
return response
|
|
109
|
+
response.tls = tls
|
|
110
|
+
response.error = response.error or "TLS handshake failed"
|
|
111
|
+
return response
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _parse(data: bytes, response: Response) -> None:
|
|
115
|
+
head, _, body = data.partition(b"\r\n\r\n")
|
|
116
|
+
lines = head.decode("iso-8859-1").split("\r\n")
|
|
117
|
+
if lines and lines[0].startswith("HTTP/"):
|
|
118
|
+
parts = lines[0].split(" ", 2)
|
|
119
|
+
response.status = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else 0
|
|
120
|
+
for line in lines[1:]:
|
|
121
|
+
key, sep, value = line.partition(":")
|
|
122
|
+
if sep:
|
|
123
|
+
response.headers[key.strip().lower()] = value.strip()
|
|
124
|
+
response.body = body.decode("utf-8", errors="replace")
|