targetzero 0.3.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. targetzero/__init__.py +4 -0
  2. targetzero/__main__.py +6 -0
  3. targetzero/admins.py +109 -0
  4. targetzero/cli.py +366 -0
  5. targetzero/common/__init__.py +2 -0
  6. targetzero/common/changes.py +46 -0
  7. targetzero/common/config.py +83 -0
  8. targetzero/common/ids.py +17 -0
  9. targetzero/common/output.py +91 -0
  10. targetzero/common/resources.py +32 -0
  11. targetzero/common/roles.py +24 -0
  12. targetzero/common/schema.py +45 -0
  13. targetzero/config/approved_exposure.example.csv +4 -0
  14. targetzero/config/privileged_roles.yaml +50 -0
  15. targetzero/config/targetzero.example.yaml +57 -0
  16. targetzero/external/__init__.py +2 -0
  17. targetzero/external/dangling.py +106 -0
  18. targetzero/external/discover.py +181 -0
  19. targetzero/external/dnsinfo.py +54 -0
  20. targetzero/external/enrich.py +137 -0
  21. targetzero/external/http.py +33 -0
  22. targetzero/external/kev.py +222 -0
  23. targetzero/external/ports.py +63 -0
  24. targetzero/external/scan.py +260 -0
  25. targetzero/external/scope.py +54 -0
  26. targetzero/external/waf.py +202 -0
  27. targetzero/external/web.py +124 -0
  28. targetzero/gws/__init__.py +2 -0
  29. targetzero/gws/admins.py +230 -0
  30. targetzero/gws/changes.py +190 -0
  31. targetzero/gws/client.py +94 -0
  32. targetzero/gws/domains.py +36 -0
  33. targetzero/registers/__init__.py +2 -0
  34. targetzero/registers/exceptions.py +85 -0
  35. targetzero/registers/findings.py +108 -0
  36. targetzero/registers/inventory.py +142 -0
  37. targetzero/registers/manual_admins.py +43 -0
  38. targetzero/schemas/admins.schema.json +27 -0
  39. targetzero/schemas/alert_tests.schema.json +19 -0
  40. targetzero/schemas/config.schema.json +71 -0
  41. targetzero/schemas/exceptions.schema.json +20 -0
  42. targetzero/schemas/findings.schema.json +23 -0
  43. targetzero/schemas/inventory.schema.json +26 -0
  44. targetzero/schemas/summary.schema.json +36 -0
  45. targetzero/scorecard/__init__.py +2 -0
  46. targetzero/scorecard/scorecard.py +201 -0
  47. targetzero/templates/alert_tests.csv +1 -0
  48. targetzero/templates/exceptions.csv +1 -0
  49. targetzero/templates/findings.csv +1 -0
  50. targetzero/templates/inventory.csv +1 -0
  51. targetzero/templates/manual_admins.csv +1 -0
  52. targetzero/templates/origins.csv +1 -0
  53. targetzero/templates/recovery_page.md +51 -0
  54. targetzero/vendors/__init__.py +2 -0
  55. targetzero/vendors/cisco_asa.py +151 -0
  56. targetzero/vendors/common.py +84 -0
  57. targetzero/vendors/fortigate.py +176 -0
  58. targetzero/vendors/panos.py +141 -0
  59. targetzero/vendors/redfish.py +124 -0
  60. targetzero/vendors/vsphere.py +108 -0
  61. targetzero-0.3.2.dist-info/METADATA +116 -0
  62. targetzero-0.3.2.dist-info/RECORD +66 -0
  63. targetzero-0.3.2.dist-info/WHEEL +4 -0
  64. targetzero-0.3.2.dist-info/entry_points.txt +2 -0
  65. targetzero-0.3.2.dist-info/licenses/LICENSE +202 -0
  66. targetzero-0.3.2.dist-info/licenses/NOTICE +7 -0
@@ -0,0 +1,137 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Inventory enrichment (Zero 5): suggest product, version and support status per asset.
3
+
4
+ Reads the inventory, probes each in-scope web asset once, and writes suggestions to
5
+ 5_external_enrich.csv. The register itself is not changed: the owner decides.
6
+
7
+ Sources: HTTP Server / X-Powered-By headers, TLS certificate and version, CPEs from Shodan
8
+ InternetDB (when `external scan --internetdb` ran today), and support dates from endoflife.date.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import json
14
+ import re
15
+ import urllib.error
16
+ from collections.abc import Mapping
17
+ from datetime import date
18
+ from pathlib import Path
19
+ from typing import Any
20
+
21
+ from targetzero.common.config import Config
22
+ from targetzero.common.output import run_dir, write_csv
23
+ from targetzero.external import web
24
+ from targetzero.external.http import cached_json
25
+ from targetzero.external.scope import Scope
26
+ from targetzero.registers import inventory
27
+
28
+ FIELDS = (
29
+ "id",
30
+ "asset",
31
+ "https",
32
+ "tls_version",
33
+ "certificate_valid",
34
+ "certificate_issuer",
35
+ "certificate_expiry",
36
+ "server",
37
+ "powered_by",
38
+ "suggested_product",
39
+ "suggested_version",
40
+ "suggested_vendor_supported",
41
+ "eol_date",
42
+ "evidence",
43
+ )
44
+ # Header product name -> endoflife.date product id.
45
+ EOL_PRODUCTS = {
46
+ "nginx": "nginx",
47
+ "apache": "apache-http-server",
48
+ "php": "php",
49
+ "tomcat": "tomcat",
50
+ "openssl": "openssl",
51
+ "node.js": "nodejs",
52
+ }
53
+ PRODUCT_PATTERN = re.compile(r"(?P<product>[A-Za-z][A-Za-z.\-]*)/(?P<version>\d+(?:\.\d+)*)")
54
+
55
+
56
+ def products_from_headers(headers: Mapping[str, str]) -> list[tuple[str, str]]:
57
+ found = []
58
+ for key in ("server", "x-powered-by"):
59
+ for match in PRODUCT_PATTERN.finditer(headers.get(key, "")):
60
+ found.append((match["product"].lower(), match["version"]))
61
+ return found
62
+
63
+
64
+ def products_from_cpes(cpes: list[str]) -> list[tuple[str, str]]:
65
+ found = []
66
+ for cpe in cpes:
67
+ parts = cpe.split(":") # cpe:/a:vendor:product:version
68
+ if len(parts) >= 5 and parts[4]:
69
+ found.append((parts[3].lower(), parts[4]))
70
+ return found
71
+
72
+
73
+ def support_status(product: str, version: str, cache: Path) -> tuple[str, str]:
74
+ """('Y'|'N'|'?', eol date) from endoflife.date."""
75
+ slug = EOL_PRODUCTS.get(product)
76
+ if not slug:
77
+ return "?", ""
78
+ try:
79
+ cycles = cached_json(
80
+ f"https://endoflife.date/api/{slug}.json", cache / f"eol-{slug}.json", max_age_hours=24
81
+ )
82
+ except (urllib.error.URLError, TimeoutError, ValueError):
83
+ return "?", ""
84
+ for cycle in sorted(cycles, key=lambda c: -len(str(c.get("cycle", "")))):
85
+ name = str(cycle.get("cycle", ""))
86
+ if version == name or version.startswith(name + "."):
87
+ eol: Any = cycle.get("eol")
88
+ if isinstance(eol, bool):
89
+ return ("N" if eol else "Y"), ""
90
+ return ("N" if str(eol) <= date.today().isoformat() else "Y"), str(eol)
91
+ return "?", ""
92
+
93
+
94
+ def run(config: Config, run_date: date) -> list[Path]:
95
+ scope = Scope.from_config(config)
96
+ cache = config.output_dir / "cache"
97
+ idb_file = config.output_dir / run_date.isoformat() / "external_internetdb.json"
98
+ idb: dict[str, Any] = json.loads(idb_file.read_text()) if idb_file.exists() else {}
99
+ records = []
100
+ for row in inventory.load(config):
101
+ asset = row["asset"]
102
+ in_scope = (
103
+ scope.contains_ip(asset) if row.get("type") == "ip" else scope.contains_name(asset)
104
+ )
105
+ if row.get("status") == "retired" or not in_scope:
106
+ continue
107
+ record = {f: "" for f in FIELDS} | {"id": row["id"], "asset": asset}
108
+ response = web.get(asset, port=443, use_tls=True)
109
+ if not response.status:
110
+ response = web.get(asset, port=80, use_tls=False)
111
+ else:
112
+ record["https"] = "Y"
113
+ if response.tls and record["https"]:
114
+ record["tls_version"] = response.tls.version
115
+ record["certificate_valid"] = "Y" if response.tls.verified else "N"
116
+ record["certificate_issuer"] = response.tls.issuer
117
+ record["certificate_expiry"] = response.tls.not_after
118
+ if response.tls.error:
119
+ record["evidence"] = f"certificate: {response.tls.error}"
120
+ record["server"] = response.headers.get("server", "")
121
+ record["powered_by"] = response.headers.get("x-powered-by", "")
122
+ cpes = [
123
+ str(c) for ip in row.get("ip", "").split(";") for c in idb.get(ip, {}).get("cpes", [])
124
+ ]
125
+ candidates = products_from_headers(response.headers) + products_from_cpes(cpes)
126
+ if candidates:
127
+ product, version = candidates[0]
128
+ supported, eol = support_status(product, version, cache)
129
+ record |= {
130
+ "suggested_product": product,
131
+ "suggested_version": version,
132
+ "suggested_vendor_supported": supported,
133
+ "eol_date": eol,
134
+ }
135
+ records.append(record)
136
+ directory = run_dir(config.output_dir, run_date)
137
+ return [write_csv(directory / "5_external_enrich.csv", records, FIELDS)]
@@ -0,0 +1,33 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Minimal HTTPS JSON client for public reference data, with a one-day file cache."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import json
7
+ import time
8
+ import urllib.request
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ from targetzero import __version__
13
+
14
+ USER_AGENT = f"TargetZero/{__version__} (+https://github.com/wiresecurity/TargetZero)"
15
+
16
+
17
+ def get_json(url: str, *, timeout: float = 30, headers: dict[str, str] | None = None) -> Any:
18
+ if not url.startswith("https://"):
19
+ raise ValueError(f"Only HTTPS URLs are allowed: {url}")
20
+ request = urllib.request.Request( # noqa: S310 (https only, checked above)
21
+ url, headers={"User-Agent": USER_AGENT, **(headers or {})}
22
+ )
23
+ with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 (https only)
24
+ return json.loads(response.read().decode("utf-8"))
25
+
26
+
27
+ def cached_json(url: str, cache_file: Path, max_age_hours: float = 24, **kwargs: Any) -> Any:
28
+ if cache_file.exists() and time.time() - cache_file.stat().st_mtime < max_age_hours * 3600:
29
+ return json.loads(cache_file.read_text(encoding="utf-8"))
30
+ data = get_json(url, **kwargs)
31
+ cache_file.parent.mkdir(parents=True, exist_ok=True)
32
+ cache_file.write_text(json.dumps(data), encoding="utf-8")
33
+ return data
@@ -0,0 +1,222 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Known exploited vulnerabilities (Zero 6): match the CISA KEV catalog against the inventory.
3
+
4
+ KEV entries name a vendor and product but no versions. When the inventory row
5
+ has a version, the NVD API gives the affected version ranges and entries that
6
+ do not apply are dropped. Without a version, only entries added recently are
7
+ reported, with a note to fill in the version.
8
+
9
+ Results are merged into registers/findings.csv; rows people already updated
10
+ are never changed.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import json
16
+ import os
17
+ import re
18
+ import time
19
+ import urllib.error
20
+ from collections.abc import Callable, Iterable, Mapping
21
+ from datetime import UTC, date, datetime, timedelta
22
+ from pathlib import Path
23
+ from typing import Any
24
+
25
+ from targetzero.common.config import Config
26
+ from targetzero.common.ids import make_id
27
+ from targetzero.common.output import read_csv
28
+ from targetzero.external.http import cached_json
29
+ from targetzero.registers import findings, inventory
30
+
31
+ KEV_URL = "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"
32
+ NVD_URL = "https://services.nvd.nist.gov/rest/json/cves/2.0?cveId={}"
33
+ GENERIC_PRODUCTS = {"multiple products", "multiple", "unknown"}
34
+
35
+
36
+ def norm(text: str) -> str:
37
+ return " ".join(re.sub(r"[^a-z0-9]+", " ", text.lower()).split())
38
+
39
+
40
+ def product_matches(kev: Mapping[str, str], inventory_product: str) -> bool:
41
+ product = norm(kev.get("product", ""))
42
+ if not product or product in GENERIC_PRODUCTS:
43
+ return False
44
+ return f" {product} " in f" {norm(inventory_product)} "
45
+
46
+
47
+ def version_tuple(version: str) -> tuple[int, ...]:
48
+ return tuple(int(p) for p in re.findall(r"\d+", version))
49
+
50
+
51
+ def version_affected(nvd: Mapping[str, Any], product: str, version: str) -> bool | None:
52
+ """True/False when NVD has ranges for this product, None when it cannot tell."""
53
+ wanted = version_tuple(version)
54
+ if not wanted:
55
+ return None
56
+ product_key = norm(product).replace(" ", "")
57
+ seen_product = False
58
+ for vuln in nvd.get("vulnerabilities", []):
59
+ for config in vuln.get("cve", {}).get("configurations", []):
60
+ for node in config.get("nodes", []):
61
+ for match in node.get("cpeMatch", []):
62
+ if not match.get("vulnerable"):
63
+ continue
64
+ parts = match.get("criteria", "").split(":")
65
+ if len(parts) < 6 or norm(parts[4]).replace(" ", "") != product_key:
66
+ continue
67
+ seen_product = True
68
+ if _in_range(wanted, parts[5], match):
69
+ return True
70
+ return False if seen_product else None
71
+
72
+
73
+ def _in_range(wanted: tuple[int, ...], exact: str, match: Mapping[str, str]) -> bool:
74
+ bounds = {k: version_tuple(v) for k, v in match.items() if k.startswith("version")}
75
+ if exact not in ("*", "-", ""):
76
+ return version_tuple(exact) == wanted
77
+ if not bounds:
78
+ return True
79
+ checks: tuple[tuple[str, Callable[[tuple[int, ...]], bool]], ...] = (
80
+ ("versionStartIncluding", lambda b: wanted >= b),
81
+ ("versionStartExcluding", lambda b: wanted > b),
82
+ ("versionEndIncluding", lambda b: wanted <= b),
83
+ ("versionEndExcluding", lambda b: wanted < b),
84
+ )
85
+ return all(check(bounds[key]) for key, check in checks if key in bounds)
86
+
87
+
88
+ class Nvd:
89
+ """NVD API 2.0 with a disk cache and the public rate limit (5 requests / 30 s)."""
90
+
91
+ def __init__(self, cache_dir: Path) -> None:
92
+ self.cache_dir = cache_dir
93
+ self.api_key = os.environ.get("NVD_API_KEY", "")
94
+ self.delay = 0.7 if self.api_key else 6.5
95
+ self._last = 0.0
96
+
97
+ def cve(self, cve_id: str) -> Mapping[str, Any] | None:
98
+ cache = self.cache_dir / f"{cve_id}.json"
99
+ if cache.exists() and time.time() - cache.stat().st_mtime < 7 * 86400:
100
+ data: Mapping[str, Any] = json.loads(cache.read_text(encoding="utf-8"))
101
+ return data
102
+ wait = self._last + self.delay - time.monotonic()
103
+ if wait > 0:
104
+ time.sleep(wait)
105
+ self._last = time.monotonic()
106
+ headers = {"apiKey": self.api_key} if self.api_key else {}
107
+ try:
108
+ fetched: Mapping[str, Any] = cached_json(
109
+ NVD_URL.format(cve_id), cache, max_age_hours=0, headers=headers
110
+ )
111
+ return fetched
112
+ except (urllib.error.URLError, TimeoutError, ValueError):
113
+ return None
114
+
115
+
116
+ def kev_deadline(date_added: str, hours: int) -> str:
117
+ start = datetime.fromisoformat(date_added).replace(tzinfo=UTC)
118
+ return (start + timedelta(hours=hours)).isoformat()
119
+
120
+
121
+ def match(
122
+ catalog: Iterable[Mapping[str, str]],
123
+ rows: Iterable[Mapping[str, str]],
124
+ *,
125
+ run_date: date,
126
+ deadline_hours: int,
127
+ recent_days: int,
128
+ nvd: Nvd | None,
129
+ internetdb: Mapping[str, Mapping[str, Any]] | None = None,
130
+ ) -> tuple[list[dict[str, str]], list[str]]:
131
+ catalog = list(catalog)
132
+ by_cve = {v["cveID"]: v for v in catalog}
133
+ recent = (run_date - timedelta(days=recent_days)).isoformat()
134
+ found: dict[str, dict[str, str]] = {}
135
+ notes: list[str] = []
136
+
137
+ def add(row: Mapping[str, str], kev: Mapping[str, str], basis: str) -> None:
138
+ asset = row["id"].split(":", 1)[-1]
139
+ finding_id = make_id("kev", asset, kev["cveID"])
140
+ ransomware = kev.get("knownRansomwareCampaignUse") == "Known"
141
+ found.setdefault(
142
+ finding_id,
143
+ {
144
+ "id": finding_id,
145
+ "asset_id": row["id"],
146
+ "vulnerability": kev["cveID"],
147
+ "title": kev.get("vulnerabilityName", ""),
148
+ "severity": "kev",
149
+ "published": kev["dateAdded"],
150
+ "deadline": kev_deadline(kev["dateAdded"], deadline_hours),
151
+ "status": "open",
152
+ "owner": row.get("owner", ""),
153
+ "notes": basis + ("; used in ransomware campaigns" if ransomware else ""),
154
+ },
155
+ )
156
+
157
+ for row in rows:
158
+ if row.get("status") == "retired" or not row.get("product"):
159
+ continue
160
+ version = row.get("version", "")
161
+ for kev in catalog:
162
+ if not product_matches(kev, row["product"]):
163
+ continue
164
+ affected = nvd.cve(kev["cveID"]) if nvd and version else None
165
+ verdict = version_affected(affected, kev["product"], version) if affected else None
166
+ if verdict is True:
167
+ add(row, kev, f"version {version} is in the affected range (NVD)")
168
+ elif verdict is None and kev["dateAdded"] >= recent:
169
+ add(row, kev, "product match; version could not be checked - confirm")
170
+ if internetdb:
171
+ for ip in filter(None, row.get("ip", "").split(";")):
172
+ for cve in internetdb.get(ip, {}).get("vulns", []):
173
+ if cve in by_cve:
174
+ add(row, by_cve[cve], f"reported for {ip} by Shodan InternetDB - confirm")
175
+ if nvd is None:
176
+ notes.append(
177
+ f"NVD lookups off: only KEV entries from the last {recent_days} days are matched"
178
+ )
179
+ return sorted(found.values(), key=lambda f: f["id"]), notes
180
+
181
+
182
+ def run(
183
+ config: Config, run_date: date, *, use_nvd: bool = True, recent_days: int = 30
184
+ ) -> list[Path]:
185
+ cache = config.output_dir / "cache"
186
+ notes: list[str] = []
187
+ complete = True
188
+ try:
189
+ catalog = cached_json(KEV_URL, cache / "kev.json", max_age_hours=6, timeout=60)
190
+ entries = catalog.get("vulnerabilities", [])
191
+ notes.append(
192
+ f"CISA KEV catalog {catalog.get('catalogVersion', '?')}, {len(entries)} entries"
193
+ )
194
+ except (urllib.error.URLError, TimeoutError, ValueError) as exc:
195
+ entries, complete = [], False
196
+ notes.append(f"Could not download the CISA KEV catalog: {exc}")
197
+
198
+ idb_file = config.output_dir / run_date.isoformat() / "external_internetdb.json"
199
+ idb = json.loads(idb_file.read_text(encoding="utf-8")) if idb_file.exists() else None
200
+ rows = inventory.load(config)
201
+ # Firewalls and VPNs found by the vendor collectors today, with exact firmware versions.
202
+ for devices_file in sorted((config.output_dir / run_date.isoformat()).glob("devices_*.csv")):
203
+ rows += read_csv(devices_file)
204
+ if not any(r.get("product") for r in rows):
205
+ notes.append("No inventory row has a product filled in: nothing to match")
206
+ complete = False
207
+ new, match_notes = match(
208
+ entries,
209
+ rows,
210
+ run_date=run_date,
211
+ deadline_hours=int(
212
+ config.data.get("thresholds", {}).get("deadline_hours", {}).get("kev", 8)
213
+ ),
214
+ recent_days=recent_days,
215
+ nvd=Nvd(cache / "nvd") if use_nvd else None,
216
+ internetdb=idb,
217
+ )
218
+ merged, added = findings.merge(findings.load(config), new)
219
+ notes += [*match_notes, f"{len(new)} KEV match(es), {added} new in registers/findings.csv"]
220
+ paths = [findings.save(config, merged)]
221
+ paths += findings.count(config, run_date, notes=notes, complete=complete)
222
+ return paths
@@ -0,0 +1,63 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Ports checked by the exposure scan, grouped by why they matter."""
3
+
4
+ from __future__ import annotations
5
+
6
+ PORTS: dict[int, tuple[str, str]] = {
7
+ # port: (category, service)
8
+ 21: ("file", "ftp"),
9
+ 22: ("management", "ssh"),
10
+ 23: ("management", "telnet"),
11
+ 25: ("mail", "smtp"),
12
+ 53: ("other", "dns"),
13
+ 80: ("web", "http"),
14
+ 110: ("mail", "pop3"),
15
+ 111: ("other", "rpcbind"),
16
+ 135: ("management", "msrpc"),
17
+ 139: ("file", "netbios"),
18
+ 143: ("mail", "imap"),
19
+ 161: ("management", "snmp"),
20
+ 389: ("directory", "ldap"),
21
+ 443: ("web", "https"),
22
+ 445: ("file", "smb"),
23
+ 465: ("mail", "smtps"),
24
+ 514: ("management", "syslog/rsh"),
25
+ 587: ("mail", "submission"),
26
+ 623: ("management", "ipmi"),
27
+ 636: ("directory", "ldaps"),
28
+ 873: ("file", "rsync"),
29
+ 993: ("mail", "imaps"),
30
+ 995: ("mail", "pop3s"),
31
+ 1194: ("vpn", "openvpn"),
32
+ 1433: ("database", "mssql"),
33
+ 1521: ("database", "oracle"),
34
+ 1723: ("vpn", "pptp"),
35
+ 2049: ("file", "nfs"),
36
+ 2375: ("management", "docker"),
37
+ 2376: ("management", "docker-tls"),
38
+ 3268: ("directory", "ldap-gc"),
39
+ 3306: ("database", "mysql"),
40
+ 3389: ("management", "rdp"),
41
+ 4443: ("management", "admin-https"),
42
+ 5432: ("database", "postgresql"),
43
+ 5900: ("management", "vnc"),
44
+ 5985: ("management", "winrm"),
45
+ 5986: ("management", "winrm-https"),
46
+ 6379: ("database", "redis"),
47
+ 6443: ("management", "kubernetes-api"),
48
+ 8000: ("web", "http-alt"),
49
+ 8006: ("management", "proxmox"),
50
+ 8080: ("web", "http-alt"),
51
+ 8443: ("management", "admin-https"),
52
+ 9000: ("management", "admin-web"),
53
+ 9200: ("database", "elasticsearch"),
54
+ 9443: ("management", "admin-https"),
55
+ 10000: ("management", "webmin/backup"),
56
+ 10443: ("vpn", "ssl-vpn/admin"),
57
+ 11211: ("database", "memcached"),
58
+ 27017: ("database", "mongodb"),
59
+ }
60
+
61
+
62
+ def describe(port: int) -> tuple[str, str]:
63
+ return PORTS.get(port, ("other", ""))