targetzero 0.3.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. targetzero/__init__.py +4 -0
  2. targetzero/__main__.py +6 -0
  3. targetzero/admins.py +109 -0
  4. targetzero/cli.py +366 -0
  5. targetzero/common/__init__.py +2 -0
  6. targetzero/common/changes.py +46 -0
  7. targetzero/common/config.py +83 -0
  8. targetzero/common/ids.py +17 -0
  9. targetzero/common/output.py +91 -0
  10. targetzero/common/resources.py +32 -0
  11. targetzero/common/roles.py +24 -0
  12. targetzero/common/schema.py +45 -0
  13. targetzero/config/approved_exposure.example.csv +4 -0
  14. targetzero/config/privileged_roles.yaml +50 -0
  15. targetzero/config/targetzero.example.yaml +57 -0
  16. targetzero/external/__init__.py +2 -0
  17. targetzero/external/dangling.py +106 -0
  18. targetzero/external/discover.py +181 -0
  19. targetzero/external/dnsinfo.py +54 -0
  20. targetzero/external/enrich.py +137 -0
  21. targetzero/external/http.py +33 -0
  22. targetzero/external/kev.py +222 -0
  23. targetzero/external/ports.py +63 -0
  24. targetzero/external/scan.py +260 -0
  25. targetzero/external/scope.py +54 -0
  26. targetzero/external/waf.py +202 -0
  27. targetzero/external/web.py +124 -0
  28. targetzero/gws/__init__.py +2 -0
  29. targetzero/gws/admins.py +230 -0
  30. targetzero/gws/changes.py +190 -0
  31. targetzero/gws/client.py +94 -0
  32. targetzero/gws/domains.py +36 -0
  33. targetzero/registers/__init__.py +2 -0
  34. targetzero/registers/exceptions.py +85 -0
  35. targetzero/registers/findings.py +108 -0
  36. targetzero/registers/inventory.py +142 -0
  37. targetzero/registers/manual_admins.py +43 -0
  38. targetzero/schemas/admins.schema.json +27 -0
  39. targetzero/schemas/alert_tests.schema.json +19 -0
  40. targetzero/schemas/config.schema.json +71 -0
  41. targetzero/schemas/exceptions.schema.json +20 -0
  42. targetzero/schemas/findings.schema.json +23 -0
  43. targetzero/schemas/inventory.schema.json +26 -0
  44. targetzero/schemas/summary.schema.json +36 -0
  45. targetzero/scorecard/__init__.py +2 -0
  46. targetzero/scorecard/scorecard.py +201 -0
  47. targetzero/templates/alert_tests.csv +1 -0
  48. targetzero/templates/exceptions.csv +1 -0
  49. targetzero/templates/findings.csv +1 -0
  50. targetzero/templates/inventory.csv +1 -0
  51. targetzero/templates/manual_admins.csv +1 -0
  52. targetzero/templates/origins.csv +1 -0
  53. targetzero/templates/recovery_page.md +51 -0
  54. targetzero/vendors/__init__.py +2 -0
  55. targetzero/vendors/cisco_asa.py +151 -0
  56. targetzero/vendors/common.py +84 -0
  57. targetzero/vendors/fortigate.py +176 -0
  58. targetzero/vendors/panos.py +141 -0
  59. targetzero/vendors/redfish.py +124 -0
  60. targetzero/vendors/vsphere.py +108 -0
  61. targetzero-0.3.2.dist-info/METADATA +116 -0
  62. targetzero-0.3.2.dist-info/RECORD +66 -0
  63. targetzero-0.3.2.dist-info/WHEEL +4 -0
  64. targetzero-0.3.2.dist-info/entry_points.txt +2 -0
  65. targetzero-0.3.2.dist-info/licenses/LICENSE +202 -0
  66. targetzero-0.3.2.dist-info/licenses/NOTICE +7 -0
@@ -0,0 +1,91 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Write records (CSV) and count summaries (JSON) in the TargetZero output contract.
3
+
4
+ Layout: <output_dir>/<YYYY-MM-DD>/<zero>_<source>_<name>.csv
5
+ <output_dir>/<YYYY-MM-DD>/<zero>_<source>_<name>.summary.json
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import csv
11
+ import json
12
+ from collections.abc import Iterable, Mapping, Sequence
13
+ from datetime import UTC, date, datetime
14
+ from pathlib import Path
15
+ from typing import Any
16
+
17
+ from targetzero import __version__
18
+ from targetzero.common.schema import validate_summary
19
+
20
+
21
+ def run_dir(output_dir: Path, run_date: date) -> Path:
22
+ path = output_dir / run_date.isoformat()
23
+ path.mkdir(parents=True, exist_ok=True)
24
+ return path
25
+
26
+
27
+ def base_name(zero: int, source: str, name: str) -> str:
28
+ return f"{zero}_{source}_{name}"
29
+
30
+
31
+ def delimiter_of(header: str) -> str:
32
+ """Excel with Belgian, Dutch, French or German settings saves CSV with semicolons."""
33
+ return ";" if header.count(";") > header.count(",") else ","
34
+
35
+
36
+ def read_csv(path: Path) -> list[dict[str, str]]:
37
+ # utf-8-sig: files saved by Excel start with a byte order mark.
38
+ with path.open(newline="", encoding="utf-8-sig") as handle:
39
+ delimiter = delimiter_of(handle.readline())
40
+ handle.seek(0) # utf-8-sig skips the byte order mark again after a seek to 0
41
+ rows = csv.DictReader(handle, delimiter=delimiter)
42
+ return [{k: (v or "").strip() for k, v in row.items() if k} for row in rows]
43
+
44
+
45
+ def write_csv(path: Path, rows: Iterable[Mapping[str, Any]], fieldnames: Sequence[str]) -> Path:
46
+ path.parent.mkdir(parents=True, exist_ok=True)
47
+ with path.open("w", newline="", encoding="utf-8") as handle:
48
+ writer = csv.DictWriter(handle, fieldnames=list(fieldnames), extrasaction="raise")
49
+ writer.writeheader()
50
+ for row in rows:
51
+ writer.writerow({k: "" if v is None else v for k, v in row.items()})
52
+ return path
53
+
54
+
55
+ def write_summary(
56
+ directory: Path,
57
+ *,
58
+ zero: int,
59
+ source: str,
60
+ name: str,
61
+ items: Sequence[Mapping[str, str]],
62
+ tool: str,
63
+ run_date: date,
64
+ complete: bool = True,
65
+ notes: Sequence[str] = (),
66
+ record_file: str | None = None,
67
+ tool_version: str = __version__,
68
+ ) -> Path:
69
+ """Validate and write a count summary. Raises ValueError when it breaks the contract."""
70
+ summary: dict[str, Any] = {
71
+ "schema_version": "1.0",
72
+ "zero": zero,
73
+ "source": source,
74
+ "name": name,
75
+ "tool": tool,
76
+ "tool_version": tool_version,
77
+ "run_date": run_date.isoformat(),
78
+ "generated_at": datetime.now(UTC).isoformat(timespec="seconds"),
79
+ "count": len(items),
80
+ "complete": complete,
81
+ "notes": list(notes),
82
+ "items": [{"id": i["id"], "reason": i["reason"]} for i in items],
83
+ }
84
+ if record_file:
85
+ summary["record_file"] = record_file
86
+ problems = validate_summary(summary)
87
+ if problems:
88
+ raise ValueError("Summary does not match the schema:\n " + "\n ".join(problems))
89
+ path = directory / f"{base_name(zero, source, name)}.summary.json"
90
+ path.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
91
+ return path
@@ -0,0 +1,32 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Locate files that ship with TargetZero (schemas, templates, example config).
3
+
4
+ In an installed wheel they live inside the package. In a source checkout they
5
+ live at the repository root, so both places are tried.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from pathlib import Path
11
+
12
+ _PACKAGE_DIR = Path(__file__).resolve().parent.parent
13
+ _REPO_ROOT = _PACKAGE_DIR.parent.parent
14
+
15
+
16
+ def _find(packaged: str, in_repo: str) -> Path:
17
+ for candidate in (_PACKAGE_DIR / packaged, _REPO_ROOT / in_repo):
18
+ if candidate.exists():
19
+ return candidate
20
+ raise FileNotFoundError(f"TargetZero resource not found: {in_repo}")
21
+
22
+
23
+ def schema_dir() -> Path:
24
+ return _find("schemas", "schema")
25
+
26
+
27
+ def templates_dir() -> Path:
28
+ return _find("templates", "templates")
29
+
30
+
31
+ def example_config_dir() -> Path:
32
+ return _find("config", "config")
@@ -0,0 +1,24 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Load privileged_roles.yaml: which roles count as full admin."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from pathlib import Path
7
+
8
+ import yaml
9
+
10
+ from targetzero.common.config import Config
11
+ from targetzero.common.resources import example_config_dir
12
+
13
+ FILE_NAME = "privileged_roles.yaml"
14
+
15
+
16
+ def roles_file(config: Config) -> Path:
17
+ """Prefer the copy next to targetzero.yaml, so users can change it."""
18
+ local = config.path.parent / FILE_NAME
19
+ return local if local.exists() else example_config_dir() / FILE_NAME
20
+
21
+
22
+ def privileged_roles(config: Config, key: str) -> set[str]:
23
+ data = yaml.safe_load(roles_file(config).read_text(encoding="utf-8")) or {}
24
+ return {str(r).casefold() for r in data.get(key, [])}
@@ -0,0 +1,45 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Validate data against the JSON schemas in schema/."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import json
7
+ from functools import cache
8
+ from typing import Any
9
+
10
+ from jsonschema import Draft202012Validator
11
+
12
+ from targetzero.common.resources import schema_dir
13
+
14
+ SCHEMAS = ("summary", "admins", "inventory", "findings", "exceptions", "alert_tests", "config")
15
+
16
+
17
+ @cache
18
+ def _validator(name: str) -> Draft202012Validator:
19
+ if name not in SCHEMAS:
20
+ raise ValueError(f"Unknown schema '{name}'. Known: {', '.join(SCHEMAS)}")
21
+ path = schema_dir() / f"{name}.schema.json"
22
+ return Draft202012Validator(json.loads(path.read_text(encoding="utf-8")))
23
+
24
+
25
+ def validate(instance: Any, name: str) -> list[str]:
26
+ """Return a readable list of problems; empty when the instance is valid."""
27
+ problems = []
28
+ for error in sorted(_validator(name).iter_errors(instance), key=lambda e: list(e.path)):
29
+ location = "/".join(str(part) for part in error.path) or "(root)"
30
+ problems.append(f"{location}: {error.message}")
31
+ return problems
32
+
33
+
34
+ def validate_summary(summary: Any) -> list[str]:
35
+ """Schema check plus the rules a schema cannot express."""
36
+ problems = validate(summary, "summary")
37
+ if problems or not isinstance(summary, dict):
38
+ return problems
39
+ ids = [item["id"] for item in summary["items"]]
40
+ if summary["count"] != len(ids):
41
+ problems.append(f"count is {summary['count']} but there are {len(ids)} items")
42
+ duplicates = sorted({i for i in ids if ids.count(i) > 1})
43
+ if duplicates:
44
+ problems.append(f"duplicate item ids: {', '.join(duplicates)}")
45
+ return problems
@@ -0,0 +1,4 @@
1
+ host,port,protocol,justification,owner,approved_by,review_date
2
+ www.example.com,443,tcp,Public website behind CDN,Jane Doe,John Smith (CEO),2027-02-19
3
+ vpn.example.com,443,tcp,SSL VPN for remote staff (MFA enforced),Jane Doe,John Smith (CEO),2027-02-19
4
+ mail.example.com,25,tcp,Inbound mail relay,Jane Doe,John Smith (CEO),2027-02-19
@@ -0,0 +1,50 @@
1
+ # Roles that count as "full admin" for Zero 1 and Zero 2.
2
+ # Roles not listed here are still reported, with privileged = N.
3
+ # Change this list to match your own risk view; keep the names exactly as the
4
+ # platform shows them.
5
+
6
+ m365:
7
+ # Entra ID built-in roles (display names)
8
+ - Global Administrator
9
+ - Privileged Role Administrator
10
+ - Privileged Authentication Administrator
11
+ - Security Administrator
12
+ - Conditional Access Administrator
13
+ - Exchange Administrator
14
+ - SharePoint Administrator
15
+ - Teams Administrator
16
+ - User Administrator
17
+ - Authentication Administrator
18
+ - Application Administrator
19
+ - Cloud Application Administrator
20
+ - Intune Administrator
21
+ - Hybrid Identity Administrator
22
+ - Domain Name Administrator
23
+ - Partner Tier2 Support
24
+ - Directory Synchronization Accounts
25
+
26
+ # Microsoft Graph application permissions that give an app admin-level power.
27
+ # Apps holding one of these are reported as hidden admins (account_type = service).
28
+ m365_app_permissions:
29
+ - RoleManagement.ReadWrite.Directory
30
+ - AppRoleAssignment.ReadWrite.All
31
+ - Application.ReadWrite.All
32
+ - Directory.ReadWrite.All
33
+ - User.ReadWrite.All
34
+ - Group.ReadWrite.All
35
+ - UserAuthenticationMethod.ReadWrite.All
36
+ - Policy.ReadWrite.ConditionalAccess
37
+ - Domain.ReadWrite.All
38
+ - Mail.ReadWrite
39
+ - Mail.Send
40
+ - Files.ReadWrite.All
41
+ - Sites.FullControl.All
42
+ - Sites.ReadWrite.All
43
+
44
+ gws:
45
+ # Google Workspace system roles
46
+ - Super Admin
47
+ - Groups Admin
48
+ - User Management Admin
49
+ - Services Admin
50
+ - Mobile Admin
@@ -0,0 +1,57 @@
1
+ # TargetZero configuration.
2
+ # Copy this file to targetzero.yaml (which git ignores) and fill it in.
3
+ # Relative paths are resolved against the folder that holds this file.
4
+
5
+ organisation: Example Ltd
6
+
7
+ # Where scripts write their results (output/<YYYY-MM-DD>/...) and where the
8
+ # hand-maintained registers live (inventory, findings, exceptions, ...).
9
+ output_dir: output
10
+ registers_dir: registers
11
+
12
+ # One owner per Target 0 count. Shown on the scorecard.
13
+ owners:
14
+ "1": "Jane Doe, IT lead"
15
+ "2": "Jane Doe, IT lead"
16
+ "3": "Managed SOC / IT provider"
17
+ "4": "Jane Doe, IT lead"
18
+ "5": "Jane Doe, IT lead"
19
+ "6": "Jane Doe, IT lead"
20
+ "7": "Application owners"
21
+
22
+ # Emergency ("break-glass") accounts. Never changed by any script and left
23
+ # out of the Zero 1 count. Keep exactly two per platform.
24
+ break_glass:
25
+ m365:
26
+ - bg-admin-01@example.onmicrosoft.com
27
+ - bg-admin-02@example.onmicrosoft.com
28
+ gws:
29
+ - bg-admin-01@example.com
30
+ - bg-admin-02@example.com
31
+
32
+ # What you own and are authorised to scan. External checks never go outside
33
+ # this list, not even for hosts they discover themselves.
34
+ scope:
35
+ domains:
36
+ - example.com
37
+ ip_ranges:
38
+ - 203.0.113.0/28 # documentation range, replace with your own
39
+
40
+ thresholds:
41
+ exception_max_days: 90 # Target 0: at most 90 days
42
+ inventory_review_days: 90 # inventory rows older than this count as outdated
43
+ alert_test_max_age_days: 35 # monthly test, with a few days of slack
44
+ deadline_hours:
45
+ kev: 8
46
+ critical: 24
47
+ high: 72
48
+
49
+ m365:
50
+ tenant_id: "" # e.g. example.onmicrosoft.com or the tenant GUID
51
+ client_id: "" # only for unattended runs
52
+ certificate_thumbprint: ""
53
+
54
+ gws:
55
+ customer_id: my_customer
56
+ admin_subject: "" # admin account to impersonate
57
+ credentials_file: "" # keep outside the repository
@@ -0,0 +1,2 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Checks run from outside: exposure scan, asset discovery, known exploited vulnerabilities."""
@@ -0,0 +1,106 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Dangling DNS (Zero 5): names that point to things you no longer use.
3
+
4
+ A name counts when its CNAME target does not exist, or when the service it points to says
5
+ the site is gone (an unclaimed bucket, app or page that someone else could take over).
6
+ A records pointing to released cloud addresses cannot be detected this way.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from datetime import date
12
+ from pathlib import Path
13
+
14
+ from targetzero.common.config import Config
15
+ from targetzero.common.ids import make_id
16
+ from targetzero.common.output import read_csv, run_dir, write_csv, write_summary
17
+ from targetzero.external import dnsinfo, web
18
+ from targetzero.external.scope import Scope
19
+ from targetzero.registers import inventory
20
+
21
+ FIELDS = ("name", "cname", "target_exists", "service", "evidence", "result")
22
+
23
+ # (service, CNAME suffixes, text in the response that shows the resource is unclaimed)
24
+ FINGERPRINTS = (
25
+ ("Amazon S3", ("s3.amazonaws.com", "amazonaws.com"), "NoSuchBucket"),
26
+ ("GitHub Pages", ("github.io",), "There isn't a GitHub Pages site here"),
27
+ ("Heroku", ("herokuapp.com", "herokudns.com"), "no-such-app"),
28
+ ("Azure App Service", ("azurewebsites.net",), "404 Web Site not found"),
29
+ ("Shopify", ("myshopify.com",), "Sorry, this shop is currently unavailable"),
30
+ ("Zendesk", ("zendesk.com",), "Help Center Closed"),
31
+ ("Fastly", ("fastly.net",), "Fastly error: unknown domain"),
32
+ ("Surge", ("surge.sh",), "project not found"),
33
+ )
34
+
35
+
36
+ def candidate_names(config: Config, scope: Scope, run_date: date) -> list[str]:
37
+ names = {
38
+ r["asset"].lower()
39
+ for r in inventory.load(config)
40
+ if r.get("type") in ("domain", "subdomain")
41
+ }
42
+ discovery = config.output_dir / run_date.isoformat() / "5_external_discovery.csv"
43
+ if discovery.exists():
44
+ names |= {r["name"].lower() for r in read_csv(discovery)}
45
+ return sorted(n for n in names if scope.contains_name(n))
46
+
47
+
48
+ def check_name(name: str, res: object) -> dict[str, str] | None:
49
+ target = dnsinfo.cname_target(name, res) # type: ignore[arg-type]
50
+ if not target:
51
+ return None
52
+ record = {
53
+ "name": name,
54
+ "cname": target,
55
+ "target_exists": "",
56
+ "service": "",
57
+ "evidence": "",
58
+ "result": "",
59
+ }
60
+ found = dnsinfo.lookup(target, res) # type: ignore[arg-type]
61
+ record["target_exists"] = "N" if found.nxdomain or not found.ips else "Y"
62
+ if record["target_exists"] == "N":
63
+ record["result"] = "dangling"
64
+ record["evidence"] = "CNAME target does not resolve"
65
+ return record
66
+ for service, suffixes, text in FINGERPRINTS:
67
+ if any(target == s or target.endswith("." + s) for s in suffixes):
68
+ record["service"] = service
69
+ for use_tls, port in ((True, 443), (False, 80)):
70
+ response = web.get(name, port=port, use_tls=use_tls, timeout=8)
71
+ if text in response.body:
72
+ record["result"] = "dangling"
73
+ record["evidence"] = f"{service} answers '{text}'"
74
+ return record
75
+ record["result"] = "ok"
76
+ return record
77
+
78
+
79
+ def run(config: Config, run_date: date) -> list[Path]:
80
+ scope = Scope.from_config(config)
81
+ res = dnsinfo.resolver()
82
+ records = [r for n in candidate_names(config, scope, run_date) if (r := check_name(n, res))]
83
+ items = [
84
+ {
85
+ "id": make_id("external", r["name"], "dangling"),
86
+ "reason": f"{r['name']} -> {r['cname']}: {r['evidence']}; delete the record",
87
+ }
88
+ for r in records
89
+ if r["result"] == "dangling"
90
+ ]
91
+ directory = run_dir(config.output_dir, run_date)
92
+ record_file = "5_external_dangling-dns.csv"
93
+ return [
94
+ write_csv(directory / record_file, records, FIELDS),
95
+ write_summary(
96
+ directory,
97
+ zero=5,
98
+ source="external",
99
+ name="dangling-dns",
100
+ items=items,
101
+ tool="targetzero external dangling-dns",
102
+ run_date=run_date,
103
+ notes=["Only CNAME records are checked"],
104
+ record_file=record_file,
105
+ ),
106
+ ]
@@ -0,0 +1,181 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Asset discovery (Zero 5): find your public names and addresses and merge them into
3
+ registers/inventory.csv.
4
+
5
+ Sources: scope.domains, domain lists written today by the M365 and Google
6
+ Workspace scripts (5_*_domains.csv), DNS zone files, and certificate
7
+ transparency logs (crt.sh). Names that no longer resolve are listed but not added.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import time
13
+ import urllib.error
14
+ import urllib.parse
15
+ from collections.abc import Iterable
16
+ from dataclasses import dataclass, field
17
+ from datetime import date
18
+ from pathlib import Path
19
+
20
+ import dns.exception
21
+ import dns.rdatatype
22
+ import dns.resolver
23
+ import dns.zone
24
+
25
+ from targetzero.common.config import Config
26
+ from targetzero.common.ids import make_id
27
+ from targetzero.common.output import read_csv, run_dir, write_csv
28
+ from targetzero.external.http import get_json
29
+ from targetzero.external.scope import Scope
30
+ from targetzero.registers import inventory
31
+
32
+ FIELDS = ("name", "resolves", "ips", "cname", "in_scope", "sources")
33
+
34
+
35
+ @dataclass
36
+ class Name:
37
+ name: str
38
+ sources: set[str] = field(default_factory=set)
39
+ ips: set[str] = field(default_factory=set)
40
+ cname: str = ""
41
+
42
+
43
+ def crtsh_names(domain: str, timeout: float = 90) -> set[str]:
44
+ query = urllib.parse.quote(f"%.{domain}")
45
+ url = f"https://crt.sh/?q={query}&output=json"
46
+ try:
47
+ entries = get_json(url, timeout=timeout)
48
+ except urllib.error.HTTPError as exc:
49
+ if exc.code < 500:
50
+ raise
51
+ time.sleep(10) # crt.sh often answers 502/503 under load; one retry
52
+ entries = get_json(url, timeout=timeout)
53
+ names = set()
54
+ for entry in entries:
55
+ for value in str(entry.get("name_value", "")).splitlines():
56
+ value = value.strip().lower().removeprefix("*.")
57
+ if value and " " not in value and "@" not in value:
58
+ names.add(value)
59
+ return names
60
+
61
+
62
+ def zone_names(path: Path, origin: str | None) -> set[str]:
63
+ zone = dns.zone.from_file(str(path), origin=origin, relativize=False, check_origin=False)
64
+ wanted = {dns.rdatatype.A, dns.rdatatype.AAAA, dns.rdatatype.CNAME}
65
+ return {
66
+ str(name).rstrip(".").lower()
67
+ for name, node in zone.nodes.items()
68
+ if any(rdataset.rdtype in wanted for rdataset in node.rdatasets)
69
+ and not str(name).startswith("*")
70
+ }
71
+
72
+
73
+ def platform_domains(directory: Path) -> dict[str, str]:
74
+ """Domains written today by Get-T0M365Domain or `targetzero gws domains`."""
75
+ found = {}
76
+ for path in sorted(directory.glob("5_*_domains.csv")):
77
+ source = path.stem.split("_")[1]
78
+ for row in read_csv(path):
79
+ if row.get("domain"):
80
+ found[row["domain"].lower()] = source
81
+ return found
82
+
83
+
84
+ def resolve(name: Name, resolver: dns.resolver.Resolver) -> None:
85
+ for rdtype in ("A", "AAAA"):
86
+ try:
87
+ answer = resolver.resolve(name.name, rdtype)
88
+ except (
89
+ dns.resolver.NXDOMAIN,
90
+ dns.resolver.NoAnswer,
91
+ dns.resolver.NoNameservers,
92
+ dns.exception.Timeout,
93
+ ):
94
+ continue
95
+ name.ips |= {r.to_text() for r in answer}
96
+ if answer.canonical_name.to_text().rstrip(".").lower() != name.name:
97
+ name.cname = answer.canonical_name.to_text().rstrip(".").lower()
98
+
99
+
100
+ def discovered_rows(names: Iterable[Name], apexes: set[str], scope: Scope) -> list[dict[str, str]]:
101
+ rows: list[dict[str, str]] = []
102
+ ips: set[str] = set()
103
+ for n in names:
104
+ if not n.ips:
105
+ continue
106
+ rows.append(
107
+ {
108
+ "id": make_id("inventory", n.name),
109
+ "asset": n.name,
110
+ "type": "domain" if n.name in apexes else "subdomain",
111
+ "ip": ";".join(sorted(n.ips)),
112
+ }
113
+ )
114
+ ips |= {ip for ip in n.ips if scope.contains_ip(ip)}
115
+ rows += [
116
+ {"id": make_id("inventory", ip), "asset": ip, "type": "ip", "ip": ip} for ip in sorted(ips)
117
+ ]
118
+ return rows
119
+
120
+
121
+ def run(
122
+ config: Config,
123
+ run_date: date,
124
+ *,
125
+ use_crtsh: bool = True,
126
+ zone_files: Iterable[tuple[Path, str | None]] = (),
127
+ ) -> list[Path]:
128
+ scope = Scope.from_config(config)
129
+ directory = run_dir(config.output_dir, run_date)
130
+ notes: list[str] = []
131
+ complete = True
132
+ names: dict[str, Name] = {}
133
+
134
+ def add(name: str, source: str) -> None:
135
+ name = name.lower().strip(".")
136
+ names.setdefault(name, Name(name)).sources.add(source)
137
+
138
+ for domain in scope.domains:
139
+ add(domain, "config")
140
+ for domain, source in platform_domains(directory).items():
141
+ add(domain, source)
142
+ if not scope.contains_name(domain):
143
+ notes.append(f"{domain} ({source}) is not in scope.domains: add it if you own it")
144
+ for path, origin in zone_files:
145
+ for name in zone_names(path, origin):
146
+ add(name, "zone")
147
+ apexes = set(names)
148
+ if use_crtsh:
149
+ for domain in sorted(apexes):
150
+ try:
151
+ for name in crtsh_names(domain):
152
+ if name == domain or name.endswith("." + domain):
153
+ add(name, "crt.sh")
154
+ except (urllib.error.URLError, TimeoutError, ValueError) as exc:
155
+ notes.append(f"crt.sh failed for {domain}: {exc}")
156
+ complete = False
157
+
158
+ resolver = dns.resolver.Resolver()
159
+ resolver.lifetime = 5
160
+ for entry in names.values():
161
+ resolve(entry, resolver)
162
+
163
+ rows, added, updated = inventory.merge(
164
+ inventory.load(config), discovered_rows(names.values(), apexes, scope)
165
+ )
166
+ paths = [inventory.save(config, rows)]
167
+ notes.append(f"Inventory: {added} new row(s) marked unreviewed, {updated} address update(s)")
168
+ records = [
169
+ {
170
+ "name": n.name,
171
+ "resolves": "Y" if n.ips else "N",
172
+ "ips": ";".join(sorted(n.ips)),
173
+ "cname": n.cname,
174
+ "in_scope": "Y" if scope.contains_name(n.name) else "N",
175
+ "sources": ";".join(sorted(n.sources)),
176
+ }
177
+ for n in sorted(names.values(), key=lambda n: n.name)
178
+ ]
179
+ paths.append(write_csv(directory / "5_external_discovery.csv", records, FIELDS))
180
+ paths += inventory.count(config, run_date, notes=notes, complete=complete)
181
+ return paths
@@ -0,0 +1,54 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """DNS lookups shared by the external checks: addresses plus the CNAME chain."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from dataclasses import dataclass, field
7
+
8
+ import dns.exception
9
+ import dns.rdatatype
10
+ import dns.resolver
11
+
12
+ NOT_FOUND = (dns.resolver.NoAnswer, dns.resolver.NoNameservers, dns.exception.Timeout)
13
+
14
+
15
+ @dataclass
16
+ class Lookup:
17
+ name: str
18
+ ips: list[str] = field(default_factory=list)
19
+ cnames: list[str] = field(default_factory=list)
20
+ nxdomain: bool = False
21
+
22
+
23
+ def resolver(timeout: float = 5) -> dns.resolver.Resolver:
24
+ r = dns.resolver.Resolver()
25
+ r.lifetime = timeout
26
+ return r
27
+
28
+
29
+ def lookup(name: str, res: dns.resolver.Resolver) -> Lookup:
30
+ result = Lookup(name.lower().rstrip("."))
31
+ for rdtype in ("A", "AAAA"):
32
+ try:
33
+ answer = res.resolve(result.name, rdtype)
34
+ except dns.resolver.NXDOMAIN:
35
+ result.nxdomain = True
36
+ break
37
+ except NOT_FOUND:
38
+ continue
39
+ result.ips += [r.to_text() for r in answer]
40
+ if answer.response is not None:
41
+ for rrset in answer.response.answer:
42
+ if rrset.rdtype == dns.rdatatype.CNAME:
43
+ result.cnames += [r.target.to_text().rstrip(".").lower() for r in rrset]
44
+ result.cnames = list(dict.fromkeys(result.cnames))
45
+ return result
46
+
47
+
48
+ def cname_target(name: str, res: dns.resolver.Resolver) -> str:
49
+ """The direct CNAME target of a name, or '' when it has none."""
50
+ try:
51
+ answer = res.resolve(name, "CNAME")
52
+ except (dns.resolver.NXDOMAIN, *NOT_FOUND):
53
+ return ""
54
+ return str(answer[0].target.to_text()).rstrip(".").lower()