targetzero 0.3.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- targetzero/__init__.py +4 -0
- targetzero/__main__.py +6 -0
- targetzero/admins.py +109 -0
- targetzero/cli.py +366 -0
- targetzero/common/__init__.py +2 -0
- targetzero/common/changes.py +46 -0
- targetzero/common/config.py +83 -0
- targetzero/common/ids.py +17 -0
- targetzero/common/output.py +91 -0
- targetzero/common/resources.py +32 -0
- targetzero/common/roles.py +24 -0
- targetzero/common/schema.py +45 -0
- targetzero/config/approved_exposure.example.csv +4 -0
- targetzero/config/privileged_roles.yaml +50 -0
- targetzero/config/targetzero.example.yaml +57 -0
- targetzero/external/__init__.py +2 -0
- targetzero/external/dangling.py +106 -0
- targetzero/external/discover.py +181 -0
- targetzero/external/dnsinfo.py +54 -0
- targetzero/external/enrich.py +137 -0
- targetzero/external/http.py +33 -0
- targetzero/external/kev.py +222 -0
- targetzero/external/ports.py +63 -0
- targetzero/external/scan.py +260 -0
- targetzero/external/scope.py +54 -0
- targetzero/external/waf.py +202 -0
- targetzero/external/web.py +124 -0
- targetzero/gws/__init__.py +2 -0
- targetzero/gws/admins.py +230 -0
- targetzero/gws/changes.py +190 -0
- targetzero/gws/client.py +94 -0
- targetzero/gws/domains.py +36 -0
- targetzero/registers/__init__.py +2 -0
- targetzero/registers/exceptions.py +85 -0
- targetzero/registers/findings.py +108 -0
- targetzero/registers/inventory.py +142 -0
- targetzero/registers/manual_admins.py +43 -0
- targetzero/schemas/admins.schema.json +27 -0
- targetzero/schemas/alert_tests.schema.json +19 -0
- targetzero/schemas/config.schema.json +71 -0
- targetzero/schemas/exceptions.schema.json +20 -0
- targetzero/schemas/findings.schema.json +23 -0
- targetzero/schemas/inventory.schema.json +26 -0
- targetzero/schemas/summary.schema.json +36 -0
- targetzero/scorecard/__init__.py +2 -0
- targetzero/scorecard/scorecard.py +201 -0
- targetzero/templates/alert_tests.csv +1 -0
- targetzero/templates/exceptions.csv +1 -0
- targetzero/templates/findings.csv +1 -0
- targetzero/templates/inventory.csv +1 -0
- targetzero/templates/manual_admins.csv +1 -0
- targetzero/templates/origins.csv +1 -0
- targetzero/templates/recovery_page.md +51 -0
- targetzero/vendors/__init__.py +2 -0
- targetzero/vendors/cisco_asa.py +151 -0
- targetzero/vendors/common.py +84 -0
- targetzero/vendors/fortigate.py +176 -0
- targetzero/vendors/panos.py +141 -0
- targetzero/vendors/redfish.py +124 -0
- targetzero/vendors/vsphere.py +108 -0
- targetzero-0.3.2.dist-info/METADATA +116 -0
- targetzero-0.3.2.dist-info/RECORD +66 -0
- targetzero-0.3.2.dist-info/WHEEL +4 -0
- targetzero-0.3.2.dist-info/entry_points.txt +2 -0
- targetzero-0.3.2.dist-info/licenses/LICENSE +202 -0
- targetzero-0.3.2.dist-info/licenses/NOTICE +7 -0
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Write records (CSV) and count summaries (JSON) in the TargetZero output contract.
|
|
3
|
+
|
|
4
|
+
Layout: <output_dir>/<YYYY-MM-DD>/<zero>_<source>_<name>.csv
|
|
5
|
+
<output_dir>/<YYYY-MM-DD>/<zero>_<source>_<name>.summary.json
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import csv
|
|
11
|
+
import json
|
|
12
|
+
from collections.abc import Iterable, Mapping, Sequence
|
|
13
|
+
from datetime import UTC, date, datetime
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from targetzero import __version__
|
|
18
|
+
from targetzero.common.schema import validate_summary
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def run_dir(output_dir: Path, run_date: date) -> Path:
|
|
22
|
+
path = output_dir / run_date.isoformat()
|
|
23
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
24
|
+
return path
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def base_name(zero: int, source: str, name: str) -> str:
|
|
28
|
+
return f"{zero}_{source}_{name}"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def delimiter_of(header: str) -> str:
|
|
32
|
+
"""Excel with Belgian, Dutch, French or German settings saves CSV with semicolons."""
|
|
33
|
+
return ";" if header.count(";") > header.count(",") else ","
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def read_csv(path: Path) -> list[dict[str, str]]:
|
|
37
|
+
# utf-8-sig: files saved by Excel start with a byte order mark.
|
|
38
|
+
with path.open(newline="", encoding="utf-8-sig") as handle:
|
|
39
|
+
delimiter = delimiter_of(handle.readline())
|
|
40
|
+
handle.seek(0) # utf-8-sig skips the byte order mark again after a seek to 0
|
|
41
|
+
rows = csv.DictReader(handle, delimiter=delimiter)
|
|
42
|
+
return [{k: (v or "").strip() for k, v in row.items() if k} for row in rows]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def write_csv(path: Path, rows: Iterable[Mapping[str, Any]], fieldnames: Sequence[str]) -> Path:
|
|
46
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
47
|
+
with path.open("w", newline="", encoding="utf-8") as handle:
|
|
48
|
+
writer = csv.DictWriter(handle, fieldnames=list(fieldnames), extrasaction="raise")
|
|
49
|
+
writer.writeheader()
|
|
50
|
+
for row in rows:
|
|
51
|
+
writer.writerow({k: "" if v is None else v for k, v in row.items()})
|
|
52
|
+
return path
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def write_summary(
|
|
56
|
+
directory: Path,
|
|
57
|
+
*,
|
|
58
|
+
zero: int,
|
|
59
|
+
source: str,
|
|
60
|
+
name: str,
|
|
61
|
+
items: Sequence[Mapping[str, str]],
|
|
62
|
+
tool: str,
|
|
63
|
+
run_date: date,
|
|
64
|
+
complete: bool = True,
|
|
65
|
+
notes: Sequence[str] = (),
|
|
66
|
+
record_file: str | None = None,
|
|
67
|
+
tool_version: str = __version__,
|
|
68
|
+
) -> Path:
|
|
69
|
+
"""Validate and write a count summary. Raises ValueError when it breaks the contract."""
|
|
70
|
+
summary: dict[str, Any] = {
|
|
71
|
+
"schema_version": "1.0",
|
|
72
|
+
"zero": zero,
|
|
73
|
+
"source": source,
|
|
74
|
+
"name": name,
|
|
75
|
+
"tool": tool,
|
|
76
|
+
"tool_version": tool_version,
|
|
77
|
+
"run_date": run_date.isoformat(),
|
|
78
|
+
"generated_at": datetime.now(UTC).isoformat(timespec="seconds"),
|
|
79
|
+
"count": len(items),
|
|
80
|
+
"complete": complete,
|
|
81
|
+
"notes": list(notes),
|
|
82
|
+
"items": [{"id": i["id"], "reason": i["reason"]} for i in items],
|
|
83
|
+
}
|
|
84
|
+
if record_file:
|
|
85
|
+
summary["record_file"] = record_file
|
|
86
|
+
problems = validate_summary(summary)
|
|
87
|
+
if problems:
|
|
88
|
+
raise ValueError("Summary does not match the schema:\n " + "\n ".join(problems))
|
|
89
|
+
path = directory / f"{base_name(zero, source, name)}.summary.json"
|
|
90
|
+
path.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
|
91
|
+
return path
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Locate files that ship with TargetZero (schemas, templates, example config).
|
|
3
|
+
|
|
4
|
+
In an installed wheel they live inside the package. In a source checkout they
|
|
5
|
+
live at the repository root, so both places are tried.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
_PACKAGE_DIR = Path(__file__).resolve().parent.parent
|
|
13
|
+
_REPO_ROOT = _PACKAGE_DIR.parent.parent
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _find(packaged: str, in_repo: str) -> Path:
|
|
17
|
+
for candidate in (_PACKAGE_DIR / packaged, _REPO_ROOT / in_repo):
|
|
18
|
+
if candidate.exists():
|
|
19
|
+
return candidate
|
|
20
|
+
raise FileNotFoundError(f"TargetZero resource not found: {in_repo}")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def schema_dir() -> Path:
|
|
24
|
+
return _find("schemas", "schema")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def templates_dir() -> Path:
|
|
28
|
+
return _find("templates", "templates")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def example_config_dir() -> Path:
|
|
32
|
+
return _find("config", "config")
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Load privileged_roles.yaml: which roles count as full admin."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import yaml
|
|
9
|
+
|
|
10
|
+
from targetzero.common.config import Config
|
|
11
|
+
from targetzero.common.resources import example_config_dir
|
|
12
|
+
|
|
13
|
+
FILE_NAME = "privileged_roles.yaml"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def roles_file(config: Config) -> Path:
|
|
17
|
+
"""Prefer the copy next to targetzero.yaml, so users can change it."""
|
|
18
|
+
local = config.path.parent / FILE_NAME
|
|
19
|
+
return local if local.exists() else example_config_dir() / FILE_NAME
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def privileged_roles(config: Config, key: str) -> set[str]:
|
|
23
|
+
data = yaml.safe_load(roles_file(config).read_text(encoding="utf-8")) or {}
|
|
24
|
+
return {str(r).casefold() for r in data.get(key, [])}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Validate data against the JSON schemas in schema/."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
from functools import cache
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from jsonschema import Draft202012Validator
|
|
11
|
+
|
|
12
|
+
from targetzero.common.resources import schema_dir
|
|
13
|
+
|
|
14
|
+
SCHEMAS = ("summary", "admins", "inventory", "findings", "exceptions", "alert_tests", "config")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@cache
|
|
18
|
+
def _validator(name: str) -> Draft202012Validator:
|
|
19
|
+
if name not in SCHEMAS:
|
|
20
|
+
raise ValueError(f"Unknown schema '{name}'. Known: {', '.join(SCHEMAS)}")
|
|
21
|
+
path = schema_dir() / f"{name}.schema.json"
|
|
22
|
+
return Draft202012Validator(json.loads(path.read_text(encoding="utf-8")))
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def validate(instance: Any, name: str) -> list[str]:
|
|
26
|
+
"""Return a readable list of problems; empty when the instance is valid."""
|
|
27
|
+
problems = []
|
|
28
|
+
for error in sorted(_validator(name).iter_errors(instance), key=lambda e: list(e.path)):
|
|
29
|
+
location = "/".join(str(part) for part in error.path) or "(root)"
|
|
30
|
+
problems.append(f"{location}: {error.message}")
|
|
31
|
+
return problems
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def validate_summary(summary: Any) -> list[str]:
|
|
35
|
+
"""Schema check plus the rules a schema cannot express."""
|
|
36
|
+
problems = validate(summary, "summary")
|
|
37
|
+
if problems or not isinstance(summary, dict):
|
|
38
|
+
return problems
|
|
39
|
+
ids = [item["id"] for item in summary["items"]]
|
|
40
|
+
if summary["count"] != len(ids):
|
|
41
|
+
problems.append(f"count is {summary['count']} but there are {len(ids)} items")
|
|
42
|
+
duplicates = sorted({i for i in ids if ids.count(i) > 1})
|
|
43
|
+
if duplicates:
|
|
44
|
+
problems.append(f"duplicate item ids: {', '.join(duplicates)}")
|
|
45
|
+
return problems
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
host,port,protocol,justification,owner,approved_by,review_date
|
|
2
|
+
www.example.com,443,tcp,Public website behind CDN,Jane Doe,John Smith (CEO),2027-02-19
|
|
3
|
+
vpn.example.com,443,tcp,SSL VPN for remote staff (MFA enforced),Jane Doe,John Smith (CEO),2027-02-19
|
|
4
|
+
mail.example.com,25,tcp,Inbound mail relay,Jane Doe,John Smith (CEO),2027-02-19
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Roles that count as "full admin" for Zero 1 and Zero 2.
|
|
2
|
+
# Roles not listed here are still reported, with privileged = N.
|
|
3
|
+
# Change this list to match your own risk view; keep the names exactly as the
|
|
4
|
+
# platform shows them.
|
|
5
|
+
|
|
6
|
+
m365:
|
|
7
|
+
# Entra ID built-in roles (display names)
|
|
8
|
+
- Global Administrator
|
|
9
|
+
- Privileged Role Administrator
|
|
10
|
+
- Privileged Authentication Administrator
|
|
11
|
+
- Security Administrator
|
|
12
|
+
- Conditional Access Administrator
|
|
13
|
+
- Exchange Administrator
|
|
14
|
+
- SharePoint Administrator
|
|
15
|
+
- Teams Administrator
|
|
16
|
+
- User Administrator
|
|
17
|
+
- Authentication Administrator
|
|
18
|
+
- Application Administrator
|
|
19
|
+
- Cloud Application Administrator
|
|
20
|
+
- Intune Administrator
|
|
21
|
+
- Hybrid Identity Administrator
|
|
22
|
+
- Domain Name Administrator
|
|
23
|
+
- Partner Tier2 Support
|
|
24
|
+
- Directory Synchronization Accounts
|
|
25
|
+
|
|
26
|
+
# Microsoft Graph application permissions that give an app admin-level power.
|
|
27
|
+
# Apps holding one of these are reported as hidden admins (account_type = service).
|
|
28
|
+
m365_app_permissions:
|
|
29
|
+
- RoleManagement.ReadWrite.Directory
|
|
30
|
+
- AppRoleAssignment.ReadWrite.All
|
|
31
|
+
- Application.ReadWrite.All
|
|
32
|
+
- Directory.ReadWrite.All
|
|
33
|
+
- User.ReadWrite.All
|
|
34
|
+
- Group.ReadWrite.All
|
|
35
|
+
- UserAuthenticationMethod.ReadWrite.All
|
|
36
|
+
- Policy.ReadWrite.ConditionalAccess
|
|
37
|
+
- Domain.ReadWrite.All
|
|
38
|
+
- Mail.ReadWrite
|
|
39
|
+
- Mail.Send
|
|
40
|
+
- Files.ReadWrite.All
|
|
41
|
+
- Sites.FullControl.All
|
|
42
|
+
- Sites.ReadWrite.All
|
|
43
|
+
|
|
44
|
+
gws:
|
|
45
|
+
# Google Workspace system roles
|
|
46
|
+
- Super Admin
|
|
47
|
+
- Groups Admin
|
|
48
|
+
- User Management Admin
|
|
49
|
+
- Services Admin
|
|
50
|
+
- Mobile Admin
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# TargetZero configuration.
|
|
2
|
+
# Copy this file to targetzero.yaml (which git ignores) and fill it in.
|
|
3
|
+
# Relative paths are resolved against the folder that holds this file.
|
|
4
|
+
|
|
5
|
+
organisation: Example Ltd
|
|
6
|
+
|
|
7
|
+
# Where scripts write their results (output/<YYYY-MM-DD>/...) and where the
|
|
8
|
+
# hand-maintained registers live (inventory, findings, exceptions, ...).
|
|
9
|
+
output_dir: output
|
|
10
|
+
registers_dir: registers
|
|
11
|
+
|
|
12
|
+
# One owner per Target 0 count. Shown on the scorecard.
|
|
13
|
+
owners:
|
|
14
|
+
"1": "Jane Doe, IT lead"
|
|
15
|
+
"2": "Jane Doe, IT lead"
|
|
16
|
+
"3": "Managed SOC / IT provider"
|
|
17
|
+
"4": "Jane Doe, IT lead"
|
|
18
|
+
"5": "Jane Doe, IT lead"
|
|
19
|
+
"6": "Jane Doe, IT lead"
|
|
20
|
+
"7": "Application owners"
|
|
21
|
+
|
|
22
|
+
# Emergency ("break-glass") accounts. Never changed by any script and left
|
|
23
|
+
# out of the Zero 1 count. Keep exactly two per platform.
|
|
24
|
+
break_glass:
|
|
25
|
+
m365:
|
|
26
|
+
- bg-admin-01@example.onmicrosoft.com
|
|
27
|
+
- bg-admin-02@example.onmicrosoft.com
|
|
28
|
+
gws:
|
|
29
|
+
- bg-admin-01@example.com
|
|
30
|
+
- bg-admin-02@example.com
|
|
31
|
+
|
|
32
|
+
# What you own and are authorised to scan. External checks never go outside
|
|
33
|
+
# this list, not even for hosts they discover themselves.
|
|
34
|
+
scope:
|
|
35
|
+
domains:
|
|
36
|
+
- example.com
|
|
37
|
+
ip_ranges:
|
|
38
|
+
- 203.0.113.0/28 # documentation range, replace with your own
|
|
39
|
+
|
|
40
|
+
thresholds:
|
|
41
|
+
exception_max_days: 90 # Target 0: at most 90 days
|
|
42
|
+
inventory_review_days: 90 # inventory rows older than this count as outdated
|
|
43
|
+
alert_test_max_age_days: 35 # monthly test, with a few days of slack
|
|
44
|
+
deadline_hours:
|
|
45
|
+
kev: 8
|
|
46
|
+
critical: 24
|
|
47
|
+
high: 72
|
|
48
|
+
|
|
49
|
+
m365:
|
|
50
|
+
tenant_id: "" # e.g. example.onmicrosoft.com or the tenant GUID
|
|
51
|
+
client_id: "" # only for unattended runs
|
|
52
|
+
certificate_thumbprint: ""
|
|
53
|
+
|
|
54
|
+
gws:
|
|
55
|
+
customer_id: my_customer
|
|
56
|
+
admin_subject: "" # admin account to impersonate
|
|
57
|
+
credentials_file: "" # keep outside the repository
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Dangling DNS (Zero 5): names that point to things you no longer use.
|
|
3
|
+
|
|
4
|
+
A name counts when its CNAME target does not exist, or when the service it points to says
|
|
5
|
+
the site is gone (an unclaimed bucket, app or page that someone else could take over).
|
|
6
|
+
A records pointing to released cloud addresses cannot be detected this way.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from datetime import date
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from targetzero.common.config import Config
|
|
15
|
+
from targetzero.common.ids import make_id
|
|
16
|
+
from targetzero.common.output import read_csv, run_dir, write_csv, write_summary
|
|
17
|
+
from targetzero.external import dnsinfo, web
|
|
18
|
+
from targetzero.external.scope import Scope
|
|
19
|
+
from targetzero.registers import inventory
|
|
20
|
+
|
|
21
|
+
FIELDS = ("name", "cname", "target_exists", "service", "evidence", "result")
|
|
22
|
+
|
|
23
|
+
# (service, CNAME suffixes, text in the response that shows the resource is unclaimed)
|
|
24
|
+
FINGERPRINTS = (
|
|
25
|
+
("Amazon S3", ("s3.amazonaws.com", "amazonaws.com"), "NoSuchBucket"),
|
|
26
|
+
("GitHub Pages", ("github.io",), "There isn't a GitHub Pages site here"),
|
|
27
|
+
("Heroku", ("herokuapp.com", "herokudns.com"), "no-such-app"),
|
|
28
|
+
("Azure App Service", ("azurewebsites.net",), "404 Web Site not found"),
|
|
29
|
+
("Shopify", ("myshopify.com",), "Sorry, this shop is currently unavailable"),
|
|
30
|
+
("Zendesk", ("zendesk.com",), "Help Center Closed"),
|
|
31
|
+
("Fastly", ("fastly.net",), "Fastly error: unknown domain"),
|
|
32
|
+
("Surge", ("surge.sh",), "project not found"),
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def candidate_names(config: Config, scope: Scope, run_date: date) -> list[str]:
|
|
37
|
+
names = {
|
|
38
|
+
r["asset"].lower()
|
|
39
|
+
for r in inventory.load(config)
|
|
40
|
+
if r.get("type") in ("domain", "subdomain")
|
|
41
|
+
}
|
|
42
|
+
discovery = config.output_dir / run_date.isoformat() / "5_external_discovery.csv"
|
|
43
|
+
if discovery.exists():
|
|
44
|
+
names |= {r["name"].lower() for r in read_csv(discovery)}
|
|
45
|
+
return sorted(n for n in names if scope.contains_name(n))
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def check_name(name: str, res: object) -> dict[str, str] | None:
|
|
49
|
+
target = dnsinfo.cname_target(name, res) # type: ignore[arg-type]
|
|
50
|
+
if not target:
|
|
51
|
+
return None
|
|
52
|
+
record = {
|
|
53
|
+
"name": name,
|
|
54
|
+
"cname": target,
|
|
55
|
+
"target_exists": "",
|
|
56
|
+
"service": "",
|
|
57
|
+
"evidence": "",
|
|
58
|
+
"result": "",
|
|
59
|
+
}
|
|
60
|
+
found = dnsinfo.lookup(target, res) # type: ignore[arg-type]
|
|
61
|
+
record["target_exists"] = "N" if found.nxdomain or not found.ips else "Y"
|
|
62
|
+
if record["target_exists"] == "N":
|
|
63
|
+
record["result"] = "dangling"
|
|
64
|
+
record["evidence"] = "CNAME target does not resolve"
|
|
65
|
+
return record
|
|
66
|
+
for service, suffixes, text in FINGERPRINTS:
|
|
67
|
+
if any(target == s or target.endswith("." + s) for s in suffixes):
|
|
68
|
+
record["service"] = service
|
|
69
|
+
for use_tls, port in ((True, 443), (False, 80)):
|
|
70
|
+
response = web.get(name, port=port, use_tls=use_tls, timeout=8)
|
|
71
|
+
if text in response.body:
|
|
72
|
+
record["result"] = "dangling"
|
|
73
|
+
record["evidence"] = f"{service} answers '{text}'"
|
|
74
|
+
return record
|
|
75
|
+
record["result"] = "ok"
|
|
76
|
+
return record
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def run(config: Config, run_date: date) -> list[Path]:
|
|
80
|
+
scope = Scope.from_config(config)
|
|
81
|
+
res = dnsinfo.resolver()
|
|
82
|
+
records = [r for n in candidate_names(config, scope, run_date) if (r := check_name(n, res))]
|
|
83
|
+
items = [
|
|
84
|
+
{
|
|
85
|
+
"id": make_id("external", r["name"], "dangling"),
|
|
86
|
+
"reason": f"{r['name']} -> {r['cname']}: {r['evidence']}; delete the record",
|
|
87
|
+
}
|
|
88
|
+
for r in records
|
|
89
|
+
if r["result"] == "dangling"
|
|
90
|
+
]
|
|
91
|
+
directory = run_dir(config.output_dir, run_date)
|
|
92
|
+
record_file = "5_external_dangling-dns.csv"
|
|
93
|
+
return [
|
|
94
|
+
write_csv(directory / record_file, records, FIELDS),
|
|
95
|
+
write_summary(
|
|
96
|
+
directory,
|
|
97
|
+
zero=5,
|
|
98
|
+
source="external",
|
|
99
|
+
name="dangling-dns",
|
|
100
|
+
items=items,
|
|
101
|
+
tool="targetzero external dangling-dns",
|
|
102
|
+
run_date=run_date,
|
|
103
|
+
notes=["Only CNAME records are checked"],
|
|
104
|
+
record_file=record_file,
|
|
105
|
+
),
|
|
106
|
+
]
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Asset discovery (Zero 5): find your public names and addresses and merge them into
|
|
3
|
+
registers/inventory.csv.
|
|
4
|
+
|
|
5
|
+
Sources: scope.domains, domain lists written today by the M365 and Google
|
|
6
|
+
Workspace scripts (5_*_domains.csv), DNS zone files, and certificate
|
|
7
|
+
transparency logs (crt.sh). Names that no longer resolve are listed but not added.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import time
|
|
13
|
+
import urllib.error
|
|
14
|
+
import urllib.parse
|
|
15
|
+
from collections.abc import Iterable
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from datetime import date
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
import dns.exception
|
|
21
|
+
import dns.rdatatype
|
|
22
|
+
import dns.resolver
|
|
23
|
+
import dns.zone
|
|
24
|
+
|
|
25
|
+
from targetzero.common.config import Config
|
|
26
|
+
from targetzero.common.ids import make_id
|
|
27
|
+
from targetzero.common.output import read_csv, run_dir, write_csv
|
|
28
|
+
from targetzero.external.http import get_json
|
|
29
|
+
from targetzero.external.scope import Scope
|
|
30
|
+
from targetzero.registers import inventory
|
|
31
|
+
|
|
32
|
+
FIELDS = ("name", "resolves", "ips", "cname", "in_scope", "sources")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass
|
|
36
|
+
class Name:
|
|
37
|
+
name: str
|
|
38
|
+
sources: set[str] = field(default_factory=set)
|
|
39
|
+
ips: set[str] = field(default_factory=set)
|
|
40
|
+
cname: str = ""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def crtsh_names(domain: str, timeout: float = 90) -> set[str]:
|
|
44
|
+
query = urllib.parse.quote(f"%.{domain}")
|
|
45
|
+
url = f"https://crt.sh/?q={query}&output=json"
|
|
46
|
+
try:
|
|
47
|
+
entries = get_json(url, timeout=timeout)
|
|
48
|
+
except urllib.error.HTTPError as exc:
|
|
49
|
+
if exc.code < 500:
|
|
50
|
+
raise
|
|
51
|
+
time.sleep(10) # crt.sh often answers 502/503 under load; one retry
|
|
52
|
+
entries = get_json(url, timeout=timeout)
|
|
53
|
+
names = set()
|
|
54
|
+
for entry in entries:
|
|
55
|
+
for value in str(entry.get("name_value", "")).splitlines():
|
|
56
|
+
value = value.strip().lower().removeprefix("*.")
|
|
57
|
+
if value and " " not in value and "@" not in value:
|
|
58
|
+
names.add(value)
|
|
59
|
+
return names
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def zone_names(path: Path, origin: str | None) -> set[str]:
|
|
63
|
+
zone = dns.zone.from_file(str(path), origin=origin, relativize=False, check_origin=False)
|
|
64
|
+
wanted = {dns.rdatatype.A, dns.rdatatype.AAAA, dns.rdatatype.CNAME}
|
|
65
|
+
return {
|
|
66
|
+
str(name).rstrip(".").lower()
|
|
67
|
+
for name, node in zone.nodes.items()
|
|
68
|
+
if any(rdataset.rdtype in wanted for rdataset in node.rdatasets)
|
|
69
|
+
and not str(name).startswith("*")
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def platform_domains(directory: Path) -> dict[str, str]:
|
|
74
|
+
"""Domains written today by Get-T0M365Domain or `targetzero gws domains`."""
|
|
75
|
+
found = {}
|
|
76
|
+
for path in sorted(directory.glob("5_*_domains.csv")):
|
|
77
|
+
source = path.stem.split("_")[1]
|
|
78
|
+
for row in read_csv(path):
|
|
79
|
+
if row.get("domain"):
|
|
80
|
+
found[row["domain"].lower()] = source
|
|
81
|
+
return found
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def resolve(name: Name, resolver: dns.resolver.Resolver) -> None:
|
|
85
|
+
for rdtype in ("A", "AAAA"):
|
|
86
|
+
try:
|
|
87
|
+
answer = resolver.resolve(name.name, rdtype)
|
|
88
|
+
except (
|
|
89
|
+
dns.resolver.NXDOMAIN,
|
|
90
|
+
dns.resolver.NoAnswer,
|
|
91
|
+
dns.resolver.NoNameservers,
|
|
92
|
+
dns.exception.Timeout,
|
|
93
|
+
):
|
|
94
|
+
continue
|
|
95
|
+
name.ips |= {r.to_text() for r in answer}
|
|
96
|
+
if answer.canonical_name.to_text().rstrip(".").lower() != name.name:
|
|
97
|
+
name.cname = answer.canonical_name.to_text().rstrip(".").lower()
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def discovered_rows(names: Iterable[Name], apexes: set[str], scope: Scope) -> list[dict[str, str]]:
|
|
101
|
+
rows: list[dict[str, str]] = []
|
|
102
|
+
ips: set[str] = set()
|
|
103
|
+
for n in names:
|
|
104
|
+
if not n.ips:
|
|
105
|
+
continue
|
|
106
|
+
rows.append(
|
|
107
|
+
{
|
|
108
|
+
"id": make_id("inventory", n.name),
|
|
109
|
+
"asset": n.name,
|
|
110
|
+
"type": "domain" if n.name in apexes else "subdomain",
|
|
111
|
+
"ip": ";".join(sorted(n.ips)),
|
|
112
|
+
}
|
|
113
|
+
)
|
|
114
|
+
ips |= {ip for ip in n.ips if scope.contains_ip(ip)}
|
|
115
|
+
rows += [
|
|
116
|
+
{"id": make_id("inventory", ip), "asset": ip, "type": "ip", "ip": ip} for ip in sorted(ips)
|
|
117
|
+
]
|
|
118
|
+
return rows
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def run(
|
|
122
|
+
config: Config,
|
|
123
|
+
run_date: date,
|
|
124
|
+
*,
|
|
125
|
+
use_crtsh: bool = True,
|
|
126
|
+
zone_files: Iterable[tuple[Path, str | None]] = (),
|
|
127
|
+
) -> list[Path]:
|
|
128
|
+
scope = Scope.from_config(config)
|
|
129
|
+
directory = run_dir(config.output_dir, run_date)
|
|
130
|
+
notes: list[str] = []
|
|
131
|
+
complete = True
|
|
132
|
+
names: dict[str, Name] = {}
|
|
133
|
+
|
|
134
|
+
def add(name: str, source: str) -> None:
|
|
135
|
+
name = name.lower().strip(".")
|
|
136
|
+
names.setdefault(name, Name(name)).sources.add(source)
|
|
137
|
+
|
|
138
|
+
for domain in scope.domains:
|
|
139
|
+
add(domain, "config")
|
|
140
|
+
for domain, source in platform_domains(directory).items():
|
|
141
|
+
add(domain, source)
|
|
142
|
+
if not scope.contains_name(domain):
|
|
143
|
+
notes.append(f"{domain} ({source}) is not in scope.domains: add it if you own it")
|
|
144
|
+
for path, origin in zone_files:
|
|
145
|
+
for name in zone_names(path, origin):
|
|
146
|
+
add(name, "zone")
|
|
147
|
+
apexes = set(names)
|
|
148
|
+
if use_crtsh:
|
|
149
|
+
for domain in sorted(apexes):
|
|
150
|
+
try:
|
|
151
|
+
for name in crtsh_names(domain):
|
|
152
|
+
if name == domain or name.endswith("." + domain):
|
|
153
|
+
add(name, "crt.sh")
|
|
154
|
+
except (urllib.error.URLError, TimeoutError, ValueError) as exc:
|
|
155
|
+
notes.append(f"crt.sh failed for {domain}: {exc}")
|
|
156
|
+
complete = False
|
|
157
|
+
|
|
158
|
+
resolver = dns.resolver.Resolver()
|
|
159
|
+
resolver.lifetime = 5
|
|
160
|
+
for entry in names.values():
|
|
161
|
+
resolve(entry, resolver)
|
|
162
|
+
|
|
163
|
+
rows, added, updated = inventory.merge(
|
|
164
|
+
inventory.load(config), discovered_rows(names.values(), apexes, scope)
|
|
165
|
+
)
|
|
166
|
+
paths = [inventory.save(config, rows)]
|
|
167
|
+
notes.append(f"Inventory: {added} new row(s) marked unreviewed, {updated} address update(s)")
|
|
168
|
+
records = [
|
|
169
|
+
{
|
|
170
|
+
"name": n.name,
|
|
171
|
+
"resolves": "Y" if n.ips else "N",
|
|
172
|
+
"ips": ";".join(sorted(n.ips)),
|
|
173
|
+
"cname": n.cname,
|
|
174
|
+
"in_scope": "Y" if scope.contains_name(n.name) else "N",
|
|
175
|
+
"sources": ";".join(sorted(n.sources)),
|
|
176
|
+
}
|
|
177
|
+
for n in sorted(names.values(), key=lambda n: n.name)
|
|
178
|
+
]
|
|
179
|
+
paths.append(write_csv(directory / "5_external_discovery.csv", records, FIELDS))
|
|
180
|
+
paths += inventory.count(config, run_date, notes=notes, complete=complete)
|
|
181
|
+
return paths
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""DNS lookups shared by the external checks: addresses plus the CNAME chain."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
|
|
8
|
+
import dns.exception
|
|
9
|
+
import dns.rdatatype
|
|
10
|
+
import dns.resolver
|
|
11
|
+
|
|
12
|
+
NOT_FOUND = (dns.resolver.NoAnswer, dns.resolver.NoNameservers, dns.exception.Timeout)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass
|
|
16
|
+
class Lookup:
|
|
17
|
+
name: str
|
|
18
|
+
ips: list[str] = field(default_factory=list)
|
|
19
|
+
cnames: list[str] = field(default_factory=list)
|
|
20
|
+
nxdomain: bool = False
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def resolver(timeout: float = 5) -> dns.resolver.Resolver:
|
|
24
|
+
r = dns.resolver.Resolver()
|
|
25
|
+
r.lifetime = timeout
|
|
26
|
+
return r
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def lookup(name: str, res: dns.resolver.Resolver) -> Lookup:
|
|
30
|
+
result = Lookup(name.lower().rstrip("."))
|
|
31
|
+
for rdtype in ("A", "AAAA"):
|
|
32
|
+
try:
|
|
33
|
+
answer = res.resolve(result.name, rdtype)
|
|
34
|
+
except dns.resolver.NXDOMAIN:
|
|
35
|
+
result.nxdomain = True
|
|
36
|
+
break
|
|
37
|
+
except NOT_FOUND:
|
|
38
|
+
continue
|
|
39
|
+
result.ips += [r.to_text() for r in answer]
|
|
40
|
+
if answer.response is not None:
|
|
41
|
+
for rrset in answer.response.answer:
|
|
42
|
+
if rrset.rdtype == dns.rdatatype.CNAME:
|
|
43
|
+
result.cnames += [r.target.to_text().rstrip(".").lower() for r in rrset]
|
|
44
|
+
result.cnames = list(dict.fromkeys(result.cnames))
|
|
45
|
+
return result
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def cname_target(name: str, res: dns.resolver.Resolver) -> str:
|
|
49
|
+
"""The direct CNAME target of a name, or '' when it has none."""
|
|
50
|
+
try:
|
|
51
|
+
answer = res.resolve(name, "CNAME")
|
|
52
|
+
except (dns.resolver.NXDOMAIN, *NOT_FOUND):
|
|
53
|
+
return ""
|
|
54
|
+
return str(answer[0].target.to_text()).rstrip(".").lower()
|