rimscan-cli 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rimscan_cli/__init__.py +3 -0
- rimscan_cli/__main__.py +4 -0
- rimscan_cli/banner.py +48 -0
- rimscan_cli/cli.py +158 -0
- rimscan_cli/data/fingerprints.json +932 -0
- rimscan_cli/recon/__init__.py +0 -0
- rimscan_cli/recon/_common.py +62 -0
- rimscan_cli/recon/_http.py +65 -0
- rimscan_cli/recon/cert_audit.py +131 -0
- rimscan_cli/recon/dns_audit.py +196 -0
- rimscan_cli/recon/enumeration.py +192 -0
- rimscan_cli/recon/resolver.py +232 -0
- rimscan_cli/recon/secret_scan.py +241 -0
- rimscan_cli/recon/severity.py +15 -0
- rimscan_cli/recon/takeover.py +302 -0
- rimscan_cli/scanner.py +305 -0
- rimscan_cli-1.0.0.dist-info/METADATA +250 -0
- rimscan_cli-1.0.0.dist-info/RECORD +23 -0
- rimscan_cli-1.0.0.dist-info/WHEEL +5 -0
- rimscan_cli-1.0.0.dist-info/entry_points.txt +2 -0
- rimscan_cli-1.0.0.dist-info/licenses/LICENSE +202 -0
- rimscan_cli-1.0.0.dist-info/licenses/NOTICE +8 -0
- rimscan_cli-1.0.0.dist-info/top_level.txt +1 -0
|
File without changes
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Small helpers shared by the recon modules, the scanner and the CLI."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ipaddress
|
|
6
|
+
import re
|
|
7
|
+
from typing import Iterable
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"HOST_LABEL",
|
|
11
|
+
"all_public",
|
|
12
|
+
"is_public_address",
|
|
13
|
+
"is_valid_hostname",
|
|
14
|
+
"normalize_domain",
|
|
15
|
+
"normalize_name",
|
|
16
|
+
]
|
|
17
|
+
|
|
18
|
+
HOST_LABEL = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def normalize_name(name: str) -> str:
|
|
22
|
+
"""Lowercase a DNS name and strip whitespace and leading/trailing dots."""
|
|
23
|
+
return name.strip().lower().strip(".")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def is_valid_hostname(name: str) -> bool:
|
|
27
|
+
"""True if every label is a valid LDH label and the whole name fits in 253 characters."""
|
|
28
|
+
return len(name) <= 253 and all(HOST_LABEL.fullmatch(label) for label in name.split("."))
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def normalize_domain(value: str) -> str:
|
|
32
|
+
"""Return a lowercase ASCII hostname, or raise ValueError if `value` is not a domain name."""
|
|
33
|
+
domain = value.strip().lower().rstrip(".")
|
|
34
|
+
if not domain or any(char in domain for char in "/:@ ?#"):
|
|
35
|
+
raise ValueError("domain must be a bare hostname such as example.com (no scheme, path, port or user)")
|
|
36
|
+
try:
|
|
37
|
+
ipaddress.ip_address(domain)
|
|
38
|
+
except ValueError:
|
|
39
|
+
pass
|
|
40
|
+
else:
|
|
41
|
+
raise ValueError("IP addresses are not supported; pass a domain name")
|
|
42
|
+
try:
|
|
43
|
+
domain = domain.encode("idna").decode("ascii")
|
|
44
|
+
except UnicodeError:
|
|
45
|
+
raise ValueError(f"{value!r} is not a valid domain name") from None
|
|
46
|
+
if len(domain.split(".")) < 2 or not is_valid_hostname(domain):
|
|
47
|
+
raise ValueError(f"{value!r} is not a valid domain name")
|
|
48
|
+
return domain
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def is_public_address(value: str) -> bool:
|
|
52
|
+
"""True for a globally routable IP address (not loopback, private, link-local, ...)."""
|
|
53
|
+
try:
|
|
54
|
+
return ipaddress.ip_address(value).is_global
|
|
55
|
+
except ValueError:
|
|
56
|
+
return False
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def all_public(addresses: Iterable[str]) -> bool:
|
|
60
|
+
"""True if there is at least one address and every one of them is globally routable."""
|
|
61
|
+
addresses = list(addresses)
|
|
62
|
+
return bool(addresses) and all(is_public_address(address) for address in addresses)
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Shared HTTP session: one honest User-Agent and one retry policy."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
|
|
7
|
+
import requests
|
|
8
|
+
from requests.adapters import HTTPAdapter
|
|
9
|
+
from urllib3.util.retry import Retry
|
|
10
|
+
|
|
11
|
+
from rimscan_cli import __version__
|
|
12
|
+
|
|
13
|
+
USER_AGENT = f"rimscan-cli/{__version__} (+https://rimscan.cloud)"
|
|
14
|
+
|
|
15
|
+
_CHARSET = re.compile(r"charset=\s*[\"']?([\w.:-]+)", re.IGNORECASE)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def make_session(retries: int = 2) -> requests.Session:
|
|
19
|
+
"""Build a session. Pass ``retries=0`` when each request must be sent at most once.
|
|
20
|
+
|
|
21
|
+
A server-supplied ``Retry-After`` header is ignored: urllib3 would sleep for as long as
|
|
22
|
+
the server asks (the request timeout does not cover that sleep), so a scanned host could
|
|
23
|
+
stall a worker thread for hours. Retries use the short exponential backoff instead.
|
|
24
|
+
"""
|
|
25
|
+
retry = Retry(
|
|
26
|
+
total=retries,
|
|
27
|
+
backoff_factor=1.0,
|
|
28
|
+
status_forcelist=(429, 500, 502, 503, 504),
|
|
29
|
+
allowed_methods=("GET", "HEAD"),
|
|
30
|
+
raise_on_status=False,
|
|
31
|
+
respect_retry_after_header=False,
|
|
32
|
+
)
|
|
33
|
+
session = requests.Session()
|
|
34
|
+
session.headers["User-Agent"] = USER_AGENT
|
|
35
|
+
session.max_redirects = 5
|
|
36
|
+
adapter = HTTPAdapter(max_retries=retry, pool_maxsize=20)
|
|
37
|
+
session.mount("https://", adapter)
|
|
38
|
+
session.mount("http://", adapter)
|
|
39
|
+
return session
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _declared_charset(response: requests.Response) -> str:
|
|
43
|
+
"""Charset from the Content-Type header, or UTF-8.
|
|
44
|
+
|
|
45
|
+
``response.encoding`` is not used: for ``text/*`` without a charset, requests guesses
|
|
46
|
+
ISO-8859-1, which garbles the non-Latin fingerprints and secrets we match against.
|
|
47
|
+
"""
|
|
48
|
+
headers = getattr(response, "headers", None) or {}
|
|
49
|
+
match = _CHARSET.search(headers.get("Content-Type", ""))
|
|
50
|
+
return match.group(1).strip("\"'") if match else "utf-8"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def read_capped(response: requests.Response, limit: int) -> str:
|
|
54
|
+
"""Read at most `limit` bytes of a streamed response and decode them."""
|
|
55
|
+
body = bytearray()
|
|
56
|
+
for chunk in response.iter_content(chunk_size=16 * 1024):
|
|
57
|
+
body.extend(chunk)
|
|
58
|
+
if len(body) >= limit:
|
|
59
|
+
break
|
|
60
|
+
response.close()
|
|
61
|
+
data = bytes(body[:limit])
|
|
62
|
+
try:
|
|
63
|
+
return data.decode(_declared_charset(response), errors="replace")
|
|
64
|
+
except LookupError:
|
|
65
|
+
return data.decode("utf-8", errors="replace")
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""TLS certificate checks: validity, expiry and protocol support."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import functools
|
|
6
|
+
import logging
|
|
7
|
+
import socket
|
|
8
|
+
import ssl
|
|
9
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from datetime import datetime, timezone
|
|
12
|
+
from typing import List, Optional, Sequence
|
|
13
|
+
|
|
14
|
+
import certifi
|
|
15
|
+
|
|
16
|
+
from rimscan_cli.recon._common import normalize_name
|
|
17
|
+
from rimscan_cli.recon.severity import Severity
|
|
18
|
+
|
|
19
|
+
__all__ = ["CertAuditFinding", "CertAuditor", "CertSeverity", "MAX_HOSTS"]
|
|
20
|
+
|
|
21
|
+
# Kept so existing imports keep working; there is a single Severity enum.
|
|
22
|
+
CertSeverity = Severity
|
|
23
|
+
|
|
24
|
+
logger = logging.getLogger(__name__)
|
|
25
|
+
|
|
26
|
+
MAX_HOSTS = 100
|
|
27
|
+
|
|
28
|
+
# OpenSSL X509_V_ERR_* codes reported by ssl.SSLCertVerificationError.verify_code.
|
|
29
|
+
_VERIFY_FINDINGS = {
|
|
30
|
+
10: ("expired", Severity.CRITICAL, "Certificate has expired"),
|
|
31
|
+
9: ("not_yet_valid", Severity.HIGH, "Certificate is not yet valid"),
|
|
32
|
+
18: ("self_signed", Severity.HIGH, "Certificate is self-signed"),
|
|
33
|
+
19: ("self_signed", Severity.HIGH, "Certificate chain ends in an untrusted self-signed root"),
|
|
34
|
+
20: ("untrusted_chain", Severity.HIGH, "Certificate issuer is not trusted (missing intermediate or unknown CA)"),
|
|
35
|
+
21: ("untrusted_chain", Severity.HIGH, "Certificate chain is incomplete"),
|
|
36
|
+
62: ("hostname_mismatch", Severity.HIGH, "Certificate does not match the hostname"),
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
_PROTOCOL_REASONS = ("UNSUPPORTED_PROTOCOL", "NO_PROTOCOLS_AVAILABLE", "PROTOCOL_VERSION")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass
|
|
43
|
+
class CertAuditFinding:
|
|
44
|
+
check: str
|
|
45
|
+
severity: Severity
|
|
46
|
+
hostname: str
|
|
47
|
+
detail: str
|
|
48
|
+
expires: Optional[str] = None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@functools.lru_cache(maxsize=1)
|
|
52
|
+
def _context() -> ssl.SSLContext:
|
|
53
|
+
"""A client context whose results do not depend on the Python version or the OS.
|
|
54
|
+
|
|
55
|
+
- the CA bundle is certifi's, so a slim image without ca-certificates does not
|
|
56
|
+
report every host as untrusted;
|
|
57
|
+
- TLS 1.2 is the floor on every Python version, so the "protocol" check is stable;
|
|
58
|
+
- strict X.509 checking (default since Python 3.13) is off, so the same certificate
|
|
59
|
+
gives the same verdict on 3.10 and 3.13.
|
|
60
|
+
|
|
61
|
+
The context is built once (loading the CA bundle is slow) and shared by all handshakes.
|
|
62
|
+
"""
|
|
63
|
+
context = ssl.create_default_context(cafile=certifi.where())
|
|
64
|
+
context.minimum_version = ssl.TLSVersion.TLSv1_2
|
|
65
|
+
context.verify_flags &= ~ssl.VERIFY_X509_STRICT
|
|
66
|
+
return context
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class CertAuditor:
|
|
70
|
+
def __init__(self, timeout: float = 5.0, max_workers: int = 10):
|
|
71
|
+
self.timeout = timeout
|
|
72
|
+
self.max_workers = max_workers
|
|
73
|
+
|
|
74
|
+
def _handshake(self, hostname: str, port: int) -> dict:
|
|
75
|
+
context = _context()
|
|
76
|
+
with socket.create_connection((hostname, port), timeout=self.timeout) as sock:
|
|
77
|
+
with context.wrap_socket(sock, server_hostname=hostname) as tls:
|
|
78
|
+
return tls.getpeercert() or {}
|
|
79
|
+
|
|
80
|
+
def audit_host(self, hostname: str, port: int = 443) -> List[CertAuditFinding]:
|
|
81
|
+
hostname = normalize_name(hostname)
|
|
82
|
+
try:
|
|
83
|
+
cert = self._handshake(hostname, port)
|
|
84
|
+
except ssl.SSLCertVerificationError as exc:
|
|
85
|
+
check, severity, detail = _VERIFY_FINDINGS.get(
|
|
86
|
+
exc.verify_code, ("invalid", Severity.HIGH, f"Certificate failed verification: {exc.verify_message}")
|
|
87
|
+
)
|
|
88
|
+
return [CertAuditFinding(check, severity, hostname, detail)]
|
|
89
|
+
except ssl.SSLError as exc:
|
|
90
|
+
reason = getattr(exc, "reason", "") or ""
|
|
91
|
+
if any(marker in reason for marker in _PROTOCOL_REASONS):
|
|
92
|
+
return [CertAuditFinding(
|
|
93
|
+
"protocol", Severity.HIGH, hostname, "Server does not support TLS 1.2 or newer"
|
|
94
|
+
)]
|
|
95
|
+
logger.debug("%s: TLS handshake failed (%s)", hostname, reason)
|
|
96
|
+
return []
|
|
97
|
+
except OSError as exc:
|
|
98
|
+
# No listener on 443, filtered port, DNS failure: the host simply has no HTTPS.
|
|
99
|
+
logger.debug("%s: connection failed (%s)", hostname, type(exc).__name__)
|
|
100
|
+
return []
|
|
101
|
+
return self._check_expiry(hostname, cert)
|
|
102
|
+
|
|
103
|
+
@staticmethod
|
|
104
|
+
def _check_expiry(hostname: str, cert: dict) -> List[CertAuditFinding]:
|
|
105
|
+
not_after = cert.get("notAfter")
|
|
106
|
+
if not not_after:
|
|
107
|
+
return []
|
|
108
|
+
try:
|
|
109
|
+
expires = datetime.fromtimestamp(ssl.cert_time_to_seconds(not_after), tz=timezone.utc)
|
|
110
|
+
except (ValueError, OverflowError, OSError):
|
|
111
|
+
logger.debug("%s: unreadable notAfter %r", hostname, not_after)
|
|
112
|
+
return []
|
|
113
|
+
days_left = (expires - datetime.now(timezone.utc)).days
|
|
114
|
+
if days_left <= 14:
|
|
115
|
+
severity = Severity.HIGH
|
|
116
|
+
elif days_left <= 30:
|
|
117
|
+
severity = Severity.MEDIUM
|
|
118
|
+
else:
|
|
119
|
+
return []
|
|
120
|
+
return [CertAuditFinding(
|
|
121
|
+
"expiring", severity, hostname, f"Certificate expires in {days_left} days", expires.isoformat()
|
|
122
|
+
)]
|
|
123
|
+
|
|
124
|
+
def audit_many(self, hostnames: Sequence[str], max_hosts: int = MAX_HOSTS) -> List[CertAuditFinding]:
|
|
125
|
+
hosts = list(dict.fromkeys(hostnames))
|
|
126
|
+
if len(hosts) > max_hosts:
|
|
127
|
+
logger.warning("Certificate audit limited to %d of %d hosts", max_hosts, len(hosts))
|
|
128
|
+
hosts = hosts[:max_hosts]
|
|
129
|
+
with ThreadPoolExecutor(max_workers=self.max_workers) as pool:
|
|
130
|
+
results = pool.map(self.audit_host, hosts)
|
|
131
|
+
return [finding for host_findings in results for finding in host_findings]
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
"""DNS misconfiguration checks: SPF, DMARC and zone-transfer exposure."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from typing import List, Optional, Sequence
|
|
8
|
+
|
|
9
|
+
import dns.exception
|
|
10
|
+
import dns.query
|
|
11
|
+
import dns.rcode
|
|
12
|
+
import dns.resolver
|
|
13
|
+
|
|
14
|
+
from rimscan_cli.recon._common import is_public_address, normalize_name
|
|
15
|
+
from rimscan_cli.recon.severity import Severity
|
|
16
|
+
|
|
17
|
+
__all__ = ["DNSAuditFinding", "DNSAuditor", "AuditSeverity"]
|
|
18
|
+
|
|
19
|
+
# Kept so existing imports keep working; there is a single Severity enum.
|
|
20
|
+
AuditSeverity = Severity
|
|
21
|
+
|
|
22
|
+
_SPF_ALL = re.compile(r"^([+\-~?]?)all$", re.IGNORECASE)
|
|
23
|
+
_DMARC_START = re.compile(r"^v=dmarc1\s*(?:;|$)", re.IGNORECASE)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _is_spf(record: str) -> bool:
|
|
27
|
+
text = record.strip().lower()
|
|
28
|
+
return text == "v=spf1" or text.startswith("v=spf1 ")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _dmarc_tag(record: str, tag: str) -> Optional[str]:
|
|
32
|
+
match = re.search(rf"(?:^|;)\s*{re.escape(tag)}\s*=\s*(\w+)", record, re.IGNORECASE)
|
|
33
|
+
return match.group(1).lower() if match else None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass
|
|
37
|
+
class DNSAuditFinding:
|
|
38
|
+
check: str
|
|
39
|
+
severity: Severity
|
|
40
|
+
domain: str
|
|
41
|
+
detail: str
|
|
42
|
+
record: str = ""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class DNSAuditor:
|
|
46
|
+
def __init__(self, timeout: float = 5.0, nameservers: Optional[Sequence[str]] = None):
|
|
47
|
+
self.timeout = timeout
|
|
48
|
+
self.resolver = dns.resolver.Resolver()
|
|
49
|
+
self.resolver.timeout = timeout
|
|
50
|
+
self.resolver.lifetime = timeout
|
|
51
|
+
if nameservers:
|
|
52
|
+
self.resolver.nameservers = list(nameservers)
|
|
53
|
+
# Lookups that failed (timeout, SERVFAIL, ...). A failed lookup produces no finding,
|
|
54
|
+
# so callers use this to tell "clean" apart from "could not check".
|
|
55
|
+
self.errors: List[str] = []
|
|
56
|
+
|
|
57
|
+
def audit(self, domain: str, include_zone_transfer: bool = False) -> List[DNSAuditFinding]:
|
|
58
|
+
domain = normalize_name(domain)
|
|
59
|
+
self.errors = []
|
|
60
|
+
findings = self.check_spf(domain) + self.check_dmarc(domain)
|
|
61
|
+
if include_zone_transfer:
|
|
62
|
+
findings += self.check_zone_transfer(domain)
|
|
63
|
+
return findings
|
|
64
|
+
|
|
65
|
+
def _txt_records(self, name: str) -> Optional[List[str]]:
|
|
66
|
+
"""TXT strings at `name`; [] if none exist, None if the lookup itself failed."""
|
|
67
|
+
try:
|
|
68
|
+
answers = self.resolver.resolve(name, "TXT")
|
|
69
|
+
except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN):
|
|
70
|
+
return []
|
|
71
|
+
except dns.exception.DNSException:
|
|
72
|
+
self.errors.append(f"TXT lookup for {name} failed")
|
|
73
|
+
return None
|
|
74
|
+
return [b"".join(rdata.strings).decode("utf-8", "replace") for rdata in answers]
|
|
75
|
+
|
|
76
|
+
def check_spf(self, domain: str) -> List[DNSAuditFinding]:
|
|
77
|
+
records = self._txt_records(domain)
|
|
78
|
+
if records is None:
|
|
79
|
+
return []
|
|
80
|
+
spf_records = [r for r in records if _is_spf(r)]
|
|
81
|
+
if not spf_records:
|
|
82
|
+
return [DNSAuditFinding("spf", Severity.MEDIUM, domain, "No SPF record found")]
|
|
83
|
+
if len(spf_records) > 1:
|
|
84
|
+
return [DNSAuditFinding(
|
|
85
|
+
"spf", Severity.HIGH, domain,
|
|
86
|
+
"Multiple SPF records; receivers treat this as a permanent error", spf_records[0],
|
|
87
|
+
)]
|
|
88
|
+
|
|
89
|
+
spf = spf_records[0]
|
|
90
|
+
tokens = spf.split()[1:]
|
|
91
|
+
for token in tokens:
|
|
92
|
+
match = _SPF_ALL.match(token)
|
|
93
|
+
if not match:
|
|
94
|
+
continue
|
|
95
|
+
qualifier = match.group(1) or "+"
|
|
96
|
+
if qualifier in "+?":
|
|
97
|
+
return [DNSAuditFinding(
|
|
98
|
+
"spf", Severity.HIGH, domain,
|
|
99
|
+
f"SPF allows any sender ({token.lower()})", spf,
|
|
100
|
+
)]
|
|
101
|
+
if qualifier == "~":
|
|
102
|
+
return [DNSAuditFinding(
|
|
103
|
+
"spf", Severity.MEDIUM, domain, "SPF soft-fail policy (~all)", spf
|
|
104
|
+
)]
|
|
105
|
+
return []
|
|
106
|
+
# RFC 7208: without an "all" mechanism, redirect= supplies the policy instead.
|
|
107
|
+
if any(token.lower().startswith("redirect=") for token in tokens):
|
|
108
|
+
return []
|
|
109
|
+
return [DNSAuditFinding(
|
|
110
|
+
"spf", Severity.MEDIUM, domain, "SPF record has no terminating 'all' mechanism", spf
|
|
111
|
+
)]
|
|
112
|
+
|
|
113
|
+
def check_dmarc(self, domain: str) -> List[DNSAuditFinding]:
|
|
114
|
+
"""Look for DMARC at the domain, then at each parent (subdomains inherit the policy).
|
|
115
|
+
|
|
116
|
+
Parents are walked down to two labels because there is no public-suffix list here.
|
|
117
|
+
"""
|
|
118
|
+
labels = domain.split(".")
|
|
119
|
+
candidates = [".".join(labels[i:]) for i in range(max(len(labels) - 1, 1))]
|
|
120
|
+
|
|
121
|
+
record = ""
|
|
122
|
+
found_at = domain
|
|
123
|
+
for name in candidates:
|
|
124
|
+
records = self._txt_records(f"_dmarc.{name}")
|
|
125
|
+
if records is None:
|
|
126
|
+
return []
|
|
127
|
+
matching = [r for r in records if _DMARC_START.match(r.strip())]
|
|
128
|
+
if matching:
|
|
129
|
+
record, found_at = matching[0], name
|
|
130
|
+
break
|
|
131
|
+
else:
|
|
132
|
+
return [DNSAuditFinding("dmarc", Severity.HIGH, domain, "No DMARC record found")]
|
|
133
|
+
|
|
134
|
+
policy = _dmarc_tag(record, "p") or "none"
|
|
135
|
+
suffix = ""
|
|
136
|
+
if found_at != domain:
|
|
137
|
+
policy = _dmarc_tag(record, "sp") or policy
|
|
138
|
+
suffix = f" (inherited from {found_at})"
|
|
139
|
+
if policy == "none":
|
|
140
|
+
return [DNSAuditFinding(
|
|
141
|
+
"dmarc", Severity.HIGH, domain, f"DMARC policy is p=none (monitoring only){suffix}", record
|
|
142
|
+
)]
|
|
143
|
+
if policy == "quarantine":
|
|
144
|
+
return [DNSAuditFinding(
|
|
145
|
+
"dmarc", Severity.MEDIUM, domain, f"DMARC policy is p=quarantine{suffix}", record
|
|
146
|
+
)]
|
|
147
|
+
return []
|
|
148
|
+
|
|
149
|
+
def check_zone_transfer(self, domain: str) -> List[DNSAuditFinding]:
|
|
150
|
+
"""Ask each nameserver for an AXFR and stop reading after the first reply message.
|
|
151
|
+
|
|
152
|
+
A failed NS or nameserver-address lookup is recorded in `errors`, so a run that could
|
|
153
|
+
not test a nameserver is reported as incomplete instead of looking clean. A
|
|
154
|
+
nameserver's A records are used; if it has none, its AAAA records are tried.
|
|
155
|
+
"""
|
|
156
|
+
try:
|
|
157
|
+
nameservers = self.resolver.resolve(domain, "NS")
|
|
158
|
+
except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN):
|
|
159
|
+
return []
|
|
160
|
+
except dns.exception.DNSException:
|
|
161
|
+
self.errors.append(f"NS lookup for {domain} failed; zone transfer not checked")
|
|
162
|
+
return []
|
|
163
|
+
|
|
164
|
+
findings: List[DNSAuditFinding] = []
|
|
165
|
+
for ns in nameservers:
|
|
166
|
+
ns_host = str(ns.target).rstrip(".")
|
|
167
|
+
for address in self._nameserver_addresses(ns_host):
|
|
168
|
+
if not is_public_address(address):
|
|
169
|
+
continue # never send a transfer request to an internal address
|
|
170
|
+
if self._axfr_allowed(address, domain):
|
|
171
|
+
findings.append(DNSAuditFinding(
|
|
172
|
+
"zone_transfer", Severity.CRITICAL, domain,
|
|
173
|
+
f"Zone transfer (AXFR) allowed on {ns_host} ({address})", ns_host,
|
|
174
|
+
))
|
|
175
|
+
break
|
|
176
|
+
return findings
|
|
177
|
+
|
|
178
|
+
def _nameserver_addresses(self, ns_host: str) -> List[str]:
|
|
179
|
+
"""A records of a nameserver, or its AAAA records when it has no A record."""
|
|
180
|
+
for rdtype in ("A", "AAAA"):
|
|
181
|
+
try:
|
|
182
|
+
return [str(a) for a in self.resolver.resolve(ns_host, rdtype)]
|
|
183
|
+
except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN):
|
|
184
|
+
continue
|
|
185
|
+
except dns.exception.DNSException:
|
|
186
|
+
self.errors.append(f"{rdtype} lookup for nameserver {ns_host} failed; zone transfer not checked on it")
|
|
187
|
+
return []
|
|
188
|
+
return []
|
|
189
|
+
|
|
190
|
+
def _axfr_allowed(self, address: str, domain: str) -> bool:
|
|
191
|
+
try:
|
|
192
|
+
for message in dns.query.xfr(address, domain, lifetime=self.timeout):
|
|
193
|
+
return message.rcode() == dns.rcode.NOERROR and bool(message.answer)
|
|
194
|
+
except (dns.exception.DNSException, OSError, EOFError):
|
|
195
|
+
return False
|
|
196
|
+
return False
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
"""Subdomain enumeration from public passive-DNS and certificate-transparency sources."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
import re
|
|
8
|
+
import threading
|
|
9
|
+
import time
|
|
10
|
+
from typing import Callable, Dict, Iterable, List, Optional, Set
|
|
11
|
+
from urllib.parse import urlparse
|
|
12
|
+
|
|
13
|
+
from rimscan_cli.recon._common import is_valid_hostname, normalize_name
|
|
14
|
+
from rimscan_cli.recon._http import make_session, read_capped
|
|
15
|
+
from rimscan_cli.recon.resolver import DNSResolver
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
__all__ = ["SubdomainEnumerator", "DEFAULT_SOURCES", "BRUTEFORCE_WORDLIST"]
|
|
20
|
+
|
|
21
|
+
DEFAULT_SOURCES = ("crt.sh", "certspotter", "otx", "rapiddns", "wayback", "anubis")
|
|
22
|
+
|
|
23
|
+
BRUTEFORCE_WORDLIST = (
|
|
24
|
+
"www", "mail", "ftp", "dev", "staging", "api", "admin", "test", "portal", "vpn",
|
|
25
|
+
"app", "blog", "shop", "cdn", "static", "assets", "beta", "demo", "internal",
|
|
26
|
+
"secure", "cloud", "docs", "support", "help", "status", "monitor", "git",
|
|
27
|
+
"gitlab", "jenkins", "jira", "backup", "old", "new", "m", "mobile", "webmail",
|
|
28
|
+
"smtp", "ns1", "ns2", "db", "dashboard", "auth", "sso", "login", "stage",
|
|
29
|
+
"uat", "qa", "preview",
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
_WAYBACK_MAX_BYTES = 25 * 1024 * 1024
|
|
33
|
+
# crt.sh answers with one JSON array for the whole domain, which can be huge. Past this size
|
|
34
|
+
# the body is cut off, the JSON no longer parses, and the source is reported as failed
|
|
35
|
+
# instead of the whole array being held in memory.
|
|
36
|
+
_CRTSH_MAX_BYTES = 50 * 1024 * 1024
|
|
37
|
+
_RAPIDDNS_LINK = re.compile(r'target="_blank">([^<]+)</a>')
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class SubdomainEnumerator:
|
|
41
|
+
def __init__(self, domain: str, timeout: float = 10.0):
|
|
42
|
+
self.domain = normalize_name(domain)
|
|
43
|
+
self.timeout = (min(timeout, 5.0), timeout)
|
|
44
|
+
self.session = make_session()
|
|
45
|
+
self.source_status: Dict[str, str] = {}
|
|
46
|
+
self._sources: Dict[str, Callable[[], Set[str]]] = {
|
|
47
|
+
"crt.sh": self.from_crtsh,
|
|
48
|
+
"certspotter": self.from_certspotter,
|
|
49
|
+
"otx": self.from_otx,
|
|
50
|
+
"rapiddns": self.from_rapiddns,
|
|
51
|
+
"wayback": self.from_wayback,
|
|
52
|
+
"anubis": self.from_anubis,
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
# -- helpers -----------------------------------------------------------
|
|
56
|
+
|
|
57
|
+
def _accept(self, name: str) -> Optional[str]:
|
|
58
|
+
"""Normalize a candidate; return it only if it is a valid subdomain of the target."""
|
|
59
|
+
name = normalize_name(name)
|
|
60
|
+
if name.startswith("*."):
|
|
61
|
+
name = name[2:]
|
|
62
|
+
if not name.endswith("." + self.domain) or not is_valid_hostname(name):
|
|
63
|
+
return None
|
|
64
|
+
return name
|
|
65
|
+
|
|
66
|
+
def _collect(self, names: Iterable[str]) -> Set[str]:
|
|
67
|
+
return {valid for valid in map(self._accept, names) if valid}
|
|
68
|
+
|
|
69
|
+
def _get_json(self, url: str, **kwargs):
|
|
70
|
+
response = self.session.get(url, timeout=self.timeout, **kwargs)
|
|
71
|
+
response.raise_for_status()
|
|
72
|
+
return response.json()
|
|
73
|
+
|
|
74
|
+
# -- sources -----------------------------------------------------------
|
|
75
|
+
|
|
76
|
+
def from_crtsh(self) -> Set[str]:
|
|
77
|
+
response = self.session.get(
|
|
78
|
+
f"https://crt.sh/?q=%25.{self.domain}&output=json", timeout=self.timeout, stream=True
|
|
79
|
+
)
|
|
80
|
+
response.raise_for_status()
|
|
81
|
+
data = json.loads(read_capped(response, _CRTSH_MAX_BYTES))
|
|
82
|
+
return self._collect(
|
|
83
|
+
name for entry in data for name in entry.get("name_value", "").split("\n")
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
def from_certspotter(self, max_pages: int = 5) -> Set[str]:
|
|
87
|
+
url = "https://api.certspotter.com/v1/issuances"
|
|
88
|
+
params = {"domain": self.domain, "include_subdomains": "true", "expand": "dns_names"}
|
|
89
|
+
names: Set[str] = set()
|
|
90
|
+
for _ in range(max_pages):
|
|
91
|
+
response = self.session.get(url, params=params, timeout=self.timeout)
|
|
92
|
+
response.raise_for_status()
|
|
93
|
+
page = response.json()
|
|
94
|
+
if not page:
|
|
95
|
+
break
|
|
96
|
+
for issuance in page:
|
|
97
|
+
names |= self._collect(issuance.get("dns_names", []))
|
|
98
|
+
params["after"] = page[-1]["id"]
|
|
99
|
+
return names
|
|
100
|
+
|
|
101
|
+
def from_otx(self) -> Set[str]:
|
|
102
|
+
data = self._get_json(
|
|
103
|
+
f"https://otx.alienvault.com/api/v1/indicators/domain/{self.domain}/passive_dns"
|
|
104
|
+
)
|
|
105
|
+
return self._collect(record.get("hostname", "") for record in data.get("passive_dns", []))
|
|
106
|
+
|
|
107
|
+
def from_rapiddns(self) -> Set[str]:
|
|
108
|
+
response = self.session.get(
|
|
109
|
+
f"https://rapiddns.io/subdomain/{self.domain}", params={"full": 1}, timeout=self.timeout
|
|
110
|
+
)
|
|
111
|
+
response.raise_for_status()
|
|
112
|
+
return self._collect(_RAPIDDNS_LINK.findall(response.text))
|
|
113
|
+
|
|
114
|
+
def from_anubis(self) -> Set[str]:
|
|
115
|
+
return self._collect(self._get_json(f"https://jldc.me/anubis/subdomains/{self.domain}"))
|
|
116
|
+
|
|
117
|
+
def from_wayback(self) -> Set[str]:
|
|
118
|
+
response = self.session.get(
|
|
119
|
+
"https://web.archive.org/cdx/search/cdx",
|
|
120
|
+
params={
|
|
121
|
+
"url": f"*.{self.domain}", "output": "json", "fl": "original",
|
|
122
|
+
"collapse": "urlkey", "limit": 100000,
|
|
123
|
+
},
|
|
124
|
+
timeout=self.timeout,
|
|
125
|
+
stream=True,
|
|
126
|
+
)
|
|
127
|
+
response.raise_for_status()
|
|
128
|
+
rows = json.loads(read_capped(response, _WAYBACK_MAX_BYTES))
|
|
129
|
+
hosts = (urlparse(row[0] if "//" in row[0] else "//" + row[0]).hostname or "" for row in rows[1:] if row)
|
|
130
|
+
return self._collect(hosts)
|
|
131
|
+
|
|
132
|
+
def from_bruteforce(self, resolver: DNSResolver, wordlist: Iterable[str] = BRUTEFORCE_WORDLIST) -> Set[str]:
|
|
133
|
+
"""Resolve common prefixes. The resolver must already have run wildcard detection."""
|
|
134
|
+
candidates = [f"{word.strip().lower()}.{self.domain}" for word in wordlist if word.strip()]
|
|
135
|
+
return {r.hostname for r in resolver.resolve_many(candidates) if r.is_live}
|
|
136
|
+
|
|
137
|
+
# -- orchestration -----------------------------------------------------
|
|
138
|
+
|
|
139
|
+
def run(
|
|
140
|
+
self,
|
|
141
|
+
sources: Optional[List[str]] = None,
|
|
142
|
+
source_timeout: float = 60.0,
|
|
143
|
+
on_source_done: Optional[Callable[[str, int, str], None]] = None,
|
|
144
|
+
) -> Set[str]:
|
|
145
|
+
"""Query the selected sources concurrently and return the subdomains found.
|
|
146
|
+
|
|
147
|
+
A slow or failing source never blocks the others; its outcome is recorded in
|
|
148
|
+
`source_status`. The target domain itself is not included in the result.
|
|
149
|
+
|
|
150
|
+
Each source runs in a daemon thread, so one that is still stuck after
|
|
151
|
+
`source_timeout` cannot keep the interpreter alive at exit. Any exception a source
|
|
152
|
+
raises (including a malformed response) is contained to that source.
|
|
153
|
+
"""
|
|
154
|
+
chosen = list(sources) if sources else list(DEFAULT_SOURCES)
|
|
155
|
+
unknown = [name for name in chosen if name not in self._sources]
|
|
156
|
+
if unknown:
|
|
157
|
+
raise ValueError(f"unknown source(s): {', '.join(unknown)}")
|
|
158
|
+
|
|
159
|
+
results: Dict[str, Set[str]] = {}
|
|
160
|
+
errors: Dict[str, BaseException] = {}
|
|
161
|
+
|
|
162
|
+
def worker(name: str) -> None:
|
|
163
|
+
try:
|
|
164
|
+
results[name] = set(self._sources[name]())
|
|
165
|
+
except Exception as exc: # noqa: BLE001 - one bad source must not stop the scan
|
|
166
|
+
errors[name] = exc
|
|
167
|
+
|
|
168
|
+
threads = {
|
|
169
|
+
name: threading.Thread(target=worker, args=(name,), name=f"enum-{name}", daemon=True)
|
|
170
|
+
for name in chosen
|
|
171
|
+
}
|
|
172
|
+
for thread in threads.values():
|
|
173
|
+
thread.start()
|
|
174
|
+
deadline = time.monotonic() + source_timeout
|
|
175
|
+
for thread in threads.values():
|
|
176
|
+
thread.join(max(0.0, deadline - time.monotonic()))
|
|
177
|
+
|
|
178
|
+
found: Set[str] = set()
|
|
179
|
+
for name in chosen:
|
|
180
|
+
names: Set[str] = set()
|
|
181
|
+
if threads[name].is_alive():
|
|
182
|
+
self.source_status[name] = "timed out"
|
|
183
|
+
elif name in errors:
|
|
184
|
+
self.source_status[name] = f"failed ({type(errors[name]).__name__})"
|
|
185
|
+
logger.debug("%s failed", name, exc_info=errors[name])
|
|
186
|
+
else:
|
|
187
|
+
names = results.get(name, set())
|
|
188
|
+
self.source_status[name] = "ok"
|
|
189
|
+
found |= names
|
|
190
|
+
if on_source_done:
|
|
191
|
+
on_source_done(name, len(names), self.source_status[name])
|
|
192
|
+
return found
|