rimscan-cli 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
File without changes
@@ -0,0 +1,62 @@
1
+ """Small helpers shared by the recon modules, the scanner and the CLI."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ipaddress
6
+ import re
7
+ from typing import Iterable
8
+
9
+ __all__ = [
10
+ "HOST_LABEL",
11
+ "all_public",
12
+ "is_public_address",
13
+ "is_valid_hostname",
14
+ "normalize_domain",
15
+ "normalize_name",
16
+ ]
17
+
18
+ HOST_LABEL = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?")
19
+
20
+
21
+ def normalize_name(name: str) -> str:
22
+ """Lowercase a DNS name and strip whitespace and leading/trailing dots."""
23
+ return name.strip().lower().strip(".")
24
+
25
+
26
+ def is_valid_hostname(name: str) -> bool:
27
+ """True if every label is a valid LDH label and the whole name fits in 253 characters."""
28
+ return len(name) <= 253 and all(HOST_LABEL.fullmatch(label) for label in name.split("."))
29
+
30
+
31
+ def normalize_domain(value: str) -> str:
32
+ """Return a lowercase ASCII hostname, or raise ValueError if `value` is not a domain name."""
33
+ domain = value.strip().lower().rstrip(".")
34
+ if not domain or any(char in domain for char in "/:@ ?#"):
35
+ raise ValueError("domain must be a bare hostname such as example.com (no scheme, path, port or user)")
36
+ try:
37
+ ipaddress.ip_address(domain)
38
+ except ValueError:
39
+ pass
40
+ else:
41
+ raise ValueError("IP addresses are not supported; pass a domain name")
42
+ try:
43
+ domain = domain.encode("idna").decode("ascii")
44
+ except UnicodeError:
45
+ raise ValueError(f"{value!r} is not a valid domain name") from None
46
+ if len(domain.split(".")) < 2 or not is_valid_hostname(domain):
47
+ raise ValueError(f"{value!r} is not a valid domain name")
48
+ return domain
49
+
50
+
51
+ def is_public_address(value: str) -> bool:
52
+ """True for a globally routable IP address (not loopback, private, link-local, ...)."""
53
+ try:
54
+ return ipaddress.ip_address(value).is_global
55
+ except ValueError:
56
+ return False
57
+
58
+
59
+ def all_public(addresses: Iterable[str]) -> bool:
60
+ """True if there is at least one address and every one of them is globally routable."""
61
+ addresses = list(addresses)
62
+ return bool(addresses) and all(is_public_address(address) for address in addresses)
@@ -0,0 +1,65 @@
1
+ """Shared HTTP session: one honest User-Agent and one retry policy."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+
7
+ import requests
8
+ from requests.adapters import HTTPAdapter
9
+ from urllib3.util.retry import Retry
10
+
11
+ from rimscan_cli import __version__
12
+
13
+ USER_AGENT = f"rimscan-cli/{__version__} (+https://rimscan.cloud)"
14
+
15
+ _CHARSET = re.compile(r"charset=\s*[\"']?([\w.:-]+)", re.IGNORECASE)
16
+
17
+
18
+ def make_session(retries: int = 2) -> requests.Session:
19
+ """Build a session. Pass ``retries=0`` when each request must be sent at most once.
20
+
21
+ A server-supplied ``Retry-After`` header is ignored: urllib3 would sleep for as long as
22
+ the server asks (the request timeout does not cover that sleep), so a scanned host could
23
+ stall a worker thread for hours. Retries use the short exponential backoff instead.
24
+ """
25
+ retry = Retry(
26
+ total=retries,
27
+ backoff_factor=1.0,
28
+ status_forcelist=(429, 500, 502, 503, 504),
29
+ allowed_methods=("GET", "HEAD"),
30
+ raise_on_status=False,
31
+ respect_retry_after_header=False,
32
+ )
33
+ session = requests.Session()
34
+ session.headers["User-Agent"] = USER_AGENT
35
+ session.max_redirects = 5
36
+ adapter = HTTPAdapter(max_retries=retry, pool_maxsize=20)
37
+ session.mount("https://", adapter)
38
+ session.mount("http://", adapter)
39
+ return session
40
+
41
+
42
+ def _declared_charset(response: requests.Response) -> str:
43
+ """Charset from the Content-Type header, or UTF-8.
44
+
45
+ ``response.encoding`` is not used: for ``text/*`` without a charset, requests guesses
46
+ ISO-8859-1, which garbles the non-Latin fingerprints and secrets we match against.
47
+ """
48
+ headers = getattr(response, "headers", None) or {}
49
+ match = _CHARSET.search(headers.get("Content-Type", ""))
50
+ return match.group(1).strip("\"'") if match else "utf-8"
51
+
52
+
53
+ def read_capped(response: requests.Response, limit: int) -> str:
54
+ """Read at most `limit` bytes of a streamed response and decode them."""
55
+ body = bytearray()
56
+ for chunk in response.iter_content(chunk_size=16 * 1024):
57
+ body.extend(chunk)
58
+ if len(body) >= limit:
59
+ break
60
+ response.close()
61
+ data = bytes(body[:limit])
62
+ try:
63
+ return data.decode(_declared_charset(response), errors="replace")
64
+ except LookupError:
65
+ return data.decode("utf-8", errors="replace")
@@ -0,0 +1,131 @@
1
+ """TLS certificate checks: validity, expiry and protocol support."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import functools
6
+ import logging
7
+ import socket
8
+ import ssl
9
+ from concurrent.futures import ThreadPoolExecutor
10
+ from dataclasses import dataclass
11
+ from datetime import datetime, timezone
12
+ from typing import List, Optional, Sequence
13
+
14
+ import certifi
15
+
16
+ from rimscan_cli.recon._common import normalize_name
17
+ from rimscan_cli.recon.severity import Severity
18
+
19
+ __all__ = ["CertAuditFinding", "CertAuditor", "CertSeverity", "MAX_HOSTS"]
20
+
21
+ # Kept so existing imports keep working; there is a single Severity enum.
22
+ CertSeverity = Severity
23
+
24
+ logger = logging.getLogger(__name__)
25
+
26
+ MAX_HOSTS = 100
27
+
28
+ # OpenSSL X509_V_ERR_* codes reported by ssl.SSLCertVerificationError.verify_code.
29
+ _VERIFY_FINDINGS = {
30
+ 10: ("expired", Severity.CRITICAL, "Certificate has expired"),
31
+ 9: ("not_yet_valid", Severity.HIGH, "Certificate is not yet valid"),
32
+ 18: ("self_signed", Severity.HIGH, "Certificate is self-signed"),
33
+ 19: ("self_signed", Severity.HIGH, "Certificate chain ends in an untrusted self-signed root"),
34
+ 20: ("untrusted_chain", Severity.HIGH, "Certificate issuer is not trusted (missing intermediate or unknown CA)"),
35
+ 21: ("untrusted_chain", Severity.HIGH, "Certificate chain is incomplete"),
36
+ 62: ("hostname_mismatch", Severity.HIGH, "Certificate does not match the hostname"),
37
+ }
38
+
39
+ _PROTOCOL_REASONS = ("UNSUPPORTED_PROTOCOL", "NO_PROTOCOLS_AVAILABLE", "PROTOCOL_VERSION")
40
+
41
+
42
+ @dataclass
43
+ class CertAuditFinding:
44
+ check: str
45
+ severity: Severity
46
+ hostname: str
47
+ detail: str
48
+ expires: Optional[str] = None
49
+
50
+
51
+ @functools.lru_cache(maxsize=1)
52
+ def _context() -> ssl.SSLContext:
53
+ """A client context whose results do not depend on the Python version or the OS.
54
+
55
+ - the CA bundle is certifi's, so a slim image without ca-certificates does not
56
+ report every host as untrusted;
57
+ - TLS 1.2 is the floor on every Python version, so the "protocol" check is stable;
58
+ - strict X.509 checking (default since Python 3.13) is off, so the same certificate
59
+ gives the same verdict on 3.10 and 3.13.
60
+
61
+ The context is built once (loading the CA bundle is slow) and shared by all handshakes.
62
+ """
63
+ context = ssl.create_default_context(cafile=certifi.where())
64
+ context.minimum_version = ssl.TLSVersion.TLSv1_2
65
+ context.verify_flags &= ~ssl.VERIFY_X509_STRICT
66
+ return context
67
+
68
+
69
+ class CertAuditor:
70
+ def __init__(self, timeout: float = 5.0, max_workers: int = 10):
71
+ self.timeout = timeout
72
+ self.max_workers = max_workers
73
+
74
+ def _handshake(self, hostname: str, port: int) -> dict:
75
+ context = _context()
76
+ with socket.create_connection((hostname, port), timeout=self.timeout) as sock:
77
+ with context.wrap_socket(sock, server_hostname=hostname) as tls:
78
+ return tls.getpeercert() or {}
79
+
80
+ def audit_host(self, hostname: str, port: int = 443) -> List[CertAuditFinding]:
81
+ hostname = normalize_name(hostname)
82
+ try:
83
+ cert = self._handshake(hostname, port)
84
+ except ssl.SSLCertVerificationError as exc:
85
+ check, severity, detail = _VERIFY_FINDINGS.get(
86
+ exc.verify_code, ("invalid", Severity.HIGH, f"Certificate failed verification: {exc.verify_message}")
87
+ )
88
+ return [CertAuditFinding(check, severity, hostname, detail)]
89
+ except ssl.SSLError as exc:
90
+ reason = getattr(exc, "reason", "") or ""
91
+ if any(marker in reason for marker in _PROTOCOL_REASONS):
92
+ return [CertAuditFinding(
93
+ "protocol", Severity.HIGH, hostname, "Server does not support TLS 1.2 or newer"
94
+ )]
95
+ logger.debug("%s: TLS handshake failed (%s)", hostname, reason)
96
+ return []
97
+ except OSError as exc:
98
+ # No listener on 443, filtered port, DNS failure: the host simply has no HTTPS.
99
+ logger.debug("%s: connection failed (%s)", hostname, type(exc).__name__)
100
+ return []
101
+ return self._check_expiry(hostname, cert)
102
+
103
+ @staticmethod
104
+ def _check_expiry(hostname: str, cert: dict) -> List[CertAuditFinding]:
105
+ not_after = cert.get("notAfter")
106
+ if not not_after:
107
+ return []
108
+ try:
109
+ expires = datetime.fromtimestamp(ssl.cert_time_to_seconds(not_after), tz=timezone.utc)
110
+ except (ValueError, OverflowError, OSError):
111
+ logger.debug("%s: unreadable notAfter %r", hostname, not_after)
112
+ return []
113
+ days_left = (expires - datetime.now(timezone.utc)).days
114
+ if days_left <= 14:
115
+ severity = Severity.HIGH
116
+ elif days_left <= 30:
117
+ severity = Severity.MEDIUM
118
+ else:
119
+ return []
120
+ return [CertAuditFinding(
121
+ "expiring", severity, hostname, f"Certificate expires in {days_left} days", expires.isoformat()
122
+ )]
123
+
124
+ def audit_many(self, hostnames: Sequence[str], max_hosts: int = MAX_HOSTS) -> List[CertAuditFinding]:
125
+ hosts = list(dict.fromkeys(hostnames))
126
+ if len(hosts) > max_hosts:
127
+ logger.warning("Certificate audit limited to %d of %d hosts", max_hosts, len(hosts))
128
+ hosts = hosts[:max_hosts]
129
+ with ThreadPoolExecutor(max_workers=self.max_workers) as pool:
130
+ results = pool.map(self.audit_host, hosts)
131
+ return [finding for host_findings in results for finding in host_findings]
@@ -0,0 +1,196 @@
1
+ """DNS misconfiguration checks: SPF, DMARC and zone-transfer exposure."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from dataclasses import dataclass
7
+ from typing import List, Optional, Sequence
8
+
9
+ import dns.exception
10
+ import dns.query
11
+ import dns.rcode
12
+ import dns.resolver
13
+
14
+ from rimscan_cli.recon._common import is_public_address, normalize_name
15
+ from rimscan_cli.recon.severity import Severity
16
+
17
+ __all__ = ["DNSAuditFinding", "DNSAuditor", "AuditSeverity"]
18
+
19
+ # Kept so existing imports keep working; there is a single Severity enum.
20
+ AuditSeverity = Severity
21
+
22
+ _SPF_ALL = re.compile(r"^([+\-~?]?)all$", re.IGNORECASE)
23
+ _DMARC_START = re.compile(r"^v=dmarc1\s*(?:;|$)", re.IGNORECASE)
24
+
25
+
26
+ def _is_spf(record: str) -> bool:
27
+ text = record.strip().lower()
28
+ return text == "v=spf1" or text.startswith("v=spf1 ")
29
+
30
+
31
+ def _dmarc_tag(record: str, tag: str) -> Optional[str]:
32
+ match = re.search(rf"(?:^|;)\s*{re.escape(tag)}\s*=\s*(\w+)", record, re.IGNORECASE)
33
+ return match.group(1).lower() if match else None
34
+
35
+
36
+ @dataclass
37
+ class DNSAuditFinding:
38
+ check: str
39
+ severity: Severity
40
+ domain: str
41
+ detail: str
42
+ record: str = ""
43
+
44
+
45
+ class DNSAuditor:
46
+ def __init__(self, timeout: float = 5.0, nameservers: Optional[Sequence[str]] = None):
47
+ self.timeout = timeout
48
+ self.resolver = dns.resolver.Resolver()
49
+ self.resolver.timeout = timeout
50
+ self.resolver.lifetime = timeout
51
+ if nameservers:
52
+ self.resolver.nameservers = list(nameservers)
53
+ # Lookups that failed (timeout, SERVFAIL, ...). A failed lookup produces no finding,
54
+ # so callers use this to tell "clean" apart from "could not check".
55
+ self.errors: List[str] = []
56
+
57
+ def audit(self, domain: str, include_zone_transfer: bool = False) -> List[DNSAuditFinding]:
58
+ domain = normalize_name(domain)
59
+ self.errors = []
60
+ findings = self.check_spf(domain) + self.check_dmarc(domain)
61
+ if include_zone_transfer:
62
+ findings += self.check_zone_transfer(domain)
63
+ return findings
64
+
65
+ def _txt_records(self, name: str) -> Optional[List[str]]:
66
+ """TXT strings at `name`; [] if none exist, None if the lookup itself failed."""
67
+ try:
68
+ answers = self.resolver.resolve(name, "TXT")
69
+ except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN):
70
+ return []
71
+ except dns.exception.DNSException:
72
+ self.errors.append(f"TXT lookup for {name} failed")
73
+ return None
74
+ return [b"".join(rdata.strings).decode("utf-8", "replace") for rdata in answers]
75
+
76
+ def check_spf(self, domain: str) -> List[DNSAuditFinding]:
77
+ records = self._txt_records(domain)
78
+ if records is None:
79
+ return []
80
+ spf_records = [r for r in records if _is_spf(r)]
81
+ if not spf_records:
82
+ return [DNSAuditFinding("spf", Severity.MEDIUM, domain, "No SPF record found")]
83
+ if len(spf_records) > 1:
84
+ return [DNSAuditFinding(
85
+ "spf", Severity.HIGH, domain,
86
+ "Multiple SPF records; receivers treat this as a permanent error", spf_records[0],
87
+ )]
88
+
89
+ spf = spf_records[0]
90
+ tokens = spf.split()[1:]
91
+ for token in tokens:
92
+ match = _SPF_ALL.match(token)
93
+ if not match:
94
+ continue
95
+ qualifier = match.group(1) or "+"
96
+ if qualifier in "+?":
97
+ return [DNSAuditFinding(
98
+ "spf", Severity.HIGH, domain,
99
+ f"SPF allows any sender ({token.lower()})", spf,
100
+ )]
101
+ if qualifier == "~":
102
+ return [DNSAuditFinding(
103
+ "spf", Severity.MEDIUM, domain, "SPF soft-fail policy (~all)", spf
104
+ )]
105
+ return []
106
+ # RFC 7208: without an "all" mechanism, redirect= supplies the policy instead.
107
+ if any(token.lower().startswith("redirect=") for token in tokens):
108
+ return []
109
+ return [DNSAuditFinding(
110
+ "spf", Severity.MEDIUM, domain, "SPF record has no terminating 'all' mechanism", spf
111
+ )]
112
+
113
+ def check_dmarc(self, domain: str) -> List[DNSAuditFinding]:
114
+ """Look for DMARC at the domain, then at each parent (subdomains inherit the policy).
115
+
116
+ Parents are walked down to two labels because there is no public-suffix list here.
117
+ """
118
+ labels = domain.split(".")
119
+ candidates = [".".join(labels[i:]) for i in range(max(len(labels) - 1, 1))]
120
+
121
+ record = ""
122
+ found_at = domain
123
+ for name in candidates:
124
+ records = self._txt_records(f"_dmarc.{name}")
125
+ if records is None:
126
+ return []
127
+ matching = [r for r in records if _DMARC_START.match(r.strip())]
128
+ if matching:
129
+ record, found_at = matching[0], name
130
+ break
131
+ else:
132
+ return [DNSAuditFinding("dmarc", Severity.HIGH, domain, "No DMARC record found")]
133
+
134
+ policy = _dmarc_tag(record, "p") or "none"
135
+ suffix = ""
136
+ if found_at != domain:
137
+ policy = _dmarc_tag(record, "sp") or policy
138
+ suffix = f" (inherited from {found_at})"
139
+ if policy == "none":
140
+ return [DNSAuditFinding(
141
+ "dmarc", Severity.HIGH, domain, f"DMARC policy is p=none (monitoring only){suffix}", record
142
+ )]
143
+ if policy == "quarantine":
144
+ return [DNSAuditFinding(
145
+ "dmarc", Severity.MEDIUM, domain, f"DMARC policy is p=quarantine{suffix}", record
146
+ )]
147
+ return []
148
+
149
+ def check_zone_transfer(self, domain: str) -> List[DNSAuditFinding]:
150
+ """Ask each nameserver for an AXFR and stop reading after the first reply message.
151
+
152
+ A failed NS or nameserver-address lookup is recorded in `errors`, so a run that could
153
+ not test a nameserver is reported as incomplete instead of looking clean. A
154
+ nameserver's A records are used; if it has none, its AAAA records are tried.
155
+ """
156
+ try:
157
+ nameservers = self.resolver.resolve(domain, "NS")
158
+ except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN):
159
+ return []
160
+ except dns.exception.DNSException:
161
+ self.errors.append(f"NS lookup for {domain} failed; zone transfer not checked")
162
+ return []
163
+
164
+ findings: List[DNSAuditFinding] = []
165
+ for ns in nameservers:
166
+ ns_host = str(ns.target).rstrip(".")
167
+ for address in self._nameserver_addresses(ns_host):
168
+ if not is_public_address(address):
169
+ continue # never send a transfer request to an internal address
170
+ if self._axfr_allowed(address, domain):
171
+ findings.append(DNSAuditFinding(
172
+ "zone_transfer", Severity.CRITICAL, domain,
173
+ f"Zone transfer (AXFR) allowed on {ns_host} ({address})", ns_host,
174
+ ))
175
+ break
176
+ return findings
177
+
178
+ def _nameserver_addresses(self, ns_host: str) -> List[str]:
179
+ """A records of a nameserver, or its AAAA records when it has no A record."""
180
+ for rdtype in ("A", "AAAA"):
181
+ try:
182
+ return [str(a) for a in self.resolver.resolve(ns_host, rdtype)]
183
+ except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN):
184
+ continue
185
+ except dns.exception.DNSException:
186
+ self.errors.append(f"{rdtype} lookup for nameserver {ns_host} failed; zone transfer not checked on it")
187
+ return []
188
+ return []
189
+
190
+ def _axfr_allowed(self, address: str, domain: str) -> bool:
191
+ try:
192
+ for message in dns.query.xfr(address, domain, lifetime=self.timeout):
193
+ return message.rcode() == dns.rcode.NOERROR and bool(message.answer)
194
+ except (dns.exception.DNSException, OSError, EOFError):
195
+ return False
196
+ return False
@@ -0,0 +1,192 @@
1
+ """Subdomain enumeration from public passive-DNS and certificate-transparency sources."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import logging
7
+ import re
8
+ import threading
9
+ import time
10
+ from typing import Callable, Dict, Iterable, List, Optional, Set
11
+ from urllib.parse import urlparse
12
+
13
+ from rimscan_cli.recon._common import is_valid_hostname, normalize_name
14
+ from rimscan_cli.recon._http import make_session, read_capped
15
+ from rimscan_cli.recon.resolver import DNSResolver
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+ __all__ = ["SubdomainEnumerator", "DEFAULT_SOURCES", "BRUTEFORCE_WORDLIST"]
20
+
21
+ DEFAULT_SOURCES = ("crt.sh", "certspotter", "otx", "rapiddns", "wayback", "anubis")
22
+
23
+ BRUTEFORCE_WORDLIST = (
24
+ "www", "mail", "ftp", "dev", "staging", "api", "admin", "test", "portal", "vpn",
25
+ "app", "blog", "shop", "cdn", "static", "assets", "beta", "demo", "internal",
26
+ "secure", "cloud", "docs", "support", "help", "status", "monitor", "git",
27
+ "gitlab", "jenkins", "jira", "backup", "old", "new", "m", "mobile", "webmail",
28
+ "smtp", "ns1", "ns2", "db", "dashboard", "auth", "sso", "login", "stage",
29
+ "uat", "qa", "preview",
30
+ )
31
+
32
+ _WAYBACK_MAX_BYTES = 25 * 1024 * 1024
33
+ # crt.sh answers with one JSON array for the whole domain, which can be huge. Past this size
34
+ # the body is cut off, the JSON no longer parses, and the source is reported as failed
35
+ # instead of the whole array being held in memory.
36
+ _CRTSH_MAX_BYTES = 50 * 1024 * 1024
37
+ _RAPIDDNS_LINK = re.compile(r'target="_blank">([^<]+)</a>')
38
+
39
+
40
+ class SubdomainEnumerator:
41
+ def __init__(self, domain: str, timeout: float = 10.0):
42
+ self.domain = normalize_name(domain)
43
+ self.timeout = (min(timeout, 5.0), timeout)
44
+ self.session = make_session()
45
+ self.source_status: Dict[str, str] = {}
46
+ self._sources: Dict[str, Callable[[], Set[str]]] = {
47
+ "crt.sh": self.from_crtsh,
48
+ "certspotter": self.from_certspotter,
49
+ "otx": self.from_otx,
50
+ "rapiddns": self.from_rapiddns,
51
+ "wayback": self.from_wayback,
52
+ "anubis": self.from_anubis,
53
+ }
54
+
55
+ # -- helpers -----------------------------------------------------------
56
+
57
+ def _accept(self, name: str) -> Optional[str]:
58
+ """Normalize a candidate; return it only if it is a valid subdomain of the target."""
59
+ name = normalize_name(name)
60
+ if name.startswith("*."):
61
+ name = name[2:]
62
+ if not name.endswith("." + self.domain) or not is_valid_hostname(name):
63
+ return None
64
+ return name
65
+
66
+ def _collect(self, names: Iterable[str]) -> Set[str]:
67
+ return {valid for valid in map(self._accept, names) if valid}
68
+
69
+ def _get_json(self, url: str, **kwargs):
70
+ response = self.session.get(url, timeout=self.timeout, **kwargs)
71
+ response.raise_for_status()
72
+ return response.json()
73
+
74
+ # -- sources -----------------------------------------------------------
75
+
76
+ def from_crtsh(self) -> Set[str]:
77
+ response = self.session.get(
78
+ f"https://crt.sh/?q=%25.{self.domain}&output=json", timeout=self.timeout, stream=True
79
+ )
80
+ response.raise_for_status()
81
+ data = json.loads(read_capped(response, _CRTSH_MAX_BYTES))
82
+ return self._collect(
83
+ name for entry in data for name in entry.get("name_value", "").split("\n")
84
+ )
85
+
86
+ def from_certspotter(self, max_pages: int = 5) -> Set[str]:
87
+ url = "https://api.certspotter.com/v1/issuances"
88
+ params = {"domain": self.domain, "include_subdomains": "true", "expand": "dns_names"}
89
+ names: Set[str] = set()
90
+ for _ in range(max_pages):
91
+ response = self.session.get(url, params=params, timeout=self.timeout)
92
+ response.raise_for_status()
93
+ page = response.json()
94
+ if not page:
95
+ break
96
+ for issuance in page:
97
+ names |= self._collect(issuance.get("dns_names", []))
98
+ params["after"] = page[-1]["id"]
99
+ return names
100
+
101
+ def from_otx(self) -> Set[str]:
102
+ data = self._get_json(
103
+ f"https://otx.alienvault.com/api/v1/indicators/domain/{self.domain}/passive_dns"
104
+ )
105
+ return self._collect(record.get("hostname", "") for record in data.get("passive_dns", []))
106
+
107
+ def from_rapiddns(self) -> Set[str]:
108
+ response = self.session.get(
109
+ f"https://rapiddns.io/subdomain/{self.domain}", params={"full": 1}, timeout=self.timeout
110
+ )
111
+ response.raise_for_status()
112
+ return self._collect(_RAPIDDNS_LINK.findall(response.text))
113
+
114
+ def from_anubis(self) -> Set[str]:
115
+ return self._collect(self._get_json(f"https://jldc.me/anubis/subdomains/{self.domain}"))
116
+
117
+ def from_wayback(self) -> Set[str]:
118
+ response = self.session.get(
119
+ "https://web.archive.org/cdx/search/cdx",
120
+ params={
121
+ "url": f"*.{self.domain}", "output": "json", "fl": "original",
122
+ "collapse": "urlkey", "limit": 100000,
123
+ },
124
+ timeout=self.timeout,
125
+ stream=True,
126
+ )
127
+ response.raise_for_status()
128
+ rows = json.loads(read_capped(response, _WAYBACK_MAX_BYTES))
129
+ hosts = (urlparse(row[0] if "//" in row[0] else "//" + row[0]).hostname or "" for row in rows[1:] if row)
130
+ return self._collect(hosts)
131
+
132
+ def from_bruteforce(self, resolver: DNSResolver, wordlist: Iterable[str] = BRUTEFORCE_WORDLIST) -> Set[str]:
133
+ """Resolve common prefixes. The resolver must already have run wildcard detection."""
134
+ candidates = [f"{word.strip().lower()}.{self.domain}" for word in wordlist if word.strip()]
135
+ return {r.hostname for r in resolver.resolve_many(candidates) if r.is_live}
136
+
137
+ # -- orchestration -----------------------------------------------------
138
+
139
+ def run(
140
+ self,
141
+ sources: Optional[List[str]] = None,
142
+ source_timeout: float = 60.0,
143
+ on_source_done: Optional[Callable[[str, int, str], None]] = None,
144
+ ) -> Set[str]:
145
+ """Query the selected sources concurrently and return the subdomains found.
146
+
147
+ A slow or failing source never blocks the others; its outcome is recorded in
148
+ `source_status`. The target domain itself is not included in the result.
149
+
150
+ Each source runs in a daemon thread, so one that is still stuck after
151
+ `source_timeout` cannot keep the interpreter alive at exit. Any exception a source
152
+ raises (including a malformed response) is contained to that source.
153
+ """
154
+ chosen = list(sources) if sources else list(DEFAULT_SOURCES)
155
+ unknown = [name for name in chosen if name not in self._sources]
156
+ if unknown:
157
+ raise ValueError(f"unknown source(s): {', '.join(unknown)}")
158
+
159
+ results: Dict[str, Set[str]] = {}
160
+ errors: Dict[str, BaseException] = {}
161
+
162
+ def worker(name: str) -> None:
163
+ try:
164
+ results[name] = set(self._sources[name]())
165
+ except Exception as exc: # noqa: BLE001 - one bad source must not stop the scan
166
+ errors[name] = exc
167
+
168
+ threads = {
169
+ name: threading.Thread(target=worker, args=(name,), name=f"enum-{name}", daemon=True)
170
+ for name in chosen
171
+ }
172
+ for thread in threads.values():
173
+ thread.start()
174
+ deadline = time.monotonic() + source_timeout
175
+ for thread in threads.values():
176
+ thread.join(max(0.0, deadline - time.monotonic()))
177
+
178
+ found: Set[str] = set()
179
+ for name in chosen:
180
+ names: Set[str] = set()
181
+ if threads[name].is_alive():
182
+ self.source_status[name] = "timed out"
183
+ elif name in errors:
184
+ self.source_status[name] = f"failed ({type(errors[name]).__name__})"
185
+ logger.debug("%s failed", name, exc_info=errors[name])
186
+ else:
187
+ names = results.get(name, set())
188
+ self.source_status[name] = "ok"
189
+ found |= names
190
+ if on_source_done:
191
+ on_source_done(name, len(names), self.source_status[name])
192
+ return found