webrecon-cli 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- webrecon/__init__.py +7 -0
- webrecon/__main__.py +6 -0
- webrecon/cli.py +520 -0
- webrecon/config.py +63 -0
- webrecon/context.py +61 -0
- webrecon/data/subdomains-small.txt +140 -0
- webrecon/models.py +121 -0
- webrecon/modules/__init__.py +1 -0
- webrecon/modules/dns_enum.py +408 -0
- webrecon/modules/http_probe.py +407 -0
- webrecon/modules/port_scan.py +134 -0
- webrecon/modules/report.py +408 -0
- webrecon/modules/security_headers.py +69 -0
- webrecon/modules/tech_detect.py +125 -0
- webrecon/modules/tls_info.py +208 -0
- webrecon/py.typed +0 -0
- webrecon/ratelimit.py +61 -0
- webrecon/scanner.py +264 -0
- webrecon/targets.py +234 -0
- webrecon_cli-0.2.1.dist-info/METADATA +290 -0
- webrecon_cli-0.2.1.dist-info/RECORD +25 -0
- webrecon_cli-0.2.1.dist-info/WHEEL +5 -0
- webrecon_cli-0.2.1.dist-info/entry_points.txt +2 -0
- webrecon_cli-0.2.1.dist-info/licenses/LICENSE +21 -0
- webrecon_cli-0.2.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
# Small starter wordlist for subdomain brute-force.
|
|
2
|
+
# One label per line; the target domain is appended automatically.
|
|
3
|
+
www
|
|
4
|
+
mail
|
|
5
|
+
smtp
|
|
6
|
+
imap
|
|
7
|
+
pop
|
|
8
|
+
webmail
|
|
9
|
+
ftp
|
|
10
|
+
sftp
|
|
11
|
+
ns1
|
|
12
|
+
ns2
|
|
13
|
+
dns
|
|
14
|
+
api
|
|
15
|
+
api-dev
|
|
16
|
+
dev
|
|
17
|
+
devel
|
|
18
|
+
staging
|
|
19
|
+
stage
|
|
20
|
+
test
|
|
21
|
+
testing
|
|
22
|
+
qa
|
|
23
|
+
uat
|
|
24
|
+
sandbox
|
|
25
|
+
demo
|
|
26
|
+
beta
|
|
27
|
+
alpha
|
|
28
|
+
preview
|
|
29
|
+
admin
|
|
30
|
+
administrator
|
|
31
|
+
panel
|
|
32
|
+
cpanel
|
|
33
|
+
dashboard
|
|
34
|
+
portal
|
|
35
|
+
console
|
|
36
|
+
manage
|
|
37
|
+
app
|
|
38
|
+
apps
|
|
39
|
+
web
|
|
40
|
+
web1
|
|
41
|
+
web2
|
|
42
|
+
blog
|
|
43
|
+
news
|
|
44
|
+
shop
|
|
45
|
+
store
|
|
46
|
+
cart
|
|
47
|
+
checkout
|
|
48
|
+
payment
|
|
49
|
+
pay
|
|
50
|
+
billing
|
|
51
|
+
support
|
|
52
|
+
help
|
|
53
|
+
helpdesk
|
|
54
|
+
docs
|
|
55
|
+
doc
|
|
56
|
+
wiki
|
|
57
|
+
kb
|
|
58
|
+
status
|
|
59
|
+
monitor
|
|
60
|
+
monitoring
|
|
61
|
+
metrics
|
|
62
|
+
grafana
|
|
63
|
+
kibana
|
|
64
|
+
prometheus
|
|
65
|
+
jenkins
|
|
66
|
+
ci
|
|
67
|
+
build
|
|
68
|
+
git
|
|
69
|
+
gitlab
|
|
70
|
+
github
|
|
71
|
+
svn
|
|
72
|
+
repo
|
|
73
|
+
registry
|
|
74
|
+
docker
|
|
75
|
+
k8s
|
|
76
|
+
vpn
|
|
77
|
+
remote
|
|
78
|
+
rdp
|
|
79
|
+
ssh
|
|
80
|
+
gateway
|
|
81
|
+
proxy
|
|
82
|
+
cdn
|
|
83
|
+
static
|
|
84
|
+
assets
|
|
85
|
+
media
|
|
86
|
+
img
|
|
87
|
+
images
|
|
88
|
+
video
|
|
89
|
+
files
|
|
90
|
+
download
|
|
91
|
+
uploads
|
|
92
|
+
backup
|
|
93
|
+
backups
|
|
94
|
+
db
|
|
95
|
+
database
|
|
96
|
+
mysql
|
|
97
|
+
postgres
|
|
98
|
+
redis
|
|
99
|
+
mongo
|
|
100
|
+
elastic
|
|
101
|
+
search
|
|
102
|
+
auth
|
|
103
|
+
login
|
|
104
|
+
sso
|
|
105
|
+
oauth
|
|
106
|
+
id
|
|
107
|
+
identity
|
|
108
|
+
accounts
|
|
109
|
+
account
|
|
110
|
+
user
|
|
111
|
+
users
|
|
112
|
+
crm
|
|
113
|
+
erp
|
|
114
|
+
intranet
|
|
115
|
+
internal
|
|
116
|
+
corp
|
|
117
|
+
office
|
|
118
|
+
exchange
|
|
119
|
+
owa
|
|
120
|
+
lync
|
|
121
|
+
teams
|
|
122
|
+
chat
|
|
123
|
+
forum
|
|
124
|
+
community
|
|
125
|
+
events
|
|
126
|
+
careers
|
|
127
|
+
jobs
|
|
128
|
+
partners
|
|
129
|
+
legacy
|
|
130
|
+
old
|
|
131
|
+
new
|
|
132
|
+
m
|
|
133
|
+
mobile
|
|
134
|
+
secure
|
|
135
|
+
ssl
|
|
136
|
+
edge
|
|
137
|
+
origin
|
|
138
|
+
lb
|
|
139
|
+
node1
|
|
140
|
+
node2
|
webrecon/models.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""Structured result types shared by every recon module.
|
|
2
|
+
|
|
3
|
+
Every module returns one of these dataclasses instead of a raw dict, so the
|
|
4
|
+
report layer never has to guess at a schema.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from dataclasses import asdict, dataclass, field
|
|
10
|
+
from datetime import datetime, timezone
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _utcnow() -> str:
|
|
15
|
+
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class DnsRecords:
|
|
20
|
+
domain: str
|
|
21
|
+
records: dict[str, list[str]] = field(default_factory=dict)
|
|
22
|
+
# Per query outcome: OK, NOANSWER, NXDOMAIN, TIMEOUT, SERVFAIL or ERROR.
|
|
23
|
+
status: dict[str, str] = field(default_factory=dict)
|
|
24
|
+
findings: list[str] = field(default_factory=list)
|
|
25
|
+
error: str | None = None
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class Subdomain:
|
|
30
|
+
name: str
|
|
31
|
+
addresses: list[str] = field(default_factory=list)
|
|
32
|
+
source: str = "bruteforce"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass
|
|
36
|
+
class Redirect:
|
|
37
|
+
status: int
|
|
38
|
+
location: str
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass
|
|
42
|
+
class HttpResult:
|
|
43
|
+
url: str
|
|
44
|
+
reachable: bool
|
|
45
|
+
status: int | None = None
|
|
46
|
+
title: str | None = None
|
|
47
|
+
server: str | None = None
|
|
48
|
+
content_length: int | None = None
|
|
49
|
+
headers: dict[str, str] = field(default_factory=dict)
|
|
50
|
+
redirects: list[Redirect] = field(default_factory=list)
|
|
51
|
+
final_url: str | None = None
|
|
52
|
+
technologies: list[str] = field(default_factory=list)
|
|
53
|
+
interesting_paths: list[str] = field(default_factory=list)
|
|
54
|
+
findings: list[str] = field(default_factory=list)
|
|
55
|
+
error: str | None = None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass
|
|
59
|
+
class TlsInfo:
|
|
60
|
+
host: str
|
|
61
|
+
port: int
|
|
62
|
+
valid: bool
|
|
63
|
+
issuer: str | None = None
|
|
64
|
+
subject: str | None = None
|
|
65
|
+
not_before: str | None = None
|
|
66
|
+
not_after: str | None = None
|
|
67
|
+
days_until_expiry: int | None = None
|
|
68
|
+
san: list[str] = field(default_factory=list)
|
|
69
|
+
# True when the chain and the hostname verified against the system trust
|
|
70
|
+
# store, False when they did not (see verify_error), None when unknown.
|
|
71
|
+
trusted: bool | None = None
|
|
72
|
+
verify_error: str | None = None
|
|
73
|
+
protocol: str | None = None
|
|
74
|
+
cipher: str | None = None
|
|
75
|
+
findings: list[str] = field(default_factory=list)
|
|
76
|
+
error: str | None = None
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass
|
|
80
|
+
class PortResult:
|
|
81
|
+
port: int
|
|
82
|
+
open: bool
|
|
83
|
+
service: str | None = None
|
|
84
|
+
banner: str | None = None
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@dataclass
|
|
88
|
+
class SkippedHost:
|
|
89
|
+
host: str
|
|
90
|
+
reason: str
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@dataclass
|
|
94
|
+
class ScanReport:
|
|
95
|
+
target: str
|
|
96
|
+
started_at: str = field(default_factory=_utcnow)
|
|
97
|
+
finished_at: str | None = None
|
|
98
|
+
complete: bool = False
|
|
99
|
+
incomplete_reason: str | None = None
|
|
100
|
+
scope: list[str] = field(default_factory=list)
|
|
101
|
+
authorization: dict[str, Any] | None = None
|
|
102
|
+
settings: dict[str, Any] = field(default_factory=dict)
|
|
103
|
+
dns: DnsRecords | None = None
|
|
104
|
+
subdomains: list[Subdomain] = field(default_factory=list)
|
|
105
|
+
http: list[HttpResult] = field(default_factory=list)
|
|
106
|
+
tls: list[TlsInfo] = field(default_factory=list)
|
|
107
|
+
ports: list[PortResult] = field(default_factory=list)
|
|
108
|
+
skipped: list[SkippedHost] = field(default_factory=list)
|
|
109
|
+
notes: list[str] = field(default_factory=list)
|
|
110
|
+
errors: list[str] = field(default_factory=list)
|
|
111
|
+
|
|
112
|
+
def to_dict(self) -> dict[str, Any]:
|
|
113
|
+
return asdict(self)
|
|
114
|
+
|
|
115
|
+
def anything_reachable(self) -> bool:
|
|
116
|
+
"""True when at least one host answered on any probed channel."""
|
|
117
|
+
return (
|
|
118
|
+
any(h.reachable for h in self.http)
|
|
119
|
+
or any(p.open for p in self.ports)
|
|
120
|
+
or any(t.error is None for t in self.tls)
|
|
121
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Recon modules. Each returns dataclasses from webrecon.models."""
|
|
@@ -0,0 +1,408 @@
|
|
|
1
|
+
"""DNS record lookup, passive subdomain discovery and wordlist brute-force."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import json
|
|
7
|
+
import secrets
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import TYPE_CHECKING
|
|
11
|
+
|
|
12
|
+
import dns.asyncresolver
|
|
13
|
+
import dns.exception
|
|
14
|
+
import dns.query
|
|
15
|
+
import dns.resolver
|
|
16
|
+
import dns.zone
|
|
17
|
+
import httpx
|
|
18
|
+
|
|
19
|
+
from ..config import USER_AGENT
|
|
20
|
+
from ..context import ScanContext
|
|
21
|
+
from ..models import DnsRecords, Subdomain
|
|
22
|
+
from ..ratelimit import Throttle
|
|
23
|
+
from ..targets import is_valid_hostname
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING:
|
|
26
|
+
if sys.version_info >= (3, 11):
|
|
27
|
+
from importlib.resources.abc import Traversable
|
|
28
|
+
else:
|
|
29
|
+
from importlib.abc import Traversable
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
RECORD_TYPES = ("A", "AAAA", "MX", "NS", "TXT", "CNAME", "SOA", "CAA")
|
|
33
|
+
|
|
34
|
+
# Common DKIM selectors; DKIM keys cannot be enumerated, only guessed.
|
|
35
|
+
DKIM_SELECTORS = ("default", "google", "selector1", "selector2", "k1", "mail", "dkim")
|
|
36
|
+
|
|
37
|
+
CRT_SH_URL = "https://crt.sh/"
|
|
38
|
+
|
|
39
|
+
# crt.sh is slow; its lookup gets at least this long whatever -t says.
|
|
40
|
+
CRT_SH_MIN_TIMEOUT = 15.0
|
|
41
|
+
|
|
42
|
+
WILDCARD_PROBES = 3
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def make_resolver(
|
|
46
|
+
timeout: float, nameservers: list[str] | None = None, port: int = 53
|
|
47
|
+
) -> dns.asyncresolver.Resolver:
|
|
48
|
+
"""Build a resolver; raises dns.resolver.NoResolverConfiguration when none exists."""
|
|
49
|
+
if nameservers:
|
|
50
|
+
resolver = dns.asyncresolver.Resolver(configure=False)
|
|
51
|
+
resolver.nameservers = list(nameservers)
|
|
52
|
+
resolver.port = port
|
|
53
|
+
else:
|
|
54
|
+
resolver = dns.asyncresolver.Resolver()
|
|
55
|
+
resolver.timeout = timeout
|
|
56
|
+
resolver.lifetime = timeout
|
|
57
|
+
return resolver
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def query_status(exc: BaseException) -> str:
|
|
61
|
+
"""Name the outcome of a failed query so NXDOMAIN, timeouts and SERVFAIL differ."""
|
|
62
|
+
if isinstance(exc, dns.resolver.NXDOMAIN):
|
|
63
|
+
return "NXDOMAIN"
|
|
64
|
+
if isinstance(exc, dns.resolver.NoAnswer):
|
|
65
|
+
return "NOANSWER"
|
|
66
|
+
if isinstance(exc, dns.exception.Timeout):
|
|
67
|
+
return "TIMEOUT"
|
|
68
|
+
if isinstance(exc, dns.resolver.NoNameservers):
|
|
69
|
+
return "SERVFAIL"
|
|
70
|
+
return "ERROR"
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
async def _query(
|
|
74
|
+
resolver: dns.asyncresolver.Resolver, name: str, rtype: str, throttle: Throttle
|
|
75
|
+
) -> tuple[str, list[str]]:
|
|
76
|
+
"""One throttled query. Returns ``(status, sorted values)``; never raises DNS errors."""
|
|
77
|
+
async with throttle:
|
|
78
|
+
try:
|
|
79
|
+
answer = await resolver.resolve(name, rtype)
|
|
80
|
+
except dns.exception.DNSException as exc:
|
|
81
|
+
return query_status(exc), []
|
|
82
|
+
return "OK", sorted(rdata.to_text() for rdata in answer)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def summarise_failure(status: dict[str, str]) -> str:
|
|
86
|
+
values = set(status.values())
|
|
87
|
+
if values == {"NXDOMAIN"}:
|
|
88
|
+
return "domain does not exist (NXDOMAIN)"
|
|
89
|
+
if "TIMEOUT" in values and "OK" not in values and "NOANSWER" not in values:
|
|
90
|
+
return "DNS queries timed out"
|
|
91
|
+
if "SERVFAIL" in values and "OK" not in values and "NOANSWER" not in values:
|
|
92
|
+
return "DNS server failure (SERVFAIL or REFUSED)"
|
|
93
|
+
return "no DNS records resolved"
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
async def fetch_records(
|
|
97
|
+
domain: str,
|
|
98
|
+
timeout: float = 5.0,
|
|
99
|
+
throttle: Throttle | None = None,
|
|
100
|
+
nameservers: list[str] | None = None,
|
|
101
|
+
port: int = 53,
|
|
102
|
+
axfr: bool = False,
|
|
103
|
+
) -> DnsRecords:
|
|
104
|
+
"""Query the common record types for a domain, tolerating partial failure.
|
|
105
|
+
|
|
106
|
+
Also looks up DMARC and common DKIM selectors when the domain handles mail,
|
|
107
|
+
flags missing SPF/DMARC, and optionally attempts a zone transfer.
|
|
108
|
+
"""
|
|
109
|
+
result = DnsRecords(domain=domain)
|
|
110
|
+
throttle = throttle or Throttle(len(RECORD_TYPES))
|
|
111
|
+
try:
|
|
112
|
+
resolver = make_resolver(timeout, nameservers, port)
|
|
113
|
+
except dns.exception.DNSException as exc:
|
|
114
|
+
result.error = f"no usable DNS resolver: {exc}"
|
|
115
|
+
return result
|
|
116
|
+
|
|
117
|
+
answers = await asyncio.gather(
|
|
118
|
+
*(_query(resolver, domain, rtype, throttle) for rtype in RECORD_TYPES)
|
|
119
|
+
)
|
|
120
|
+
for rtype, (status, values) in zip(RECORD_TYPES, answers, strict=True):
|
|
121
|
+
result.status[rtype] = status
|
|
122
|
+
if values:
|
|
123
|
+
result.records[rtype] = values
|
|
124
|
+
|
|
125
|
+
if not result.records:
|
|
126
|
+
result.error = summarise_failure(result.status)
|
|
127
|
+
return result
|
|
128
|
+
|
|
129
|
+
if "MX" in result.records:
|
|
130
|
+
await _mail_hints(result, resolver, throttle)
|
|
131
|
+
|
|
132
|
+
if axfr and result.records.get("NS"):
|
|
133
|
+
await _attempt_axfr(result, resolver, timeout, throttle)
|
|
134
|
+
|
|
135
|
+
return result
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
async def _mail_hints(
|
|
139
|
+
result: DnsRecords, resolver: dns.asyncresolver.Resolver, throttle: Throttle
|
|
140
|
+
) -> None:
|
|
141
|
+
domain = result.domain
|
|
142
|
+
spf = [t for t in result.records.get("TXT", []) if "v=spf1" in t.lower()]
|
|
143
|
+
if not spf:
|
|
144
|
+
result.findings.append("domain receives mail but publishes no SPF record")
|
|
145
|
+
elif any("+all" in t.lower() for t in spf):
|
|
146
|
+
result.findings.append("SPF record ends in +all (any sender is authorised)")
|
|
147
|
+
|
|
148
|
+
status, values = await _query(resolver, f"_dmarc.{domain}", "TXT", throttle)
|
|
149
|
+
result.status["DMARC"] = status
|
|
150
|
+
dmarc = [v for v in values if "v=dmarc1" in v.lower()]
|
|
151
|
+
if dmarc:
|
|
152
|
+
result.records["DMARC"] = dmarc
|
|
153
|
+
if any("p=none" in v.lower().replace(" ", "") for v in dmarc):
|
|
154
|
+
result.findings.append("DMARC policy is p=none (monitoring only)")
|
|
155
|
+
else:
|
|
156
|
+
result.findings.append("domain receives mail but publishes no DMARC record")
|
|
157
|
+
|
|
158
|
+
answers = await asyncio.gather(
|
|
159
|
+
*(_query(resolver, f"{s}._domainkey.{domain}", "TXT", throttle) for s in DKIM_SELECTORS)
|
|
160
|
+
)
|
|
161
|
+
dkim = [
|
|
162
|
+
f"{selector}: {value}"
|
|
163
|
+
for selector, (_, values) in zip(DKIM_SELECTORS, answers, strict=True)
|
|
164
|
+
for value in values
|
|
165
|
+
]
|
|
166
|
+
if dkim:
|
|
167
|
+
result.records["DKIM"] = dkim
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _axfr_count(address: str, domain: str, timeout: float) -> int:
|
|
171
|
+
zone = dns.zone.from_xfr(dns.query.xfr(address, domain, timeout=timeout, lifetime=timeout))
|
|
172
|
+
return sum(1 for _ in zone.iterate_rdatas())
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
async def _attempt_axfr(
|
|
176
|
+
result: DnsRecords,
|
|
177
|
+
resolver: dns.asyncresolver.Resolver,
|
|
178
|
+
timeout: float,
|
|
179
|
+
throttle: Throttle,
|
|
180
|
+
) -> None:
|
|
181
|
+
"""Try a zone transfer against each authoritative server; a success is a finding."""
|
|
182
|
+
outcomes: list[str] = []
|
|
183
|
+
for ns in result.records.get("NS", []):
|
|
184
|
+
ns = ns.rstrip(".")
|
|
185
|
+
_, addresses = await _query(resolver, ns, "A", throttle)
|
|
186
|
+
if not addresses:
|
|
187
|
+
continue
|
|
188
|
+
try:
|
|
189
|
+
async with throttle:
|
|
190
|
+
count = await asyncio.wait_for(
|
|
191
|
+
asyncio.to_thread(_axfr_count, addresses[0], result.domain, timeout),
|
|
192
|
+
timeout * 2,
|
|
193
|
+
)
|
|
194
|
+
except (dns.exception.DNSException, OSError, EOFError, asyncio.TimeoutError):
|
|
195
|
+
outcomes.append("REFUSED")
|
|
196
|
+
continue
|
|
197
|
+
outcomes.append("ALLOWED")
|
|
198
|
+
result.findings.append(f"zone transfer (AXFR) allowed by {ns}: {count} records")
|
|
199
|
+
if outcomes:
|
|
200
|
+
result.status["AXFR"] = "ALLOWED" if "ALLOWED" in outcomes else "REFUSED"
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
async def _resolve_host(
|
|
204
|
+
resolver: dns.asyncresolver.Resolver, host: str, throttle: Throttle
|
|
205
|
+
) -> tuple[list[str], bool]:
|
|
206
|
+
"""Return the A/AAAA addresses for a host and whether a lookup failed abnormally.
|
|
207
|
+
|
|
208
|
+
Each query takes its own throttle slot. "Abnormal" means a timeout or
|
|
209
|
+
server failure rather than a clean NXDOMAIN/no-answer.
|
|
210
|
+
"""
|
|
211
|
+
addresses: list[str] = []
|
|
212
|
+
failed = False
|
|
213
|
+
for rtype in ("A", "AAAA"):
|
|
214
|
+
status, values = await _query(resolver, host, rtype, throttle)
|
|
215
|
+
addresses.extend(values)
|
|
216
|
+
if status in ("TIMEOUT", "SERVFAIL", "ERROR"):
|
|
217
|
+
failed = True
|
|
218
|
+
if status == "NXDOMAIN":
|
|
219
|
+
break
|
|
220
|
+
return sorted(set(addresses)), failed
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def load_wordlist(path: Path | Traversable) -> list[str]:
|
|
224
|
+
"""Read a subdomain wordlist, skipping blanks, comments and invalid labels.
|
|
225
|
+
|
|
226
|
+
``#`` starts a comment anywhere on a line; a UTF-8 byte order mark is
|
|
227
|
+
ignored; entries that are not valid hostname labels (``foo bar``, ``*.x``)
|
|
228
|
+
are dropped.
|
|
229
|
+
"""
|
|
230
|
+
text = path.read_bytes().decode("utf-8-sig", errors="ignore")
|
|
231
|
+
words: list[str] = []
|
|
232
|
+
seen: set[str] = set()
|
|
233
|
+
for raw in text.splitlines():
|
|
234
|
+
word = raw.split("#", 1)[0].strip().lower().strip(".")
|
|
235
|
+
if not word or word in seen or not is_valid_hostname(word):
|
|
236
|
+
continue
|
|
237
|
+
seen.add(word)
|
|
238
|
+
words.append(word)
|
|
239
|
+
return words
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def parse_crtsh(payload: str, domain: str) -> set[str]:
|
|
243
|
+
"""Extract unique subdomains from a crt.sh JSON response.
|
|
244
|
+
|
|
245
|
+
Certificate name fields may hold several newline-separated names and
|
|
246
|
+
wildcard entries, both of which are normalised here. Anything that is not
|
|
247
|
+
a valid hostname (e-mail addresses, ``?`` placeholders, spaces, slashes)
|
|
248
|
+
is discarded.
|
|
249
|
+
"""
|
|
250
|
+
try:
|
|
251
|
+
entries = json.loads(payload)
|
|
252
|
+
except json.JSONDecodeError:
|
|
253
|
+
return set()
|
|
254
|
+
if not isinstance(entries, list):
|
|
255
|
+
return set()
|
|
256
|
+
|
|
257
|
+
found: set[str] = set()
|
|
258
|
+
domain = domain.lower()
|
|
259
|
+
suffix = "." + domain
|
|
260
|
+
for entry in entries:
|
|
261
|
+
if not isinstance(entry, dict):
|
|
262
|
+
continue
|
|
263
|
+
for key in ("name_value", "common_name"):
|
|
264
|
+
value = entry.get(key)
|
|
265
|
+
if not isinstance(value, str):
|
|
266
|
+
continue
|
|
267
|
+
for name in value.split("\n"):
|
|
268
|
+
name = name.strip().lower()
|
|
269
|
+
if name.startswith("*."):
|
|
270
|
+
name = name[2:]
|
|
271
|
+
name = name.rstrip(".")
|
|
272
|
+
if not is_valid_hostname(name):
|
|
273
|
+
continue
|
|
274
|
+
if name == domain or name.endswith(suffix):
|
|
275
|
+
found.add(name)
|
|
276
|
+
return found
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
async def passive_subdomains(
|
|
280
|
+
domain: str,
|
|
281
|
+
timeout: float = CRT_SH_MIN_TIMEOUT,
|
|
282
|
+
user_agent: str = USER_AGENT,
|
|
283
|
+
throttle: Throttle | None = None,
|
|
284
|
+
proxy: str | None = None,
|
|
285
|
+
) -> set[str]:
|
|
286
|
+
"""Query certificate transparency logs via crt.sh.
|
|
287
|
+
|
|
288
|
+
Raises ``httpx.HTTPError`` (or ``ValueError`` for a non-JSON body) so the
|
|
289
|
+
caller can record the failure instead of mistaking it for "no names".
|
|
290
|
+
"""
|
|
291
|
+
throttle = throttle or Throttle(1)
|
|
292
|
+
async with httpx.AsyncClient(
|
|
293
|
+
timeout=timeout,
|
|
294
|
+
headers={"User-Agent": user_agent},
|
|
295
|
+
proxy=proxy,
|
|
296
|
+
trust_env=False,
|
|
297
|
+
) as client:
|
|
298
|
+
async with throttle:
|
|
299
|
+
response = await client.get(
|
|
300
|
+
CRT_SH_URL, params={"q": f"%.{domain}", "output": "json"}
|
|
301
|
+
)
|
|
302
|
+
response.raise_for_status()
|
|
303
|
+
if response.text.strip() and not response.text.lstrip().startswith("["):
|
|
304
|
+
raise ValueError("crt.sh returned a non-JSON response")
|
|
305
|
+
return parse_crtsh(response.text, domain)
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def select_candidates(
|
|
309
|
+
candidates: dict[str, str], domain: str, limit: int
|
|
310
|
+
) -> list[str]:
|
|
311
|
+
"""Trim the candidate set to ``limit`` hosts without losing the good ones.
|
|
312
|
+
|
|
313
|
+
Plain alphabetical truncation silently drops the apex domain and every
|
|
314
|
+
passively discovered name that sorts late, which are the hosts most likely
|
|
315
|
+
to exist. Order by confidence first: the target, then names seen in
|
|
316
|
+
certificate transparency, then wordlist guesses.
|
|
317
|
+
"""
|
|
318
|
+
priority = {"target": 0, "crt.sh": 1, "bruteforce": 2}
|
|
319
|
+
ordered = sorted(
|
|
320
|
+
candidates, key=lambda host: (priority.get(candidates[host], 3), host)
|
|
321
|
+
)
|
|
322
|
+
return ordered[:limit] if limit > 0 else ordered
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
async def detect_wildcard(
|
|
326
|
+
resolver: dns.asyncresolver.Resolver, domain: str, throttle: Throttle
|
|
327
|
+
) -> set[str]:
|
|
328
|
+
"""Resolve a few random labels; any answer means the zone has a wildcard."""
|
|
329
|
+
labels = [f"wr-{secrets.token_hex(8)}.{domain}" for _ in range(WILDCARD_PROBES)]
|
|
330
|
+
answers = await asyncio.gather(*(_resolve_host(resolver, h, throttle) for h in labels))
|
|
331
|
+
return {address for addresses, _ in answers for address in addresses}
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
async def enumerate_subdomains(ctx: ScanContext) -> list[Subdomain]:
|
|
335
|
+
"""Discover subdomains passively and/or by brute-force, then resolve them."""
|
|
336
|
+
config = ctx.config
|
|
337
|
+
domain = config.target
|
|
338
|
+
resolver = make_resolver(config.timeout, config.nameservers, config.dns_port)
|
|
339
|
+
|
|
340
|
+
candidates: dict[str, str] = {}
|
|
341
|
+
|
|
342
|
+
if config.passive:
|
|
343
|
+
try:
|
|
344
|
+
names = await passive_subdomains(
|
|
345
|
+
domain,
|
|
346
|
+
max(config.timeout, CRT_SH_MIN_TIMEOUT),
|
|
347
|
+
config.user_agent,
|
|
348
|
+
ctx.throttle,
|
|
349
|
+
config.proxy,
|
|
350
|
+
)
|
|
351
|
+
except (httpx.HTTPError, ValueError) as exc:
|
|
352
|
+
ctx.error(f"subdomains: crt.sh lookup failed: {type(exc).__name__}: {exc}")
|
|
353
|
+
names = set()
|
|
354
|
+
for name in names:
|
|
355
|
+
candidates.setdefault(name, "crt.sh")
|
|
356
|
+
|
|
357
|
+
if config.wordlist is not None:
|
|
358
|
+
for word in load_wordlist(config.wordlist):
|
|
359
|
+
candidates.setdefault(f"{word}.{domain}", "bruteforce")
|
|
360
|
+
|
|
361
|
+
# The apex always ranks as the target, even when crt.sh listed it too.
|
|
362
|
+
candidates[domain] = "target"
|
|
363
|
+
|
|
364
|
+
for name in list(candidates):
|
|
365
|
+
if not ctx.scope.allows_name(name):
|
|
366
|
+
ctx.skip(name, "out of scope")
|
|
367
|
+
del candidates[name]
|
|
368
|
+
|
|
369
|
+
wildcard = await detect_wildcard(resolver, domain, ctx.throttle)
|
|
370
|
+
ordered = select_candidates(candidates, domain, config.max_subdomains)
|
|
371
|
+
|
|
372
|
+
discarded = 0
|
|
373
|
+
failures = 0
|
|
374
|
+
|
|
375
|
+
async def check(host: str) -> Subdomain | None:
|
|
376
|
+
nonlocal discarded, failures
|
|
377
|
+
addresses, failed = await _resolve_host(resolver, host, ctx.throttle)
|
|
378
|
+
if failed and not addresses:
|
|
379
|
+
failures += 1
|
|
380
|
+
if not addresses:
|
|
381
|
+
return None
|
|
382
|
+
source = candidates[host]
|
|
383
|
+
if wildcard and source == "bruteforce" and set(addresses) <= wildcard:
|
|
384
|
+
discarded += 1
|
|
385
|
+
return None
|
|
386
|
+
sub = Subdomain(name=host, addresses=addresses, source=source)
|
|
387
|
+
ctx.emit("subdomain", sub)
|
|
388
|
+
return sub
|
|
389
|
+
|
|
390
|
+
results = await asyncio.gather(*(check(h) for h in ordered), return_exceptions=True)
|
|
391
|
+
found: list[Subdomain] = []
|
|
392
|
+
for host, outcome in zip(ordered, results, strict=True):
|
|
393
|
+
if isinstance(outcome, BaseException):
|
|
394
|
+
ctx.error(f"subdomains: {host}: {type(outcome).__name__}: {outcome}")
|
|
395
|
+
elif outcome is not None:
|
|
396
|
+
found.append(outcome)
|
|
397
|
+
|
|
398
|
+
if wildcard:
|
|
399
|
+
ctx.note(
|
|
400
|
+
f"wildcard DNS detected for *.{domain} ({', '.join(sorted(wildcard))}); "
|
|
401
|
+
f"{discarded} wordlist candidate(s) resolving only to those addresses discarded"
|
|
402
|
+
)
|
|
403
|
+
if failures:
|
|
404
|
+
ctx.note(
|
|
405
|
+
f"{failures} subdomain candidate(s) could not be resolved because of DNS "
|
|
406
|
+
"timeouts or server failures"
|
|
407
|
+
)
|
|
408
|
+
return sorted(found, key=lambda s: s.name)
|