webrecon-cli 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,140 @@
1
+ # Small starter wordlist for subdomain brute-force.
2
+ # One label per line; the target domain is appended automatically.
3
+ www
4
+ mail
5
+ smtp
6
+ imap
7
+ pop
8
+ webmail
9
+ ftp
10
+ sftp
11
+ ns1
12
+ ns2
13
+ dns
14
+ api
15
+ api-dev
16
+ dev
17
+ devel
18
+ staging
19
+ stage
20
+ test
21
+ testing
22
+ qa
23
+ uat
24
+ sandbox
25
+ demo
26
+ beta
27
+ alpha
28
+ preview
29
+ admin
30
+ administrator
31
+ panel
32
+ cpanel
33
+ dashboard
34
+ portal
35
+ console
36
+ manage
37
+ app
38
+ apps
39
+ web
40
+ web1
41
+ web2
42
+ blog
43
+ news
44
+ shop
45
+ store
46
+ cart
47
+ checkout
48
+ payment
49
+ pay
50
+ billing
51
+ support
52
+ help
53
+ helpdesk
54
+ docs
55
+ doc
56
+ wiki
57
+ kb
58
+ status
59
+ monitor
60
+ monitoring
61
+ metrics
62
+ grafana
63
+ kibana
64
+ prometheus
65
+ jenkins
66
+ ci
67
+ build
68
+ git
69
+ gitlab
70
+ github
71
+ svn
72
+ repo
73
+ registry
74
+ docker
75
+ k8s
76
+ vpn
77
+ remote
78
+ rdp
79
+ ssh
80
+ gateway
81
+ proxy
82
+ cdn
83
+ static
84
+ assets
85
+ media
86
+ img
87
+ images
88
+ video
89
+ files
90
+ download
91
+ uploads
92
+ backup
93
+ backups
94
+ db
95
+ database
96
+ mysql
97
+ postgres
98
+ redis
99
+ mongo
100
+ elastic
101
+ search
102
+ auth
103
+ login
104
+ sso
105
+ oauth
106
+ id
107
+ identity
108
+ accounts
109
+ account
110
+ user
111
+ users
112
+ crm
113
+ erp
114
+ intranet
115
+ internal
116
+ corp
117
+ office
118
+ exchange
119
+ owa
120
+ lync
121
+ teams
122
+ chat
123
+ forum
124
+ community
125
+ events
126
+ careers
127
+ jobs
128
+ partners
129
+ legacy
130
+ old
131
+ new
132
+ m
133
+ mobile
134
+ secure
135
+ ssl
136
+ edge
137
+ origin
138
+ lb
139
+ node1
140
+ node2
webrecon/models.py ADDED
@@ -0,0 +1,121 @@
1
+ """Structured result types shared by every recon module.
2
+
3
+ Every module returns one of these dataclasses instead of a raw dict, so the
4
+ report layer never has to guess at a schema.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from dataclasses import asdict, dataclass, field
10
+ from datetime import datetime, timezone
11
+ from typing import Any
12
+
13
+
14
+ def _utcnow() -> str:
15
+ return datetime.now(timezone.utc).isoformat(timespec="seconds")
16
+
17
+
18
+ @dataclass
19
+ class DnsRecords:
20
+ domain: str
21
+ records: dict[str, list[str]] = field(default_factory=dict)
22
+ # Per query outcome: OK, NOANSWER, NXDOMAIN, TIMEOUT, SERVFAIL or ERROR.
23
+ status: dict[str, str] = field(default_factory=dict)
24
+ findings: list[str] = field(default_factory=list)
25
+ error: str | None = None
26
+
27
+
28
+ @dataclass
29
+ class Subdomain:
30
+ name: str
31
+ addresses: list[str] = field(default_factory=list)
32
+ source: str = "bruteforce"
33
+
34
+
35
+ @dataclass
36
+ class Redirect:
37
+ status: int
38
+ location: str
39
+
40
+
41
+ @dataclass
42
+ class HttpResult:
43
+ url: str
44
+ reachable: bool
45
+ status: int | None = None
46
+ title: str | None = None
47
+ server: str | None = None
48
+ content_length: int | None = None
49
+ headers: dict[str, str] = field(default_factory=dict)
50
+ redirects: list[Redirect] = field(default_factory=list)
51
+ final_url: str | None = None
52
+ technologies: list[str] = field(default_factory=list)
53
+ interesting_paths: list[str] = field(default_factory=list)
54
+ findings: list[str] = field(default_factory=list)
55
+ error: str | None = None
56
+
57
+
58
+ @dataclass
59
+ class TlsInfo:
60
+ host: str
61
+ port: int
62
+ valid: bool
63
+ issuer: str | None = None
64
+ subject: str | None = None
65
+ not_before: str | None = None
66
+ not_after: str | None = None
67
+ days_until_expiry: int | None = None
68
+ san: list[str] = field(default_factory=list)
69
+ # True when the chain and the hostname verified against the system trust
70
+ # store, False when they did not (see verify_error), None when unknown.
71
+ trusted: bool | None = None
72
+ verify_error: str | None = None
73
+ protocol: str | None = None
74
+ cipher: str | None = None
75
+ findings: list[str] = field(default_factory=list)
76
+ error: str | None = None
77
+
78
+
79
+ @dataclass
80
+ class PortResult:
81
+ port: int
82
+ open: bool
83
+ service: str | None = None
84
+ banner: str | None = None
85
+
86
+
87
+ @dataclass
88
+ class SkippedHost:
89
+ host: str
90
+ reason: str
91
+
92
+
93
+ @dataclass
94
+ class ScanReport:
95
+ target: str
96
+ started_at: str = field(default_factory=_utcnow)
97
+ finished_at: str | None = None
98
+ complete: bool = False
99
+ incomplete_reason: str | None = None
100
+ scope: list[str] = field(default_factory=list)
101
+ authorization: dict[str, Any] | None = None
102
+ settings: dict[str, Any] = field(default_factory=dict)
103
+ dns: DnsRecords | None = None
104
+ subdomains: list[Subdomain] = field(default_factory=list)
105
+ http: list[HttpResult] = field(default_factory=list)
106
+ tls: list[TlsInfo] = field(default_factory=list)
107
+ ports: list[PortResult] = field(default_factory=list)
108
+ skipped: list[SkippedHost] = field(default_factory=list)
109
+ notes: list[str] = field(default_factory=list)
110
+ errors: list[str] = field(default_factory=list)
111
+
112
+ def to_dict(self) -> dict[str, Any]:
113
+ return asdict(self)
114
+
115
+ def anything_reachable(self) -> bool:
116
+ """True when at least one host answered on any probed channel."""
117
+ return (
118
+ any(h.reachable for h in self.http)
119
+ or any(p.open for p in self.ports)
120
+ or any(t.error is None for t in self.tls)
121
+ )
@@ -0,0 +1 @@
1
+ """Recon modules. Each returns dataclasses from webrecon.models."""
@@ -0,0 +1,408 @@
1
+ """DNS record lookup, passive subdomain discovery and wordlist brute-force."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import json
7
+ import secrets
8
+ import sys
9
+ from pathlib import Path
10
+ from typing import TYPE_CHECKING
11
+
12
+ import dns.asyncresolver
13
+ import dns.exception
14
+ import dns.query
15
+ import dns.resolver
16
+ import dns.zone
17
+ import httpx
18
+
19
+ from ..config import USER_AGENT
20
+ from ..context import ScanContext
21
+ from ..models import DnsRecords, Subdomain
22
+ from ..ratelimit import Throttle
23
+ from ..targets import is_valid_hostname
24
+
25
+ if TYPE_CHECKING:
26
+ if sys.version_info >= (3, 11):
27
+ from importlib.resources.abc import Traversable
28
+ else:
29
+ from importlib.abc import Traversable
30
+
31
+
32
+ RECORD_TYPES = ("A", "AAAA", "MX", "NS", "TXT", "CNAME", "SOA", "CAA")
33
+
34
+ # Common DKIM selectors; DKIM keys cannot be enumerated, only guessed.
35
+ DKIM_SELECTORS = ("default", "google", "selector1", "selector2", "k1", "mail", "dkim")
36
+
37
+ CRT_SH_URL = "https://crt.sh/"
38
+
39
+ # crt.sh is slow; its lookup gets at least this long whatever -t says.
40
+ CRT_SH_MIN_TIMEOUT = 15.0
41
+
42
+ WILDCARD_PROBES = 3
43
+
44
+
45
+ def make_resolver(
46
+ timeout: float, nameservers: list[str] | None = None, port: int = 53
47
+ ) -> dns.asyncresolver.Resolver:
48
+ """Build a resolver; raises dns.resolver.NoResolverConfiguration when none exists."""
49
+ if nameservers:
50
+ resolver = dns.asyncresolver.Resolver(configure=False)
51
+ resolver.nameservers = list(nameservers)
52
+ resolver.port = port
53
+ else:
54
+ resolver = dns.asyncresolver.Resolver()
55
+ resolver.timeout = timeout
56
+ resolver.lifetime = timeout
57
+ return resolver
58
+
59
+
60
+ def query_status(exc: BaseException) -> str:
61
+ """Name the outcome of a failed query so NXDOMAIN, timeouts and SERVFAIL differ."""
62
+ if isinstance(exc, dns.resolver.NXDOMAIN):
63
+ return "NXDOMAIN"
64
+ if isinstance(exc, dns.resolver.NoAnswer):
65
+ return "NOANSWER"
66
+ if isinstance(exc, dns.exception.Timeout):
67
+ return "TIMEOUT"
68
+ if isinstance(exc, dns.resolver.NoNameservers):
69
+ return "SERVFAIL"
70
+ return "ERROR"
71
+
72
+
73
+ async def _query(
74
+ resolver: dns.asyncresolver.Resolver, name: str, rtype: str, throttle: Throttle
75
+ ) -> tuple[str, list[str]]:
76
+ """One throttled query. Returns ``(status, sorted values)``; never raises DNS errors."""
77
+ async with throttle:
78
+ try:
79
+ answer = await resolver.resolve(name, rtype)
80
+ except dns.exception.DNSException as exc:
81
+ return query_status(exc), []
82
+ return "OK", sorted(rdata.to_text() for rdata in answer)
83
+
84
+
85
+ def summarise_failure(status: dict[str, str]) -> str:
86
+ values = set(status.values())
87
+ if values == {"NXDOMAIN"}:
88
+ return "domain does not exist (NXDOMAIN)"
89
+ if "TIMEOUT" in values and "OK" not in values and "NOANSWER" not in values:
90
+ return "DNS queries timed out"
91
+ if "SERVFAIL" in values and "OK" not in values and "NOANSWER" not in values:
92
+ return "DNS server failure (SERVFAIL or REFUSED)"
93
+ return "no DNS records resolved"
94
+
95
+
96
+ async def fetch_records(
97
+ domain: str,
98
+ timeout: float = 5.0,
99
+ throttle: Throttle | None = None,
100
+ nameservers: list[str] | None = None,
101
+ port: int = 53,
102
+ axfr: bool = False,
103
+ ) -> DnsRecords:
104
+ """Query the common record types for a domain, tolerating partial failure.
105
+
106
+ Also looks up DMARC and common DKIM selectors when the domain handles mail,
107
+ flags missing SPF/DMARC, and optionally attempts a zone transfer.
108
+ """
109
+ result = DnsRecords(domain=domain)
110
+ throttle = throttle or Throttle(len(RECORD_TYPES))
111
+ try:
112
+ resolver = make_resolver(timeout, nameservers, port)
113
+ except dns.exception.DNSException as exc:
114
+ result.error = f"no usable DNS resolver: {exc}"
115
+ return result
116
+
117
+ answers = await asyncio.gather(
118
+ *(_query(resolver, domain, rtype, throttle) for rtype in RECORD_TYPES)
119
+ )
120
+ for rtype, (status, values) in zip(RECORD_TYPES, answers, strict=True):
121
+ result.status[rtype] = status
122
+ if values:
123
+ result.records[rtype] = values
124
+
125
+ if not result.records:
126
+ result.error = summarise_failure(result.status)
127
+ return result
128
+
129
+ if "MX" in result.records:
130
+ await _mail_hints(result, resolver, throttle)
131
+
132
+ if axfr and result.records.get("NS"):
133
+ await _attempt_axfr(result, resolver, timeout, throttle)
134
+
135
+ return result
136
+
137
+
138
+ async def _mail_hints(
139
+ result: DnsRecords, resolver: dns.asyncresolver.Resolver, throttle: Throttle
140
+ ) -> None:
141
+ domain = result.domain
142
+ spf = [t for t in result.records.get("TXT", []) if "v=spf1" in t.lower()]
143
+ if not spf:
144
+ result.findings.append("domain receives mail but publishes no SPF record")
145
+ elif any("+all" in t.lower() for t in spf):
146
+ result.findings.append("SPF record ends in +all (any sender is authorised)")
147
+
148
+ status, values = await _query(resolver, f"_dmarc.{domain}", "TXT", throttle)
149
+ result.status["DMARC"] = status
150
+ dmarc = [v for v in values if "v=dmarc1" in v.lower()]
151
+ if dmarc:
152
+ result.records["DMARC"] = dmarc
153
+ if any("p=none" in v.lower().replace(" ", "") for v in dmarc):
154
+ result.findings.append("DMARC policy is p=none (monitoring only)")
155
+ else:
156
+ result.findings.append("domain receives mail but publishes no DMARC record")
157
+
158
+ answers = await asyncio.gather(
159
+ *(_query(resolver, f"{s}._domainkey.{domain}", "TXT", throttle) for s in DKIM_SELECTORS)
160
+ )
161
+ dkim = [
162
+ f"{selector}: {value}"
163
+ for selector, (_, values) in zip(DKIM_SELECTORS, answers, strict=True)
164
+ for value in values
165
+ ]
166
+ if dkim:
167
+ result.records["DKIM"] = dkim
168
+
169
+
170
+ def _axfr_count(address: str, domain: str, timeout: float) -> int:
171
+ zone = dns.zone.from_xfr(dns.query.xfr(address, domain, timeout=timeout, lifetime=timeout))
172
+ return sum(1 for _ in zone.iterate_rdatas())
173
+
174
+
175
+ async def _attempt_axfr(
176
+ result: DnsRecords,
177
+ resolver: dns.asyncresolver.Resolver,
178
+ timeout: float,
179
+ throttle: Throttle,
180
+ ) -> None:
181
+ """Try a zone transfer against each authoritative server; a success is a finding."""
182
+ outcomes: list[str] = []
183
+ for ns in result.records.get("NS", []):
184
+ ns = ns.rstrip(".")
185
+ _, addresses = await _query(resolver, ns, "A", throttle)
186
+ if not addresses:
187
+ continue
188
+ try:
189
+ async with throttle:
190
+ count = await asyncio.wait_for(
191
+ asyncio.to_thread(_axfr_count, addresses[0], result.domain, timeout),
192
+ timeout * 2,
193
+ )
194
+ except (dns.exception.DNSException, OSError, EOFError, asyncio.TimeoutError):
195
+ outcomes.append("REFUSED")
196
+ continue
197
+ outcomes.append("ALLOWED")
198
+ result.findings.append(f"zone transfer (AXFR) allowed by {ns}: {count} records")
199
+ if outcomes:
200
+ result.status["AXFR"] = "ALLOWED" if "ALLOWED" in outcomes else "REFUSED"
201
+
202
+
203
+ async def _resolve_host(
204
+ resolver: dns.asyncresolver.Resolver, host: str, throttle: Throttle
205
+ ) -> tuple[list[str], bool]:
206
+ """Return the A/AAAA addresses for a host and whether a lookup failed abnormally.
207
+
208
+ Each query takes its own throttle slot. "Abnormal" means a timeout or
209
+ server failure rather than a clean NXDOMAIN/no-answer.
210
+ """
211
+ addresses: list[str] = []
212
+ failed = False
213
+ for rtype in ("A", "AAAA"):
214
+ status, values = await _query(resolver, host, rtype, throttle)
215
+ addresses.extend(values)
216
+ if status in ("TIMEOUT", "SERVFAIL", "ERROR"):
217
+ failed = True
218
+ if status == "NXDOMAIN":
219
+ break
220
+ return sorted(set(addresses)), failed
221
+
222
+
223
+ def load_wordlist(path: Path | Traversable) -> list[str]:
224
+ """Read a subdomain wordlist, skipping blanks, comments and invalid labels.
225
+
226
+ ``#`` starts a comment anywhere on a line; a UTF-8 byte order mark is
227
+ ignored; entries that are not valid hostname labels (``foo bar``, ``*.x``)
228
+ are dropped.
229
+ """
230
+ text = path.read_bytes().decode("utf-8-sig", errors="ignore")
231
+ words: list[str] = []
232
+ seen: set[str] = set()
233
+ for raw in text.splitlines():
234
+ word = raw.split("#", 1)[0].strip().lower().strip(".")
235
+ if not word or word in seen or not is_valid_hostname(word):
236
+ continue
237
+ seen.add(word)
238
+ words.append(word)
239
+ return words
240
+
241
+
242
+ def parse_crtsh(payload: str, domain: str) -> set[str]:
243
+ """Extract unique subdomains from a crt.sh JSON response.
244
+
245
+ Certificate name fields may hold several newline-separated names and
246
+ wildcard entries, both of which are normalised here. Anything that is not
247
+ a valid hostname (e-mail addresses, ``?`` placeholders, spaces, slashes)
248
+ is discarded.
249
+ """
250
+ try:
251
+ entries = json.loads(payload)
252
+ except json.JSONDecodeError:
253
+ return set()
254
+ if not isinstance(entries, list):
255
+ return set()
256
+
257
+ found: set[str] = set()
258
+ domain = domain.lower()
259
+ suffix = "." + domain
260
+ for entry in entries:
261
+ if not isinstance(entry, dict):
262
+ continue
263
+ for key in ("name_value", "common_name"):
264
+ value = entry.get(key)
265
+ if not isinstance(value, str):
266
+ continue
267
+ for name in value.split("\n"):
268
+ name = name.strip().lower()
269
+ if name.startswith("*."):
270
+ name = name[2:]
271
+ name = name.rstrip(".")
272
+ if not is_valid_hostname(name):
273
+ continue
274
+ if name == domain or name.endswith(suffix):
275
+ found.add(name)
276
+ return found
277
+
278
+
279
+ async def passive_subdomains(
280
+ domain: str,
281
+ timeout: float = CRT_SH_MIN_TIMEOUT,
282
+ user_agent: str = USER_AGENT,
283
+ throttle: Throttle | None = None,
284
+ proxy: str | None = None,
285
+ ) -> set[str]:
286
+ """Query certificate transparency logs via crt.sh.
287
+
288
+ Raises ``httpx.HTTPError`` (or ``ValueError`` for a non-JSON body) so the
289
+ caller can record the failure instead of mistaking it for "no names".
290
+ """
291
+ throttle = throttle or Throttle(1)
292
+ async with httpx.AsyncClient(
293
+ timeout=timeout,
294
+ headers={"User-Agent": user_agent},
295
+ proxy=proxy,
296
+ trust_env=False,
297
+ ) as client:
298
+ async with throttle:
299
+ response = await client.get(
300
+ CRT_SH_URL, params={"q": f"%.{domain}", "output": "json"}
301
+ )
302
+ response.raise_for_status()
303
+ if response.text.strip() and not response.text.lstrip().startswith("["):
304
+ raise ValueError("crt.sh returned a non-JSON response")
305
+ return parse_crtsh(response.text, domain)
306
+
307
+
308
+ def select_candidates(
309
+ candidates: dict[str, str], domain: str, limit: int
310
+ ) -> list[str]:
311
+ """Trim the candidate set to ``limit`` hosts without losing the good ones.
312
+
313
+ Plain alphabetical truncation silently drops the apex domain and every
314
+ passively discovered name that sorts late, which are the hosts most likely
315
+ to exist. Order by confidence first: the target, then names seen in
316
+ certificate transparency, then wordlist guesses.
317
+ """
318
+ priority = {"target": 0, "crt.sh": 1, "bruteforce": 2}
319
+ ordered = sorted(
320
+ candidates, key=lambda host: (priority.get(candidates[host], 3), host)
321
+ )
322
+ return ordered[:limit] if limit > 0 else ordered
323
+
324
+
325
+ async def detect_wildcard(
326
+ resolver: dns.asyncresolver.Resolver, domain: str, throttle: Throttle
327
+ ) -> set[str]:
328
+ """Resolve a few random labels; any answer means the zone has a wildcard."""
329
+ labels = [f"wr-{secrets.token_hex(8)}.{domain}" for _ in range(WILDCARD_PROBES)]
330
+ answers = await asyncio.gather(*(_resolve_host(resolver, h, throttle) for h in labels))
331
+ return {address for addresses, _ in answers for address in addresses}
332
+
333
+
334
+ async def enumerate_subdomains(ctx: ScanContext) -> list[Subdomain]:
335
+ """Discover subdomains passively and/or by brute-force, then resolve them."""
336
+ config = ctx.config
337
+ domain = config.target
338
+ resolver = make_resolver(config.timeout, config.nameservers, config.dns_port)
339
+
340
+ candidates: dict[str, str] = {}
341
+
342
+ if config.passive:
343
+ try:
344
+ names = await passive_subdomains(
345
+ domain,
346
+ max(config.timeout, CRT_SH_MIN_TIMEOUT),
347
+ config.user_agent,
348
+ ctx.throttle,
349
+ config.proxy,
350
+ )
351
+ except (httpx.HTTPError, ValueError) as exc:
352
+ ctx.error(f"subdomains: crt.sh lookup failed: {type(exc).__name__}: {exc}")
353
+ names = set()
354
+ for name in names:
355
+ candidates.setdefault(name, "crt.sh")
356
+
357
+ if config.wordlist is not None:
358
+ for word in load_wordlist(config.wordlist):
359
+ candidates.setdefault(f"{word}.{domain}", "bruteforce")
360
+
361
+ # The apex always ranks as the target, even when crt.sh listed it too.
362
+ candidates[domain] = "target"
363
+
364
+ for name in list(candidates):
365
+ if not ctx.scope.allows_name(name):
366
+ ctx.skip(name, "out of scope")
367
+ del candidates[name]
368
+
369
+ wildcard = await detect_wildcard(resolver, domain, ctx.throttle)
370
+ ordered = select_candidates(candidates, domain, config.max_subdomains)
371
+
372
+ discarded = 0
373
+ failures = 0
374
+
375
+ async def check(host: str) -> Subdomain | None:
376
+ nonlocal discarded, failures
377
+ addresses, failed = await _resolve_host(resolver, host, ctx.throttle)
378
+ if failed and not addresses:
379
+ failures += 1
380
+ if not addresses:
381
+ return None
382
+ source = candidates[host]
383
+ if wildcard and source == "bruteforce" and set(addresses) <= wildcard:
384
+ discarded += 1
385
+ return None
386
+ sub = Subdomain(name=host, addresses=addresses, source=source)
387
+ ctx.emit("subdomain", sub)
388
+ return sub
389
+
390
+ results = await asyncio.gather(*(check(h) for h in ordered), return_exceptions=True)
391
+ found: list[Subdomain] = []
392
+ for host, outcome in zip(ordered, results, strict=True):
393
+ if isinstance(outcome, BaseException):
394
+ ctx.error(f"subdomains: {host}: {type(outcome).__name__}: {outcome}")
395
+ elif outcome is not None:
396
+ found.append(outcome)
397
+
398
+ if wildcard:
399
+ ctx.note(
400
+ f"wildcard DNS detected for *.{domain} ({', '.join(sorted(wildcard))}); "
401
+ f"{discarded} wordlist candidate(s) resolving only to those addresses discarded"
402
+ )
403
+ if failures:
404
+ ctx.note(
405
+ f"{failures} subdomain candidate(s) could not be resolved because of DNS "
406
+ "timeouts or server failures"
407
+ )
408
+ return sorted(found, key=lambda s: s.name)