webrecon-cli 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,208 @@
1
+ """TLS certificate inspection.
2
+
3
+ A first handshake verifies the chain and the hostname against the system
4
+ trust store. If that fails (self-signed, expired, hostname mismatch), the
5
+ reason is recorded and a second handshake without verification retrieves the
6
+ certificate anyway: an anomalous certificate is exactly the finding worth
7
+ reporting, so refusing the handshake would hide it. The certificate is read in
8
+ DER form and decoded with ``cryptography``, because ``getpeercert()`` returns
9
+ an empty dict when verification is disabled.
10
+
11
+ Handshakes run on the event loop (no threads), so a timeout or Ctrl-C cancels
12
+ them immediately.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import asyncio
18
+ import contextlib
19
+ import ssl
20
+ from collections.abc import Iterable
21
+ from datetime import datetime, timezone
22
+ from typing import Any
23
+
24
+ from cryptography import x509
25
+
26
+ from ..context import ScanContext
27
+ from ..models import TlsInfo
28
+ from ..ratelimit import Throttle
29
+
30
+ EXPIRY_WARNING_DAYS = 30
31
+
32
+
33
+ def _flatten_name(name: Any) -> str | None:
34
+ """Turn the nested tuple form Python uses for X.509 names into a string."""
35
+ if not name:
36
+ return None
37
+ parts = []
38
+ for rdn in name:
39
+ for key, value in rdn:
40
+ parts.append(f"{key}={value}")
41
+ return ", ".join(parts) or None
42
+
43
+
44
+ def _x509_name(name: x509.Name) -> str | None:
45
+ parts = [f"{attr.rfc4514_attribute_name}={attr.value!s}" for attr in name]
46
+ return ", ".join(parts) or None
47
+
48
+
49
+ def _apply_validity(
50
+ info: TlsInfo, not_before: datetime | None, not_after: datetime | None
51
+ ) -> None:
52
+ now = datetime.now(timezone.utc)
53
+ if not_before is not None:
54
+ info.not_before = not_before.isoformat(timespec="seconds")
55
+ if not_after is not None:
56
+ info.not_after = not_after.isoformat(timespec="seconds")
57
+ info.days_until_expiry = (not_after - now).days
58
+ info.valid = (
59
+ not_after is not None
60
+ and now <= not_after
61
+ and (not_before is None or not_before <= now)
62
+ )
63
+ if not_before is not None and now < not_before:
64
+ info.findings.append(f"certificate not valid until {info.not_before}")
65
+ if not_after is not None:
66
+ if now > not_after:
67
+ info.findings.append(f"certificate expired on {info.not_after}")
68
+ elif info.days_until_expiry is not None and info.days_until_expiry < EXPIRY_WARNING_DAYS:
69
+ info.findings.append(f"certificate expires in {info.days_until_expiry} day(s)")
70
+
71
+
72
+ def _parse_cert_time(raw: Any) -> datetime | None:
73
+ if not isinstance(raw, str):
74
+ return None
75
+ try:
76
+ # cert_time_to_seconds does not depend on the locale, unlike %b.
77
+ return datetime.fromtimestamp(ssl.cert_time_to_seconds(raw), timezone.utc)
78
+ except ValueError:
79
+ return None
80
+
81
+
82
+ def parse_certificate(cert: dict[str, Any], host: str, port: int) -> TlsInfo:
83
+ """Convert a ``getpeercert()`` dict into a TlsInfo, computing validity."""
84
+ info = TlsInfo(host=host, port=port, valid=False)
85
+ info.issuer = _flatten_name(cert.get("issuer"))
86
+ info.subject = _flatten_name(cert.get("subject"))
87
+ info.san = sorted(
88
+ value
89
+ for kind, value in cert.get("subjectAltName", ())
90
+ if kind.lower() in ("dns", "ip address")
91
+ )
92
+
93
+ not_before = _parse_cert_time(cert.get("notBefore"))
94
+ not_after = _parse_cert_time(cert.get("notAfter"))
95
+ _apply_validity(info, not_before, not_after)
96
+ # Keep an unparseable date visible rather than dropping it.
97
+ if not_after is None and cert.get("notAfter"):
98
+ info.not_after = str(cert["notAfter"])
99
+ if not_before is None and cert.get("notBefore"):
100
+ info.not_before = str(cert["notBefore"])
101
+ return info
102
+
103
+
104
+ def parse_der(der: bytes, host: str, port: int) -> TlsInfo:
105
+ """Decode a DER certificate into a TlsInfo, computing validity."""
106
+ cert = x509.load_der_x509_certificate(der)
107
+ info = TlsInfo(host=host, port=port, valid=False)
108
+ info.issuer = _x509_name(cert.issuer)
109
+ info.subject = _x509_name(cert.subject)
110
+ try:
111
+ san = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value
112
+ except x509.ExtensionNotFound:
113
+ names: list[str] = []
114
+ else:
115
+ names = list(san.get_values_for_type(x509.DNSName))
116
+ names += [str(ip) for ip in san.get_values_for_type(x509.IPAddress)]
117
+ info.san = sorted(names)
118
+ _apply_validity(info, cert.not_valid_before_utc, cert.not_valid_after_utc)
119
+ if cert.issuer == cert.subject:
120
+ info.findings.append("self-signed certificate")
121
+ return info
122
+
123
+
124
+ def describe_error(exc: BaseException) -> str:
125
+ if isinstance(exc, asyncio.TimeoutError):
126
+ return "TLS handshake timed out"
127
+ text = str(exc) or type(exc).__name__
128
+ return f"{type(exc).__name__}: {text}"
129
+
130
+
131
+ async def _handshake(
132
+ host: str, port: int, timeout: float, verify: bool
133
+ ) -> tuple[bytes | None, str | None, str | None]:
134
+ """Connect and complete a TLS handshake; returns (DER, protocol, cipher)."""
135
+ context = ssl.create_default_context()
136
+ if not verify:
137
+ context.check_hostname = False
138
+ context.verify_mode = ssl.CERT_NONE
139
+ _, writer = await asyncio.wait_for(
140
+ asyncio.open_connection(
141
+ host, port, ssl=context, server_hostname=host, ssl_handshake_timeout=timeout
142
+ ),
143
+ timeout,
144
+ )
145
+ try:
146
+ ssl_object = writer.get_extra_info("ssl_object")
147
+ der = ssl_object.getpeercert(binary_form=True)
148
+ cipher = ssl_object.cipher()
149
+ return der, ssl_object.version(), cipher[0] if cipher else None
150
+ finally:
151
+ writer.close()
152
+ with contextlib.suppress(Exception):
153
+ await asyncio.wait_for(writer.wait_closed(), 1.0)
154
+
155
+
156
+ async def inspect(
157
+ host: str, port: int = 443, timeout: float = 5.0, throttle: Throttle | None = None
158
+ ) -> TlsInfo:
159
+ """Inspect the TLS certificate served by host:port. Does not raise network errors."""
160
+ throttle = throttle or Throttle(1)
161
+ verify_error: str | None = None
162
+ try:
163
+ try:
164
+ async with throttle:
165
+ der, protocol, cipher = await _handshake(host, port, timeout, verify=True)
166
+ except ssl.SSLCertVerificationError as exc:
167
+ verify_error = exc.verify_message or str(exc)
168
+ async with throttle:
169
+ der, protocol, cipher = await _handshake(host, port, timeout, verify=False)
170
+ except (ssl.SSLError, OSError, asyncio.TimeoutError, UnicodeError, ValueError) as exc:
171
+ return TlsInfo(host=host, port=port, valid=False, error=describe_error(exc))
172
+
173
+ if not der:
174
+ return TlsInfo(host=host, port=port, valid=False, error="no certificate retrieved")
175
+ try:
176
+ info = parse_der(der, host, port)
177
+ except ValueError as exc:
178
+ return TlsInfo(host=host, port=port, valid=False, error=f"unparseable certificate: {exc}")
179
+
180
+ info.trusted = verify_error is None
181
+ info.verify_error = verify_error
182
+ info.protocol = protocol
183
+ info.cipher = cipher
184
+ if verify_error:
185
+ reason = f"certificate verification failed: {verify_error}"
186
+ if reason not in info.findings:
187
+ info.findings.insert(0, reason)
188
+ if protocol in ("TLSv1", "TLSv1.1", "SSLv3"):
189
+ info.findings.append(f"legacy protocol negotiated: {protocol}")
190
+ return info
191
+
192
+
193
+ async def inspect_hosts(endpoints: Iterable[tuple[str, int]], ctx: ScanContext) -> list[TlsInfo]:
194
+ """Inspect every (host, port) endpoint through the scan's shared throttle."""
195
+
196
+ async def one(host: str, port: int) -> TlsInfo:
197
+ reason = ctx.scope.refusal(host)
198
+ if reason:
199
+ ctx.skip(host, reason)
200
+ return TlsInfo(host=host, port=port, valid=False, error=f"skipped: {reason}")
201
+ try:
202
+ info = await inspect(host, port, ctx.config.timeout, ctx.throttle)
203
+ except Exception as exc: # keep the other hosts' results
204
+ info = TlsInfo(host=host, port=port, valid=False, error=describe_error(exc))
205
+ ctx.emit("tls", info)
206
+ return info
207
+
208
+ return list(await asyncio.gather(*(one(h, p) for h, p in endpoints)))
webrecon/py.typed ADDED
File without changes
webrecon/ratelimit.py ADDED
@@ -0,0 +1,61 @@
1
+ """Concurrency and pacing controls.
2
+
3
+ Recon tooling that fires unbounded concurrent requests is indistinguishable
4
+ from a denial-of-service attempt. Every outbound call in this project passes
5
+ through a Throttle so both the in-flight count and the request rate stay
6
+ bounded.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import asyncio
12
+ import time
13
+ from types import TracebackType
14
+
15
+
16
+ class Throttle:
17
+ """Caps in-flight work and enforces a minimum spacing between starts.
18
+
19
+ Combines a semaphore (how many at once) with a token bucket (how many per
20
+ second). Used as an async context manager::
21
+
22
+ async with throttle:
23
+ await do_request()
24
+ """
25
+
26
+ def __init__(self, concurrency: int = 20, rate_per_second: float = 0.0) -> None:
27
+ if concurrency < 1:
28
+ raise ValueError("concurrency must be >= 1")
29
+ if rate_per_second < 0:
30
+ raise ValueError("rate_per_second must be >= 0")
31
+ self._sem = asyncio.Semaphore(concurrency)
32
+ self._min_interval = 1.0 / rate_per_second if rate_per_second else 0.0
33
+ self._lock = asyncio.Lock()
34
+ self._next_slot = 0.0
35
+
36
+ async def _wait_for_slot(self) -> None:
37
+ if not self._min_interval:
38
+ return
39
+ async with self._lock:
40
+ now = time.monotonic()
41
+ wait = self._next_slot - now
42
+ self._next_slot = max(now, self._next_slot) + self._min_interval
43
+ if wait > 0:
44
+ await asyncio.sleep(wait)
45
+
46
+ async def __aenter__(self) -> Throttle:
47
+ await self._sem.acquire()
48
+ try:
49
+ await self._wait_for_slot()
50
+ except BaseException:
51
+ self._sem.release()
52
+ raise
53
+ return self
54
+
55
+ async def __aexit__(
56
+ self,
57
+ exc_type: type[BaseException] | None,
58
+ exc: BaseException | None,
59
+ tb: TracebackType | None,
60
+ ) -> None:
61
+ self._sem.release()
webrecon/scanner.py ADDED
@@ -0,0 +1,264 @@
1
+ """Scan orchestration: runs the enabled modules and assembles the report.
2
+
3
+ One :class:`~webrecon.context.ScanContext` carries a single shared throttle and
4
+ scope for every stage. Each stage is isolated: an exception is recorded in
5
+ ``report.errors`` and the scan moves on, so results gathered earlier are never
6
+ lost. ``config.max_time`` bounds the whole scan; when it is reached the report
7
+ is returned marked as incomplete.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import asyncio
13
+ import socket
14
+ from collections.abc import Awaitable, Callable
15
+ from datetime import datetime, timezone
16
+ from typing import Any, TypeVar
17
+ from urllib.parse import urlsplit
18
+
19
+ from .config import WEB_PORTS, ScanConfig
20
+ from .context import Emit, ScanContext
21
+ from .models import ScanReport
22
+ from .modules import dns_enum, http_probe, port_scan, tls_info
23
+ from .targets import is_ip
24
+
25
+ T = TypeVar("T")
26
+
27
+ ACTIVE_MODULES = frozenset({"subdomains", "http", "tls", "ports"})
28
+ TLS_PORTS = frozenset({443, 8443, 9443})
29
+
30
+
31
+ def _now() -> str:
32
+ return datetime.now(timezone.utc).isoformat(timespec="seconds")
33
+
34
+
35
+ def redact_proxy(proxy: str | None) -> str | None:
36
+ """Drop credentials from a proxy URL before it is written to a report."""
37
+ if not proxy:
38
+ return None
39
+ parts = urlsplit(proxy)
40
+ if parts.username or parts.password:
41
+ host = parts.hostname or ""
42
+ netloc = f"{host}:{parts.port}" if parts.port else host
43
+ return parts._replace(netloc=f"<redacted>@{netloc}").geturl()
44
+ return proxy
45
+
46
+
47
+ def describe_settings(config: ScanConfig) -> dict[str, Any]:
48
+ wordlist = config.wordlist
49
+ return {
50
+ "modules": sorted(config.modules),
51
+ "port": config.port,
52
+ "concurrency": config.concurrency,
53
+ "rate_per_second": config.rate_per_second,
54
+ "timeout": config.timeout,
55
+ "max_time": config.max_time,
56
+ "max_requests_per_host": config.max_requests_per_host,
57
+ "verify_tls": config.verify_tls,
58
+ "proxy": redact_proxy(config.proxy),
59
+ "proxy_applies_to": "HTTP probes and crt.sh only; DNS, TLS and port scans connect directly"
60
+ if config.proxy
61
+ else None,
62
+ "user_agent": config.user_agent,
63
+ "passive_lookups": "crt.sh" if config.passive and config.enabled("subdomains") else None,
64
+ "wordlist": getattr(wordlist, "name", None) if wordlist is not None else None,
65
+ "allow_private": config.allow_private,
66
+ "redact_secrets": config.redact,
67
+ }
68
+
69
+
70
+ async def run_scan(
71
+ config: ScanConfig,
72
+ on_stage: Callable[[str], None] | None = None,
73
+ report: ScanReport | None = None,
74
+ emit: Emit | None = None,
75
+ ) -> ScanReport:
76
+ """Execute a scan, filling ``report`` (created if omitted) in place.
77
+
78
+ ``on_stage`` is an optional callable invoked with a short status string
79
+ before each stage, so a CLI can show progress without this module knowing
80
+ anything about the terminal. ``emit(kind, payload)`` receives each result
81
+ as soon as it exists.
82
+ """
83
+ ctx = ScanContext.create(config, report, emit)
84
+ report = ctx.report
85
+ report.complete = False
86
+ report.scope = ctx.scope.describe()
87
+ report.settings = describe_settings(config)
88
+
89
+ def stage(message: str) -> None:
90
+ if on_stage:
91
+ on_stage(message)
92
+
93
+ try:
94
+ if config.max_time:
95
+ await asyncio.wait_for(_run_stages(ctx, stage), config.max_time)
96
+ else:
97
+ await _run_stages(ctx, stage)
98
+ report.complete = True
99
+ except asyncio.TimeoutError:
100
+ report.incomplete_reason = f"maximum scan time of {config.max_time:g}s reached"
101
+ ctx.error(f"scan stopped: {report.incomplete_reason}")
102
+ finally:
103
+ report.finished_at = _now()
104
+ ctx.emit(
105
+ "summary",
106
+ {
107
+ "complete": report.complete,
108
+ "incomplete_reason": report.incomplete_reason,
109
+ "errors": len(report.errors),
110
+ },
111
+ )
112
+ return report
113
+
114
+
115
+ async def _guarded(ctx: ScanContext, name: str, work: Awaitable[T]) -> T | None:
116
+ """Run one stage; an exception becomes an error entry instead of a traceback."""
117
+ try:
118
+ return await work
119
+ except Exception as exc:
120
+ ctx.error(f"{name}: {type(exc).__name__}: {exc}")
121
+ return None
122
+
123
+
124
+ async def _target_refusal(ctx: ScanContext) -> str | None:
125
+ """Why the target itself must not be actively probed, or None."""
126
+ target = ctx.config.target
127
+ reason = ctx.scope.refusal(target)
128
+ if reason or is_ip(target) or ctx.scope.allow_private:
129
+ return reason
130
+ loop = asyncio.get_running_loop()
131
+ try:
132
+ async with ctx.throttle:
133
+ infos = await asyncio.wait_for(
134
+ loop.getaddrinfo(target, None, type=socket.SOCK_STREAM), ctx.config.timeout
135
+ )
136
+ except (OSError, UnicodeError, asyncio.TimeoutError):
137
+ return None # unresolvable: the modules report it themselves
138
+ addresses = {str(info[4][0]) for info in infos}
139
+ if addresses and all(ctx.scope.address_refusal(a) for a in addresses):
140
+ return f"resolves only to non-public addresses ({', '.join(sorted(addresses))}); " \
141
+ "use --allow-private"
142
+ return None
143
+
144
+
145
+ async def _run_stages(ctx: ScanContext, stage: Callable[[str], None]) -> None:
146
+ config = ctx.config
147
+ report = ctx.report
148
+ target = config.target
149
+
150
+ if config.enabled("dns"):
151
+ stage("resolving DNS records")
152
+ dns_result = await _guarded(
153
+ ctx,
154
+ "dns",
155
+ dns_enum.fetch_records(
156
+ target,
157
+ config.timeout,
158
+ ctx.throttle,
159
+ config.nameservers or None,
160
+ config.dns_port,
161
+ config.axfr,
162
+ ),
163
+ )
164
+ if dns_result is not None:
165
+ report.dns = dns_result
166
+ ctx.emit("dns", dns_result)
167
+ if dns_result.error:
168
+ ctx.error(f"dns: {dns_result.error}")
169
+
170
+ if not config.modules & ACTIVE_MODULES:
171
+ return
172
+
173
+ refusal = await _target_refusal(ctx)
174
+ if refusal:
175
+ ctx.skip(target, refusal)
176
+ ctx.error(f"{target}: active modules skipped ({refusal})")
177
+ return
178
+
179
+ hosts = [target]
180
+ if config.enabled("subdomains"):
181
+ if is_ip(target):
182
+ ctx.note("subdomain enumeration skipped: the target is an IP address")
183
+ else:
184
+ stage("enumerating subdomains")
185
+ subs = await _guarded(ctx, "subdomains", dns_enum.enumerate_subdomains(ctx))
186
+ report.subdomains = subs or []
187
+ names = []
188
+ for sub in report.subdomains:
189
+ refused = [a for a in sub.addresses if ctx.scope.address_refusal(a)]
190
+ if sub.addresses and len(refused) == len(sub.addresses):
191
+ ctx.skip(sub.name, "resolves only to non-public addresses")
192
+ continue
193
+ names.append(sub.name)
194
+ hosts = names or hosts
195
+
196
+ open_web_ports: list[int] = []
197
+ if config.enabled("ports"):
198
+ stage(f"scanning {len(config.ports)} ports")
199
+ open_ports = await _guarded(
200
+ ctx,
201
+ "ports",
202
+ port_scan.scan(
203
+ target,
204
+ config.ports,
205
+ config.timeout,
206
+ ctx.throttle,
207
+ config.banners,
208
+ on_result=lambda r: ctx.emit("port", r),
209
+ ),
210
+ )
211
+ report.ports = open_ports or []
212
+ open_web_ports = [p.port for p in report.ports if p.port in WEB_PORTS]
213
+
214
+ if config.enabled("http"):
215
+ endpoints = _http_endpoints(config, hosts, open_web_ports)
216
+ stage(f"probing HTTP on {len(endpoints)} endpoint(s)")
217
+ report.http = (
218
+ await _guarded(ctx, "http", http_probe.probe_hosts(endpoints, ctx)) or []
219
+ )
220
+
221
+ if config.enabled("tls"):
222
+ endpoints_tls = _tls_endpoints(config, report, open_web_ports)
223
+ stage(f"inspecting TLS on {len(endpoints_tls)} endpoint(s)")
224
+ report.tls = (
225
+ await _guarded(ctx, "tls", tls_info.inspect_hosts(endpoints_tls, ctx)) or []
226
+ )
227
+
228
+
229
+ def _http_endpoints(
230
+ config: ScanConfig, hosts: list[str], open_web_ports: list[int]
231
+ ) -> list[tuple[str, int | None, str | None]]:
232
+ endpoints: list[tuple[str, int | None, str | None]] = []
233
+ for host in hosts:
234
+ if host == config.target and (config.port or config.scheme):
235
+ endpoints.append((host, config.port, config.scheme))
236
+ else:
237
+ endpoints.append((host, None, None))
238
+ # Open web ports found by the port scan get their own probe; 80 and 443
239
+ # are already covered by the default HTTPS-then-HTTP probe.
240
+ covered = {config.port} if config.port else {80, 443}
241
+ for port in open_web_ports:
242
+ if port not in covered:
243
+ endpoints.append((config.target, port, None))
244
+ return list(dict.fromkeys(endpoints))
245
+
246
+
247
+ def _tls_endpoints(
248
+ config: ScanConfig, report: ScanReport, open_web_ports: list[int]
249
+ ) -> list[tuple[str, int]]:
250
+ endpoints: list[tuple[str, int]] = []
251
+ for result in report.http:
252
+ if not result.reachable:
253
+ continue
254
+ for url in (result.url, result.final_url):
255
+ parts = urlsplit(url or "")
256
+ if parts.scheme == "https" and parts.hostname:
257
+ endpoints.append((parts.hostname, parts.port or 443))
258
+ break
259
+ if not endpoints:
260
+ endpoints.append((config.target, config.port or 443))
261
+ for port in open_web_ports:
262
+ if port in TLS_PORTS:
263
+ endpoints.append((config.target, port))
264
+ return list(dict.fromkeys(endpoints))