webrecon-cli 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,407 @@
1
+ """HTTP probing: reachability, redirect chain, title, headers, content discovery."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import re
7
+ import secrets
8
+ from collections.abc import Callable
9
+ from html.parser import HTMLParser
10
+ from urllib.parse import urljoin, urlsplit
11
+
12
+ import httpx
13
+
14
+ from ..context import ScanContext
15
+ from ..models import HttpResult, Redirect
16
+ from ..ratelimit import Throttle
17
+ from ..targets import Scope, format_netloc
18
+ from . import security_headers, tech_detect
19
+
20
+ MAX_BODY_BYTES = 256_000
21
+ MAX_TITLE_SCAN = 64_000
22
+ MAX_REDIRECTS = 5
23
+
24
+ # Paths worth reporting when present. Nothing here is an exploit attempt; these
25
+ # are files servers publish or commonly leave world-readable by mistake.
26
+ COMMON_PATHS = (
27
+ "/robots.txt",
28
+ "/sitemap.xml",
29
+ "/.well-known/security.txt",
30
+ "/.git/HEAD",
31
+ "/.env",
32
+ )
33
+
34
+ # Response headers whose values are credentials or session tokens.
35
+ SENSITIVE_HEADERS = ("set-cookie", "authorization", "proxy-authorization")
36
+ REDACTED = "<redacted>"
37
+
38
+ # Ports on which plain HTTP is the more likely protocol, so it is tried first.
39
+ PLAIN_HTTP_PORTS = frozenset({80, 3000, 5000, 8000, 8008, 8080, 8888})
40
+
41
+ _GIT_HEAD_RE = re.compile(r"^(?:ref: refs/\S+|[0-9a-f]{40}|[0-9a-f]{64})\s*$")
42
+ _ENV_RE = re.compile(r"^[ \t]*(?:export[ \t]+)?[A-Za-z_][A-Za-z0-9_]*[ \t]*=", re.MULTILINE)
43
+ _ROBOTS_RE = re.compile(
44
+ r"^[ \t]*(?:user-agent|disallow|allow|sitemap|crawl-delay)[ \t]*:", re.I | re.M
45
+ )
46
+ _SECURITY_TXT_RE = re.compile(r"^[ \t]*contact[ \t]*:", re.I | re.M)
47
+ _SITEMAP_RE = re.compile(r"<(?:[\w-]+:)?(?:urlset|sitemapindex)\b", re.I)
48
+ _HTML_RE = re.compile(r"<(?:!doctype\s+html|html|head|body)\b", re.I)
49
+
50
+ SkipCallback = Callable[[str, str], None]
51
+
52
+
53
+ class _TitleParser(HTMLParser):
54
+ """Find the document title, ignoring <title> inside SVG, script and the like."""
55
+
56
+ SKIP = frozenset({"svg", "math", "script", "style", "template", "noscript"})
57
+
58
+ def __init__(self) -> None:
59
+ super().__init__(convert_charrefs=True)
60
+ self.skip_depth = 0
61
+ self.in_title = False
62
+ self.parts: list[str] = []
63
+ self.title: str | None = None
64
+
65
+ @property
66
+ def done(self) -> bool:
67
+ return self.title is not None
68
+
69
+ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
70
+ if self.done:
71
+ return
72
+ if tag in self.SKIP:
73
+ self.skip_depth += 1
74
+ elif tag == "title" and not self.skip_depth and not self.in_title:
75
+ self.in_title = True
76
+ self.parts = []
77
+
78
+ def handle_endtag(self, tag: str) -> None:
79
+ if self.done:
80
+ return
81
+ if tag in self.SKIP and self.skip_depth:
82
+ self.skip_depth -= 1
83
+ elif tag == "title" and self.in_title:
84
+ self.in_title = False
85
+ self.title = "".join(self.parts)
86
+
87
+ def handle_data(self, data: str) -> None:
88
+ if self.in_title and not self.done:
89
+ self.parts.append(data)
90
+
91
+
92
+ def extract_title(body: str) -> str | None:
93
+ """Pull the document <title> text out of an HTML page, entities decoded."""
94
+ parser = _TitleParser()
95
+ try:
96
+ parser.feed(body[:MAX_TITLE_SCAN])
97
+ parser.close()
98
+ except (AssertionError, ValueError):
99
+ pass
100
+ raw = parser.title if parser.title is not None else ("".join(parser.parts) or None)
101
+ if raw is None:
102
+ return None
103
+ title = re.sub(r"\s+", " ", raw).strip()
104
+ return title[:200] or None
105
+
106
+
107
+ def parse_robots(body: str) -> list[str]:
108
+ """Return the paths named by Disallow/Allow/Sitemap directives in robots.txt."""
109
+ paths: list[str] = []
110
+ seen: set[str] = set()
111
+ for line in body.splitlines():
112
+ line = line.split("#", 1)[0].strip()
113
+ if not line:
114
+ continue
115
+ directive, _, value = line.partition(":")
116
+ if directive.strip().lower() not in {"disallow", "allow", "sitemap"}:
117
+ continue
118
+ value = value.strip()
119
+ if value and value != "/" and value not in seen:
120
+ seen.add(value)
121
+ paths.append(value)
122
+ return paths
123
+
124
+
125
+ def redact_cookie(raw: str) -> str:
126
+ """Replace a Set-Cookie value, keeping its name and attributes."""
127
+ pair, sep, attrs = raw.partition(";")
128
+ name = pair.split("=", 1)[0].strip()
129
+ return f"{name}={REDACTED}{sep}{attrs}"
130
+
131
+
132
+ def collect_headers(headers: httpx.Headers, redact: bool = True) -> dict[str, str]:
133
+ """Flatten response headers to a dict; repeated Set-Cookie values are newline-joined.
134
+
135
+ httpx joins repeated headers with ``", "``, which corrupts Set-Cookie
136
+ (``Expires`` dates contain commas), so each header is read with
137
+ ``get_list``. Credentials and session tokens are redacted unless
138
+ ``redact`` is False.
139
+ """
140
+ out: dict[str, str] = {}
141
+ for name in headers.keys():
142
+ key = name.lower()
143
+ values = headers.get_list(key)
144
+ if redact and key == "set-cookie":
145
+ values = [redact_cookie(v) for v in values]
146
+ elif redact and key in SENSITIVE_HEADERS:
147
+ values = [REDACTED for _ in values]
148
+ out[key] = ("\n" if key == "set-cookie" else ", ").join(values)
149
+ return out
150
+
151
+
152
+ def _looks_like_html(body: str, content_type: str) -> bool:
153
+ return "html" in content_type or bool(_HTML_RE.search(body[:2000]))
154
+
155
+
156
+ def _looks_like_hit(response: httpx.Response, path: str, baseline: str | None = None) -> bool:
157
+ """Guard against servers that answer 200 to everything with an error page.
158
+
159
+ Each path must have the shape of the real file, and a body identical to
160
+ the one served for a random non-existent path never counts.
161
+ """
162
+ if response.status_code != 200 or not response.content:
163
+ return False
164
+ headers = getattr(response, "headers", None) or {}
165
+ content_type = str(headers.get("content-type", "")).lower()
166
+ body = response.text[:4000]
167
+ stripped = body.strip()
168
+ if baseline is not None and stripped == baseline.strip():
169
+ return False
170
+ html_like = _looks_like_html(body, content_type)
171
+
172
+ if path == "/.git/HEAD":
173
+ return bool(_GIT_HEAD_RE.match(stripped))
174
+ if path == "/.env":
175
+ return (
176
+ not html_like
177
+ and not stripped.startswith(("<", "{", "["))
178
+ and bool(_ENV_RE.search(body))
179
+ )
180
+ if path == "/robots.txt":
181
+ return not html_like and bool(_ROBOTS_RE.search(body))
182
+ if path.endswith("security.txt"):
183
+ return not html_like and bool(_SECURITY_TXT_RE.search(body))
184
+ if path.endswith(".xml"):
185
+ return bool(_SITEMAP_RE.search(body))
186
+ return not html_like
187
+
188
+
189
+ class RequestBudget:
190
+ """Caps how many requests each host may receive; 0 means unlimited."""
191
+
192
+ def __init__(self, per_host: int = 0) -> None:
193
+ self.per_host = per_host
194
+ self._used: dict[str, int] = {}
195
+
196
+ def take(self, url: str) -> bool:
197
+ if self.per_host <= 0:
198
+ return True
199
+ host = urlsplit(url).netloc.lower()
200
+ used = self._used.get(host, 0)
201
+ if used >= self.per_host:
202
+ return False
203
+ self._used[host] = used + 1
204
+ return True
205
+
206
+
207
+ async def _get(
208
+ client: httpx.AsyncClient, url: str, throttle: Throttle, budget: RequestBudget | None
209
+ ) -> httpx.Response | None:
210
+ if budget is not None and not budget.take(url):
211
+ return None
212
+ async with throttle:
213
+ return await client.get(url)
214
+
215
+
216
+ async def probe_url(
217
+ client: httpx.AsyncClient,
218
+ url: str,
219
+ throttle: Throttle,
220
+ discover_paths: bool = True,
221
+ scope: Scope | None = None,
222
+ budget: RequestBudget | None = None,
223
+ redact: bool = True,
224
+ on_skip: SkipCallback | None = None,
225
+ ) -> HttpResult:
226
+ """Fetch a URL, following redirects manually so the chain is recorded.
227
+
228
+ Redirects to hosts outside ``scope`` are recorded but not followed; loops
229
+ and chains longer than MAX_REDIRECTS are reported as errors.
230
+ """
231
+ result = HttpResult(url=url, reachable=False)
232
+ current = url
233
+ seen = {url}
234
+
235
+ try:
236
+ for _ in range(MAX_REDIRECTS + 1):
237
+ response = await _get(client, current, throttle, budget)
238
+ if response is None:
239
+ result.error = "per-host request budget exhausted"
240
+ return result
241
+
242
+ if not response.is_redirect:
243
+ break
244
+ location = response.headers.get("location", "")
245
+ result.redirects.append(Redirect(status=response.status_code, location=location))
246
+ if not location:
247
+ break
248
+ target = urljoin(current, location)
249
+ parts = urlsplit(target)
250
+ if parts.scheme not in ("http", "https"):
251
+ break
252
+ if scope is not None and parts.hostname:
253
+ reason = scope.refusal(parts.hostname)
254
+ if reason:
255
+ result.findings.append(
256
+ f"redirect to {parts.hostname} not followed ({reason})"
257
+ )
258
+ if on_skip:
259
+ on_skip(parts.hostname, f"{reason} (redirect target)")
260
+ break
261
+ if target in seen:
262
+ result.error = f"redirect loop at {target}"
263
+ return result
264
+ seen.add(target)
265
+ current = target
266
+ else:
267
+ result.error = "too many redirects"
268
+ return result
269
+
270
+ body = response.text[:MAX_BODY_BYTES] if response.content else ""
271
+ raw_cookies = response.headers.get_list("set-cookie")
272
+
273
+ result.reachable = True
274
+ result.status = response.status_code
275
+ result.final_url = str(response.url)
276
+ result.title = extract_title(body)
277
+ result.server = response.headers.get("server")
278
+ result.headers = collect_headers(response.headers, redact)
279
+ result.technologies = tech_detect.detect(result.headers, body, raw_cookies)
280
+ result.findings.extend(
281
+ security_headers.analyse(
282
+ result.headers, raw_cookies, https=result.final_url.startswith("https://")
283
+ )
284
+ )
285
+
286
+ length = response.headers.get("content-length")
287
+ result.content_length = (
288
+ int(length) if length and length.isascii() and length.isdigit()
289
+ else len(response.content)
290
+ )
291
+
292
+ if discover_paths:
293
+ result.interesting_paths = await discover_content(client, current, throttle, budget)
294
+
295
+ except httpx.HTTPError as exc:
296
+ result.error = f"{type(exc).__name__}: {exc}"
297
+ except (httpx.InvalidURL, UnicodeError, ValueError) as exc:
298
+ result.error = f"{type(exc).__name__}: {exc}"
299
+
300
+ return result
301
+
302
+
303
+ async def discover_content(
304
+ client: httpx.AsyncClient,
305
+ base_url: str,
306
+ throttle: Throttle,
307
+ budget: RequestBudget | None = None,
308
+ ) -> list[str]:
309
+ """Check the well-known paths and expand robots.txt into its listed paths.
310
+
311
+ A request for a random path first records what this server sends for
312
+ something that does not exist; identical answers are soft 404s.
313
+ """
314
+ parsed = urlsplit(base_url)
315
+ root = f"{parsed.scheme}://{parsed.netloc}"
316
+ found: list[str] = []
317
+
318
+ async def fetch(path: str) -> httpx.Response | None:
319
+ try:
320
+ return await _get(client, root + path, throttle, budget)
321
+ except httpx.HTTPError:
322
+ return None
323
+
324
+ baseline_response = await fetch(f"/wr-{secrets.token_hex(8)}")
325
+ baseline = (
326
+ baseline_response.text[:4000]
327
+ if baseline_response is not None and baseline_response.status_code == 200
328
+ else None
329
+ )
330
+
331
+ responses = await asyncio.gather(*(fetch(p) for p in COMMON_PATHS))
332
+ for path, response in zip(COMMON_PATHS, responses, strict=True):
333
+ if response is None or not _looks_like_hit(response, path, baseline):
334
+ continue
335
+ found.append(path)
336
+ if path == "/robots.txt":
337
+ found.extend(f"robots: {p}" for p in parse_robots(response.text)[:20])
338
+
339
+ return found
340
+
341
+
342
+ def _schemes_for(port: int | None, scheme: str | None) -> list[str]:
343
+ if scheme:
344
+ return [scheme]
345
+ if port in PLAIN_HTTP_PORTS:
346
+ return ["http", "https"]
347
+ return ["https", "http"]
348
+
349
+
350
+ async def probe_hosts(
351
+ endpoints: list[tuple[str, int | None, str | None]], ctx: ScanContext
352
+ ) -> list[HttpResult]:
353
+ """Probe each (host, port, scheme) endpoint, trying HTTPS and HTTP as needed.
354
+
355
+ Without an explicit scheme, HTTPS is tried first except on ports where
356
+ plain HTTP is the norm. The first reachable answer wins.
357
+ """
358
+ config = ctx.config
359
+ budget = RequestBudget(config.max_requests_per_host)
360
+ limits = httpx.Limits(max_connections=config.concurrency)
361
+
362
+ async with httpx.AsyncClient(
363
+ timeout=config.timeout,
364
+ follow_redirects=False,
365
+ verify=config.verify_tls,
366
+ limits=limits,
367
+ headers={"User-Agent": config.user_agent},
368
+ proxy=config.proxy,
369
+ trust_env=False,
370
+ ) as client:
371
+
372
+ async def probe_endpoint(endpoint: tuple[str, int | None, str | None]) -> HttpResult:
373
+ host, port, scheme = endpoint
374
+ netloc = format_netloc(host, port)
375
+ first: HttpResult | None = None
376
+ for candidate in _schemes_for(port, scheme):
377
+ result = await probe_url(
378
+ client,
379
+ f"{candidate}://{netloc}",
380
+ ctx.throttle,
381
+ scope=ctx.scope,
382
+ budget=budget,
383
+ redact=config.redact,
384
+ on_skip=ctx.skip,
385
+ )
386
+ if result.reachable:
387
+ return result
388
+ first = first or result
389
+ assert first is not None
390
+ return first
391
+
392
+ async def guarded(endpoint: tuple[str, int | None, str | None]) -> HttpResult:
393
+ try:
394
+ result = await probe_endpoint(endpoint)
395
+ except Exception as exc: # one broken host must not end the scan
396
+ host, port, _ = endpoint
397
+ result = HttpResult(
398
+ url=format_netloc(host, port),
399
+ reachable=False,
400
+ error=f"{type(exc).__name__}: {exc}",
401
+ )
402
+ ctx.emit("http", result)
403
+ return result
404
+
405
+ results = await asyncio.gather(*(guarded(e) for e in endpoints))
406
+
407
+ return list(results)
@@ -0,0 +1,134 @@
1
+ """TCP connect scanning of a small, explicit port list.
2
+
3
+ This is a connect() scan, not a stealth scan: it completes the handshake and
4
+ closes cleanly. Slower and fully logged by the target, which is the correct
5
+ behaviour for a tool run against systems you own or are authorised to test.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import asyncio
11
+ import contextlib
12
+ import socket
13
+ from collections.abc import Callable
14
+
15
+ from ..models import PortResult
16
+ from ..ratelimit import Throttle
17
+ from ..targets import is_ip
18
+
19
+ COMMON_SERVICES = {
20
+ 21: "ftp",
21
+ 22: "ssh",
22
+ 23: "telnet",
23
+ 25: "smtp",
24
+ 53: "dns",
25
+ 80: "http",
26
+ 110: "pop3",
27
+ 143: "imap",
28
+ 443: "https",
29
+ 445: "smb",
30
+ 3306: "mysql",
31
+ 3389: "rdp",
32
+ 5432: "postgresql",
33
+ 6379: "redis",
34
+ 8000: "http-alt",
35
+ 8080: "http-proxy",
36
+ 8443: "https-alt",
37
+ 27017: "mongodb",
38
+ }
39
+
40
+ BANNER_BYTES = 256
41
+ BANNER_TIMEOUT = 2.0
42
+ BANNER_MAX_CHARS = 160
43
+
44
+
45
+ class HostResolutionError(OSError):
46
+ """The host to scan has no address; distinct from "every port closed"."""
47
+
48
+
49
+ def sanitise_banner(data: bytes) -> str | None:
50
+ """Make untrusted banner bytes safe to display: printable text, bounded length."""
51
+ text = data[:BANNER_BYTES].decode("utf-8", errors="replace")
52
+ cleaned = "".join(ch if ch.isprintable() and ch != "�" else " " for ch in text)
53
+ cleaned = " ".join(cleaned.split())
54
+ if not cleaned:
55
+ return None
56
+ if len(cleaned) > BANNER_MAX_CHARS:
57
+ cleaned = cleaned[: BANNER_MAX_CHARS - 1] + "…"
58
+ return cleaned
59
+
60
+
61
+ async def resolve(host: str, timeout: float, throttle: Throttle) -> str:
62
+ """Resolve the host once, so every port connects to the same address."""
63
+ if is_ip(host):
64
+ return host
65
+ loop = asyncio.get_running_loop()
66
+ try:
67
+ async with throttle:
68
+ infos = await asyncio.wait_for(
69
+ loop.getaddrinfo(host, None, type=socket.SOCK_STREAM), timeout
70
+ )
71
+ except (OSError, UnicodeError, ValueError, asyncio.TimeoutError) as exc:
72
+ detail = "timed out" if isinstance(exc, asyncio.TimeoutError) else str(exc)
73
+ raise HostResolutionError(f"cannot resolve {host}: {detail}") from exc
74
+ if not infos:
75
+ raise HostResolutionError(f"cannot resolve {host}: no addresses")
76
+ # Prefer IPv4, which is what most single-stack scanners can reach.
77
+ infos.sort(key=lambda info: info[0] != socket.AF_INET)
78
+ return str(infos[0][4][0])
79
+
80
+
81
+ async def _read_banner(reader: asyncio.StreamReader, timeout: float) -> str | None:
82
+ try:
83
+ data = await asyncio.wait_for(reader.read(BANNER_BYTES), timeout)
84
+ except (OSError, asyncio.TimeoutError):
85
+ return None
86
+ return sanitise_banner(data)
87
+
88
+
89
+ async def check_port(
90
+ host: str, port: int, timeout: float, throttle: Throttle, banner: bool = False
91
+ ) -> PortResult:
92
+ """Attempt a TCP connection, reporting the port as open only on success."""
93
+ service = COMMON_SERVICES.get(port)
94
+ async with throttle:
95
+ writer = None
96
+ try:
97
+ reader, writer = await asyncio.wait_for(
98
+ asyncio.open_connection(host, port), timeout=timeout
99
+ )
100
+ text = await _read_banner(reader, min(timeout, BANNER_TIMEOUT)) if banner else None
101
+ return PortResult(port=port, open=True, service=service, banner=text)
102
+ except (asyncio.TimeoutError, OSError, UnicodeError, ValueError):
103
+ return PortResult(port=port, open=False, service=service)
104
+ finally:
105
+ if writer is not None:
106
+ writer.close()
107
+ with contextlib.suppress(Exception):
108
+ await asyncio.wait_for(writer.wait_closed(), 1.0)
109
+
110
+
111
+ async def scan(
112
+ host: str,
113
+ ports: list[int],
114
+ timeout: float = 3.0,
115
+ throttle: Throttle | None = None,
116
+ banners: bool = False,
117
+ on_result: Callable[[PortResult], None] | None = None,
118
+ ) -> list[PortResult]:
119
+ """Scan the given ports on a host and return only the open ones.
120
+
121
+ Raises :class:`HostResolutionError` when the host has no address, so an
122
+ unresolvable host is reported as an error instead of "no open ports".
123
+ """
124
+ throttle = throttle or Throttle(50)
125
+ address = await resolve(host, timeout, throttle)
126
+
127
+ async def one(port: int) -> PortResult:
128
+ result = await check_port(address, port, timeout, throttle, banners)
129
+ if result.open and on_result is not None:
130
+ on_result(result)
131
+ return result
132
+
133
+ results = await asyncio.gather(*(one(port) for port in ports))
134
+ return sorted((r for r in results if r.open), key=lambda r: r.port)