webrecon-cli 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- webrecon/__init__.py +7 -0
- webrecon/__main__.py +6 -0
- webrecon/cli.py +520 -0
- webrecon/config.py +63 -0
- webrecon/context.py +61 -0
- webrecon/data/subdomains-small.txt +140 -0
- webrecon/models.py +121 -0
- webrecon/modules/__init__.py +1 -0
- webrecon/modules/dns_enum.py +408 -0
- webrecon/modules/http_probe.py +407 -0
- webrecon/modules/port_scan.py +134 -0
- webrecon/modules/report.py +408 -0
- webrecon/modules/security_headers.py +69 -0
- webrecon/modules/tech_detect.py +125 -0
- webrecon/modules/tls_info.py +208 -0
- webrecon/py.typed +0 -0
- webrecon/ratelimit.py +61 -0
- webrecon/scanner.py +264 -0
- webrecon/targets.py +234 -0
- webrecon_cli-0.2.1.dist-info/METADATA +290 -0
- webrecon_cli-0.2.1.dist-info/RECORD +25 -0
- webrecon_cli-0.2.1.dist-info/WHEEL +5 -0
- webrecon_cli-0.2.1.dist-info/entry_points.txt +2 -0
- webrecon_cli-0.2.1.dist-info/licenses/LICENSE +21 -0
- webrecon_cli-0.2.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,407 @@
|
|
|
1
|
+
"""HTTP probing: reachability, redirect chain, title, headers, content discovery."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import re
|
|
7
|
+
import secrets
|
|
8
|
+
from collections.abc import Callable
|
|
9
|
+
from html.parser import HTMLParser
|
|
10
|
+
from urllib.parse import urljoin, urlsplit
|
|
11
|
+
|
|
12
|
+
import httpx
|
|
13
|
+
|
|
14
|
+
from ..context import ScanContext
|
|
15
|
+
from ..models import HttpResult, Redirect
|
|
16
|
+
from ..ratelimit import Throttle
|
|
17
|
+
from ..targets import Scope, format_netloc
|
|
18
|
+
from . import security_headers, tech_detect
|
|
19
|
+
|
|
20
|
+
MAX_BODY_BYTES = 256_000
|
|
21
|
+
MAX_TITLE_SCAN = 64_000
|
|
22
|
+
MAX_REDIRECTS = 5
|
|
23
|
+
|
|
24
|
+
# Paths worth reporting when present. Nothing here is an exploit attempt; these
|
|
25
|
+
# are files servers publish or commonly leave world-readable by mistake.
|
|
26
|
+
COMMON_PATHS = (
|
|
27
|
+
"/robots.txt",
|
|
28
|
+
"/sitemap.xml",
|
|
29
|
+
"/.well-known/security.txt",
|
|
30
|
+
"/.git/HEAD",
|
|
31
|
+
"/.env",
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
# Response headers whose values are credentials or session tokens.
|
|
35
|
+
SENSITIVE_HEADERS = ("set-cookie", "authorization", "proxy-authorization")
|
|
36
|
+
REDACTED = "<redacted>"
|
|
37
|
+
|
|
38
|
+
# Ports on which plain HTTP is the more likely protocol, so it is tried first.
|
|
39
|
+
PLAIN_HTTP_PORTS = frozenset({80, 3000, 5000, 8000, 8008, 8080, 8888})
|
|
40
|
+
|
|
41
|
+
_GIT_HEAD_RE = re.compile(r"^(?:ref: refs/\S+|[0-9a-f]{40}|[0-9a-f]{64})\s*$")
|
|
42
|
+
_ENV_RE = re.compile(r"^[ \t]*(?:export[ \t]+)?[A-Za-z_][A-Za-z0-9_]*[ \t]*=", re.MULTILINE)
|
|
43
|
+
_ROBOTS_RE = re.compile(
|
|
44
|
+
r"^[ \t]*(?:user-agent|disallow|allow|sitemap|crawl-delay)[ \t]*:", re.I | re.M
|
|
45
|
+
)
|
|
46
|
+
_SECURITY_TXT_RE = re.compile(r"^[ \t]*contact[ \t]*:", re.I | re.M)
|
|
47
|
+
_SITEMAP_RE = re.compile(r"<(?:[\w-]+:)?(?:urlset|sitemapindex)\b", re.I)
|
|
48
|
+
_HTML_RE = re.compile(r"<(?:!doctype\s+html|html|head|body)\b", re.I)
|
|
49
|
+
|
|
50
|
+
SkipCallback = Callable[[str, str], None]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class _TitleParser(HTMLParser):
|
|
54
|
+
"""Find the document title, ignoring <title> inside SVG, script and the like."""
|
|
55
|
+
|
|
56
|
+
SKIP = frozenset({"svg", "math", "script", "style", "template", "noscript"})
|
|
57
|
+
|
|
58
|
+
def __init__(self) -> None:
|
|
59
|
+
super().__init__(convert_charrefs=True)
|
|
60
|
+
self.skip_depth = 0
|
|
61
|
+
self.in_title = False
|
|
62
|
+
self.parts: list[str] = []
|
|
63
|
+
self.title: str | None = None
|
|
64
|
+
|
|
65
|
+
@property
|
|
66
|
+
def done(self) -> bool:
|
|
67
|
+
return self.title is not None
|
|
68
|
+
|
|
69
|
+
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
70
|
+
if self.done:
|
|
71
|
+
return
|
|
72
|
+
if tag in self.SKIP:
|
|
73
|
+
self.skip_depth += 1
|
|
74
|
+
elif tag == "title" and not self.skip_depth and not self.in_title:
|
|
75
|
+
self.in_title = True
|
|
76
|
+
self.parts = []
|
|
77
|
+
|
|
78
|
+
def handle_endtag(self, tag: str) -> None:
|
|
79
|
+
if self.done:
|
|
80
|
+
return
|
|
81
|
+
if tag in self.SKIP and self.skip_depth:
|
|
82
|
+
self.skip_depth -= 1
|
|
83
|
+
elif tag == "title" and self.in_title:
|
|
84
|
+
self.in_title = False
|
|
85
|
+
self.title = "".join(self.parts)
|
|
86
|
+
|
|
87
|
+
def handle_data(self, data: str) -> None:
|
|
88
|
+
if self.in_title and not self.done:
|
|
89
|
+
self.parts.append(data)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def extract_title(body: str) -> str | None:
|
|
93
|
+
"""Pull the document <title> text out of an HTML page, entities decoded."""
|
|
94
|
+
parser = _TitleParser()
|
|
95
|
+
try:
|
|
96
|
+
parser.feed(body[:MAX_TITLE_SCAN])
|
|
97
|
+
parser.close()
|
|
98
|
+
except (AssertionError, ValueError):
|
|
99
|
+
pass
|
|
100
|
+
raw = parser.title if parser.title is not None else ("".join(parser.parts) or None)
|
|
101
|
+
if raw is None:
|
|
102
|
+
return None
|
|
103
|
+
title = re.sub(r"\s+", " ", raw).strip()
|
|
104
|
+
return title[:200] or None
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def parse_robots(body: str) -> list[str]:
|
|
108
|
+
"""Return the paths named by Disallow/Allow/Sitemap directives in robots.txt."""
|
|
109
|
+
paths: list[str] = []
|
|
110
|
+
seen: set[str] = set()
|
|
111
|
+
for line in body.splitlines():
|
|
112
|
+
line = line.split("#", 1)[0].strip()
|
|
113
|
+
if not line:
|
|
114
|
+
continue
|
|
115
|
+
directive, _, value = line.partition(":")
|
|
116
|
+
if directive.strip().lower() not in {"disallow", "allow", "sitemap"}:
|
|
117
|
+
continue
|
|
118
|
+
value = value.strip()
|
|
119
|
+
if value and value != "/" and value not in seen:
|
|
120
|
+
seen.add(value)
|
|
121
|
+
paths.append(value)
|
|
122
|
+
return paths
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def redact_cookie(raw: str) -> str:
|
|
126
|
+
"""Replace a Set-Cookie value, keeping its name and attributes."""
|
|
127
|
+
pair, sep, attrs = raw.partition(";")
|
|
128
|
+
name = pair.split("=", 1)[0].strip()
|
|
129
|
+
return f"{name}={REDACTED}{sep}{attrs}"
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def collect_headers(headers: httpx.Headers, redact: bool = True) -> dict[str, str]:
|
|
133
|
+
"""Flatten response headers to a dict; repeated Set-Cookie values are newline-joined.
|
|
134
|
+
|
|
135
|
+
httpx joins repeated headers with ``", "``, which corrupts Set-Cookie
|
|
136
|
+
(``Expires`` dates contain commas), so each header is read with
|
|
137
|
+
``get_list``. Credentials and session tokens are redacted unless
|
|
138
|
+
``redact`` is False.
|
|
139
|
+
"""
|
|
140
|
+
out: dict[str, str] = {}
|
|
141
|
+
for name in headers.keys():
|
|
142
|
+
key = name.lower()
|
|
143
|
+
values = headers.get_list(key)
|
|
144
|
+
if redact and key == "set-cookie":
|
|
145
|
+
values = [redact_cookie(v) for v in values]
|
|
146
|
+
elif redact and key in SENSITIVE_HEADERS:
|
|
147
|
+
values = [REDACTED for _ in values]
|
|
148
|
+
out[key] = ("\n" if key == "set-cookie" else ", ").join(values)
|
|
149
|
+
return out
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _looks_like_html(body: str, content_type: str) -> bool:
|
|
153
|
+
return "html" in content_type or bool(_HTML_RE.search(body[:2000]))
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _looks_like_hit(response: httpx.Response, path: str, baseline: str | None = None) -> bool:
|
|
157
|
+
"""Guard against servers that answer 200 to everything with an error page.
|
|
158
|
+
|
|
159
|
+
Each path must have the shape of the real file, and a body identical to
|
|
160
|
+
the one served for a random non-existent path never counts.
|
|
161
|
+
"""
|
|
162
|
+
if response.status_code != 200 or not response.content:
|
|
163
|
+
return False
|
|
164
|
+
headers = getattr(response, "headers", None) or {}
|
|
165
|
+
content_type = str(headers.get("content-type", "")).lower()
|
|
166
|
+
body = response.text[:4000]
|
|
167
|
+
stripped = body.strip()
|
|
168
|
+
if baseline is not None and stripped == baseline.strip():
|
|
169
|
+
return False
|
|
170
|
+
html_like = _looks_like_html(body, content_type)
|
|
171
|
+
|
|
172
|
+
if path == "/.git/HEAD":
|
|
173
|
+
return bool(_GIT_HEAD_RE.match(stripped))
|
|
174
|
+
if path == "/.env":
|
|
175
|
+
return (
|
|
176
|
+
not html_like
|
|
177
|
+
and not stripped.startswith(("<", "{", "["))
|
|
178
|
+
and bool(_ENV_RE.search(body))
|
|
179
|
+
)
|
|
180
|
+
if path == "/robots.txt":
|
|
181
|
+
return not html_like and bool(_ROBOTS_RE.search(body))
|
|
182
|
+
if path.endswith("security.txt"):
|
|
183
|
+
return not html_like and bool(_SECURITY_TXT_RE.search(body))
|
|
184
|
+
if path.endswith(".xml"):
|
|
185
|
+
return bool(_SITEMAP_RE.search(body))
|
|
186
|
+
return not html_like
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
class RequestBudget:
|
|
190
|
+
"""Caps how many requests each host may receive; 0 means unlimited."""
|
|
191
|
+
|
|
192
|
+
def __init__(self, per_host: int = 0) -> None:
|
|
193
|
+
self.per_host = per_host
|
|
194
|
+
self._used: dict[str, int] = {}
|
|
195
|
+
|
|
196
|
+
def take(self, url: str) -> bool:
|
|
197
|
+
if self.per_host <= 0:
|
|
198
|
+
return True
|
|
199
|
+
host = urlsplit(url).netloc.lower()
|
|
200
|
+
used = self._used.get(host, 0)
|
|
201
|
+
if used >= self.per_host:
|
|
202
|
+
return False
|
|
203
|
+
self._used[host] = used + 1
|
|
204
|
+
return True
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
async def _get(
|
|
208
|
+
client: httpx.AsyncClient, url: str, throttle: Throttle, budget: RequestBudget | None
|
|
209
|
+
) -> httpx.Response | None:
|
|
210
|
+
if budget is not None and not budget.take(url):
|
|
211
|
+
return None
|
|
212
|
+
async with throttle:
|
|
213
|
+
return await client.get(url)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
async def probe_url(
|
|
217
|
+
client: httpx.AsyncClient,
|
|
218
|
+
url: str,
|
|
219
|
+
throttle: Throttle,
|
|
220
|
+
discover_paths: bool = True,
|
|
221
|
+
scope: Scope | None = None,
|
|
222
|
+
budget: RequestBudget | None = None,
|
|
223
|
+
redact: bool = True,
|
|
224
|
+
on_skip: SkipCallback | None = None,
|
|
225
|
+
) -> HttpResult:
|
|
226
|
+
"""Fetch a URL, following redirects manually so the chain is recorded.
|
|
227
|
+
|
|
228
|
+
Redirects to hosts outside ``scope`` are recorded but not followed; loops
|
|
229
|
+
and chains longer than MAX_REDIRECTS are reported as errors.
|
|
230
|
+
"""
|
|
231
|
+
result = HttpResult(url=url, reachable=False)
|
|
232
|
+
current = url
|
|
233
|
+
seen = {url}
|
|
234
|
+
|
|
235
|
+
try:
|
|
236
|
+
for _ in range(MAX_REDIRECTS + 1):
|
|
237
|
+
response = await _get(client, current, throttle, budget)
|
|
238
|
+
if response is None:
|
|
239
|
+
result.error = "per-host request budget exhausted"
|
|
240
|
+
return result
|
|
241
|
+
|
|
242
|
+
if not response.is_redirect:
|
|
243
|
+
break
|
|
244
|
+
location = response.headers.get("location", "")
|
|
245
|
+
result.redirects.append(Redirect(status=response.status_code, location=location))
|
|
246
|
+
if not location:
|
|
247
|
+
break
|
|
248
|
+
target = urljoin(current, location)
|
|
249
|
+
parts = urlsplit(target)
|
|
250
|
+
if parts.scheme not in ("http", "https"):
|
|
251
|
+
break
|
|
252
|
+
if scope is not None and parts.hostname:
|
|
253
|
+
reason = scope.refusal(parts.hostname)
|
|
254
|
+
if reason:
|
|
255
|
+
result.findings.append(
|
|
256
|
+
f"redirect to {parts.hostname} not followed ({reason})"
|
|
257
|
+
)
|
|
258
|
+
if on_skip:
|
|
259
|
+
on_skip(parts.hostname, f"{reason} (redirect target)")
|
|
260
|
+
break
|
|
261
|
+
if target in seen:
|
|
262
|
+
result.error = f"redirect loop at {target}"
|
|
263
|
+
return result
|
|
264
|
+
seen.add(target)
|
|
265
|
+
current = target
|
|
266
|
+
else:
|
|
267
|
+
result.error = "too many redirects"
|
|
268
|
+
return result
|
|
269
|
+
|
|
270
|
+
body = response.text[:MAX_BODY_BYTES] if response.content else ""
|
|
271
|
+
raw_cookies = response.headers.get_list("set-cookie")
|
|
272
|
+
|
|
273
|
+
result.reachable = True
|
|
274
|
+
result.status = response.status_code
|
|
275
|
+
result.final_url = str(response.url)
|
|
276
|
+
result.title = extract_title(body)
|
|
277
|
+
result.server = response.headers.get("server")
|
|
278
|
+
result.headers = collect_headers(response.headers, redact)
|
|
279
|
+
result.technologies = tech_detect.detect(result.headers, body, raw_cookies)
|
|
280
|
+
result.findings.extend(
|
|
281
|
+
security_headers.analyse(
|
|
282
|
+
result.headers, raw_cookies, https=result.final_url.startswith("https://")
|
|
283
|
+
)
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
length = response.headers.get("content-length")
|
|
287
|
+
result.content_length = (
|
|
288
|
+
int(length) if length and length.isascii() and length.isdigit()
|
|
289
|
+
else len(response.content)
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
if discover_paths:
|
|
293
|
+
result.interesting_paths = await discover_content(client, current, throttle, budget)
|
|
294
|
+
|
|
295
|
+
except httpx.HTTPError as exc:
|
|
296
|
+
result.error = f"{type(exc).__name__}: {exc}"
|
|
297
|
+
except (httpx.InvalidURL, UnicodeError, ValueError) as exc:
|
|
298
|
+
result.error = f"{type(exc).__name__}: {exc}"
|
|
299
|
+
|
|
300
|
+
return result
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
async def discover_content(
|
|
304
|
+
client: httpx.AsyncClient,
|
|
305
|
+
base_url: str,
|
|
306
|
+
throttle: Throttle,
|
|
307
|
+
budget: RequestBudget | None = None,
|
|
308
|
+
) -> list[str]:
|
|
309
|
+
"""Check the well-known paths and expand robots.txt into its listed paths.
|
|
310
|
+
|
|
311
|
+
A request for a random path first records what this server sends for
|
|
312
|
+
something that does not exist; identical answers are soft 404s.
|
|
313
|
+
"""
|
|
314
|
+
parsed = urlsplit(base_url)
|
|
315
|
+
root = f"{parsed.scheme}://{parsed.netloc}"
|
|
316
|
+
found: list[str] = []
|
|
317
|
+
|
|
318
|
+
async def fetch(path: str) -> httpx.Response | None:
|
|
319
|
+
try:
|
|
320
|
+
return await _get(client, root + path, throttle, budget)
|
|
321
|
+
except httpx.HTTPError:
|
|
322
|
+
return None
|
|
323
|
+
|
|
324
|
+
baseline_response = await fetch(f"/wr-{secrets.token_hex(8)}")
|
|
325
|
+
baseline = (
|
|
326
|
+
baseline_response.text[:4000]
|
|
327
|
+
if baseline_response is not None and baseline_response.status_code == 200
|
|
328
|
+
else None
|
|
329
|
+
)
|
|
330
|
+
|
|
331
|
+
responses = await asyncio.gather(*(fetch(p) for p in COMMON_PATHS))
|
|
332
|
+
for path, response in zip(COMMON_PATHS, responses, strict=True):
|
|
333
|
+
if response is None or not _looks_like_hit(response, path, baseline):
|
|
334
|
+
continue
|
|
335
|
+
found.append(path)
|
|
336
|
+
if path == "/robots.txt":
|
|
337
|
+
found.extend(f"robots: {p}" for p in parse_robots(response.text)[:20])
|
|
338
|
+
|
|
339
|
+
return found
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _schemes_for(port: int | None, scheme: str | None) -> list[str]:
|
|
343
|
+
if scheme:
|
|
344
|
+
return [scheme]
|
|
345
|
+
if port in PLAIN_HTTP_PORTS:
|
|
346
|
+
return ["http", "https"]
|
|
347
|
+
return ["https", "http"]
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
async def probe_hosts(
|
|
351
|
+
endpoints: list[tuple[str, int | None, str | None]], ctx: ScanContext
|
|
352
|
+
) -> list[HttpResult]:
|
|
353
|
+
"""Probe each (host, port, scheme) endpoint, trying HTTPS and HTTP as needed.
|
|
354
|
+
|
|
355
|
+
Without an explicit scheme, HTTPS is tried first except on ports where
|
|
356
|
+
plain HTTP is the norm. The first reachable answer wins.
|
|
357
|
+
"""
|
|
358
|
+
config = ctx.config
|
|
359
|
+
budget = RequestBudget(config.max_requests_per_host)
|
|
360
|
+
limits = httpx.Limits(max_connections=config.concurrency)
|
|
361
|
+
|
|
362
|
+
async with httpx.AsyncClient(
|
|
363
|
+
timeout=config.timeout,
|
|
364
|
+
follow_redirects=False,
|
|
365
|
+
verify=config.verify_tls,
|
|
366
|
+
limits=limits,
|
|
367
|
+
headers={"User-Agent": config.user_agent},
|
|
368
|
+
proxy=config.proxy,
|
|
369
|
+
trust_env=False,
|
|
370
|
+
) as client:
|
|
371
|
+
|
|
372
|
+
async def probe_endpoint(endpoint: tuple[str, int | None, str | None]) -> HttpResult:
|
|
373
|
+
host, port, scheme = endpoint
|
|
374
|
+
netloc = format_netloc(host, port)
|
|
375
|
+
first: HttpResult | None = None
|
|
376
|
+
for candidate in _schemes_for(port, scheme):
|
|
377
|
+
result = await probe_url(
|
|
378
|
+
client,
|
|
379
|
+
f"{candidate}://{netloc}",
|
|
380
|
+
ctx.throttle,
|
|
381
|
+
scope=ctx.scope,
|
|
382
|
+
budget=budget,
|
|
383
|
+
redact=config.redact,
|
|
384
|
+
on_skip=ctx.skip,
|
|
385
|
+
)
|
|
386
|
+
if result.reachable:
|
|
387
|
+
return result
|
|
388
|
+
first = first or result
|
|
389
|
+
assert first is not None
|
|
390
|
+
return first
|
|
391
|
+
|
|
392
|
+
async def guarded(endpoint: tuple[str, int | None, str | None]) -> HttpResult:
|
|
393
|
+
try:
|
|
394
|
+
result = await probe_endpoint(endpoint)
|
|
395
|
+
except Exception as exc: # one broken host must not end the scan
|
|
396
|
+
host, port, _ = endpoint
|
|
397
|
+
result = HttpResult(
|
|
398
|
+
url=format_netloc(host, port),
|
|
399
|
+
reachable=False,
|
|
400
|
+
error=f"{type(exc).__name__}: {exc}",
|
|
401
|
+
)
|
|
402
|
+
ctx.emit("http", result)
|
|
403
|
+
return result
|
|
404
|
+
|
|
405
|
+
results = await asyncio.gather(*(guarded(e) for e in endpoints))
|
|
406
|
+
|
|
407
|
+
return list(results)
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""TCP connect scanning of a small, explicit port list.
|
|
2
|
+
|
|
3
|
+
This is a connect() scan, not a stealth scan: it completes the handshake and
|
|
4
|
+
closes cleanly. Slower and fully logged by the target, which is the correct
|
|
5
|
+
behaviour for a tool run against systems you own or are authorised to test.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
import contextlib
|
|
12
|
+
import socket
|
|
13
|
+
from collections.abc import Callable
|
|
14
|
+
|
|
15
|
+
from ..models import PortResult
|
|
16
|
+
from ..ratelimit import Throttle
|
|
17
|
+
from ..targets import is_ip
|
|
18
|
+
|
|
19
|
+
COMMON_SERVICES = {
|
|
20
|
+
21: "ftp",
|
|
21
|
+
22: "ssh",
|
|
22
|
+
23: "telnet",
|
|
23
|
+
25: "smtp",
|
|
24
|
+
53: "dns",
|
|
25
|
+
80: "http",
|
|
26
|
+
110: "pop3",
|
|
27
|
+
143: "imap",
|
|
28
|
+
443: "https",
|
|
29
|
+
445: "smb",
|
|
30
|
+
3306: "mysql",
|
|
31
|
+
3389: "rdp",
|
|
32
|
+
5432: "postgresql",
|
|
33
|
+
6379: "redis",
|
|
34
|
+
8000: "http-alt",
|
|
35
|
+
8080: "http-proxy",
|
|
36
|
+
8443: "https-alt",
|
|
37
|
+
27017: "mongodb",
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
BANNER_BYTES = 256
|
|
41
|
+
BANNER_TIMEOUT = 2.0
|
|
42
|
+
BANNER_MAX_CHARS = 160
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class HostResolutionError(OSError):
|
|
46
|
+
"""The host to scan has no address; distinct from "every port closed"."""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def sanitise_banner(data: bytes) -> str | None:
|
|
50
|
+
"""Make untrusted banner bytes safe to display: printable text, bounded length."""
|
|
51
|
+
text = data[:BANNER_BYTES].decode("utf-8", errors="replace")
|
|
52
|
+
cleaned = "".join(ch if ch.isprintable() and ch != "�" else " " for ch in text)
|
|
53
|
+
cleaned = " ".join(cleaned.split())
|
|
54
|
+
if not cleaned:
|
|
55
|
+
return None
|
|
56
|
+
if len(cleaned) > BANNER_MAX_CHARS:
|
|
57
|
+
cleaned = cleaned[: BANNER_MAX_CHARS - 1] + "…"
|
|
58
|
+
return cleaned
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
async def resolve(host: str, timeout: float, throttle: Throttle) -> str:
|
|
62
|
+
"""Resolve the host once, so every port connects to the same address."""
|
|
63
|
+
if is_ip(host):
|
|
64
|
+
return host
|
|
65
|
+
loop = asyncio.get_running_loop()
|
|
66
|
+
try:
|
|
67
|
+
async with throttle:
|
|
68
|
+
infos = await asyncio.wait_for(
|
|
69
|
+
loop.getaddrinfo(host, None, type=socket.SOCK_STREAM), timeout
|
|
70
|
+
)
|
|
71
|
+
except (OSError, UnicodeError, ValueError, asyncio.TimeoutError) as exc:
|
|
72
|
+
detail = "timed out" if isinstance(exc, asyncio.TimeoutError) else str(exc)
|
|
73
|
+
raise HostResolutionError(f"cannot resolve {host}: {detail}") from exc
|
|
74
|
+
if not infos:
|
|
75
|
+
raise HostResolutionError(f"cannot resolve {host}: no addresses")
|
|
76
|
+
# Prefer IPv4, which is what most single-stack scanners can reach.
|
|
77
|
+
infos.sort(key=lambda info: info[0] != socket.AF_INET)
|
|
78
|
+
return str(infos[0][4][0])
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
async def _read_banner(reader: asyncio.StreamReader, timeout: float) -> str | None:
|
|
82
|
+
try:
|
|
83
|
+
data = await asyncio.wait_for(reader.read(BANNER_BYTES), timeout)
|
|
84
|
+
except (OSError, asyncio.TimeoutError):
|
|
85
|
+
return None
|
|
86
|
+
return sanitise_banner(data)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
async def check_port(
|
|
90
|
+
host: str, port: int, timeout: float, throttle: Throttle, banner: bool = False
|
|
91
|
+
) -> PortResult:
|
|
92
|
+
"""Attempt a TCP connection, reporting the port as open only on success."""
|
|
93
|
+
service = COMMON_SERVICES.get(port)
|
|
94
|
+
async with throttle:
|
|
95
|
+
writer = None
|
|
96
|
+
try:
|
|
97
|
+
reader, writer = await asyncio.wait_for(
|
|
98
|
+
asyncio.open_connection(host, port), timeout=timeout
|
|
99
|
+
)
|
|
100
|
+
text = await _read_banner(reader, min(timeout, BANNER_TIMEOUT)) if banner else None
|
|
101
|
+
return PortResult(port=port, open=True, service=service, banner=text)
|
|
102
|
+
except (asyncio.TimeoutError, OSError, UnicodeError, ValueError):
|
|
103
|
+
return PortResult(port=port, open=False, service=service)
|
|
104
|
+
finally:
|
|
105
|
+
if writer is not None:
|
|
106
|
+
writer.close()
|
|
107
|
+
with contextlib.suppress(Exception):
|
|
108
|
+
await asyncio.wait_for(writer.wait_closed(), 1.0)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
async def scan(
|
|
112
|
+
host: str,
|
|
113
|
+
ports: list[int],
|
|
114
|
+
timeout: float = 3.0,
|
|
115
|
+
throttle: Throttle | None = None,
|
|
116
|
+
banners: bool = False,
|
|
117
|
+
on_result: Callable[[PortResult], None] | None = None,
|
|
118
|
+
) -> list[PortResult]:
|
|
119
|
+
"""Scan the given ports on a host and return only the open ones.
|
|
120
|
+
|
|
121
|
+
Raises :class:`HostResolutionError` when the host has no address, so an
|
|
122
|
+
unresolvable host is reported as an error instead of "no open ports".
|
|
123
|
+
"""
|
|
124
|
+
throttle = throttle or Throttle(50)
|
|
125
|
+
address = await resolve(host, timeout, throttle)
|
|
126
|
+
|
|
127
|
+
async def one(port: int) -> PortResult:
|
|
128
|
+
result = await check_port(address, port, timeout, throttle, banners)
|
|
129
|
+
if result.open and on_result is not None:
|
|
130
|
+
on_result(result)
|
|
131
|
+
return result
|
|
132
|
+
|
|
133
|
+
results = await asyncio.gather(*(one(port) for port in ports))
|
|
134
|
+
return sorted((r for r in results if r.open), key=lambda r: r.port)
|