webrecon-cli 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- webrecon/__init__.py +7 -0
- webrecon/__main__.py +6 -0
- webrecon/cli.py +520 -0
- webrecon/config.py +63 -0
- webrecon/context.py +61 -0
- webrecon/data/subdomains-small.txt +140 -0
- webrecon/models.py +121 -0
- webrecon/modules/__init__.py +1 -0
- webrecon/modules/dns_enum.py +408 -0
- webrecon/modules/http_probe.py +407 -0
- webrecon/modules/port_scan.py +134 -0
- webrecon/modules/report.py +408 -0
- webrecon/modules/security_headers.py +69 -0
- webrecon/modules/tech_detect.py +125 -0
- webrecon/modules/tls_info.py +208 -0
- webrecon/py.typed +0 -0
- webrecon/ratelimit.py +61 -0
- webrecon/scanner.py +264 -0
- webrecon/targets.py +234 -0
- webrecon_cli-0.2.1.dist-info/METADATA +290 -0
- webrecon_cli-0.2.1.dist-info/RECORD +25 -0
- webrecon_cli-0.2.1.dist-info/WHEEL +5 -0
- webrecon_cli-0.2.1.dist-info/entry_points.txt +2 -0
- webrecon_cli-0.2.1.dist-info/licenses/LICENSE +21 -0
- webrecon_cli-0.2.1.dist-info/top_level.txt +1 -0
webrecon/__init__.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""webrecon - a small web reconnaissance scanner."""
|
|
2
|
+
|
|
3
|
+
# Single source of truth for the version: pyproject.toml reads this attribute
|
|
4
|
+
# at build time (``[tool.setuptools.dynamic]``) and config.USER_AGENT embeds it.
|
|
5
|
+
__version__ = "0.2.1"
|
|
6
|
+
|
|
7
|
+
__all__ = ["__version__"]
|
webrecon/__main__.py
ADDED
webrecon/cli.py
ADDED
|
@@ -0,0 +1,520 @@
|
|
|
1
|
+
"""Command line interface."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import asyncio
|
|
7
|
+
import getpass
|
|
8
|
+
import ipaddress
|
|
9
|
+
import re
|
|
10
|
+
import sys
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
from datetime import datetime, timezone
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
from urllib.parse import urlsplit
|
|
16
|
+
|
|
17
|
+
from rich.console import Console
|
|
18
|
+
|
|
19
|
+
from . import __version__
|
|
20
|
+
from .config import DEFAULT_PORTS, USER_AGENT, ScanConfig, bundled_wordlist
|
|
21
|
+
from .models import ScanReport
|
|
22
|
+
from .modules import report as report_module
|
|
23
|
+
from .scanner import ACTIVE_MODULES, run_scan
|
|
24
|
+
from .targets import Scope, Target, TargetError, parse_target
|
|
25
|
+
|
|
26
|
+
ALL_MODULES = ("dns", "subdomains", "http", "tls", "ports")
|
|
27
|
+
|
|
28
|
+
BANNER = "webrecon - passive/active reconnaissance for authorised targets only"
|
|
29
|
+
|
|
30
|
+
EXIT_OK = 0
|
|
31
|
+
EXIT_ERROR = 1
|
|
32
|
+
EXIT_USAGE = 2
|
|
33
|
+
EXIT_NOTHING_REACHABLE = 3
|
|
34
|
+
EXIT_INTERRUPTED = 130
|
|
35
|
+
|
|
36
|
+
MAX_PORT = 65535
|
|
37
|
+
_DIGITS = re.compile(r"^[0-9]{1,5}$")
|
|
38
|
+
_NUMBER = re.compile(r"^-?[0-9]+(?:\.[0-9]+)?$")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _port_number(text: str, chunk: str) -> int:
|
|
42
|
+
text = text.strip()
|
|
43
|
+
if not _DIGITS.match(text):
|
|
44
|
+
raise argparse.ArgumentTypeError(f"invalid port: {chunk}")
|
|
45
|
+
port = int(text)
|
|
46
|
+
if not 1 <= port <= MAX_PORT:
|
|
47
|
+
raise argparse.ArgumentTypeError(f"ports must be between 1 and {MAX_PORT}: {chunk}")
|
|
48
|
+
return port
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def parse_ports(value: str) -> list[int]:
|
|
52
|
+
"""Parse a port specification such as ``80,443,8000-8100``.
|
|
53
|
+
|
|
54
|
+
Only ASCII digits are accepted, and bounds are checked before a range is
|
|
55
|
+
expanded, so ``1-30000000`` fails immediately instead of allocating.
|
|
56
|
+
"""
|
|
57
|
+
ports: set[int] = set()
|
|
58
|
+
for chunk in value.split(","):
|
|
59
|
+
chunk = chunk.strip()
|
|
60
|
+
if not chunk:
|
|
61
|
+
continue
|
|
62
|
+
if "-" in chunk:
|
|
63
|
+
start, _, end = chunk.partition("-")
|
|
64
|
+
lo, hi = _port_number(start, chunk), _port_number(end, chunk)
|
|
65
|
+
if lo > hi:
|
|
66
|
+
raise argparse.ArgumentTypeError(f"invalid port range: {chunk}")
|
|
67
|
+
ports.update(range(lo, hi + 1))
|
|
68
|
+
else:
|
|
69
|
+
ports.add(_port_number(chunk, chunk))
|
|
70
|
+
if not ports:
|
|
71
|
+
raise argparse.ArgumentTypeError("no ports given")
|
|
72
|
+
return sorted(ports)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _number(value: str, *, minimum: float, inclusive: bool, integer: bool) -> Any:
|
|
76
|
+
text = value.strip()
|
|
77
|
+
if not _NUMBER.match(text) or (integer and "." in text):
|
|
78
|
+
kind = "an integer" if integer else "a number"
|
|
79
|
+
raise argparse.ArgumentTypeError(f"{value!r} is not {kind}")
|
|
80
|
+
number: float = int(text) if integer else float(text)
|
|
81
|
+
if number < minimum or (not inclusive and number == minimum):
|
|
82
|
+
relation = ">=" if inclusive else ">"
|
|
83
|
+
raise argparse.ArgumentTypeError(f"must be {relation} {minimum:g}, got {value}")
|
|
84
|
+
return number
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def positive_float(value: str) -> float:
|
|
88
|
+
return float(_number(value, minimum=0, inclusive=False, integer=False))
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def non_negative_float(value: str) -> float:
|
|
92
|
+
return float(_number(value, minimum=0, inclusive=True, integer=False))
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def positive_int(value: str) -> int:
|
|
96
|
+
return int(_number(value, minimum=1, inclusive=True, integer=True))
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def non_negative_int(value: str) -> int:
|
|
100
|
+
return int(_number(value, minimum=0, inclusive=True, integer=True))
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def normalise_target(value: str) -> str:
|
|
104
|
+
"""Reduce a domain, IP, ``host:port`` or URL to the bare host the modules use."""
|
|
105
|
+
try:
|
|
106
|
+
return parse_target(value).host
|
|
107
|
+
except TargetError as exc:
|
|
108
|
+
raise argparse.ArgumentTypeError(str(exc)) from exc
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def proxy_url(value: str) -> str:
|
|
112
|
+
parts = urlsplit(value)
|
|
113
|
+
if parts.scheme not in ("http", "https") or not parts.hostname:
|
|
114
|
+
raise argparse.ArgumentTypeError("proxy must be an http:// or https:// URL")
|
|
115
|
+
return value
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def ip_literal(value: str) -> str:
|
|
119
|
+
try:
|
|
120
|
+
return ipaddress.ip_address(value.strip()).compressed
|
|
121
|
+
except ValueError as exc:
|
|
122
|
+
raise argparse.ArgumentTypeError(f"not an IP address: {value}") from exc
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
126
|
+
parser = argparse.ArgumentParser(
|
|
127
|
+
prog="webrecon",
|
|
128
|
+
description=BANNER,
|
|
129
|
+
epilog=(
|
|
130
|
+
"Only scan systems you own or have written permission to test. "
|
|
131
|
+
"Exit codes: 0 success, 1 runtime error, 2 usage error or authorization not "
|
|
132
|
+
"confirmed, 3 nothing reachable (with --fail-if-unreachable), 130 interrupted."
|
|
133
|
+
),
|
|
134
|
+
)
|
|
135
|
+
parser.add_argument("-V", "--version", action="version", version=f"%(prog)s {__version__}")
|
|
136
|
+
parser.add_argument(
|
|
137
|
+
"targets",
|
|
138
|
+
nargs="*",
|
|
139
|
+
metavar="target",
|
|
140
|
+
help="domain, IP, host:port or URL to scan (several allowed)",
|
|
141
|
+
)
|
|
142
|
+
parser.add_argument(
|
|
143
|
+
"-iL", "--input-list", type=Path, metavar="FILE",
|
|
144
|
+
help="read targets from FILE, one per line ('#' comments allowed)",
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
scan = parser.add_argument_group("scan")
|
|
148
|
+
scan.add_argument(
|
|
149
|
+
"-m",
|
|
150
|
+
"--modules",
|
|
151
|
+
default="dns,subdomains,http,tls",
|
|
152
|
+
help=f"comma-separated modules to run ({', '.join(ALL_MODULES)}, or 'all')",
|
|
153
|
+
)
|
|
154
|
+
scan.add_argument(
|
|
155
|
+
"-w", "--wordlist", type=Path,
|
|
156
|
+
help="subdomain wordlist for brute-force (default: the bundled list)",
|
|
157
|
+
)
|
|
158
|
+
scan.add_argument(
|
|
159
|
+
"--no-wordlist", action="store_true", help="skip subdomain brute-force entirely"
|
|
160
|
+
)
|
|
161
|
+
scan.add_argument(
|
|
162
|
+
"-p", "--ports", type=parse_ports, default=None,
|
|
163
|
+
help="ports to scan, e.g. 80,443,8000-8100 (default: a list of 15 common ports)",
|
|
164
|
+
)
|
|
165
|
+
scan.add_argument(
|
|
166
|
+
"--max-subdomains", type=non_negative_int, default=200,
|
|
167
|
+
help="cap on how many candidate subdomains are resolved, 0 for no cap (default 200)",
|
|
168
|
+
)
|
|
169
|
+
scan.add_argument(
|
|
170
|
+
"--no-passive", action="store_true",
|
|
171
|
+
help="skip certificate transparency lookups (the target name is otherwise sent to crt.sh)",
|
|
172
|
+
)
|
|
173
|
+
scan.add_argument(
|
|
174
|
+
"--banners", action="store_true",
|
|
175
|
+
help="read a short, sanitised banner from each open port",
|
|
176
|
+
)
|
|
177
|
+
scan.add_argument("--axfr", action="store_true", help="attempt a DNS zone transfer")
|
|
178
|
+
scan.add_argument(
|
|
179
|
+
"--resolver", action="append", type=ip_literal, default=[], metavar="IP",
|
|
180
|
+
help="DNS server to query instead of the system resolver (repeatable)",
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
pacing = parser.add_argument_group("pacing and budgets")
|
|
184
|
+
pacing.add_argument(
|
|
185
|
+
"-c", "--concurrency", type=positive_int, default=20,
|
|
186
|
+
help="max concurrent requests (default 20)",
|
|
187
|
+
)
|
|
188
|
+
pacing.add_argument(
|
|
189
|
+
"-r", "--rate", type=non_negative_float, default=0.0,
|
|
190
|
+
help="max requests per second across all modules, 0 disables pacing (default 0)",
|
|
191
|
+
)
|
|
192
|
+
pacing.add_argument(
|
|
193
|
+
"-t", "--timeout", type=positive_float, default=5.0,
|
|
194
|
+
help="per-request timeout in seconds (default 5)",
|
|
195
|
+
)
|
|
196
|
+
pacing.add_argument(
|
|
197
|
+
"--max-time", type=positive_float, metavar="SECONDS",
|
|
198
|
+
help="stop the scan of each target after this long and report what was found",
|
|
199
|
+
)
|
|
200
|
+
pacing.add_argument(
|
|
201
|
+
"--max-requests-per-host", type=non_negative_int, default=0, metavar="N",
|
|
202
|
+
help="HTTP request budget per host, 0 for unlimited (default 0)",
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
safety = parser.add_argument_group("scope and authorization")
|
|
206
|
+
safety.add_argument(
|
|
207
|
+
"--scope", action="append", default=[], metavar="ENTRIES",
|
|
208
|
+
help="comma-separated domains and CIDRs that may be contacted (repeatable; "
|
|
209
|
+
"default: the target and its subdomains)",
|
|
210
|
+
)
|
|
211
|
+
safety.add_argument(
|
|
212
|
+
"--scope-file", type=Path, metavar="FILE", help="read scope entries from FILE"
|
|
213
|
+
)
|
|
214
|
+
safety.add_argument(
|
|
215
|
+
"--allow-private", action="store_true",
|
|
216
|
+
help="allow loopback, private and link-local addresses",
|
|
217
|
+
)
|
|
218
|
+
safety.add_argument(
|
|
219
|
+
"--i-have-authorization", action="store_true",
|
|
220
|
+
help="confirm you are authorised to test the targets (otherwise asked interactively)",
|
|
221
|
+
)
|
|
222
|
+
safety.add_argument(
|
|
223
|
+
"--operator", help="name recorded in the report as the person running the scan"
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
http = parser.add_argument_group("HTTP")
|
|
227
|
+
http.add_argument(
|
|
228
|
+
"--verify-tls", action="store_true",
|
|
229
|
+
help="verify certificates on HTTP probes (off by default so bad certificates "
|
|
230
|
+
"do not hide a host; the TLS module always reports verification status)",
|
|
231
|
+
)
|
|
232
|
+
http.add_argument(
|
|
233
|
+
"--proxy", type=proxy_url,
|
|
234
|
+
help="HTTP(S) proxy for HTTP probes and crt.sh; DNS, TLS and ports still go direct. "
|
|
235
|
+
"Environment proxy variables are ignored.",
|
|
236
|
+
)
|
|
237
|
+
http.add_argument("--user-agent", default=USER_AGENT, help="User-Agent for HTTP requests")
|
|
238
|
+
http.add_argument(
|
|
239
|
+
"--no-redact", action="store_true",
|
|
240
|
+
help="keep Set-Cookie and Authorization values in reports (redacted by default)",
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
output = parser.add_argument_group("output")
|
|
244
|
+
output.add_argument("-o", "--output", type=Path, help="write a JSON report to this path")
|
|
245
|
+
output.add_argument("--html", type=Path, help="write an HTML report to this path")
|
|
246
|
+
output.add_argument(
|
|
247
|
+
"--jsonl", type=Path, help="append each result to this JSONL file as it is found"
|
|
248
|
+
)
|
|
249
|
+
output.add_argument(
|
|
250
|
+
"--json", action="store_true", help="print the JSON report to stdout (for piping)"
|
|
251
|
+
)
|
|
252
|
+
output.add_argument("--force", action="store_true", help="overwrite existing report files")
|
|
253
|
+
output.add_argument("-q", "--quiet", action="store_true", help="suppress terminal report")
|
|
254
|
+
output.add_argument(
|
|
255
|
+
"--fail-if-unreachable", action="store_true",
|
|
256
|
+
help="exit with status 3 when no target answered",
|
|
257
|
+
)
|
|
258
|
+
return parser
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def resolve_modules(value: str) -> set[str]:
|
|
262
|
+
requested = {m.strip().lower() for m in value.split(",") if m.strip()}
|
|
263
|
+
if "all" in requested:
|
|
264
|
+
return set(ALL_MODULES)
|
|
265
|
+
unknown = requested - set(ALL_MODULES)
|
|
266
|
+
if unknown:
|
|
267
|
+
raise SystemExit(
|
|
268
|
+
f"unknown module(s): {', '.join(sorted(unknown))}. "
|
|
269
|
+
f"Valid modules: {', '.join(ALL_MODULES)}"
|
|
270
|
+
)
|
|
271
|
+
if not requested:
|
|
272
|
+
raise SystemExit("no modules selected")
|
|
273
|
+
return requested
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _read_lines(path: Path) -> list[str]:
|
|
277
|
+
text = path.read_bytes().decode("utf-8-sig", errors="replace")
|
|
278
|
+
return [s for s in (line.split("#", 1)[0].strip() for line in text.splitlines()) if s]
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _check_output(parser: argparse.ArgumentParser, path: Path | None, force: bool) -> None:
|
|
282
|
+
if path is None:
|
|
283
|
+
return
|
|
284
|
+
if path.is_dir():
|
|
285
|
+
parser.error(f"{path} is a directory")
|
|
286
|
+
if not path.parent.is_dir():
|
|
287
|
+
parser.error(f"directory {path.parent} does not exist (for {path})")
|
|
288
|
+
if path.exists() and not force:
|
|
289
|
+
parser.error(f"{path} already exists; use --force to overwrite it")
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _confirm(
|
|
293
|
+
console: Console, targets: list[Target], scope_entries: list[str], modules: set[str],
|
|
294
|
+
passive: bool,
|
|
295
|
+
) -> bool:
|
|
296
|
+
console.print("[bold]Active scan confirmation[/bold]")
|
|
297
|
+
console.print(f" targets: {', '.join(str(t) for t in targets)}")
|
|
298
|
+
shown_scope = ", ".join(scope_entries) or "each target and its subdomains"
|
|
299
|
+
console.print(f" scope: {shown_scope}")
|
|
300
|
+
console.print(f" modules: {', '.join(sorted(modules))}")
|
|
301
|
+
if passive and "subdomains" in modules:
|
|
302
|
+
console.print(" note: target names are sent to crt.sh (disable with --no-passive)")
|
|
303
|
+
answer = console.input("Type 'yes' if you are authorised to test these targets: ")
|
|
304
|
+
return answer.strip().lower() == "yes"
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _operator(name: str | None) -> str:
|
|
308
|
+
if name:
|
|
309
|
+
return name
|
|
310
|
+
try:
|
|
311
|
+
return getpass.getuser()
|
|
312
|
+
except Exception:
|
|
313
|
+
return "unknown"
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _warn(console: Console, message: str) -> None:
|
|
317
|
+
console.print(f"[yellow]warning:[/yellow] {message}")
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def main(argv: list[str] | None = None) -> int:
|
|
321
|
+
parser = build_parser()
|
|
322
|
+
args = parser.parse_args(argv)
|
|
323
|
+
console = Console(stderr=True)
|
|
324
|
+
|
|
325
|
+
# ---- targets
|
|
326
|
+
raw_targets = list(args.targets)
|
|
327
|
+
if args.input_list:
|
|
328
|
+
if not args.input_list.is_file():
|
|
329
|
+
parser.error(f"target list not found: {args.input_list}")
|
|
330
|
+
raw_targets += _read_lines(args.input_list)
|
|
331
|
+
if not raw_targets:
|
|
332
|
+
parser.error("no target given")
|
|
333
|
+
targets: list[Target] = []
|
|
334
|
+
for raw in raw_targets:
|
|
335
|
+
try:
|
|
336
|
+
target = parse_target(raw)
|
|
337
|
+
except TargetError as exc:
|
|
338
|
+
parser.error(f"invalid target {raw!r}: {exc}")
|
|
339
|
+
if target not in targets:
|
|
340
|
+
targets.append(target)
|
|
341
|
+
|
|
342
|
+
try:
|
|
343
|
+
modules = resolve_modules(args.modules)
|
|
344
|
+
except SystemExit as exc:
|
|
345
|
+
parser.error(str(exc))
|
|
346
|
+
|
|
347
|
+
# ---- scope
|
|
348
|
+
scope_entries = [e for e in args.scope]
|
|
349
|
+
if args.scope_file:
|
|
350
|
+
if not args.scope_file.is_file():
|
|
351
|
+
parser.error(f"scope file not found: {args.scope_file}")
|
|
352
|
+
scope_entries += _read_lines(args.scope_file)
|
|
353
|
+
try:
|
|
354
|
+
for target in targets:
|
|
355
|
+
Scope.build(target.host, scope_entries, args.allow_private)
|
|
356
|
+
except TargetError as exc:
|
|
357
|
+
parser.error(f"invalid scope entry: {exc}")
|
|
358
|
+
|
|
359
|
+
# ---- wordlist
|
|
360
|
+
wordlist: Any = None
|
|
361
|
+
if args.wordlist is not None:
|
|
362
|
+
if not args.wordlist.is_file():
|
|
363
|
+
parser.error(f"wordlist not found or not a file: {args.wordlist}")
|
|
364
|
+
wordlist = args.wordlist
|
|
365
|
+
elif not args.no_wordlist:
|
|
366
|
+
wordlist = bundled_wordlist()
|
|
367
|
+
|
|
368
|
+
# ---- outputs
|
|
369
|
+
for path in (args.output, args.html, args.jsonl):
|
|
370
|
+
_check_output(parser, path, args.force)
|
|
371
|
+
|
|
372
|
+
# ---- options that would silently do nothing
|
|
373
|
+
if args.ports is not None and "ports" not in modules:
|
|
374
|
+
_warn(console, "-p/--ports has no effect without the ports module (-m ports)")
|
|
375
|
+
if (args.wordlist or args.no_wordlist) and "subdomains" not in modules:
|
|
376
|
+
_warn(console, "wordlist options have no effect without the subdomains module")
|
|
377
|
+
if args.banners and "ports" not in modules:
|
|
378
|
+
_warn(console, "--banners has no effect without the ports module")
|
|
379
|
+
if args.axfr and "dns" not in modules:
|
|
380
|
+
_warn(console, "--axfr has no effect without the dns module")
|
|
381
|
+
if args.quiet and not (args.output or args.html or args.jsonl or args.json):
|
|
382
|
+
_warn(console, "-q without -o, --html, --jsonl or --json produces no report")
|
|
383
|
+
|
|
384
|
+
# ---- authorization gate
|
|
385
|
+
authorization: dict[str, Any] | None = None
|
|
386
|
+
if modules & ACTIVE_MODULES:
|
|
387
|
+
if args.i_have_authorization:
|
|
388
|
+
method = "flag"
|
|
389
|
+
elif sys.stdin.isatty():
|
|
390
|
+
if not _confirm(console, targets, scope_entries, modules, not args.no_passive):
|
|
391
|
+
console.print("[red]not confirmed; nothing was scanned[/red]")
|
|
392
|
+
return EXIT_USAGE
|
|
393
|
+
method = "interactive"
|
|
394
|
+
else:
|
|
395
|
+
parser.error(
|
|
396
|
+
"active modules need confirmation: pass --i-have-authorization "
|
|
397
|
+
"or run interactively"
|
|
398
|
+
)
|
|
399
|
+
authorization = {
|
|
400
|
+
"operator": _operator(args.operator),
|
|
401
|
+
"confirmed_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
|
402
|
+
"method": method,
|
|
403
|
+
"scope": scope_entries or ["each target and its subdomains"],
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
# ---- scan
|
|
407
|
+
show_progress = not args.quiet and not args.json
|
|
408
|
+
|
|
409
|
+
def on_stage(message: str) -> None:
|
|
410
|
+
if show_progress:
|
|
411
|
+
console.print(f"[dim]->[/dim] {message}")
|
|
412
|
+
|
|
413
|
+
jsonl = report_module.JsonlWriter(args.jsonl, args.force) if args.jsonl else None
|
|
414
|
+
reports: list[ScanReport] = []
|
|
415
|
+
exit_code = EXIT_OK
|
|
416
|
+
try:
|
|
417
|
+
for target in targets:
|
|
418
|
+
report = ScanReport(target=target.host)
|
|
419
|
+
report.authorization = authorization
|
|
420
|
+
reports.append(report)
|
|
421
|
+
if len(targets) > 1:
|
|
422
|
+
on_stage(f"[bold]{target}[/bold]")
|
|
423
|
+
config = ScanConfig(
|
|
424
|
+
target=target.host,
|
|
425
|
+
port=target.port,
|
|
426
|
+
scheme=target.scheme,
|
|
427
|
+
wordlist=wordlist,
|
|
428
|
+
concurrency=args.concurrency,
|
|
429
|
+
rate_per_second=args.rate,
|
|
430
|
+
timeout=args.timeout,
|
|
431
|
+
modules=modules,
|
|
432
|
+
ports=args.ports or list(DEFAULT_PORTS),
|
|
433
|
+
passive=not args.no_passive,
|
|
434
|
+
max_subdomains=args.max_subdomains,
|
|
435
|
+
user_agent=args.user_agent,
|
|
436
|
+
verify_tls=args.verify_tls,
|
|
437
|
+
proxy=args.proxy,
|
|
438
|
+
scope=scope_entries,
|
|
439
|
+
allow_private=args.allow_private,
|
|
440
|
+
max_time=args.max_time,
|
|
441
|
+
max_requests_per_host=args.max_requests_per_host,
|
|
442
|
+
banners=args.banners,
|
|
443
|
+
axfr=args.axfr,
|
|
444
|
+
redact=not args.no_redact,
|
|
445
|
+
nameservers=args.resolver,
|
|
446
|
+
)
|
|
447
|
+
emit = _jsonl_emitter(jsonl, target.host) if jsonl is not None else None
|
|
448
|
+
asyncio.run(run_scan(config, on_stage, report=report, emit=emit))
|
|
449
|
+
except KeyboardInterrupt:
|
|
450
|
+
console.print("[yellow]interrupted; writing a partial report[/yellow]")
|
|
451
|
+
_mark_unfinished(reports, "interrupted by the operator", jsonl)
|
|
452
|
+
exit_code = EXIT_INTERRUPTED
|
|
453
|
+
except Exception as exc: # last resort: never lose what was gathered
|
|
454
|
+
console.print(f"[red]scan crashed:[/red] {type(exc).__name__}: {exc}")
|
|
455
|
+
_mark_unfinished(reports, f"scan crashed: {type(exc).__name__}: {exc}", jsonl)
|
|
456
|
+
exit_code = EXIT_ERROR
|
|
457
|
+
finally:
|
|
458
|
+
if jsonl is not None:
|
|
459
|
+
jsonl.close()
|
|
460
|
+
|
|
461
|
+
# ---- output
|
|
462
|
+
if not args.quiet and not args.json:
|
|
463
|
+
report_module.render_terminal(reports, Console())
|
|
464
|
+
if args.json:
|
|
465
|
+
payload = reports[0] if len(reports) == 1 else reports
|
|
466
|
+
sys.stdout.write(report_module.to_json(payload) + "\n")
|
|
467
|
+
|
|
468
|
+
single: ScanReport | list[ScanReport] = reports[0] if len(reports) == 1 else reports
|
|
469
|
+
try:
|
|
470
|
+
if args.output:
|
|
471
|
+
report_module.write_json(single, args.output, args.force)
|
|
472
|
+
console.print(f"[green]JSON report written to[/green] {args.output}")
|
|
473
|
+
if args.html:
|
|
474
|
+
report_module.write_html(single, args.html, args.force)
|
|
475
|
+
console.print(f"[green]HTML report written to[/green] {args.html}")
|
|
476
|
+
except OSError as exc:
|
|
477
|
+
console.print(f"[red]could not write report:[/red] {exc}")
|
|
478
|
+
return EXIT_ERROR
|
|
479
|
+
|
|
480
|
+
if exit_code == EXIT_OK and args.fail_if_unreachable:
|
|
481
|
+
if not any(_reachable(r, modules) for r in reports):
|
|
482
|
+
exit_code = EXIT_NOTHING_REACHABLE
|
|
483
|
+
return exit_code
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def _jsonl_emitter(
|
|
487
|
+
writer: report_module.JsonlWriter, host: str
|
|
488
|
+
) -> Callable[[str, Any], None]:
|
|
489
|
+
def emit(kind: str, payload: Any) -> None:
|
|
490
|
+
writer.write(host, kind, payload)
|
|
491
|
+
|
|
492
|
+
return emit
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def _reachable(report: ScanReport, modules: set[str]) -> bool:
|
|
496
|
+
if modules & ACTIVE_MODULES - {"subdomains"}:
|
|
497
|
+
return report.anything_reachable()
|
|
498
|
+
if "subdomains" in modules:
|
|
499
|
+
return bool(report.subdomains)
|
|
500
|
+
return bool(report.dns and report.dns.records)
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def _mark_unfinished(
|
|
504
|
+
reports: list[ScanReport], reason: str, jsonl: report_module.JsonlWriter | None
|
|
505
|
+
) -> None:
|
|
506
|
+
now = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
507
|
+
for report in reports:
|
|
508
|
+
if report.complete:
|
|
509
|
+
continue
|
|
510
|
+
report.incomplete_reason = report.incomplete_reason or reason
|
|
511
|
+
report.finished_at = report.finished_at or now
|
|
512
|
+
if jsonl is not None:
|
|
513
|
+
jsonl.write(
|
|
514
|
+
report.target, "summary",
|
|
515
|
+
{"complete": False, "incomplete_reason": report.incomplete_reason},
|
|
516
|
+
)
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
if __name__ == "__main__":
|
|
520
|
+
sys.exit(main())
|
webrecon/config.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Runtime configuration for a scan."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import sys
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from importlib import resources
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import TYPE_CHECKING
|
|
10
|
+
|
|
11
|
+
from . import __version__
|
|
12
|
+
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
if sys.version_info >= (3, 11):
|
|
15
|
+
from importlib.resources.abc import Traversable
|
|
16
|
+
else:
|
|
17
|
+
from importlib.abc import Traversable
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
DEFAULT_PORTS = [21, 22, 25, 53, 80, 110, 143, 443, 445, 3306, 3389, 5432, 8000, 8080, 8443]
|
|
21
|
+
|
|
22
|
+
# Ports on which an open TCP port is worth an HTTP(S) probe and, for the TLS
|
|
23
|
+
# ones, a certificate inspection.
|
|
24
|
+
WEB_PORTS = (80, 443, 3000, 5000, 8000, 8008, 8080, 8443, 8888, 9443)
|
|
25
|
+
|
|
26
|
+
USER_AGENT = f"webrecon/{__version__} (+https://github.com/Just-Ipavon/web-recon)"
|
|
27
|
+
|
|
28
|
+
BUNDLED_WORDLIST_NAME = "subdomains-small.txt"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def bundled_wordlist() -> Traversable:
|
|
32
|
+
"""The wordlist shipped inside the package, usable from any install."""
|
|
33
|
+
return resources.files("webrecon") / "data" / BUNDLED_WORDLIST_NAME
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass
|
|
37
|
+
class ScanConfig:
|
|
38
|
+
target: str
|
|
39
|
+
port: int | None = None
|
|
40
|
+
scheme: str | None = None
|
|
41
|
+
wordlist: Path | Traversable | None = None
|
|
42
|
+
concurrency: int = 20
|
|
43
|
+
rate_per_second: float = 0.0
|
|
44
|
+
timeout: float = 5.0
|
|
45
|
+
modules: set[str] = field(default_factory=lambda: {"dns", "subdomains", "http", "tls"})
|
|
46
|
+
ports: list[int] = field(default_factory=lambda: list(DEFAULT_PORTS))
|
|
47
|
+
passive: bool = True
|
|
48
|
+
max_subdomains: int = 200
|
|
49
|
+
user_agent: str = USER_AGENT
|
|
50
|
+
verify_tls: bool = False
|
|
51
|
+
proxy: str | None = None
|
|
52
|
+
scope: list[str] = field(default_factory=list)
|
|
53
|
+
allow_private: bool = False
|
|
54
|
+
max_time: float | None = None
|
|
55
|
+
max_requests_per_host: int = 0
|
|
56
|
+
banners: bool = False
|
|
57
|
+
axfr: bool = False
|
|
58
|
+
redact: bool = True
|
|
59
|
+
nameservers: list[str] = field(default_factory=list)
|
|
60
|
+
dns_port: int = 53
|
|
61
|
+
|
|
62
|
+
def enabled(self, module: str) -> bool:
|
|
63
|
+
return module in self.modules
|
webrecon/context.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Shared state handed to every module during one scan."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from .config import ScanConfig
|
|
10
|
+
from .models import ScanReport, SkippedHost
|
|
11
|
+
from .ratelimit import Throttle
|
|
12
|
+
from .targets import Scope
|
|
13
|
+
|
|
14
|
+
Emit = Callable[[str, Any], None]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _discard(kind: str, payload: Any) -> None:
|
|
18
|
+
return None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class ScanContext:
|
|
23
|
+
"""One throttle, one scope and one report shared by every stage.
|
|
24
|
+
|
|
25
|
+
``emit`` receives each result as soon as it exists (used for incremental
|
|
26
|
+
JSONL output); the report itself is filled in place, so whatever was
|
|
27
|
+
gathered survives a timeout, an interrupt or a crash in a later stage.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
config: ScanConfig
|
|
31
|
+
throttle: Throttle
|
|
32
|
+
scope: Scope
|
|
33
|
+
report: ScanReport
|
|
34
|
+
emit: Emit = field(default=_discard)
|
|
35
|
+
|
|
36
|
+
@classmethod
|
|
37
|
+
def create(
|
|
38
|
+
cls, config: ScanConfig, report: ScanReport | None = None, emit: Emit | None = None
|
|
39
|
+
) -> ScanContext:
|
|
40
|
+
return cls(
|
|
41
|
+
config=config,
|
|
42
|
+
throttle=Throttle(config.concurrency, config.rate_per_second),
|
|
43
|
+
scope=Scope.build(config.target, config.scope, config.allow_private),
|
|
44
|
+
report=report or ScanReport(target=config.target),
|
|
45
|
+
emit=emit or _discard,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
def error(self, message: str) -> None:
|
|
49
|
+
self.report.errors.append(message)
|
|
50
|
+
self.emit("error", message)
|
|
51
|
+
|
|
52
|
+
def note(self, message: str) -> None:
|
|
53
|
+
self.report.notes.append(message)
|
|
54
|
+
self.emit("note", message)
|
|
55
|
+
|
|
56
|
+
def skip(self, host: str, reason: str) -> None:
|
|
57
|
+
if any(s.host == host for s in self.report.skipped):
|
|
58
|
+
return
|
|
59
|
+
entry = SkippedHost(host=host, reason=reason)
|
|
60
|
+
self.report.skipped.append(entry)
|
|
61
|
+
self.emit("skipped", entry)
|