webrecon-cli 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- webrecon/__init__.py +7 -0
- webrecon/__main__.py +6 -0
- webrecon/cli.py +520 -0
- webrecon/config.py +63 -0
- webrecon/context.py +61 -0
- webrecon/data/subdomains-small.txt +140 -0
- webrecon/models.py +121 -0
- webrecon/modules/__init__.py +1 -0
- webrecon/modules/dns_enum.py +408 -0
- webrecon/modules/http_probe.py +407 -0
- webrecon/modules/port_scan.py +134 -0
- webrecon/modules/report.py +408 -0
- webrecon/modules/security_headers.py +69 -0
- webrecon/modules/tech_detect.py +125 -0
- webrecon/modules/tls_info.py +208 -0
- webrecon/py.typed +0 -0
- webrecon/ratelimit.py +61 -0
- webrecon/scanner.py +264 -0
- webrecon/targets.py +234 -0
- webrecon_cli-0.2.1.dist-info/METADATA +290 -0
- webrecon_cli-0.2.1.dist-info/RECORD +25 -0
- webrecon_cli-0.2.1.dist-info/WHEEL +5 -0
- webrecon_cli-0.2.1.dist-info/entry_points.txt +2 -0
- webrecon_cli-0.2.1.dist-info/licenses/LICENSE +21 -0
- webrecon_cli-0.2.1.dist-info/top_level.txt +1 -0
webrecon/targets.py
ADDED
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"""Target parsing and scope enforcement.
|
|
2
|
+
|
|
3
|
+
Every host the tool contacts is checked against one :class:`Scope`, so the
|
|
4
|
+
perimeter of a scan is explicit and recorded in the report instead of being
|
|
5
|
+
whatever names happen to turn up in certificate logs or redirects.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import ipaddress
|
|
11
|
+
import re
|
|
12
|
+
from collections.abc import Iterable
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from urllib.parse import urlsplit
|
|
15
|
+
|
|
16
|
+
IPAddress = ipaddress.IPv4Address | ipaddress.IPv6Address
|
|
17
|
+
IPNetwork = ipaddress.IPv4Network | ipaddress.IPv6Network
|
|
18
|
+
|
|
19
|
+
_LABEL_RE = re.compile(r"^(?!-)[a-z0-9-]{1,63}(?<!-)$")
|
|
20
|
+
|
|
21
|
+
# Registry-controlled suffixes under which unrelated parties register names.
|
|
22
|
+
# This is not the full Public Suffix List, only the common multi-label entries
|
|
23
|
+
# that a typo or a careless invocation is likely to hit; every single-label
|
|
24
|
+
# name (``com``, ``uk``) is rejected separately.
|
|
25
|
+
PUBLIC_SUFFIXES = frozenset(
|
|
26
|
+
{
|
|
27
|
+
"ac.uk", "co.uk", "gov.uk", "ltd.uk", "me.uk", "net.uk", "org.uk", "plc.uk",
|
|
28
|
+
"com.au", "net.au", "org.au", "edu.au", "gov.au",
|
|
29
|
+
"co.nz", "org.nz", "co.jp", "ne.jp", "or.jp", "co.kr", "or.kr",
|
|
30
|
+
"co.in", "net.in", "org.in", "co.za", "org.za", "co.il",
|
|
31
|
+
"com.br", "net.br", "org.br", "com.ar", "com.mx", "com.tr", "com.cn",
|
|
32
|
+
"net.cn", "org.cn", "com.hk", "com.sg", "com.tw", "com.ua",
|
|
33
|
+
"github.io", "gitlab.io", "herokuapp.com", "appspot.com", "blogspot.com",
|
|
34
|
+
"cloudfront.net", "azurewebsites.net", "netlify.app", "vercel.app",
|
|
35
|
+
"pages.dev", "workers.dev", "web.app", "firebaseapp.com",
|
|
36
|
+
}
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class TargetError(ValueError):
|
|
41
|
+
"""Raised for input that is not a usable scan target."""
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def is_valid_hostname(name: str) -> bool:
|
|
45
|
+
"""Strict LDH check on an ASCII (punycode) hostname, without a trailing dot."""
|
|
46
|
+
if not name or len(name) > 253:
|
|
47
|
+
return False
|
|
48
|
+
return all(_LABEL_RE.match(label) for label in name.split("."))
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def is_ip(value: str) -> bool:
|
|
52
|
+
try:
|
|
53
|
+
ipaddress.ip_address(value)
|
|
54
|
+
except ValueError:
|
|
55
|
+
return False
|
|
56
|
+
return True
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def to_ascii_hostname(name: str) -> str:
|
|
60
|
+
"""Lowercase and convert an internationalised name to punycode.
|
|
61
|
+
|
|
62
|
+
Raises :class:`TargetError` when a label is empty or longer than 63
|
|
63
|
+
characters, or when the result is not a valid hostname.
|
|
64
|
+
"""
|
|
65
|
+
name = name.strip().lower().rstrip(".")
|
|
66
|
+
try:
|
|
67
|
+
ascii_name = name.encode("idna").decode("ascii").lower()
|
|
68
|
+
except UnicodeError as exc:
|
|
69
|
+
raise TargetError(f"invalid hostname {name!r}: {exc}") from exc
|
|
70
|
+
if not is_valid_hostname(ascii_name):
|
|
71
|
+
raise TargetError(f"invalid hostname {name!r}")
|
|
72
|
+
return ascii_name
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass(frozen=True)
|
|
76
|
+
class Target:
|
|
77
|
+
"""A parsed target: hostname or IP literal, plus an optional port and scheme."""
|
|
78
|
+
|
|
79
|
+
host: str
|
|
80
|
+
port: int | None = None
|
|
81
|
+
scheme: str | None = None
|
|
82
|
+
|
|
83
|
+
@property
|
|
84
|
+
def is_ip(self) -> bool:
|
|
85
|
+
return is_ip(self.host)
|
|
86
|
+
|
|
87
|
+
def __str__(self) -> str:
|
|
88
|
+
return format_netloc(self.host, self.port)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def format_netloc(host: str, port: int | None = None) -> str:
|
|
92
|
+
shown = f"[{host}]" if ":" in host else host
|
|
93
|
+
return f"{shown}:{port}" if port else shown
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def parse_target(value: str) -> Target:
|
|
97
|
+
"""Parse a domain, IP literal, ``host:port`` or URL into a :class:`Target`.
|
|
98
|
+
|
|
99
|
+
Scheme, credentials, path, query and fragment are dropped; the port is
|
|
100
|
+
kept. IDNs become punycode, bare and bracketed IPv6 literals are accepted,
|
|
101
|
+
and anything that is not a valid hostname, as well as a public suffix such
|
|
102
|
+
as ``com`` or ``co.uk``, is rejected.
|
|
103
|
+
"""
|
|
104
|
+
raw = value.strip()
|
|
105
|
+
if not raw:
|
|
106
|
+
raise TargetError("empty target")
|
|
107
|
+
|
|
108
|
+
bare = raw[1:-1] if raw.startswith("[") and raw.endswith("]") else raw
|
|
109
|
+
if is_ip(bare):
|
|
110
|
+
return Target(host=ipaddress.ip_address(bare).compressed)
|
|
111
|
+
|
|
112
|
+
try:
|
|
113
|
+
parts = urlsplit(raw if "://" in raw else "//" + raw)
|
|
114
|
+
port = parts.port
|
|
115
|
+
except ValueError as exc:
|
|
116
|
+
raise TargetError(f"invalid target {raw!r}: {exc}") from exc
|
|
117
|
+
|
|
118
|
+
scheme = parts.scheme.lower() or None
|
|
119
|
+
if scheme not in (None, "http", "https"):
|
|
120
|
+
raise TargetError(f"unsupported scheme {scheme!r}; use http or https")
|
|
121
|
+
if port == 0:
|
|
122
|
+
raise TargetError("port must be between 1 and 65535")
|
|
123
|
+
|
|
124
|
+
host = (parts.hostname or "").strip(".")
|
|
125
|
+
if not host:
|
|
126
|
+
raise TargetError("empty target")
|
|
127
|
+
if is_ip(host):
|
|
128
|
+
return Target(host=ipaddress.ip_address(host).compressed, port=port, scheme=scheme)
|
|
129
|
+
|
|
130
|
+
ascii_host = to_ascii_hostname(host)
|
|
131
|
+
if "." not in ascii_host and ascii_host != "localhost":
|
|
132
|
+
raise TargetError(
|
|
133
|
+
f"{ascii_host!r} is a single label (a top-level domain?); give a full domain name"
|
|
134
|
+
)
|
|
135
|
+
if ascii_host in PUBLIC_SUFFIXES:
|
|
136
|
+
raise TargetError(f"{ascii_host!r} is a public suffix, not a domain you can own")
|
|
137
|
+
return Target(host=ascii_host, port=port, scheme=scheme)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def is_restricted_address(address: IPAddress) -> bool:
|
|
141
|
+
"""Loopback, private, link-local and other non-public addresses."""
|
|
142
|
+
if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped:
|
|
143
|
+
address = address.ipv4_mapped
|
|
144
|
+
return (
|
|
145
|
+
address.is_private
|
|
146
|
+
or address.is_loopback
|
|
147
|
+
or address.is_link_local
|
|
148
|
+
or address.is_unspecified
|
|
149
|
+
or address.is_reserved
|
|
150
|
+
or address.is_multicast
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
@dataclass
|
|
155
|
+
class Scope:
|
|
156
|
+
"""The set of names and networks a scan may contact.
|
|
157
|
+
|
|
158
|
+
A hostname is in scope when it equals or is a subdomain of one of
|
|
159
|
+
``domains``; an IP literal when it falls inside one of ``networks``.
|
|
160
|
+
Independently, non-public addresses are refused unless
|
|
161
|
+
``allow_private`` is set.
|
|
162
|
+
"""
|
|
163
|
+
|
|
164
|
+
domains: tuple[str, ...] = ()
|
|
165
|
+
networks: tuple[IPNetwork, ...] = ()
|
|
166
|
+
allow_private: bool = False
|
|
167
|
+
explicit: bool = field(default=False, compare=False)
|
|
168
|
+
|
|
169
|
+
@classmethod
|
|
170
|
+
def build(
|
|
171
|
+
cls, target: str, entries: Iterable[str] = (), allow_private: bool = False
|
|
172
|
+
) -> Scope:
|
|
173
|
+
"""Scope from explicit entries, defaulting to the target and its subdomains."""
|
|
174
|
+
domains: list[str] = []
|
|
175
|
+
networks: list[IPNetwork] = []
|
|
176
|
+
explicit = False
|
|
177
|
+
for entry in entries:
|
|
178
|
+
for item in entry.split(","):
|
|
179
|
+
item = item.strip()
|
|
180
|
+
if not item or item.startswith("#"):
|
|
181
|
+
continue
|
|
182
|
+
explicit = True
|
|
183
|
+
try:
|
|
184
|
+
networks.append(ipaddress.ip_network(item, strict=False))
|
|
185
|
+
continue
|
|
186
|
+
except ValueError:
|
|
187
|
+
pass
|
|
188
|
+
if item.startswith("*."):
|
|
189
|
+
item = item[2:]
|
|
190
|
+
domains.append(to_ascii_hostname(item))
|
|
191
|
+
if not explicit:
|
|
192
|
+
if is_ip(target):
|
|
193
|
+
networks.append(ipaddress.ip_network(target))
|
|
194
|
+
else:
|
|
195
|
+
domains.append(target)
|
|
196
|
+
return cls(
|
|
197
|
+
domains=tuple(dict.fromkeys(domains)),
|
|
198
|
+
networks=tuple(dict.fromkeys(networks)),
|
|
199
|
+
allow_private=allow_private,
|
|
200
|
+
explicit=explicit,
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
def describe(self) -> list[str]:
|
|
204
|
+
out = [f"{d} (and subdomains)" for d in self.domains]
|
|
205
|
+
out += [str(n) for n in self.networks]
|
|
206
|
+
if self.allow_private:
|
|
207
|
+
out.append("private addresses allowed")
|
|
208
|
+
return out
|
|
209
|
+
|
|
210
|
+
def allows_name(self, host: str) -> bool:
|
|
211
|
+
host = host.lower().rstrip(".")
|
|
212
|
+
if is_ip(host):
|
|
213
|
+
return self.allows_ip_literal(host)
|
|
214
|
+
return any(host == d or host.endswith("." + d) for d in self.domains)
|
|
215
|
+
|
|
216
|
+
def allows_ip_literal(self, value: str) -> bool:
|
|
217
|
+
address = ipaddress.ip_address(value)
|
|
218
|
+
return any(address in network for network in self.networks)
|
|
219
|
+
|
|
220
|
+
def address_refusal(self, value: str) -> str | None:
|
|
221
|
+
"""Why connecting to this address is refused, or None when it is allowed."""
|
|
222
|
+
try:
|
|
223
|
+
address = ipaddress.ip_address(value)
|
|
224
|
+
except ValueError:
|
|
225
|
+
return None
|
|
226
|
+
if not self.allow_private and is_restricted_address(address):
|
|
227
|
+
return "non-public address (use --allow-private)"
|
|
228
|
+
return None
|
|
229
|
+
|
|
230
|
+
def refusal(self, host: str) -> str | None:
|
|
231
|
+
"""Why a host must not be contacted, or None when it may be."""
|
|
232
|
+
if not self.allows_name(host):
|
|
233
|
+
return "out of scope"
|
|
234
|
+
return self.address_refusal(host)
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: webrecon-cli
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Summary: A small web reconnaissance scanner: DNS, subdomains, HTTP fingerprinting, TLS and ports.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/Just-Ipavon/web-recon
|
|
7
|
+
Project-URL: Documentation, https://github.com/Just-Ipavon/web-recon/tree/main/docs
|
|
8
|
+
Project-URL: Issues, https://github.com/Just-Ipavon/web-recon/issues
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Intended Audience :: Information Technology
|
|
12
|
+
Classifier: Intended Audience :: System Administrators
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Internet :: Name Service (DNS)
|
|
21
|
+
Classifier: Topic :: Security
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
Requires-Dist: httpx>=0.27
|
|
27
|
+
Requires-Dist: dnspython>=2.6
|
|
28
|
+
Requires-Dist: rich>=13.7
|
|
29
|
+
Requires-Dist: cryptography>=42
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
32
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == "dev"
|
|
33
|
+
Requires-Dist: ruff>=0.5; extra == "dev"
|
|
34
|
+
Requires-Dist: mypy>=1.10; extra == "dev"
|
|
35
|
+
Dynamic: license-file
|
|
36
|
+
|
|
37
|
+
# webrecon
|
|
38
|
+
|
|
39
|
+
[](https://github.com/Just-Ipavon/web-recon/actions/workflows/ci.yml)
|
|
40
|
+
[](https://www.python.org/)
|
|
41
|
+
[](LICENSE)
|
|
42
|
+
|
|
43
|
+
A small asynchronous web reconnaissance scanner: DNS enumeration, passive and
|
|
44
|
+
active subdomain discovery, HTTP fingerprinting, technology detection, TLS
|
|
45
|
+
certificate inspection and TCP port scanning — with JSON, terminal and HTML
|
|
46
|
+
reports.
|
|
47
|
+
|
|
48
|
+
> **Authorised use only.** Run this against systems you own or have written
|
|
49
|
+
> permission to test. Unauthorised scanning is illegal in most jurisdictions.
|
|
50
|
+
|
|
51
|
+

|
|
52
|
+
|
|
53
|
+
## Features
|
|
54
|
+
|
|
55
|
+
| Module | What it does |
|
|
56
|
+
| --- | --- |
|
|
57
|
+
| `dns` | Resolves A, AAAA, MX, NS, TXT, CNAME, SOA and CAA records with a per-query status (NXDOMAIN, timeout, SERVFAIL); SPF/DMARC/DKIM hints for mail domains; optional zone-transfer (AXFR) check |
|
|
58
|
+
| `subdomains` | Certificate transparency lookups (crt.sh) plus wordlist brute-force with wildcard-DNS detection, then resolves each candidate |
|
|
59
|
+
| `http` | Reachability, redirect chain, status, page title, headers, security-header and cookie-flag findings, and content discovery (`robots.txt`, `sitemap.xml`, `security.txt`, exposed `.git`/`.env`) |
|
|
60
|
+
| `tls` | Issuer, subject, SANs, validity window, days until expiry, chain and hostname verification, protocol and cipher — also for self-signed and expired certificates |
|
|
61
|
+
| `ports` | TCP connect scan over an explicit port list, with optional sanitised banners; open web ports are fed to the HTTP and TLS modules |
|
|
62
|
+
|
|
63
|
+
Technology detection runs on every HTTP response and identifies ~30 servers,
|
|
64
|
+
CDNs, frameworks and CMSs from headers, cookies and body markers.
|
|
65
|
+
|
|
66
|
+
## Install
|
|
67
|
+
|
|
68
|
+
To use the tool, install it from [PyPI](https://pypi.org/project/webrecon-cli/)
|
|
69
|
+
in its own environment with [pipx](https://pipx.pypa.io/):
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
pipx install webrecon-cli
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
The package is called `webrecon-cli` (the name `webrecon` on PyPI belongs to
|
|
76
|
+
an unrelated project); the command it installs is `webrecon`. The latest
|
|
77
|
+
development version installs with
|
|
78
|
+
`pipx install git+https://github.com/Just-Ipavon/web-recon.git`.
|
|
79
|
+
|
|
80
|
+
To work on the code, clone it and install it in a virtual environment. These
|
|
81
|
+
commands work in any shell, because they call the environment's own `pip`
|
|
82
|
+
instead of activating it:
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
git clone https://github.com/Just-Ipavon/web-recon.git
|
|
86
|
+
cd web-recon
|
|
87
|
+
python3 -m venv .venv
|
|
88
|
+
.venv/bin/pip install -e ".[dev]"
|
|
89
|
+
.venv/bin/webrecon --version
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
To have `webrecon`, `pytest` and the other tools on your `PATH`, activate the
|
|
93
|
+
environment with the script for your shell:
|
|
94
|
+
|
|
95
|
+
| Shell | Command |
|
|
96
|
+
| --- | --- |
|
|
97
|
+
| bash, zsh | `source .venv/bin/activate` |
|
|
98
|
+
| fish | `source .venv/bin/activate.fish` |
|
|
99
|
+
| csh, tcsh | `source .venv/bin/activate.csh` |
|
|
100
|
+
| PowerShell | `.venv\Scripts\Activate.ps1` |
|
|
101
|
+
| Windows cmd | `.venv\Scripts\activate.bat` |
|
|
102
|
+
|
|
103
|
+
On Windows the paths in the commands above are `.venv\Scripts\pip` and
|
|
104
|
+
`.venv\Scripts\webrecon`. Do not install with the system `pip`: distributions
|
|
105
|
+
such as Arch, Debian and Fedora refuse it (PEP 668, "externally managed
|
|
106
|
+
environment").
|
|
107
|
+
|
|
108
|
+
A small subdomain wordlist ships inside the package and is used by default;
|
|
109
|
+
`-w` points at your own list and `--no-wordlist` turns brute-force off.
|
|
110
|
+
|
|
111
|
+
## Usage
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
# Default scan: DNS, subdomains, HTTP and TLS (asks you to confirm authorization)
|
|
115
|
+
webrecon example.com
|
|
116
|
+
|
|
117
|
+
# Everything, non-interactively, with JSON + HTML reports and a live JSONL log
|
|
118
|
+
webrecon example.com -m all --i-have-authorization \
|
|
119
|
+
-o report.json --html report.html --jsonl results.jsonl
|
|
120
|
+
|
|
121
|
+
# 10 requests per second across all modules, 5 at a time, at most 10 minutes
|
|
122
|
+
webrecon example.com -r 10 -c 5 --max-time 600
|
|
123
|
+
|
|
124
|
+
# A web app on a non-standard port, plus a list of further targets
|
|
125
|
+
webrecon https://app.example.com:8443 -iL targets.txt
|
|
126
|
+
|
|
127
|
+
# Limit the perimeter explicitly and allow an internal range
|
|
128
|
+
webrecon example.com --scope example.com,10.0.0.0/24 --allow-private
|
|
129
|
+
|
|
130
|
+
# Active only — skip certificate transparency lookups
|
|
131
|
+
webrecon example.com --no-passive
|
|
132
|
+
|
|
133
|
+
# Pipe the JSON report to jq
|
|
134
|
+
webrecon example.com -m dns --json | jq .dns.records
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
Targets can be a domain, an IP address (IPv4 or IPv6), `host:port` or a URL;
|
|
138
|
+
internationalised names are converted to punycode. Before any active module
|
|
139
|
+
runs, webrecon asks you to confirm that you are authorised to test the
|
|
140
|
+
targets (or takes `--i-have-authorization`), and records the operator, the
|
|
141
|
+
time and the scope in the report.
|
|
142
|
+
|
|
143
|
+
### Options
|
|
144
|
+
|
|
145
|
+
| Flag | Meaning |
|
|
146
|
+
| --- | --- |
|
|
147
|
+
| `target ...` | One or more domains, IPs, `host:port` or URLs |
|
|
148
|
+
| `-iL FILE` | Read more targets from a file, one per line |
|
|
149
|
+
| `-V, --version` | Print the version |
|
|
150
|
+
| `-m, --modules` | Modules to run: `dns`, `subdomains`, `http`, `tls`, `ports`, or `all` |
|
|
151
|
+
| `-w, --wordlist` | Subdomain wordlist for brute-force (default: the bundled list) |
|
|
152
|
+
| `--no-wordlist` | Skip subdomain brute-force |
|
|
153
|
+
| `-p, --ports` | Ports to scan, e.g. `80,443,8000-8100` |
|
|
154
|
+
| `--max-subdomains` | Cap on candidates resolved (default 200, `0` for no cap) |
|
|
155
|
+
| `--no-passive` | Skip crt.sh lookups (otherwise the target name is sent to crt.sh) |
|
|
156
|
+
| `--banners` | Read a short, sanitised banner from each open port |
|
|
157
|
+
| `--axfr` | Attempt a DNS zone transfer against each name server |
|
|
158
|
+
| `--resolver IP` | Query this DNS server instead of the system resolver |
|
|
159
|
+
| `-c, --concurrency` | Maximum concurrent requests (default 20) |
|
|
160
|
+
| `-r, --rate` | Maximum requests per second across all modules; `0` disables pacing |
|
|
161
|
+
| `-t, --timeout` | Per-request timeout in seconds (default 5) |
|
|
162
|
+
| `--max-time` | Stop each target's scan after this many seconds and report what was found |
|
|
163
|
+
| `--max-requests-per-host` | HTTP request budget per host (default unlimited) |
|
|
164
|
+
| `--scope`, `--scope-file` | Domains and CIDRs that may be contacted (default: the target and its subdomains) |
|
|
165
|
+
| `--allow-private` | Allow loopback, private and link-local addresses (refused by default) |
|
|
166
|
+
| `--i-have-authorization` | Confirm authorization without the interactive prompt |
|
|
167
|
+
| `--operator` | Name recorded in the report (default: the current user) |
|
|
168
|
+
| `--verify-tls` | Verify certificates on HTTP probes (off by default; the TLS module always reports verification status) |
|
|
169
|
+
| `--proxy URL` | HTTP(S) proxy for HTTP probes and crt.sh; DNS, TLS and port scans still connect directly. `HTTP(S)_PROXY` variables are ignored |
|
|
170
|
+
| `--user-agent` | User-Agent for HTTP requests |
|
|
171
|
+
| `--no-redact` | Keep `Set-Cookie` and `Authorization` values in reports |
|
|
172
|
+
| `-o, --output` | Write a JSON report |
|
|
173
|
+
| `--html` | Write a self-contained HTML report |
|
|
174
|
+
| `--jsonl` | Append each result to a JSONL file as soon as it is found |
|
|
175
|
+
| `--json` | Print the JSON report to stdout |
|
|
176
|
+
| `--force` | Overwrite existing report files |
|
|
177
|
+
| `-q, --quiet` | Suppress terminal output |
|
|
178
|
+
| `--fail-if-unreachable` | Exit with status 3 when no target answered |
|
|
179
|
+
|
|
180
|
+
Exit codes: `0` success, `1` runtime error, `2` usage error or authorization
|
|
181
|
+
not confirmed, `3` nothing reachable (with `--fail-if-unreachable`), `130`
|
|
182
|
+
interrupted. An interrupted or crashed scan still writes its reports, marked
|
|
183
|
+
as incomplete.
|
|
184
|
+
|
|
185
|
+
### Example output
|
|
186
|
+
|
|
187
|
+
Every scan produces the same data in three forms. The terminal report is shown
|
|
188
|
+
in the recording at the top of this page; the other two are written on request.
|
|
189
|
+
|
|
190
|
+
The JSON report is the full dataset, suitable for piping into `jq` or feeding
|
|
191
|
+
another tool:
|
|
192
|
+
|
|
193
|
+
```json
|
|
194
|
+
{
|
|
195
|
+
"target": "example.com",
|
|
196
|
+
"http": [
|
|
197
|
+
{
|
|
198
|
+
"url": "https://example.com",
|
|
199
|
+
"status": 200,
|
|
200
|
+
"title": "Example Domain",
|
|
201
|
+
"technologies": ["Cloudflare"],
|
|
202
|
+
"interesting_paths": ["/robots.txt"]
|
|
203
|
+
}
|
|
204
|
+
]
|
|
205
|
+
}
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
The HTML report is a single self-contained file — no external stylesheets,
|
|
209
|
+
scripts or fonts — so it can be archived or attached to an engagement report
|
|
210
|
+
and still render years later:
|
|
211
|
+
|
|
212
|
+

|
|
213
|
+
|
|
214
|
+
## Design notes
|
|
215
|
+
|
|
216
|
+
**Everything is rate limited.** One `Throttle`
|
|
217
|
+
([webrecon/ratelimit.py](webrecon/ratelimit.py)) is built per scan and shared by
|
|
218
|
+
every module: DNS queries, the crt.sh lookup, HTTP requests, TLS handshakes and
|
|
219
|
+
port connects each take a slot. It combines a semaphore for in-flight work with
|
|
220
|
+
a token bucket for request rate. Unbounded concurrency is what separates a
|
|
221
|
+
recon tool from a denial-of-service tool.
|
|
222
|
+
|
|
223
|
+
**Scope is explicit.** Every host the tool contacts — subdomains, redirect
|
|
224
|
+
targets, certificate endpoints — is checked against one scope (by default the
|
|
225
|
+
target and its subdomains). Out-of-scope hosts and non-public addresses are
|
|
226
|
+
listed as skipped in the report instead of being probed.
|
|
227
|
+
|
|
228
|
+
**Failures are contained, not fatal.** A DNS timeout, an unreachable host or an
|
|
229
|
+
unparseable certificate is recorded in the result and the scan continues; an
|
|
230
|
+
unexpected exception in one stage becomes an error entry and the other stages
|
|
231
|
+
still run. Errors and unreachable hosts appear in every output format. A
|
|
232
|
+
partial report beats a traceback.
|
|
233
|
+
|
|
234
|
+
**Reports are handled as sensitive.** Cookie and authorization values are
|
|
235
|
+
redacted, report files are created readable by the owner only (mode 0600) and
|
|
236
|
+
are never overwritten without `--force`, and the HTML report carries a
|
|
237
|
+
Content-Security-Policy that forbids scripts.
|
|
238
|
+
|
|
239
|
+
**Modules return dataclasses, not dicts.** Every module produces types from
|
|
240
|
+
[webrecon/models.py](webrecon/models.py), so the report layer never guesses at
|
|
241
|
+
a schema and JSON output stays stable.
|
|
242
|
+
|
|
243
|
+
**Content discovery avoids soft 404s.** Servers that answer `200` with an HTML
|
|
244
|
+
error page are common, so a path only counts as a hit when its body matches the
|
|
245
|
+
shape expected for that file — a `.git/HEAD` must hold a ref or a commit hash,
|
|
246
|
+
an `.env` must contain `KEY=value` lines and no HTML, a `robots.txt` must
|
|
247
|
+
contain directives — and differs from what the server sends for a random path.
|
|
248
|
+
|
|
249
|
+
**Candidates are ranked before truncation.** `--max-subdomains` cuts by
|
|
250
|
+
confidence (target, then certificate transparency, then wordlist guesses), not
|
|
251
|
+
alphabetically, so the hosts most likely to exist survive the cap.
|
|
252
|
+
|
|
253
|
+
## Architecture
|
|
254
|
+
|
|
255
|
+
```text
|
|
256
|
+
webrecon/
|
|
257
|
+
├── cli.py argument parsing, authorization gate, output selection
|
|
258
|
+
├── scanner.py orchestration: runs enabled modules, assembles report
|
|
259
|
+
├── context.py ScanContext: shared throttle, scope and report
|
|
260
|
+
├── targets.py target parsing and scope enforcement
|
|
261
|
+
├── config.py ScanConfig
|
|
262
|
+
├── models.py result dataclasses
|
|
263
|
+
├── ratelimit.py Throttle: concurrency + rate limiting
|
|
264
|
+
├── data/ bundled subdomain wordlist
|
|
265
|
+
└── modules/
|
|
266
|
+
├── dns_enum.py records, crt.sh, wordlist brute-force
|
|
267
|
+
├── http_probe.py probing, redirects, content discovery
|
|
268
|
+
├── security_headers.py security-header and cookie-flag findings
|
|
269
|
+
├── tech_detect.py signature-based fingerprinting
|
|
270
|
+
├── tls_info.py certificate inspection
|
|
271
|
+
├── port_scan.py TCP connect scan
|
|
272
|
+
└── report.py JSON / JSONL / terminal / HTML rendering
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
## Documentation
|
|
276
|
+
|
|
277
|
+
Full technical documentation lives in [docs/](docs/README.md): architecture and
|
|
278
|
+
design decisions, use case specifications, a function-by-function reference, and
|
|
279
|
+
the runtime behaviour — with UML diagrams throughout.
|
|
280
|
+
|
|
281
|
+
| Document | Contents |
|
|
282
|
+
| --- | --- |
|
|
283
|
+
| [01 — Architecture](docs/01-architecture.md) | Context, packages, data model, architectural decisions, deployment |
|
|
284
|
+
| [02 — Use cases](docs/02-use-cases.md) | Actors, use case diagram, eight detailed specifications, operational scenarios |
|
|
285
|
+
| [03 — Function reference](docs/03-function-reference.md) | Every function: behaviour, edge cases, error handling, covering tests |
|
|
286
|
+
| [04 — Runtime behaviour](docs/04-runtime-behaviour.md) | Sequence diagrams, lifecycle, concurrency model, error propagation |
|
|
287
|
+
|
|
288
|
+
## License
|
|
289
|
+
|
|
290
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
webrecon/__init__.py,sha256=AtPsDuXPozy0BqDAtZNTOxujPcAbn5FI3SjadZJ8qLE,262
|
|
2
|
+
webrecon/__main__.py,sha256=4JMK66Wj4uLZTKbF-sT3LAxOsr6buig77PmOkJCRRxw,83
|
|
3
|
+
webrecon/cli.py,sha256=Qr7hqdkSomZsnnSqdCwYtV-ubv0VFAN4JBl5wvTvGvI,19532
|
|
4
|
+
webrecon/config.py,sha256=YAZv05PdxhX7r20V-6RXbzclyOVgy5sa8LliQibkp5I,1947
|
|
5
|
+
webrecon/context.py,sha256=ScK7AY9cpHDKMsYuMAAn7EMsLEeai1zvAOffstpPOlc,1863
|
|
6
|
+
webrecon/models.py,sha256=Ko9MzehD735_Q0WxeFXas4Ta7v6kkH-5XH4H7oKjJDw,3482
|
|
7
|
+
webrecon/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
webrecon/ratelimit.py,sha256=tBsy42Ad4_vdtcfFV4gIhERKP9wlQ-xwdISLz4VREjs,1892
|
|
9
|
+
webrecon/scanner.py,sha256=goGp2uHLKR9jxy_g0OVTEFOIp51XqFrD3ugsulmRKOI,9530
|
|
10
|
+
webrecon/targets.py,sha256=HVVhl-HnQXFimFo7fK3_852m3krBBnNJ6KOewT-Wtd4,8397
|
|
11
|
+
webrecon/data/subdomains-small.txt,sha256=HIxFJXFXhly70q2EumQMkXR_0JpKbanJ-icIS-nZ0Vc,966
|
|
12
|
+
webrecon/modules/__init__.py,sha256=FcpdHA50RdO-j8O7y25hDonQhLGAKp626S7oupft2Ac,68
|
|
13
|
+
webrecon/modules/dns_enum.py,sha256=Es_jRVs3PI_u5BL0BYXZjoJ95ZFnZSSf0GSOZU0VkFo,14532
|
|
14
|
+
webrecon/modules/http_probe.py,sha256=fPw5oTiNWvSReE5ou-gXvIRJZrTXoCn1XyERGcszvNY,14247
|
|
15
|
+
webrecon/modules/port_scan.py,sha256=CM37pJmy5m5SN2PpYKWBMzd2wrCAFTuYr4SYSNrd6RE,4571
|
|
16
|
+
webrecon/modules/report.py,sha256=z9Prsf5mswi1Y1EyMFka0cC57c9UcXaWSCsUH08ey8M,15320
|
|
17
|
+
webrecon/modules/security_headers.py,sha256=AAQ_v7VIo6LtfuXqYgAXoOtYqU-JalSWxWihpAu3Cpc,2564
|
|
18
|
+
webrecon/modules/tech_detect.py,sha256=0laGCxxWBMPfnstJctVUVOUPrQGGeIo3U1ZCr_kyi98,5484
|
|
19
|
+
webrecon/modules/tls_info.py,sha256=egnBjCE-tZMFCk3MAZZe_X9cFPDMufdzBGGnN7GtF4s,8014
|
|
20
|
+
webrecon_cli-0.2.1.dist-info/licenses/LICENSE,sha256=2a3HKreIRCjKb0aLCV6_sGHM9yRifyoqnHSsdP1PZng,1068
|
|
21
|
+
webrecon_cli-0.2.1.dist-info/METADATA,sha256=eIoE9Nrz4WvvcVPm5tXh3D5XKiykT8SrxGE0HULFulc,13801
|
|
22
|
+
webrecon_cli-0.2.1.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
23
|
+
webrecon_cli-0.2.1.dist-info/entry_points.txt,sha256=SCVxmRB5V9h72ZTIiLEdZIYfBvAam_FcQxEskfydaYA,47
|
|
24
|
+
webrecon_cli-0.2.1.dist-info/top_level.txt,sha256=B9evSv-DItAsgXKT1tbU1pGxn_nISXg2gKDzbtWisX4,9
|
|
25
|
+
webrecon_cli-0.2.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Just-ipavon
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
webrecon
|