webrecon-cli 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- webrecon_cli-0.2.1/LICENSE +21 -0
- webrecon_cli-0.2.1/PKG-INFO +290 -0
- webrecon_cli-0.2.1/README.md +254 -0
- webrecon_cli-0.2.1/pyproject.toml +82 -0
- webrecon_cli-0.2.1/setup.cfg +4 -0
- webrecon_cli-0.2.1/tests/test_cli.py +119 -0
- webrecon_cli-0.2.1/tests/test_cli_main.py +179 -0
- webrecon_cli-0.2.1/tests/test_dns_io.py +144 -0
- webrecon_cli-0.2.1/tests/test_http_io.py +137 -0
- webrecon_cli-0.2.1/tests/test_packaging.py +31 -0
- webrecon_cli-0.2.1/tests/test_parsing.py +353 -0
- webrecon_cli-0.2.1/tests/test_ports_io.py +93 -0
- webrecon_cli-0.2.1/tests/test_ratelimit.py +79 -0
- webrecon_cli-0.2.1/tests/test_report.py +211 -0
- webrecon_cli-0.2.1/tests/test_scanner.py +217 -0
- webrecon_cli-0.2.1/tests/test_targets.py +98 -0
- webrecon_cli-0.2.1/tests/test_tls_io.py +54 -0
- webrecon_cli-0.2.1/webrecon/__init__.py +7 -0
- webrecon_cli-0.2.1/webrecon/__main__.py +6 -0
- webrecon_cli-0.2.1/webrecon/cli.py +520 -0
- webrecon_cli-0.2.1/webrecon/config.py +63 -0
- webrecon_cli-0.2.1/webrecon/context.py +61 -0
- webrecon_cli-0.2.1/webrecon/data/subdomains-small.txt +140 -0
- webrecon_cli-0.2.1/webrecon/models.py +121 -0
- webrecon_cli-0.2.1/webrecon/modules/__init__.py +1 -0
- webrecon_cli-0.2.1/webrecon/modules/dns_enum.py +408 -0
- webrecon_cli-0.2.1/webrecon/modules/http_probe.py +407 -0
- webrecon_cli-0.2.1/webrecon/modules/port_scan.py +134 -0
- webrecon_cli-0.2.1/webrecon/modules/report.py +408 -0
- webrecon_cli-0.2.1/webrecon/modules/security_headers.py +69 -0
- webrecon_cli-0.2.1/webrecon/modules/tech_detect.py +125 -0
- webrecon_cli-0.2.1/webrecon/modules/tls_info.py +208 -0
- webrecon_cli-0.2.1/webrecon/py.typed +0 -0
- webrecon_cli-0.2.1/webrecon/ratelimit.py +61 -0
- webrecon_cli-0.2.1/webrecon/scanner.py +264 -0
- webrecon_cli-0.2.1/webrecon/targets.py +234 -0
- webrecon_cli-0.2.1/webrecon_cli.egg-info/PKG-INFO +290 -0
- webrecon_cli-0.2.1/webrecon_cli.egg-info/SOURCES.txt +40 -0
- webrecon_cli-0.2.1/webrecon_cli.egg-info/dependency_links.txt +1 -0
- webrecon_cli-0.2.1/webrecon_cli.egg-info/entry_points.txt +2 -0
- webrecon_cli-0.2.1/webrecon_cli.egg-info/requires.txt +10 -0
- webrecon_cli-0.2.1/webrecon_cli.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Just-ipavon
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: webrecon-cli
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Summary: A small web reconnaissance scanner: DNS, subdomains, HTTP fingerprinting, TLS and ports.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/Just-Ipavon/web-recon
|
|
7
|
+
Project-URL: Documentation, https://github.com/Just-Ipavon/web-recon/tree/main/docs
|
|
8
|
+
Project-URL: Issues, https://github.com/Just-Ipavon/web-recon/issues
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Intended Audience :: Information Technology
|
|
12
|
+
Classifier: Intended Audience :: System Administrators
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Internet :: Name Service (DNS)
|
|
21
|
+
Classifier: Topic :: Security
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
Requires-Dist: httpx>=0.27
|
|
27
|
+
Requires-Dist: dnspython>=2.6
|
|
28
|
+
Requires-Dist: rich>=13.7
|
|
29
|
+
Requires-Dist: cryptography>=42
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
32
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == "dev"
|
|
33
|
+
Requires-Dist: ruff>=0.5; extra == "dev"
|
|
34
|
+
Requires-Dist: mypy>=1.10; extra == "dev"
|
|
35
|
+
Dynamic: license-file
|
|
36
|
+
|
|
37
|
+
# webrecon
|
|
38
|
+
|
|
39
|
+
[](https://github.com/Just-Ipavon/web-recon/actions/workflows/ci.yml)
|
|
40
|
+
[](https://www.python.org/)
|
|
41
|
+
[](LICENSE)
|
|
42
|
+
|
|
43
|
+
A small asynchronous web reconnaissance scanner: DNS enumeration, passive and
|
|
44
|
+
active subdomain discovery, HTTP fingerprinting, technology detection, TLS
|
|
45
|
+
certificate inspection and TCP port scanning — with JSON, terminal and HTML
|
|
46
|
+
reports.
|
|
47
|
+
|
|
48
|
+
> **Authorised use only.** Run this against systems you own or have written
|
|
49
|
+
> permission to test. Unauthorised scanning is illegal in most jurisdictions.
|
|
50
|
+
|
|
51
|
+

|
|
52
|
+
|
|
53
|
+
## Features
|
|
54
|
+
|
|
55
|
+
| Module | What it does |
|
|
56
|
+
| --- | --- |
|
|
57
|
+
| `dns` | Resolves A, AAAA, MX, NS, TXT, CNAME, SOA and CAA records with a per-query status (NXDOMAIN, timeout, SERVFAIL); SPF/DMARC/DKIM hints for mail domains; optional zone-transfer (AXFR) check |
|
|
58
|
+
| `subdomains` | Certificate transparency lookups (crt.sh) plus wordlist brute-force with wildcard-DNS detection, then resolves each candidate |
|
|
59
|
+
| `http` | Reachability, redirect chain, status, page title, headers, security-header and cookie-flag findings, and content discovery (`robots.txt`, `sitemap.xml`, `security.txt`, exposed `.git`/`.env`) |
|
|
60
|
+
| `tls` | Issuer, subject, SANs, validity window, days until expiry, chain and hostname verification, protocol and cipher — also for self-signed and expired certificates |
|
|
61
|
+
| `ports` | TCP connect scan over an explicit port list, with optional sanitised banners; open web ports are fed to the HTTP and TLS modules |
|
|
62
|
+
|
|
63
|
+
Technology detection runs on every HTTP response and identifies ~30 servers,
|
|
64
|
+
CDNs, frameworks and CMSs from headers, cookies and body markers.
|
|
65
|
+
|
|
66
|
+
## Install
|
|
67
|
+
|
|
68
|
+
To use the tool, install it from [PyPI](https://pypi.org/project/webrecon-cli/)
|
|
69
|
+
in its own environment with [pipx](https://pipx.pypa.io/):
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
pipx install webrecon-cli
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
The package is called `webrecon-cli` (the name `webrecon` on PyPI belongs to
|
|
76
|
+
an unrelated project); the command it installs is `webrecon`. The latest
|
|
77
|
+
development version installs with
|
|
78
|
+
`pipx install git+https://github.com/Just-Ipavon/web-recon.git`.
|
|
79
|
+
|
|
80
|
+
To work on the code, clone it and install it in a virtual environment. These
|
|
81
|
+
commands work in any shell, because they call the environment's own `pip`
|
|
82
|
+
instead of activating it:
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
git clone https://github.com/Just-Ipavon/web-recon.git
|
|
86
|
+
cd web-recon
|
|
87
|
+
python3 -m venv .venv
|
|
88
|
+
.venv/bin/pip install -e ".[dev]"
|
|
89
|
+
.venv/bin/webrecon --version
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
To have `webrecon`, `pytest` and the other tools on your `PATH`, activate the
|
|
93
|
+
environment with the script for your shell:
|
|
94
|
+
|
|
95
|
+
| Shell | Command |
|
|
96
|
+
| --- | --- |
|
|
97
|
+
| bash, zsh | `source .venv/bin/activate` |
|
|
98
|
+
| fish | `source .venv/bin/activate.fish` |
|
|
99
|
+
| csh, tcsh | `source .venv/bin/activate.csh` |
|
|
100
|
+
| PowerShell | `.venv\Scripts\Activate.ps1` |
|
|
101
|
+
| Windows cmd | `.venv\Scripts\activate.bat` |
|
|
102
|
+
|
|
103
|
+
On Windows the paths in the commands above are `.venv\Scripts\pip` and
|
|
104
|
+
`.venv\Scripts\webrecon`. Do not install with the system `pip`: distributions
|
|
105
|
+
such as Arch, Debian and Fedora refuse it (PEP 668, "externally managed
|
|
106
|
+
environment").
|
|
107
|
+
|
|
108
|
+
A small subdomain wordlist ships inside the package and is used by default;
|
|
109
|
+
`-w` points at your own list and `--no-wordlist` turns brute-force off.
|
|
110
|
+
|
|
111
|
+
## Usage
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
# Default scan: DNS, subdomains, HTTP and TLS (asks you to confirm authorization)
|
|
115
|
+
webrecon example.com
|
|
116
|
+
|
|
117
|
+
# Everything, non-interactively, with JSON + HTML reports and a live JSONL log
|
|
118
|
+
webrecon example.com -m all --i-have-authorization \
|
|
119
|
+
-o report.json --html report.html --jsonl results.jsonl
|
|
120
|
+
|
|
121
|
+
# 10 requests per second across all modules, 5 at a time, at most 10 minutes
|
|
122
|
+
webrecon example.com -r 10 -c 5 --max-time 600
|
|
123
|
+
|
|
124
|
+
# A web app on a non-standard port, plus a list of further targets
|
|
125
|
+
webrecon https://app.example.com:8443 -iL targets.txt
|
|
126
|
+
|
|
127
|
+
# Limit the perimeter explicitly and allow an internal range
|
|
128
|
+
webrecon example.com --scope example.com,10.0.0.0/24 --allow-private
|
|
129
|
+
|
|
130
|
+
# Active only — skip certificate transparency lookups
|
|
131
|
+
webrecon example.com --no-passive
|
|
132
|
+
|
|
133
|
+
# Pipe the JSON report to jq
|
|
134
|
+
webrecon example.com -m dns --json | jq .dns.records
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
Targets can be a domain, an IP address (IPv4 or IPv6), `host:port` or a URL;
|
|
138
|
+
internationalised names are converted to punycode. Before any active module
|
|
139
|
+
runs, webrecon asks you to confirm that you are authorised to test the
|
|
140
|
+
targets (or takes `--i-have-authorization`), and records the operator, the
|
|
141
|
+
time and the scope in the report.
|
|
142
|
+
|
|
143
|
+
### Options
|
|
144
|
+
|
|
145
|
+
| Flag | Meaning |
|
|
146
|
+
| --- | --- |
|
|
147
|
+
| `target ...` | One or more domains, IPs, `host:port` or URLs |
|
|
148
|
+
| `-iL FILE` | Read more targets from a file, one per line |
|
|
149
|
+
| `-V, --version` | Print the version |
|
|
150
|
+
| `-m, --modules` | Modules to run: `dns`, `subdomains`, `http`, `tls`, `ports`, or `all` |
|
|
151
|
+
| `-w, --wordlist` | Subdomain wordlist for brute-force (default: the bundled list) |
|
|
152
|
+
| `--no-wordlist` | Skip subdomain brute-force |
|
|
153
|
+
| `-p, --ports` | Ports to scan, e.g. `80,443,8000-8100` |
|
|
154
|
+
| `--max-subdomains` | Cap on candidates resolved (default 200, `0` for no cap) |
|
|
155
|
+
| `--no-passive` | Skip crt.sh lookups (otherwise the target name is sent to crt.sh) |
|
|
156
|
+
| `--banners` | Read a short, sanitised banner from each open port |
|
|
157
|
+
| `--axfr` | Attempt a DNS zone transfer against each name server |
|
|
158
|
+
| `--resolver IP` | Query this DNS server instead of the system resolver |
|
|
159
|
+
| `-c, --concurrency` | Maximum concurrent requests (default 20) |
|
|
160
|
+
| `-r, --rate` | Maximum requests per second across all modules; `0` disables pacing |
|
|
161
|
+
| `-t, --timeout` | Per-request timeout in seconds (default 5) |
|
|
162
|
+
| `--max-time` | Stop each target's scan after this many seconds and report what was found |
|
|
163
|
+
| `--max-requests-per-host` | HTTP request budget per host (default unlimited) |
|
|
164
|
+
| `--scope`, `--scope-file` | Domains and CIDRs that may be contacted (default: the target and its subdomains) |
|
|
165
|
+
| `--allow-private` | Allow loopback, private and link-local addresses (refused by default) |
|
|
166
|
+
| `--i-have-authorization` | Confirm authorization without the interactive prompt |
|
|
167
|
+
| `--operator` | Name recorded in the report (default: the current user) |
|
|
168
|
+
| `--verify-tls` | Verify certificates on HTTP probes (off by default; the TLS module always reports verification status) |
|
|
169
|
+
| `--proxy URL` | HTTP(S) proxy for HTTP probes and crt.sh; DNS, TLS and port scans still connect directly. `HTTP(S)_PROXY` variables are ignored |
|
|
170
|
+
| `--user-agent` | User-Agent for HTTP requests |
|
|
171
|
+
| `--no-redact` | Keep `Set-Cookie` and `Authorization` values in reports |
|
|
172
|
+
| `-o, --output` | Write a JSON report |
|
|
173
|
+
| `--html` | Write a self-contained HTML report |
|
|
174
|
+
| `--jsonl` | Append each result to a JSONL file as soon as it is found |
|
|
175
|
+
| `--json` | Print the JSON report to stdout |
|
|
176
|
+
| `--force` | Overwrite existing report files |
|
|
177
|
+
| `-q, --quiet` | Suppress terminal output |
|
|
178
|
+
| `--fail-if-unreachable` | Exit with status 3 when no target answered |
|
|
179
|
+
|
|
180
|
+
Exit codes: `0` success, `1` runtime error, `2` usage error or authorization
|
|
181
|
+
not confirmed, `3` nothing reachable (with `--fail-if-unreachable`), `130`
|
|
182
|
+
interrupted. An interrupted or crashed scan still writes its reports, marked
|
|
183
|
+
as incomplete.
|
|
184
|
+
|
|
185
|
+
### Example output
|
|
186
|
+
|
|
187
|
+
Every scan produces the same data in three forms. The terminal report is shown
|
|
188
|
+
in the recording at the top of this page; the other two are written on request.
|
|
189
|
+
|
|
190
|
+
The JSON report is the full dataset, suitable for piping into `jq` or feeding
|
|
191
|
+
another tool:
|
|
192
|
+
|
|
193
|
+
```json
|
|
194
|
+
{
|
|
195
|
+
"target": "example.com",
|
|
196
|
+
"http": [
|
|
197
|
+
{
|
|
198
|
+
"url": "https://example.com",
|
|
199
|
+
"status": 200,
|
|
200
|
+
"title": "Example Domain",
|
|
201
|
+
"technologies": ["Cloudflare"],
|
|
202
|
+
"interesting_paths": ["/robots.txt"]
|
|
203
|
+
}
|
|
204
|
+
]
|
|
205
|
+
}
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
The HTML report is a single self-contained file — no external stylesheets,
|
|
209
|
+
scripts or fonts — so it can be archived or attached to an engagement report
|
|
210
|
+
and still render years later:
|
|
211
|
+
|
|
212
|
+

|
|
213
|
+
|
|
214
|
+
## Design notes
|
|
215
|
+
|
|
216
|
+
**Everything is rate limited.** One `Throttle`
|
|
217
|
+
([webrecon/ratelimit.py](webrecon/ratelimit.py)) is built per scan and shared by
|
|
218
|
+
every module: DNS queries, the crt.sh lookup, HTTP requests, TLS handshakes and
|
|
219
|
+
port connects each take a slot. It combines a semaphore for in-flight work with
|
|
220
|
+
a token bucket for request rate. Unbounded concurrency is what separates a
|
|
221
|
+
recon tool from a denial-of-service tool.
|
|
222
|
+
|
|
223
|
+
**Scope is explicit.** Every host the tool contacts — subdomains, redirect
|
|
224
|
+
targets, certificate endpoints — is checked against one scope (by default the
|
|
225
|
+
target and its subdomains). Out-of-scope hosts and non-public addresses are
|
|
226
|
+
listed as skipped in the report instead of being probed.
|
|
227
|
+
|
|
228
|
+
**Failures are contained, not fatal.** A DNS timeout, an unreachable host or an
|
|
229
|
+
unparseable certificate is recorded in the result and the scan continues; an
|
|
230
|
+
unexpected exception in one stage becomes an error entry and the other stages
|
|
231
|
+
still run. Errors and unreachable hosts appear in every output format. A
|
|
232
|
+
partial report beats a traceback.
|
|
233
|
+
|
|
234
|
+
**Reports are handled as sensitive.** Cookie and authorization values are
|
|
235
|
+
redacted, report files are created readable by the owner only (mode 0600) and
|
|
236
|
+
are never overwritten without `--force`, and the HTML report carries a
|
|
237
|
+
Content-Security-Policy that forbids scripts.
|
|
238
|
+
|
|
239
|
+
**Modules return dataclasses, not dicts.** Every module produces types from
|
|
240
|
+
[webrecon/models.py](webrecon/models.py), so the report layer never guesses at
|
|
241
|
+
a schema and JSON output stays stable.
|
|
242
|
+
|
|
243
|
+
**Content discovery avoids soft 404s.** Servers that answer `200` with an HTML
|
|
244
|
+
error page are common, so a path only counts as a hit when its body matches the
|
|
245
|
+
shape expected for that file — a `.git/HEAD` must hold a ref or a commit hash,
|
|
246
|
+
an `.env` must contain `KEY=value` lines and no HTML, a `robots.txt` must
|
|
247
|
+
contain directives — and differs from what the server sends for a random path.
|
|
248
|
+
|
|
249
|
+
**Candidates are ranked before truncation.** `--max-subdomains` cuts by
|
|
250
|
+
confidence (target, then certificate transparency, then wordlist guesses), not
|
|
251
|
+
alphabetically, so the hosts most likely to exist survive the cap.
|
|
252
|
+
|
|
253
|
+
## Architecture
|
|
254
|
+
|
|
255
|
+
```text
|
|
256
|
+
webrecon/
|
|
257
|
+
├── cli.py argument parsing, authorization gate, output selection
|
|
258
|
+
├── scanner.py orchestration: runs enabled modules, assembles report
|
|
259
|
+
├── context.py ScanContext: shared throttle, scope and report
|
|
260
|
+
├── targets.py target parsing and scope enforcement
|
|
261
|
+
├── config.py ScanConfig
|
|
262
|
+
├── models.py result dataclasses
|
|
263
|
+
├── ratelimit.py Throttle: concurrency + rate limiting
|
|
264
|
+
├── data/ bundled subdomain wordlist
|
|
265
|
+
└── modules/
|
|
266
|
+
├── dns_enum.py records, crt.sh, wordlist brute-force
|
|
267
|
+
├── http_probe.py probing, redirects, content discovery
|
|
268
|
+
├── security_headers.py security-header and cookie-flag findings
|
|
269
|
+
├── tech_detect.py signature-based fingerprinting
|
|
270
|
+
├── tls_info.py certificate inspection
|
|
271
|
+
├── port_scan.py TCP connect scan
|
|
272
|
+
└── report.py JSON / JSONL / terminal / HTML rendering
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
## Documentation
|
|
276
|
+
|
|
277
|
+
Full technical documentation lives in [docs/](docs/README.md): architecture and
|
|
278
|
+
design decisions, use case specifications, a function-by-function reference, and
|
|
279
|
+
the runtime behaviour — with UML diagrams throughout.
|
|
280
|
+
|
|
281
|
+
| Document | Contents |
|
|
282
|
+
| --- | --- |
|
|
283
|
+
| [01 — Architecture](docs/01-architecture.md) | Context, packages, data model, architectural decisions, deployment |
|
|
284
|
+
| [02 — Use cases](docs/02-use-cases.md) | Actors, use case diagram, eight detailed specifications, operational scenarios |
|
|
285
|
+
| [03 — Function reference](docs/03-function-reference.md) | Every function: behaviour, edge cases, error handling, covering tests |
|
|
286
|
+
| [04 — Runtime behaviour](docs/04-runtime-behaviour.md) | Sequence diagrams, lifecycle, concurrency model, error propagation |
|
|
287
|
+
|
|
288
|
+
## License
|
|
289
|
+
|
|
290
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
# webrecon
|
|
2
|
+
|
|
3
|
+
[](https://github.com/Just-Ipavon/web-recon/actions/workflows/ci.yml)
|
|
4
|
+
[](https://www.python.org/)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
|
|
7
|
+
A small asynchronous web reconnaissance scanner: DNS enumeration, passive and
|
|
8
|
+
active subdomain discovery, HTTP fingerprinting, technology detection, TLS
|
|
9
|
+
certificate inspection and TCP port scanning — with JSON, terminal and HTML
|
|
10
|
+
reports.
|
|
11
|
+
|
|
12
|
+
> **Authorised use only.** Run this against systems you own or have written
|
|
13
|
+
> permission to test. Unauthorised scanning is illegal in most jurisdictions.
|
|
14
|
+
|
|
15
|
+

|
|
16
|
+
|
|
17
|
+
## Features
|
|
18
|
+
|
|
19
|
+
| Module | What it does |
|
|
20
|
+
| --- | --- |
|
|
21
|
+
| `dns` | Resolves A, AAAA, MX, NS, TXT, CNAME, SOA and CAA records with a per-query status (NXDOMAIN, timeout, SERVFAIL); SPF/DMARC/DKIM hints for mail domains; optional zone-transfer (AXFR) check |
|
|
22
|
+
| `subdomains` | Certificate transparency lookups (crt.sh) plus wordlist brute-force with wildcard-DNS detection, then resolves each candidate |
|
|
23
|
+
| `http` | Reachability, redirect chain, status, page title, headers, security-header and cookie-flag findings, and content discovery (`robots.txt`, `sitemap.xml`, `security.txt`, exposed `.git`/`.env`) |
|
|
24
|
+
| `tls` | Issuer, subject, SANs, validity window, days until expiry, chain and hostname verification, protocol and cipher — also for self-signed and expired certificates |
|
|
25
|
+
| `ports` | TCP connect scan over an explicit port list, with optional sanitised banners; open web ports are fed to the HTTP and TLS modules |
|
|
26
|
+
|
|
27
|
+
Technology detection runs on every HTTP response and identifies ~30 servers,
|
|
28
|
+
CDNs, frameworks and CMSs from headers, cookies and body markers.
|
|
29
|
+
|
|
30
|
+
## Install
|
|
31
|
+
|
|
32
|
+
To use the tool, install it from [PyPI](https://pypi.org/project/webrecon-cli/)
|
|
33
|
+
in its own environment with [pipx](https://pipx.pypa.io/):
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pipx install webrecon-cli
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
The package is called `webrecon-cli` (the name `webrecon` on PyPI belongs to
|
|
40
|
+
an unrelated project); the command it installs is `webrecon`. The latest
|
|
41
|
+
development version installs with
|
|
42
|
+
`pipx install git+https://github.com/Just-Ipavon/web-recon.git`.
|
|
43
|
+
|
|
44
|
+
To work on the code, clone it and install it in a virtual environment. These
|
|
45
|
+
commands work in any shell, because they call the environment's own `pip`
|
|
46
|
+
instead of activating it:
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
git clone https://github.com/Just-Ipavon/web-recon.git
|
|
50
|
+
cd web-recon
|
|
51
|
+
python3 -m venv .venv
|
|
52
|
+
.venv/bin/pip install -e ".[dev]"
|
|
53
|
+
.venv/bin/webrecon --version
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
To have `webrecon`, `pytest` and the other tools on your `PATH`, activate the
|
|
57
|
+
environment with the script for your shell:
|
|
58
|
+
|
|
59
|
+
| Shell | Command |
|
|
60
|
+
| --- | --- |
|
|
61
|
+
| bash, zsh | `source .venv/bin/activate` |
|
|
62
|
+
| fish | `source .venv/bin/activate.fish` |
|
|
63
|
+
| csh, tcsh | `source .venv/bin/activate.csh` |
|
|
64
|
+
| PowerShell | `.venv\Scripts\Activate.ps1` |
|
|
65
|
+
| Windows cmd | `.venv\Scripts\activate.bat` |
|
|
66
|
+
|
|
67
|
+
On Windows the paths in the commands above are `.venv\Scripts\pip` and
|
|
68
|
+
`.venv\Scripts\webrecon`. Do not install with the system `pip`: distributions
|
|
69
|
+
such as Arch, Debian and Fedora refuse it (PEP 668, "externally managed
|
|
70
|
+
environment").
|
|
71
|
+
|
|
72
|
+
A small subdomain wordlist ships inside the package and is used by default;
|
|
73
|
+
`-w` points at your own list and `--no-wordlist` turns brute-force off.
|
|
74
|
+
|
|
75
|
+
## Usage
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
# Default scan: DNS, subdomains, HTTP and TLS (asks you to confirm authorization)
|
|
79
|
+
webrecon example.com
|
|
80
|
+
|
|
81
|
+
# Everything, non-interactively, with JSON + HTML reports and a live JSONL log
|
|
82
|
+
webrecon example.com -m all --i-have-authorization \
|
|
83
|
+
-o report.json --html report.html --jsonl results.jsonl
|
|
84
|
+
|
|
85
|
+
# 10 requests per second across all modules, 5 at a time, at most 10 minutes
|
|
86
|
+
webrecon example.com -r 10 -c 5 --max-time 600
|
|
87
|
+
|
|
88
|
+
# A web app on a non-standard port, plus a list of further targets
|
|
89
|
+
webrecon https://app.example.com:8443 -iL targets.txt
|
|
90
|
+
|
|
91
|
+
# Limit the perimeter explicitly and allow an internal range
|
|
92
|
+
webrecon example.com --scope example.com,10.0.0.0/24 --allow-private
|
|
93
|
+
|
|
94
|
+
# Active only — skip certificate transparency lookups
|
|
95
|
+
webrecon example.com --no-passive
|
|
96
|
+
|
|
97
|
+
# Pipe the JSON report to jq
|
|
98
|
+
webrecon example.com -m dns --json | jq .dns.records
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Targets can be a domain, an IP address (IPv4 or IPv6), `host:port` or a URL;
|
|
102
|
+
internationalised names are converted to punycode. Before any active module
|
|
103
|
+
runs, webrecon asks you to confirm that you are authorised to test the
|
|
104
|
+
targets (or takes `--i-have-authorization`), and records the operator, the
|
|
105
|
+
time and the scope in the report.
|
|
106
|
+
|
|
107
|
+
### Options
|
|
108
|
+
|
|
109
|
+
| Flag | Meaning |
|
|
110
|
+
| --- | --- |
|
|
111
|
+
| `target ...` | One or more domains, IPs, `host:port` or URLs |
|
|
112
|
+
| `-iL FILE` | Read more targets from a file, one per line |
|
|
113
|
+
| `-V, --version` | Print the version |
|
|
114
|
+
| `-m, --modules` | Modules to run: `dns`, `subdomains`, `http`, `tls`, `ports`, or `all` |
|
|
115
|
+
| `-w, --wordlist` | Subdomain wordlist for brute-force (default: the bundled list) |
|
|
116
|
+
| `--no-wordlist` | Skip subdomain brute-force |
|
|
117
|
+
| `-p, --ports` | Ports to scan, e.g. `80,443,8000-8100` |
|
|
118
|
+
| `--max-subdomains` | Cap on candidates resolved (default 200, `0` for no cap) |
|
|
119
|
+
| `--no-passive` | Skip crt.sh lookups (otherwise the target name is sent to crt.sh) |
|
|
120
|
+
| `--banners` | Read a short, sanitised banner from each open port |
|
|
121
|
+
| `--axfr` | Attempt a DNS zone transfer against each name server |
|
|
122
|
+
| `--resolver IP` | Query this DNS server instead of the system resolver |
|
|
123
|
+
| `-c, --concurrency` | Maximum concurrent requests (default 20) |
|
|
124
|
+
| `-r, --rate` | Maximum requests per second across all modules; `0` disables pacing |
|
|
125
|
+
| `-t, --timeout` | Per-request timeout in seconds (default 5) |
|
|
126
|
+
| `--max-time` | Stop each target's scan after this many seconds and report what was found |
|
|
127
|
+
| `--max-requests-per-host` | HTTP request budget per host (default unlimited) |
|
|
128
|
+
| `--scope`, `--scope-file` | Domains and CIDRs that may be contacted (default: the target and its subdomains) |
|
|
129
|
+
| `--allow-private` | Allow loopback, private and link-local addresses (refused by default) |
|
|
130
|
+
| `--i-have-authorization` | Confirm authorization without the interactive prompt |
|
|
131
|
+
| `--operator` | Name recorded in the report (default: the current user) |
|
|
132
|
+
| `--verify-tls` | Verify certificates on HTTP probes (off by default; the TLS module always reports verification status) |
|
|
133
|
+
| `--proxy URL` | HTTP(S) proxy for HTTP probes and crt.sh; DNS, TLS and port scans still connect directly. `HTTP(S)_PROXY` variables are ignored |
|
|
134
|
+
| `--user-agent` | User-Agent for HTTP requests |
|
|
135
|
+
| `--no-redact` | Keep `Set-Cookie` and `Authorization` values in reports |
|
|
136
|
+
| `-o, --output` | Write a JSON report |
|
|
137
|
+
| `--html` | Write a self-contained HTML report |
|
|
138
|
+
| `--jsonl` | Append each result to a JSONL file as soon as it is found |
|
|
139
|
+
| `--json` | Print the JSON report to stdout |
|
|
140
|
+
| `--force` | Overwrite existing report files |
|
|
141
|
+
| `-q, --quiet` | Suppress terminal output |
|
|
142
|
+
| `--fail-if-unreachable` | Exit with status 3 when no target answered |
|
|
143
|
+
|
|
144
|
+
Exit codes: `0` success, `1` runtime error, `2` usage error or authorization
|
|
145
|
+
not confirmed, `3` nothing reachable (with `--fail-if-unreachable`), `130`
|
|
146
|
+
interrupted. An interrupted or crashed scan still writes its reports, marked
|
|
147
|
+
as incomplete.
|
|
148
|
+
|
|
149
|
+
### Example output
|
|
150
|
+
|
|
151
|
+
Every scan produces the same data in three forms. The terminal report is shown
|
|
152
|
+
in the recording at the top of this page; the other two are written on request.
|
|
153
|
+
|
|
154
|
+
The JSON report is the full dataset, suitable for piping into `jq` or feeding
|
|
155
|
+
another tool:
|
|
156
|
+
|
|
157
|
+
```json
|
|
158
|
+
{
|
|
159
|
+
"target": "example.com",
|
|
160
|
+
"http": [
|
|
161
|
+
{
|
|
162
|
+
"url": "https://example.com",
|
|
163
|
+
"status": 200,
|
|
164
|
+
"title": "Example Domain",
|
|
165
|
+
"technologies": ["Cloudflare"],
|
|
166
|
+
"interesting_paths": ["/robots.txt"]
|
|
167
|
+
}
|
|
168
|
+
]
|
|
169
|
+
}
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
The HTML report is a single self-contained file — no external stylesheets,
|
|
173
|
+
scripts or fonts — so it can be archived or attached to an engagement report
|
|
174
|
+
and still render years later:
|
|
175
|
+
|
|
176
|
+

|
|
177
|
+
|
|
178
|
+
## Design notes
|
|
179
|
+
|
|
180
|
+
**Everything is rate limited.** One `Throttle`
|
|
181
|
+
([webrecon/ratelimit.py](webrecon/ratelimit.py)) is built per scan and shared by
|
|
182
|
+
every module: DNS queries, the crt.sh lookup, HTTP requests, TLS handshakes and
|
|
183
|
+
port connects each take a slot. It combines a semaphore for in-flight work with
|
|
184
|
+
a token bucket for request rate. Unbounded concurrency is what separates a
|
|
185
|
+
recon tool from a denial-of-service tool.
|
|
186
|
+
|
|
187
|
+
**Scope is explicit.** Every host the tool contacts — subdomains, redirect
|
|
188
|
+
targets, certificate endpoints — is checked against one scope (by default the
|
|
189
|
+
target and its subdomains). Out-of-scope hosts and non-public addresses are
|
|
190
|
+
listed as skipped in the report instead of being probed.
|
|
191
|
+
|
|
192
|
+
**Failures are contained, not fatal.** A DNS timeout, an unreachable host or an
|
|
193
|
+
unparseable certificate is recorded in the result and the scan continues; an
|
|
194
|
+
unexpected exception in one stage becomes an error entry and the other stages
|
|
195
|
+
still run. Errors and unreachable hosts appear in every output format. A
|
|
196
|
+
partial report beats a traceback.
|
|
197
|
+
|
|
198
|
+
**Reports are handled as sensitive.** Cookie and authorization values are
|
|
199
|
+
redacted, report files are created readable by the owner only (mode 0600) and
|
|
200
|
+
are never overwritten without `--force`, and the HTML report carries a
|
|
201
|
+
Content-Security-Policy that forbids scripts.
|
|
202
|
+
|
|
203
|
+
**Modules return dataclasses, not dicts.** Every module produces types from
|
|
204
|
+
[webrecon/models.py](webrecon/models.py), so the report layer never guesses at
|
|
205
|
+
a schema and JSON output stays stable.
|
|
206
|
+
|
|
207
|
+
**Content discovery avoids soft 404s.** Servers that answer `200` with an HTML
|
|
208
|
+
error page are common, so a path only counts as a hit when its body matches the
|
|
209
|
+
shape expected for that file — a `.git/HEAD` must hold a ref or a commit hash,
|
|
210
|
+
an `.env` must contain `KEY=value` lines and no HTML, a `robots.txt` must
|
|
211
|
+
contain directives — and differs from what the server sends for a random path.
|
|
212
|
+
|
|
213
|
+
**Candidates are ranked before truncation.** `--max-subdomains` cuts by
|
|
214
|
+
confidence (target, then certificate transparency, then wordlist guesses), not
|
|
215
|
+
alphabetically, so the hosts most likely to exist survive the cap.
|
|
216
|
+
|
|
217
|
+
## Architecture
|
|
218
|
+
|
|
219
|
+
```text
|
|
220
|
+
webrecon/
|
|
221
|
+
├── cli.py argument parsing, authorization gate, output selection
|
|
222
|
+
├── scanner.py orchestration: runs enabled modules, assembles report
|
|
223
|
+
├── context.py ScanContext: shared throttle, scope and report
|
|
224
|
+
├── targets.py target parsing and scope enforcement
|
|
225
|
+
├── config.py ScanConfig
|
|
226
|
+
├── models.py result dataclasses
|
|
227
|
+
├── ratelimit.py Throttle: concurrency + rate limiting
|
|
228
|
+
├── data/ bundled subdomain wordlist
|
|
229
|
+
└── modules/
|
|
230
|
+
├── dns_enum.py records, crt.sh, wordlist brute-force
|
|
231
|
+
├── http_probe.py probing, redirects, content discovery
|
|
232
|
+
├── security_headers.py security-header and cookie-flag findings
|
|
233
|
+
├── tech_detect.py signature-based fingerprinting
|
|
234
|
+
├── tls_info.py certificate inspection
|
|
235
|
+
├── port_scan.py TCP connect scan
|
|
236
|
+
└── report.py JSON / JSONL / terminal / HTML rendering
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
## Documentation
|
|
240
|
+
|
|
241
|
+
Full technical documentation lives in [docs/](docs/README.md): architecture and
|
|
242
|
+
design decisions, use case specifications, a function-by-function reference, and
|
|
243
|
+
the runtime behaviour — with UML diagrams throughout.
|
|
244
|
+
|
|
245
|
+
| Document | Contents |
|
|
246
|
+
| --- | --- |
|
|
247
|
+
| [01 — Architecture](docs/01-architecture.md) | Context, packages, data model, architectural decisions, deployment |
|
|
248
|
+
| [02 — Use cases](docs/02-use-cases.md) | Actors, use case diagram, eight detailed specifications, operational scenarios |
|
|
249
|
+
| [03 — Function reference](docs/03-function-reference.md) | Every function: behaviour, edge cases, error handling, covering tests |
|
|
250
|
+
| [04 — Runtime behaviour](docs/04-runtime-behaviour.md) | Sequence diagrams, lifecycle, concurrency model, error propagation |
|
|
251
|
+
|
|
252
|
+
## License
|
|
253
|
+
|
|
254
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "webrecon-cli"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "A small web reconnaissance scanner: DNS, subdomains, HTTP fingerprinting, TLS and ports."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Development Status :: 4 - Beta",
|
|
15
|
+
"Environment :: Console",
|
|
16
|
+
"Intended Audience :: Information Technology",
|
|
17
|
+
"Intended Audience :: System Administrators",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
21
|
+
"Programming Language :: Python :: 3.10",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Programming Language :: Python :: 3.13",
|
|
25
|
+
"Topic :: Internet :: Name Service (DNS)",
|
|
26
|
+
"Topic :: Security",
|
|
27
|
+
"Typing :: Typed",
|
|
28
|
+
]
|
|
29
|
+
dependencies = [
|
|
30
|
+
"httpx>=0.27",
|
|
31
|
+
"dnspython>=2.6",
|
|
32
|
+
"rich>=13.7",
|
|
33
|
+
"cryptography>=42",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
[project.urls]
|
|
37
|
+
Homepage = "https://github.com/Just-Ipavon/web-recon"
|
|
38
|
+
Documentation = "https://github.com/Just-Ipavon/web-recon/tree/main/docs"
|
|
39
|
+
Issues = "https://github.com/Just-Ipavon/web-recon/issues"
|
|
40
|
+
|
|
41
|
+
[project.optional-dependencies]
|
|
42
|
+
dev = [
|
|
43
|
+
"pytest>=8.0",
|
|
44
|
+
"pytest-asyncio>=0.24",
|
|
45
|
+
"ruff>=0.5",
|
|
46
|
+
"mypy>=1.10",
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
[project.scripts]
|
|
50
|
+
webrecon = "webrecon.cli:main"
|
|
51
|
+
|
|
52
|
+
[tool.setuptools.dynamic]
|
|
53
|
+
version = { attr = "webrecon.__version__" }
|
|
54
|
+
|
|
55
|
+
[tool.setuptools.packages.find]
|
|
56
|
+
include = ["webrecon*"]
|
|
57
|
+
|
|
58
|
+
[tool.setuptools.package-data]
|
|
59
|
+
webrecon = ["data/*.txt", "py.typed"]
|
|
60
|
+
|
|
61
|
+
[tool.pytest.ini_options]
|
|
62
|
+
testpaths = ["tests"]
|
|
63
|
+
asyncio_mode = "auto"
|
|
64
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
65
|
+
|
|
66
|
+
[tool.ruff]
|
|
67
|
+
line-length = 100
|
|
68
|
+
target-version = "py310"
|
|
69
|
+
|
|
70
|
+
[tool.ruff.lint]
|
|
71
|
+
select = ["E", "F", "I", "UP", "B"]
|
|
72
|
+
|
|
73
|
+
[tool.mypy]
|
|
74
|
+
files = ["webrecon"]
|
|
75
|
+
warn_unused_ignores = true
|
|
76
|
+
warn_redundant_casts = true
|
|
77
|
+
check_untyped_defs = true
|
|
78
|
+
disallow_incomplete_defs = true
|
|
79
|
+
|
|
80
|
+
[[tool.mypy.overrides]]
|
|
81
|
+
module = ["dns.*"]
|
|
82
|
+
ignore_missing_imports = true
|