webrecon-cli 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. webrecon_cli-0.2.1/LICENSE +21 -0
  2. webrecon_cli-0.2.1/PKG-INFO +290 -0
  3. webrecon_cli-0.2.1/README.md +254 -0
  4. webrecon_cli-0.2.1/pyproject.toml +82 -0
  5. webrecon_cli-0.2.1/setup.cfg +4 -0
  6. webrecon_cli-0.2.1/tests/test_cli.py +119 -0
  7. webrecon_cli-0.2.1/tests/test_cli_main.py +179 -0
  8. webrecon_cli-0.2.1/tests/test_dns_io.py +144 -0
  9. webrecon_cli-0.2.1/tests/test_http_io.py +137 -0
  10. webrecon_cli-0.2.1/tests/test_packaging.py +31 -0
  11. webrecon_cli-0.2.1/tests/test_parsing.py +353 -0
  12. webrecon_cli-0.2.1/tests/test_ports_io.py +93 -0
  13. webrecon_cli-0.2.1/tests/test_ratelimit.py +79 -0
  14. webrecon_cli-0.2.1/tests/test_report.py +211 -0
  15. webrecon_cli-0.2.1/tests/test_scanner.py +217 -0
  16. webrecon_cli-0.2.1/tests/test_targets.py +98 -0
  17. webrecon_cli-0.2.1/tests/test_tls_io.py +54 -0
  18. webrecon_cli-0.2.1/webrecon/__init__.py +7 -0
  19. webrecon_cli-0.2.1/webrecon/__main__.py +6 -0
  20. webrecon_cli-0.2.1/webrecon/cli.py +520 -0
  21. webrecon_cli-0.2.1/webrecon/config.py +63 -0
  22. webrecon_cli-0.2.1/webrecon/context.py +61 -0
  23. webrecon_cli-0.2.1/webrecon/data/subdomains-small.txt +140 -0
  24. webrecon_cli-0.2.1/webrecon/models.py +121 -0
  25. webrecon_cli-0.2.1/webrecon/modules/__init__.py +1 -0
  26. webrecon_cli-0.2.1/webrecon/modules/dns_enum.py +408 -0
  27. webrecon_cli-0.2.1/webrecon/modules/http_probe.py +407 -0
  28. webrecon_cli-0.2.1/webrecon/modules/port_scan.py +134 -0
  29. webrecon_cli-0.2.1/webrecon/modules/report.py +408 -0
  30. webrecon_cli-0.2.1/webrecon/modules/security_headers.py +69 -0
  31. webrecon_cli-0.2.1/webrecon/modules/tech_detect.py +125 -0
  32. webrecon_cli-0.2.1/webrecon/modules/tls_info.py +208 -0
  33. webrecon_cli-0.2.1/webrecon/py.typed +0 -0
  34. webrecon_cli-0.2.1/webrecon/ratelimit.py +61 -0
  35. webrecon_cli-0.2.1/webrecon/scanner.py +264 -0
  36. webrecon_cli-0.2.1/webrecon/targets.py +234 -0
  37. webrecon_cli-0.2.1/webrecon_cli.egg-info/PKG-INFO +290 -0
  38. webrecon_cli-0.2.1/webrecon_cli.egg-info/SOURCES.txt +40 -0
  39. webrecon_cli-0.2.1/webrecon_cli.egg-info/dependency_links.txt +1 -0
  40. webrecon_cli-0.2.1/webrecon_cli.egg-info/entry_points.txt +2 -0
  41. webrecon_cli-0.2.1/webrecon_cli.egg-info/requires.txt +10 -0
  42. webrecon_cli-0.2.1/webrecon_cli.egg-info/top_level.txt +1 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Just-ipavon
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,290 @@
1
+ Metadata-Version: 2.4
2
+ Name: webrecon-cli
3
+ Version: 0.2.1
4
+ Summary: A small web reconnaissance scanner: DNS, subdomains, HTTP fingerprinting, TLS and ports.
5
+ License-Expression: MIT
6
+ Project-URL: Homepage, https://github.com/Just-Ipavon/web-recon
7
+ Project-URL: Documentation, https://github.com/Just-Ipavon/web-recon/tree/main/docs
8
+ Project-URL: Issues, https://github.com/Just-Ipavon/web-recon/issues
9
+ Classifier: Development Status :: 4 - Beta
10
+ Classifier: Environment :: Console
11
+ Classifier: Intended Audience :: Information Technology
12
+ Classifier: Intended Audience :: System Administrators
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3 :: Only
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Topic :: Internet :: Name Service (DNS)
21
+ Classifier: Topic :: Security
22
+ Classifier: Typing :: Typed
23
+ Requires-Python: >=3.10
24
+ Description-Content-Type: text/markdown
25
+ License-File: LICENSE
26
+ Requires-Dist: httpx>=0.27
27
+ Requires-Dist: dnspython>=2.6
28
+ Requires-Dist: rich>=13.7
29
+ Requires-Dist: cryptography>=42
30
+ Provides-Extra: dev
31
+ Requires-Dist: pytest>=8.0; extra == "dev"
32
+ Requires-Dist: pytest-asyncio>=0.24; extra == "dev"
33
+ Requires-Dist: ruff>=0.5; extra == "dev"
34
+ Requires-Dist: mypy>=1.10; extra == "dev"
35
+ Dynamic: license-file
36
+
37
+ # webrecon
38
+
39
+ [![CI](https://github.com/Just-Ipavon/web-recon/actions/workflows/ci.yml/badge.svg)](https://github.com/Just-Ipavon/web-recon/actions/workflows/ci.yml)
40
+ [![Python](https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12%20%7C%203.13-blue)](https://www.python.org/)
41
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
42
+
43
+ A small asynchronous web reconnaissance scanner: DNS enumeration, passive and
44
+ active subdomain discovery, HTTP fingerprinting, technology detection, TLS
45
+ certificate inspection and TCP port scanning — with JSON, terminal and HTML
46
+ reports.
47
+
48
+ > **Authorised use only.** Run this against systems you own or have written
49
+ > permission to test. Unauthorised scanning is illegal in most jurisdictions.
50
+
51
+ ![webrecon scanning example.com: DNS records, HTTP fingerprint and TLS certificate](https://raw.githubusercontent.com/Just-Ipavon/web-recon/main/assets/demo.svg)
52
+
53
+ ## Features
54
+
55
+ | Module | What it does |
56
+ | --- | --- |
57
+ | `dns` | Resolves A, AAAA, MX, NS, TXT, CNAME, SOA and CAA records with a per-query status (NXDOMAIN, timeout, SERVFAIL); SPF/DMARC/DKIM hints for mail domains; optional zone-transfer (AXFR) check |
58
+ | `subdomains` | Certificate transparency lookups (crt.sh) plus wordlist brute-force with wildcard-DNS detection, then resolves each candidate |
59
+ | `http` | Reachability, redirect chain, status, page title, headers, security-header and cookie-flag findings, and content discovery (`robots.txt`, `sitemap.xml`, `security.txt`, exposed `.git`/`.env`) |
60
+ | `tls` | Issuer, subject, SANs, validity window, days until expiry, chain and hostname verification, protocol and cipher — also for self-signed and expired certificates |
61
+ | `ports` | TCP connect scan over an explicit port list, with optional sanitised banners; open web ports are fed to the HTTP and TLS modules |
62
+
63
+ Technology detection runs on every HTTP response and identifies ~30 servers,
64
+ CDNs, frameworks and CMSs from headers, cookies and body markers.
65
+
66
+ ## Install
67
+
68
+ To use the tool, install it from [PyPI](https://pypi.org/project/webrecon-cli/)
69
+ in its own environment with [pipx](https://pipx.pypa.io/):
70
+
71
+ ```bash
72
+ pipx install webrecon-cli
73
+ ```
74
+
75
+ The package is called `webrecon-cli` (the name `webrecon` on PyPI belongs to
76
+ an unrelated project); the command it installs is `webrecon`. The latest
77
+ development version installs with
78
+ `pipx install git+https://github.com/Just-Ipavon/web-recon.git`.
79
+
80
+ To work on the code, clone it and install it in a virtual environment. These
81
+ commands work in any shell, because they call the environment's own `pip`
82
+ instead of activating it:
83
+
84
+ ```bash
85
+ git clone https://github.com/Just-Ipavon/web-recon.git
86
+ cd web-recon
87
+ python3 -m venv .venv
88
+ .venv/bin/pip install -e ".[dev]"
89
+ .venv/bin/webrecon --version
90
+ ```
91
+
92
+ To have `webrecon`, `pytest` and the other tools on your `PATH`, activate the
93
+ environment with the script for your shell:
94
+
95
+ | Shell | Command |
96
+ | --- | --- |
97
+ | bash, zsh | `source .venv/bin/activate` |
98
+ | fish | `source .venv/bin/activate.fish` |
99
+ | csh, tcsh | `source .venv/bin/activate.csh` |
100
+ | PowerShell | `.venv\Scripts\Activate.ps1` |
101
+ | Windows cmd | `.venv\Scripts\activate.bat` |
102
+
103
+ On Windows the paths in the commands above are `.venv\Scripts\pip` and
104
+ `.venv\Scripts\webrecon`. Do not install with the system `pip`: distributions
105
+ such as Arch, Debian and Fedora refuse it (PEP 668, "externally managed
106
+ environment").
107
+
108
+ A small subdomain wordlist ships inside the package and is used by default;
109
+ `-w` points at your own list and `--no-wordlist` turns brute-force off.
110
+
111
+ ## Usage
112
+
113
+ ```bash
114
+ # Default scan: DNS, subdomains, HTTP and TLS (asks you to confirm authorization)
115
+ webrecon example.com
116
+
117
+ # Everything, non-interactively, with JSON + HTML reports and a live JSONL log
118
+ webrecon example.com -m all --i-have-authorization \
119
+ -o report.json --html report.html --jsonl results.jsonl
120
+
121
+ # 10 requests per second across all modules, 5 at a time, at most 10 minutes
122
+ webrecon example.com -r 10 -c 5 --max-time 600
123
+
124
+ # A web app on a non-standard port, plus a list of further targets
125
+ webrecon https://app.example.com:8443 -iL targets.txt
126
+
127
+ # Limit the perimeter explicitly and allow an internal range
128
+ webrecon example.com --scope example.com,10.0.0.0/24 --allow-private
129
+
130
+ # Active only — skip certificate transparency lookups
131
+ webrecon example.com --no-passive
132
+
133
+ # Pipe the JSON report to jq
134
+ webrecon example.com -m dns --json | jq .dns.records
135
+ ```
136
+
137
+ Targets can be a domain, an IP address (IPv4 or IPv6), `host:port` or a URL;
138
+ internationalised names are converted to punycode. Before any active module
139
+ runs, webrecon asks you to confirm that you are authorised to test the
140
+ targets (or takes `--i-have-authorization`), and records the operator, the
141
+ time and the scope in the report.
142
+
143
+ ### Options
144
+
145
+ | Flag | Meaning |
146
+ | --- | --- |
147
+ | `target ...` | One or more domains, IPs, `host:port` or URLs |
148
+ | `-iL FILE` | Read more targets from a file, one per line |
149
+ | `-V, --version` | Print the version |
150
+ | `-m, --modules` | Modules to run: `dns`, `subdomains`, `http`, `tls`, `ports`, or `all` |
151
+ | `-w, --wordlist` | Subdomain wordlist for brute-force (default: the bundled list) |
152
+ | `--no-wordlist` | Skip subdomain brute-force |
153
+ | `-p, --ports` | Ports to scan, e.g. `80,443,8000-8100` |
154
+ | `--max-subdomains` | Cap on candidates resolved (default 200, `0` for no cap) |
155
+ | `--no-passive` | Skip crt.sh lookups (otherwise the target name is sent to crt.sh) |
156
+ | `--banners` | Read a short, sanitised banner from each open port |
157
+ | `--axfr` | Attempt a DNS zone transfer against each name server |
158
+ | `--resolver IP` | Query this DNS server instead of the system resolver |
159
+ | `-c, --concurrency` | Maximum concurrent requests (default 20) |
160
+ | `-r, --rate` | Maximum requests per second across all modules; `0` disables pacing |
161
+ | `-t, --timeout` | Per-request timeout in seconds (default 5) |
162
+ | `--max-time` | Stop each target's scan after this many seconds and report what was found |
163
+ | `--max-requests-per-host` | HTTP request budget per host (default unlimited) |
164
+ | `--scope`, `--scope-file` | Domains and CIDRs that may be contacted (default: the target and its subdomains) |
165
+ | `--allow-private` | Allow loopback, private and link-local addresses (refused by default) |
166
+ | `--i-have-authorization` | Confirm authorization without the interactive prompt |
167
+ | `--operator` | Name recorded in the report (default: the current user) |
168
+ | `--verify-tls` | Verify certificates on HTTP probes (off by default; the TLS module always reports verification status) |
169
+ | `--proxy URL` | HTTP(S) proxy for HTTP probes and crt.sh; DNS, TLS and port scans still connect directly. `HTTP(S)_PROXY` variables are ignored |
170
+ | `--user-agent` | User-Agent for HTTP requests |
171
+ | `--no-redact` | Keep `Set-Cookie` and `Authorization` values in reports |
172
+ | `-o, --output` | Write a JSON report |
173
+ | `--html` | Write a self-contained HTML report |
174
+ | `--jsonl` | Append each result to a JSONL file as soon as it is found |
175
+ | `--json` | Print the JSON report to stdout |
176
+ | `--force` | Overwrite existing report files |
177
+ | `-q, --quiet` | Suppress terminal output |
178
+ | `--fail-if-unreachable` | Exit with status 3 when no target answered |
179
+
180
+ Exit codes: `0` success, `1` runtime error, `2` usage error or authorization
181
+ not confirmed, `3` nothing reachable (with `--fail-if-unreachable`), `130`
182
+ interrupted. An interrupted or crashed scan still writes its reports, marked
183
+ as incomplete.
184
+
185
+ ### Example output
186
+
187
+ Every scan produces the same data in three forms. The terminal report is shown
188
+ in the recording at the top of this page; the other two are written on request.
189
+
190
+ The JSON report is the full dataset, suitable for piping into `jq` or feeding
191
+ another tool:
192
+
193
+ ```json
194
+ {
195
+ "target": "example.com",
196
+ "http": [
197
+ {
198
+ "url": "https://example.com",
199
+ "status": 200,
200
+ "title": "Example Domain",
201
+ "technologies": ["Cloudflare"],
202
+ "interesting_paths": ["/robots.txt"]
203
+ }
204
+ ]
205
+ }
206
+ ```
207
+
208
+ The HTML report is a single self-contained file — no external stylesheets,
209
+ scripts or fonts — so it can be archived or attached to an engagement report
210
+ and still render years later:
211
+
212
+ ![HTML report for example.com, showing DNS records, subdomains, HTTP results, TLS certificates and open ports](https://raw.githubusercontent.com/Just-Ipavon/web-recon/main/assets/reporthtml.png)
213
+
214
+ ## Design notes
215
+
216
+ **Everything is rate limited.** One `Throttle`
217
+ ([webrecon/ratelimit.py](webrecon/ratelimit.py)) is built per scan and shared by
218
+ every module: DNS queries, the crt.sh lookup, HTTP requests, TLS handshakes and
219
+ port connects each take a slot. It combines a semaphore for in-flight work with
220
+ a token bucket for request rate. Unbounded concurrency is what separates a
221
+ recon tool from a denial-of-service tool.
222
+
223
+ **Scope is explicit.** Every host the tool contacts — subdomains, redirect
224
+ targets, certificate endpoints — is checked against one scope (by default the
225
+ target and its subdomains). Out-of-scope hosts and non-public addresses are
226
+ listed as skipped in the report instead of being probed.
227
+
228
+ **Failures are contained, not fatal.** A DNS timeout, an unreachable host or an
229
+ unparseable certificate is recorded in the result and the scan continues; an
230
+ unexpected exception in one stage becomes an error entry and the other stages
231
+ still run. Errors and unreachable hosts appear in every output format. A
232
+ partial report beats a traceback.
233
+
234
+ **Reports are handled as sensitive.** Cookie and authorization values are
235
+ redacted, report files are created readable by the owner only (mode 0600) and
236
+ are never overwritten without `--force`, and the HTML report carries a
237
+ Content-Security-Policy that forbids scripts.
238
+
239
+ **Modules return dataclasses, not dicts.** Every module produces types from
240
+ [webrecon/models.py](webrecon/models.py), so the report layer never guesses at
241
+ a schema and JSON output stays stable.
242
+
243
+ **Content discovery avoids soft 404s.** Servers that answer `200` with an HTML
244
+ error page are common, so a path only counts as a hit when its body matches the
245
+ shape expected for that file — a `.git/HEAD` must hold a ref or a commit hash,
246
+ an `.env` must contain `KEY=value` lines and no HTML, a `robots.txt` must
247
+ contain directives — and differs from what the server sends for a random path.
248
+
249
+ **Candidates are ranked before truncation.** `--max-subdomains` cuts by
250
+ confidence (target, then certificate transparency, then wordlist guesses), not
251
+ alphabetically, so the hosts most likely to exist survive the cap.
252
+
253
+ ## Architecture
254
+
255
+ ```text
256
+ webrecon/
257
+ ├── cli.py argument parsing, authorization gate, output selection
258
+ ├── scanner.py orchestration: runs enabled modules, assembles report
259
+ ├── context.py ScanContext: shared throttle, scope and report
260
+ ├── targets.py target parsing and scope enforcement
261
+ ├── config.py ScanConfig
262
+ ├── models.py result dataclasses
263
+ ├── ratelimit.py Throttle: concurrency + rate limiting
264
+ ├── data/ bundled subdomain wordlist
265
+ └── modules/
266
+ ├── dns_enum.py records, crt.sh, wordlist brute-force
267
+ ├── http_probe.py probing, redirects, content discovery
268
+ ├── security_headers.py security-header and cookie-flag findings
269
+ ├── tech_detect.py signature-based fingerprinting
270
+ ├── tls_info.py certificate inspection
271
+ ├── port_scan.py TCP connect scan
272
+ └── report.py JSON / JSONL / terminal / HTML rendering
273
+ ```
274
+
275
+ ## Documentation
276
+
277
+ Full technical documentation lives in [docs/](docs/README.md): architecture and
278
+ design decisions, use case specifications, a function-by-function reference, and
279
+ the runtime behaviour — with UML diagrams throughout.
280
+
281
+ | Document | Contents |
282
+ | --- | --- |
283
+ | [01 — Architecture](docs/01-architecture.md) | Context, packages, data model, architectural decisions, deployment |
284
+ | [02 — Use cases](docs/02-use-cases.md) | Actors, use case diagram, eight detailed specifications, operational scenarios |
285
+ | [03 — Function reference](docs/03-function-reference.md) | Every function: behaviour, edge cases, error handling, covering tests |
286
+ | [04 — Runtime behaviour](docs/04-runtime-behaviour.md) | Sequence diagrams, lifecycle, concurrency model, error propagation |
287
+
288
+ ## License
289
+
290
+ MIT — see [LICENSE](LICENSE).
@@ -0,0 +1,254 @@
1
+ # webrecon
2
+
3
+ [![CI](https://github.com/Just-Ipavon/web-recon/actions/workflows/ci.yml/badge.svg)](https://github.com/Just-Ipavon/web-recon/actions/workflows/ci.yml)
4
+ [![Python](https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12%20%7C%203.13-blue)](https://www.python.org/)
5
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
6
+
7
+ A small asynchronous web reconnaissance scanner: DNS enumeration, passive and
8
+ active subdomain discovery, HTTP fingerprinting, technology detection, TLS
9
+ certificate inspection and TCP port scanning — with JSON, terminal and HTML
10
+ reports.
11
+
12
+ > **Authorised use only.** Run this against systems you own or have written
13
+ > permission to test. Unauthorised scanning is illegal in most jurisdictions.
14
+
15
+ ![webrecon scanning example.com: DNS records, HTTP fingerprint and TLS certificate](https://raw.githubusercontent.com/Just-Ipavon/web-recon/main/assets/demo.svg)
16
+
17
+ ## Features
18
+
19
+ | Module | What it does |
20
+ | --- | --- |
21
+ | `dns` | Resolves A, AAAA, MX, NS, TXT, CNAME, SOA and CAA records with a per-query status (NXDOMAIN, timeout, SERVFAIL); SPF/DMARC/DKIM hints for mail domains; optional zone-transfer (AXFR) check |
22
+ | `subdomains` | Certificate transparency lookups (crt.sh) plus wordlist brute-force with wildcard-DNS detection, then resolves each candidate |
23
+ | `http` | Reachability, redirect chain, status, page title, headers, security-header and cookie-flag findings, and content discovery (`robots.txt`, `sitemap.xml`, `security.txt`, exposed `.git`/`.env`) |
24
+ | `tls` | Issuer, subject, SANs, validity window, days until expiry, chain and hostname verification, protocol and cipher — also for self-signed and expired certificates |
25
+ | `ports` | TCP connect scan over an explicit port list, with optional sanitised banners; open web ports are fed to the HTTP and TLS modules |
26
+
27
+ Technology detection runs on every HTTP response and identifies ~30 servers,
28
+ CDNs, frameworks and CMSs from headers, cookies and body markers.
29
+
30
+ ## Install
31
+
32
+ To use the tool, install it from [PyPI](https://pypi.org/project/webrecon-cli/)
33
+ in its own environment with [pipx](https://pipx.pypa.io/):
34
+
35
+ ```bash
36
+ pipx install webrecon-cli
37
+ ```
38
+
39
+ The package is called `webrecon-cli` (the name `webrecon` on PyPI belongs to
40
+ an unrelated project); the command it installs is `webrecon`. The latest
41
+ development version installs with
42
+ `pipx install git+https://github.com/Just-Ipavon/web-recon.git`.
43
+
44
+ To work on the code, clone it and install it in a virtual environment. These
45
+ commands work in any shell, because they call the environment's own `pip`
46
+ instead of activating it:
47
+
48
+ ```bash
49
+ git clone https://github.com/Just-Ipavon/web-recon.git
50
+ cd web-recon
51
+ python3 -m venv .venv
52
+ .venv/bin/pip install -e ".[dev]"
53
+ .venv/bin/webrecon --version
54
+ ```
55
+
56
+ To have `webrecon`, `pytest` and the other tools on your `PATH`, activate the
57
+ environment with the script for your shell:
58
+
59
+ | Shell | Command |
60
+ | --- | --- |
61
+ | bash, zsh | `source .venv/bin/activate` |
62
+ | fish | `source .venv/bin/activate.fish` |
63
+ | csh, tcsh | `source .venv/bin/activate.csh` |
64
+ | PowerShell | `.venv\Scripts\Activate.ps1` |
65
+ | Windows cmd | `.venv\Scripts\activate.bat` |
66
+
67
+ On Windows the paths in the commands above are `.venv\Scripts\pip` and
68
+ `.venv\Scripts\webrecon`. Do not install with the system `pip`: distributions
69
+ such as Arch, Debian and Fedora refuse it (PEP 668, "externally managed
70
+ environment").
71
+
72
+ A small subdomain wordlist ships inside the package and is used by default;
73
+ `-w` points at your own list and `--no-wordlist` turns brute-force off.
74
+
75
+ ## Usage
76
+
77
+ ```bash
78
+ # Default scan: DNS, subdomains, HTTP and TLS (asks you to confirm authorization)
79
+ webrecon example.com
80
+
81
+ # Everything, non-interactively, with JSON + HTML reports and a live JSONL log
82
+ webrecon example.com -m all --i-have-authorization \
83
+ -o report.json --html report.html --jsonl results.jsonl
84
+
85
+ # 10 requests per second across all modules, 5 at a time, at most 10 minutes
86
+ webrecon example.com -r 10 -c 5 --max-time 600
87
+
88
+ # A web app on a non-standard port, plus a list of further targets
89
+ webrecon https://app.example.com:8443 -iL targets.txt
90
+
91
+ # Limit the perimeter explicitly and allow an internal range
92
+ webrecon example.com --scope example.com,10.0.0.0/24 --allow-private
93
+
94
+ # Active only — skip certificate transparency lookups
95
+ webrecon example.com --no-passive
96
+
97
+ # Pipe the JSON report to jq
98
+ webrecon example.com -m dns --json | jq .dns.records
99
+ ```
100
+
101
+ Targets can be a domain, an IP address (IPv4 or IPv6), `host:port` or a URL;
102
+ internationalised names are converted to punycode. Before any active module
103
+ runs, webrecon asks you to confirm that you are authorised to test the
104
+ targets (or takes `--i-have-authorization`), and records the operator, the
105
+ time and the scope in the report.
106
+
107
+ ### Options
108
+
109
+ | Flag | Meaning |
110
+ | --- | --- |
111
+ | `target ...` | One or more domains, IPs, `host:port` or URLs |
112
+ | `-iL FILE` | Read more targets from a file, one per line |
113
+ | `-V, --version` | Print the version |
114
+ | `-m, --modules` | Modules to run: `dns`, `subdomains`, `http`, `tls`, `ports`, or `all` |
115
+ | `-w, --wordlist` | Subdomain wordlist for brute-force (default: the bundled list) |
116
+ | `--no-wordlist` | Skip subdomain brute-force |
117
+ | `-p, --ports` | Ports to scan, e.g. `80,443,8000-8100` |
118
+ | `--max-subdomains` | Cap on candidates resolved (default 200, `0` for no cap) |
119
+ | `--no-passive` | Skip crt.sh lookups (otherwise the target name is sent to crt.sh) |
120
+ | `--banners` | Read a short, sanitised banner from each open port |
121
+ | `--axfr` | Attempt a DNS zone transfer against each name server |
122
+ | `--resolver IP` | Query this DNS server instead of the system resolver |
123
+ | `-c, --concurrency` | Maximum concurrent requests (default 20) |
124
+ | `-r, --rate` | Maximum requests per second across all modules; `0` disables pacing |
125
+ | `-t, --timeout` | Per-request timeout in seconds (default 5) |
126
+ | `--max-time` | Stop each target's scan after this many seconds and report what was found |
127
+ | `--max-requests-per-host` | HTTP request budget per host (default unlimited) |
128
+ | `--scope`, `--scope-file` | Domains and CIDRs that may be contacted (default: the target and its subdomains) |
129
+ | `--allow-private` | Allow loopback, private and link-local addresses (refused by default) |
130
+ | `--i-have-authorization` | Confirm authorization without the interactive prompt |
131
+ | `--operator` | Name recorded in the report (default: the current user) |
132
+ | `--verify-tls` | Verify certificates on HTTP probes (off by default; the TLS module always reports verification status) |
133
+ | `--proxy URL` | HTTP(S) proxy for HTTP probes and crt.sh; DNS, TLS and port scans still connect directly. `HTTP(S)_PROXY` variables are ignored |
134
+ | `--user-agent` | User-Agent for HTTP requests |
135
+ | `--no-redact` | Keep `Set-Cookie` and `Authorization` values in reports |
136
+ | `-o, --output` | Write a JSON report |
137
+ | `--html` | Write a self-contained HTML report |
138
+ | `--jsonl` | Append each result to a JSONL file as soon as it is found |
139
+ | `--json` | Print the JSON report to stdout |
140
+ | `--force` | Overwrite existing report files |
141
+ | `-q, --quiet` | Suppress terminal output |
142
+ | `--fail-if-unreachable` | Exit with status 3 when no target answered |
143
+
144
+ Exit codes: `0` success, `1` runtime error, `2` usage error or authorization
145
+ not confirmed, `3` nothing reachable (with `--fail-if-unreachable`), `130`
146
+ interrupted. An interrupted or crashed scan still writes its reports, marked
147
+ as incomplete.
148
+
149
+ ### Example output
150
+
151
+ Every scan produces the same data in three forms. The terminal report is shown
152
+ in the recording at the top of this page; the other two are written on request.
153
+
154
+ The JSON report is the full dataset, suitable for piping into `jq` or feeding
155
+ another tool:
156
+
157
+ ```json
158
+ {
159
+ "target": "example.com",
160
+ "http": [
161
+ {
162
+ "url": "https://example.com",
163
+ "status": 200,
164
+ "title": "Example Domain",
165
+ "technologies": ["Cloudflare"],
166
+ "interesting_paths": ["/robots.txt"]
167
+ }
168
+ ]
169
+ }
170
+ ```
171
+
172
+ The HTML report is a single self-contained file — no external stylesheets,
173
+ scripts or fonts — so it can be archived or attached to an engagement report
174
+ and still render years later:
175
+
176
+ ![HTML report for example.com, showing DNS records, subdomains, HTTP results, TLS certificates and open ports](https://raw.githubusercontent.com/Just-Ipavon/web-recon/main/assets/reporthtml.png)
177
+
178
+ ## Design notes
179
+
180
+ **Everything is rate limited.** One `Throttle`
181
+ ([webrecon/ratelimit.py](webrecon/ratelimit.py)) is built per scan and shared by
182
+ every module: DNS queries, the crt.sh lookup, HTTP requests, TLS handshakes and
183
+ port connects each take a slot. It combines a semaphore for in-flight work with
184
+ a token bucket for request rate. Unbounded concurrency is what separates a
185
+ recon tool from a denial-of-service tool.
186
+
187
+ **Scope is explicit.** Every host the tool contacts — subdomains, redirect
188
+ targets, certificate endpoints — is checked against one scope (by default the
189
+ target and its subdomains). Out-of-scope hosts and non-public addresses are
190
+ listed as skipped in the report instead of being probed.
191
+
192
+ **Failures are contained, not fatal.** A DNS timeout, an unreachable host or an
193
+ unparseable certificate is recorded in the result and the scan continues; an
194
+ unexpected exception in one stage becomes an error entry and the other stages
195
+ still run. Errors and unreachable hosts appear in every output format. A
196
+ partial report beats a traceback.
197
+
198
+ **Reports are handled as sensitive.** Cookie and authorization values are
199
+ redacted, report files are created readable by the owner only (mode 0600) and
200
+ are never overwritten without `--force`, and the HTML report carries a
201
+ Content-Security-Policy that forbids scripts.
202
+
203
+ **Modules return dataclasses, not dicts.** Every module produces types from
204
+ [webrecon/models.py](webrecon/models.py), so the report layer never guesses at
205
+ a schema and JSON output stays stable.
206
+
207
+ **Content discovery avoids soft 404s.** Servers that answer `200` with an HTML
208
+ error page are common, so a path only counts as a hit when its body matches the
209
+ shape expected for that file — a `.git/HEAD` must hold a ref or a commit hash,
210
+ an `.env` must contain `KEY=value` lines and no HTML, a `robots.txt` must
211
+ contain directives — and differs from what the server sends for a random path.
212
+
213
+ **Candidates are ranked before truncation.** `--max-subdomains` cuts by
214
+ confidence (target, then certificate transparency, then wordlist guesses), not
215
+ alphabetically, so the hosts most likely to exist survive the cap.
216
+
217
+ ## Architecture
218
+
219
+ ```text
220
+ webrecon/
221
+ ├── cli.py argument parsing, authorization gate, output selection
222
+ ├── scanner.py orchestration: runs enabled modules, assembles report
223
+ ├── context.py ScanContext: shared throttle, scope and report
224
+ ├── targets.py target parsing and scope enforcement
225
+ ├── config.py ScanConfig
226
+ ├── models.py result dataclasses
227
+ ├── ratelimit.py Throttle: concurrency + rate limiting
228
+ ├── data/ bundled subdomain wordlist
229
+ └── modules/
230
+ ├── dns_enum.py records, crt.sh, wordlist brute-force
231
+ ├── http_probe.py probing, redirects, content discovery
232
+ ├── security_headers.py security-header and cookie-flag findings
233
+ ├── tech_detect.py signature-based fingerprinting
234
+ ├── tls_info.py certificate inspection
235
+ ├── port_scan.py TCP connect scan
236
+ └── report.py JSON / JSONL / terminal / HTML rendering
237
+ ```
238
+
239
+ ## Documentation
240
+
241
+ Full technical documentation lives in [docs/](docs/README.md): architecture and
242
+ design decisions, use case specifications, a function-by-function reference, and
243
+ the runtime behaviour — with UML diagrams throughout.
244
+
245
+ | Document | Contents |
246
+ | --- | --- |
247
+ | [01 — Architecture](docs/01-architecture.md) | Context, packages, data model, architectural decisions, deployment |
248
+ | [02 — Use cases](docs/02-use-cases.md) | Actors, use case diagram, eight detailed specifications, operational scenarios |
249
+ | [03 — Function reference](docs/03-function-reference.md) | Every function: behaviour, edge cases, error handling, covering tests |
250
+ | [04 — Runtime behaviour](docs/04-runtime-behaviour.md) | Sequence diagrams, lifecycle, concurrency model, error propagation |
251
+
252
+ ## License
253
+
254
+ MIT — see [LICENSE](LICENSE).
@@ -0,0 +1,82 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "webrecon-cli"
7
+ dynamic = ["version"]
8
+ description = "A small web reconnaissance scanner: DNS, subdomains, HTTP fingerprinting, TLS and ports."
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = "MIT"
12
+ license-files = ["LICENSE"]
13
+ classifiers = [
14
+ "Development Status :: 4 - Beta",
15
+ "Environment :: Console",
16
+ "Intended Audience :: Information Technology",
17
+ "Intended Audience :: System Administrators",
18
+ "Operating System :: OS Independent",
19
+ "Programming Language :: Python :: 3",
20
+ "Programming Language :: Python :: 3 :: Only",
21
+ "Programming Language :: Python :: 3.10",
22
+ "Programming Language :: Python :: 3.11",
23
+ "Programming Language :: Python :: 3.12",
24
+ "Programming Language :: Python :: 3.13",
25
+ "Topic :: Internet :: Name Service (DNS)",
26
+ "Topic :: Security",
27
+ "Typing :: Typed",
28
+ ]
29
+ dependencies = [
30
+ "httpx>=0.27",
31
+ "dnspython>=2.6",
32
+ "rich>=13.7",
33
+ "cryptography>=42",
34
+ ]
35
+
36
+ [project.urls]
37
+ Homepage = "https://github.com/Just-Ipavon/web-recon"
38
+ Documentation = "https://github.com/Just-Ipavon/web-recon/tree/main/docs"
39
+ Issues = "https://github.com/Just-Ipavon/web-recon/issues"
40
+
41
+ [project.optional-dependencies]
42
+ dev = [
43
+ "pytest>=8.0",
44
+ "pytest-asyncio>=0.24",
45
+ "ruff>=0.5",
46
+ "mypy>=1.10",
47
+ ]
48
+
49
+ [project.scripts]
50
+ webrecon = "webrecon.cli:main"
51
+
52
+ [tool.setuptools.dynamic]
53
+ version = { attr = "webrecon.__version__" }
54
+
55
+ [tool.setuptools.packages.find]
56
+ include = ["webrecon*"]
57
+
58
+ [tool.setuptools.package-data]
59
+ webrecon = ["data/*.txt", "py.typed"]
60
+
61
+ [tool.pytest.ini_options]
62
+ testpaths = ["tests"]
63
+ asyncio_mode = "auto"
64
+ asyncio_default_fixture_loop_scope = "function"
65
+
66
+ [tool.ruff]
67
+ line-length = 100
68
+ target-version = "py310"
69
+
70
+ [tool.ruff.lint]
71
+ select = ["E", "F", "I", "UP", "B"]
72
+
73
+ [tool.mypy]
74
+ files = ["webrecon"]
75
+ warn_unused_ignores = true
76
+ warn_redundant_casts = true
77
+ check_untyped_defs = true
78
+ disallow_incomplete_defs = true
79
+
80
+ [[tool.mypy.overrides]]
81
+ module = ["dns.*"]
82
+ ignore_missing_imports = true
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+