webrecon-cli 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- webrecon/__init__.py +7 -0
- webrecon/__main__.py +6 -0
- webrecon/cli.py +520 -0
- webrecon/config.py +63 -0
- webrecon/context.py +61 -0
- webrecon/data/subdomains-small.txt +140 -0
- webrecon/models.py +121 -0
- webrecon/modules/__init__.py +1 -0
- webrecon/modules/dns_enum.py +408 -0
- webrecon/modules/http_probe.py +407 -0
- webrecon/modules/port_scan.py +134 -0
- webrecon/modules/report.py +408 -0
- webrecon/modules/security_headers.py +69 -0
- webrecon/modules/tech_detect.py +125 -0
- webrecon/modules/tls_info.py +208 -0
- webrecon/py.typed +0 -0
- webrecon/ratelimit.py +61 -0
- webrecon/scanner.py +264 -0
- webrecon/targets.py +234 -0
- webrecon_cli-0.2.1.dist-info/METADATA +290 -0
- webrecon_cli-0.2.1.dist-info/RECORD +25 -0
- webrecon_cli-0.2.1.dist-info/WHEEL +5 -0
- webrecon_cli-0.2.1.dist-info/entry_points.txt +2 -0
- webrecon_cli-0.2.1.dist-info/licenses/LICENSE +21 -0
- webrecon_cli-0.2.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,408 @@
|
|
|
1
|
+
"""Rendering of scan reports to JSON, JSONL, terminal output and standalone HTML.
|
|
2
|
+
|
|
3
|
+
Reports can contain response headers, banners and other data chosen by the
|
|
4
|
+
scanned servers. Report files are therefore created readable by the owner only
|
|
5
|
+
(mode 0600) and an existing file is never overwritten unless asked to.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import html
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
from collections.abc import Sequence
|
|
14
|
+
from dataclasses import asdict, is_dataclass
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import IO, Any
|
|
17
|
+
|
|
18
|
+
from rich.console import Console
|
|
19
|
+
from rich.markup import escape
|
|
20
|
+
from rich.panel import Panel
|
|
21
|
+
from rich.table import Table
|
|
22
|
+
|
|
23
|
+
from ..models import ScanReport
|
|
24
|
+
|
|
25
|
+
Reports = ScanReport | Sequence[ScanReport]
|
|
26
|
+
|
|
27
|
+
HTML_CSP = "default-src 'none'; style-src 'unsafe-inline'"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _as_list(reports: Reports) -> list[ScanReport]:
|
|
31
|
+
return [reports] if isinstance(reports, ScanReport) else list(reports)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def to_json(reports: Reports, indent: int = 2) -> str:
|
|
35
|
+
"""One report serialises to an object, several to an array of objects."""
|
|
36
|
+
if isinstance(reports, ScanReport):
|
|
37
|
+
payload: Any = reports.to_dict()
|
|
38
|
+
else:
|
|
39
|
+
payload = [r.to_dict() for r in reports]
|
|
40
|
+
return json.dumps(payload, indent=indent, sort_keys=False)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def open_private(path: Path, force: bool = False) -> IO[str]:
|
|
44
|
+
"""Open ``path`` for writing with mode 0600, refusing to overwrite unless ``force``."""
|
|
45
|
+
flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
|
|
46
|
+
if not force:
|
|
47
|
+
flags |= os.O_EXCL
|
|
48
|
+
fd = os.open(path, flags, 0o600)
|
|
49
|
+
if hasattr(os, "fchmod"):
|
|
50
|
+
os.fchmod(fd, 0o600) # an overwritten file keeps its old mode otherwise
|
|
51
|
+
return os.fdopen(fd, "w", encoding="utf-8")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _write(path: Path, text: str, force: bool) -> None:
|
|
55
|
+
with open_private(path, force) as fh:
|
|
56
|
+
fh.write(text)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def write_json(reports: Reports, path: Path, force: bool = False) -> None:
|
|
60
|
+
_write(path, to_json(reports) + "\n", force)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def write_html(reports: Reports, path: Path, force: bool = False) -> None:
|
|
64
|
+
_write(path, to_html(reports), force)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class JsonlWriter:
|
|
68
|
+
"""Appends one JSON object per result as the scan produces it.
|
|
69
|
+
|
|
70
|
+
Every line is flushed immediately, so an interrupted or crashed scan still
|
|
71
|
+
leaves a usable file behind.
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
def __init__(self, path: Path, force: bool = False) -> None:
|
|
75
|
+
self.path = path
|
|
76
|
+
self._fh = open_private(path, force)
|
|
77
|
+
|
|
78
|
+
def write(self, target: str, kind: str, payload: Any) -> None:
|
|
79
|
+
data = asdict(payload) if is_dataclass(payload) and not isinstance(payload, type) \
|
|
80
|
+
else payload
|
|
81
|
+
self._fh.write(json.dumps({"target": target, "type": kind, "data": data}) + "\n")
|
|
82
|
+
self._fh.flush()
|
|
83
|
+
|
|
84
|
+
def close(self) -> None:
|
|
85
|
+
self._fh.close()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
# --------------------------------------------------------------------------- terminal
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _status_line(report: ScanReport) -> str:
|
|
92
|
+
if report.complete:
|
|
93
|
+
return "[green]complete[/green]"
|
|
94
|
+
reason = escape(report.incomplete_reason or "scan did not finish")
|
|
95
|
+
return f"[bold yellow]INCOMPLETE[/bold yellow] ({reason})"
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def render_terminal(reports: Reports, console: Console | None = None) -> None:
|
|
99
|
+
"""Print a human-readable summary of each scan.
|
|
100
|
+
|
|
101
|
+
Every value that came from the network is escaped, so a page title or a
|
|
102
|
+
banner cannot inject rich markup into the terminal.
|
|
103
|
+
"""
|
|
104
|
+
console = console or Console()
|
|
105
|
+
for report in _as_list(reports):
|
|
106
|
+
_render_one(report, console)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _render_one(report: ScanReport, console: Console) -> None:
|
|
110
|
+
e = escape
|
|
111
|
+
lines = [
|
|
112
|
+
f"[bold]{e(report.target)}[/bold]",
|
|
113
|
+
f"started {e(report.started_at)}",
|
|
114
|
+
f"finished {e(report.finished_at or '-')}",
|
|
115
|
+
f"status {_status_line(report)}",
|
|
116
|
+
]
|
|
117
|
+
if report.scope:
|
|
118
|
+
lines.append(f"scope {e('; '.join(report.scope))}")
|
|
119
|
+
if report.authorization:
|
|
120
|
+
auth = report.authorization
|
|
121
|
+
lines.append(
|
|
122
|
+
f"authorised by {e(str(auth.get('operator') or '-'))} at "
|
|
123
|
+
f"{e(str(auth.get('confirmed_at') or '-'))}"
|
|
124
|
+
)
|
|
125
|
+
console.print(Panel("\n".join(lines), title="webrecon", border_style="cyan"))
|
|
126
|
+
|
|
127
|
+
if report.dns:
|
|
128
|
+
if report.dns.records:
|
|
129
|
+
table = Table(title="DNS records", header_style="bold cyan")
|
|
130
|
+
table.add_column("type")
|
|
131
|
+
table.add_column("value", overflow="fold")
|
|
132
|
+
for rtype, values in report.dns.records.items():
|
|
133
|
+
table.add_row(e(rtype), e("\n".join(values)))
|
|
134
|
+
console.print(table)
|
|
135
|
+
failed = {k: v for k, v in report.dns.status.items() if v not in ("OK", "NOANSWER")}
|
|
136
|
+
if failed:
|
|
137
|
+
console.print(
|
|
138
|
+
"[dim]DNS query status:[/dim] "
|
|
139
|
+
+ e(", ".join(f"{k}={v}" for k, v in failed.items()))
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
if report.subdomains:
|
|
143
|
+
table = Table(title=f"Subdomains ({len(report.subdomains)})", header_style="bold cyan")
|
|
144
|
+
table.add_column("host")
|
|
145
|
+
table.add_column("addresses", overflow="fold")
|
|
146
|
+
table.add_column("source")
|
|
147
|
+
for sub in report.subdomains:
|
|
148
|
+
table.add_row(e(sub.name), e(", ".join(sub.addresses)), e(sub.source))
|
|
149
|
+
console.print(table)
|
|
150
|
+
|
|
151
|
+
live = [h for h in report.http if h.reachable]
|
|
152
|
+
dead = [h for h in report.http if not h.reachable]
|
|
153
|
+
if live:
|
|
154
|
+
table = Table(title=f"HTTP ({len(live)} reachable)", header_style="bold cyan")
|
|
155
|
+
table.add_column("url", overflow="fold")
|
|
156
|
+
table.add_column("status", justify="right")
|
|
157
|
+
table.add_column("title", overflow="fold")
|
|
158
|
+
table.add_column("technologies", overflow="fold")
|
|
159
|
+
for result in live:
|
|
160
|
+
colour = "green" if result.status and result.status < 400 else "yellow"
|
|
161
|
+
table.add_row(
|
|
162
|
+
e(result.final_url or result.url),
|
|
163
|
+
f"[{colour}]{result.status}[/{colour}]",
|
|
164
|
+
e(result.title or "-"),
|
|
165
|
+
e(", ".join(result.technologies) or "-"),
|
|
166
|
+
)
|
|
167
|
+
console.print(table)
|
|
168
|
+
|
|
169
|
+
if dead:
|
|
170
|
+
table = Table(title=f"Unreachable ({len(dead)})", header_style="bold cyan")
|
|
171
|
+
table.add_column("url", overflow="fold")
|
|
172
|
+
table.add_column("error", overflow="fold")
|
|
173
|
+
for result in dead:
|
|
174
|
+
table.add_row(e(result.url), f"[red]{e(result.error or 'no response')}[/red]")
|
|
175
|
+
console.print(table)
|
|
176
|
+
|
|
177
|
+
paths = [(r.url, p) for r in live for p in r.interesting_paths]
|
|
178
|
+
if paths:
|
|
179
|
+
table = Table(title="Interesting paths", header_style="bold cyan")
|
|
180
|
+
table.add_column("host", overflow="fold")
|
|
181
|
+
table.add_column("path", overflow="fold")
|
|
182
|
+
for url, path in paths:
|
|
183
|
+
table.add_row(e(url), e(path))
|
|
184
|
+
console.print(table)
|
|
185
|
+
|
|
186
|
+
findings = _findings(report)
|
|
187
|
+
if findings:
|
|
188
|
+
table = Table(title=f"Findings ({len(findings)})", header_style="bold cyan")
|
|
189
|
+
table.add_column("where", overflow="fold")
|
|
190
|
+
table.add_column("finding", overflow="fold")
|
|
191
|
+
for where, finding in findings:
|
|
192
|
+
table.add_row(e(where), e(finding))
|
|
193
|
+
console.print(table)
|
|
194
|
+
|
|
195
|
+
if report.tls:
|
|
196
|
+
table = Table(title="TLS certificates", header_style="bold cyan")
|
|
197
|
+
table.add_column("endpoint")
|
|
198
|
+
table.add_column("issuer", overflow="fold")
|
|
199
|
+
table.add_column("expires")
|
|
200
|
+
table.add_column("days", justify="right")
|
|
201
|
+
table.add_column("trusted")
|
|
202
|
+
for cert in report.tls:
|
|
203
|
+
endpoint = e(f"{cert.host}:{cert.port}")
|
|
204
|
+
if cert.error:
|
|
205
|
+
table.add_row(endpoint, f"[red]{e(cert.error)}[/red]", "-", "-", "-")
|
|
206
|
+
continue
|
|
207
|
+
days = cert.days_until_expiry
|
|
208
|
+
colour = "red" if days is not None and days < 15 else "green"
|
|
209
|
+
trusted = {True: "[green]yes[/green]", False: "[red]no[/red]"}.get(
|
|
210
|
+
cert.trusted, "-" # type: ignore[arg-type]
|
|
211
|
+
)
|
|
212
|
+
table.add_row(
|
|
213
|
+
endpoint,
|
|
214
|
+
e(cert.issuer or "-"),
|
|
215
|
+
e(cert.not_after or "-"),
|
|
216
|
+
f"[{colour}]{days if days is not None else '-'}[/{colour}]",
|
|
217
|
+
trusted,
|
|
218
|
+
)
|
|
219
|
+
console.print(table)
|
|
220
|
+
|
|
221
|
+
if report.ports:
|
|
222
|
+
table = Table(title="Open ports", header_style="bold cyan")
|
|
223
|
+
table.add_column("port", justify="right")
|
|
224
|
+
table.add_column("service")
|
|
225
|
+
table.add_column("banner", overflow="fold")
|
|
226
|
+
for port in report.ports:
|
|
227
|
+
table.add_row(str(port.port), e(port.service or "-"), e(port.banner or "-"))
|
|
228
|
+
console.print(table)
|
|
229
|
+
|
|
230
|
+
if report.skipped:
|
|
231
|
+
table = Table(title=f"Skipped ({len(report.skipped)})", header_style="bold cyan")
|
|
232
|
+
table.add_column("host", overflow="fold")
|
|
233
|
+
table.add_column("reason", overflow="fold")
|
|
234
|
+
for skipped in report.skipped:
|
|
235
|
+
table.add_row(e(skipped.host), e(skipped.reason))
|
|
236
|
+
console.print(table)
|
|
237
|
+
|
|
238
|
+
for note in report.notes:
|
|
239
|
+
console.print(f"[cyan]note[/cyan] {e(note)}")
|
|
240
|
+
for error in report.errors:
|
|
241
|
+
console.print(f"[red]error[/red] {e(error)}")
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _findings(report: ScanReport) -> list[tuple[str, str]]:
|
|
245
|
+
out: list[tuple[str, str]] = []
|
|
246
|
+
if report.dns:
|
|
247
|
+
out += [(report.dns.domain, f) for f in report.dns.findings]
|
|
248
|
+
for result in report.http:
|
|
249
|
+
out += [(result.final_url or result.url, f) for f in result.findings]
|
|
250
|
+
for cert in report.tls:
|
|
251
|
+
out += [(f"{cert.host}:{cert.port}", f) for f in cert.findings]
|
|
252
|
+
return out
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
# ------------------------------------------------------------------------------- html
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _esc(value: object) -> str:
|
|
259
|
+
return html.escape(str(value if value not in (None, "") else "-"))
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def _section(title: str, headers: list[str], rows: list[list[Any]], css: str = "") -> str:
|
|
263
|
+
if not rows:
|
|
264
|
+
return ""
|
|
265
|
+
head = "".join(f"<th>{_esc(h)}</th>" for h in headers)
|
|
266
|
+
body = "".join(
|
|
267
|
+
"<tr>" + "".join(f"<td>{_esc(c)}</td>" for c in row) + "</tr>" for row in rows
|
|
268
|
+
)
|
|
269
|
+
cls = f" class='{css}'" if css else ""
|
|
270
|
+
return (
|
|
271
|
+
f"<h2{cls}>{_esc(title)}</h2><div class='wrap'><table>"
|
|
272
|
+
f"<thead><tr>{head}</tr></thead><tbody>{body}</tbody></table></div>"
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _html_report(report: ScanReport) -> str:
|
|
277
|
+
dns_records = report.dns.records if report.dns else {}
|
|
278
|
+
status = (
|
|
279
|
+
"<p class='ok'>Scan complete.</p>"
|
|
280
|
+
if report.complete
|
|
281
|
+
else f"<p class='warn'><strong>Incomplete report:</strong> "
|
|
282
|
+
f"{_esc(report.incomplete_reason or 'the scan did not finish')}</p>"
|
|
283
|
+
)
|
|
284
|
+
meta = [f"{_esc(report.started_at)} → {_esc(report.finished_at)}"]
|
|
285
|
+
if report.scope:
|
|
286
|
+
meta.append(f"Scope: {_esc('; '.join(report.scope))}")
|
|
287
|
+
if report.authorization:
|
|
288
|
+
auth = report.authorization
|
|
289
|
+
meta.append(
|
|
290
|
+
f"Authorised by {_esc(auth.get('operator'))} at {_esc(auth.get('confirmed_at'))} "
|
|
291
|
+
f"({_esc(auth.get('method'))})"
|
|
292
|
+
)
|
|
293
|
+
settings = report.settings
|
|
294
|
+
if settings.get("proxy"):
|
|
295
|
+
meta.append(f"Proxy: {_esc(settings['proxy'])} ({_esc(settings.get('proxy_applies_to'))})")
|
|
296
|
+
if settings:
|
|
297
|
+
meta.append(f"TLS verification: {'on' if settings.get('verify_tls') else 'off'}")
|
|
298
|
+
|
|
299
|
+
parts = [
|
|
300
|
+
f"<h1>webrecon — {_esc(report.target)}</h1>",
|
|
301
|
+
"<p class='meta'>" + "<br>".join(meta) + "</p>",
|
|
302
|
+
status,
|
|
303
|
+
_section("Errors", ["Error"], [[e] for e in report.errors], css="bad"),
|
|
304
|
+
_section(
|
|
305
|
+
"DNS records",
|
|
306
|
+
["Type", "Values"],
|
|
307
|
+
[[k, ", ".join(v)] for k, v in dns_records.items()],
|
|
308
|
+
),
|
|
309
|
+
_section(
|
|
310
|
+
"DNS query status",
|
|
311
|
+
["Query", "Status"],
|
|
312
|
+
[
|
|
313
|
+
[k, v]
|
|
314
|
+
for k, v in (report.dns.status.items() if report.dns else [])
|
|
315
|
+
if v not in ("OK", "NOANSWER")
|
|
316
|
+
],
|
|
317
|
+
),
|
|
318
|
+
_section(
|
|
319
|
+
"Subdomains",
|
|
320
|
+
["Host", "Addresses", "Source"],
|
|
321
|
+
[[s.name, ", ".join(s.addresses), s.source] for s in report.subdomains],
|
|
322
|
+
),
|
|
323
|
+
_section(
|
|
324
|
+
"HTTP",
|
|
325
|
+
["URL", "Status", "Title", "Technologies"],
|
|
326
|
+
[
|
|
327
|
+
[r.final_url or r.url, r.status, r.title, ", ".join(r.technologies)]
|
|
328
|
+
for r in report.http
|
|
329
|
+
if r.reachable
|
|
330
|
+
],
|
|
331
|
+
),
|
|
332
|
+
_section(
|
|
333
|
+
"Unreachable",
|
|
334
|
+
["URL", "Error"],
|
|
335
|
+
[[r.url, r.error or "no response"] for r in report.http if not r.reachable],
|
|
336
|
+
),
|
|
337
|
+
_section(
|
|
338
|
+
"Interesting paths",
|
|
339
|
+
["Host", "Path"],
|
|
340
|
+
[[r.url, p] for r in report.http for p in r.interesting_paths],
|
|
341
|
+
),
|
|
342
|
+
_section("Findings", ["Where", "Finding"], [list(f) for f in _findings(report)]),
|
|
343
|
+
_section(
|
|
344
|
+
"TLS",
|
|
345
|
+
["Endpoint", "Issuer", "Expires", "Days left", "Trusted", "Protocol"],
|
|
346
|
+
[
|
|
347
|
+
[
|
|
348
|
+
f"{c.host}:{c.port}",
|
|
349
|
+
c.issuer or c.error,
|
|
350
|
+
c.not_after,
|
|
351
|
+
c.days_until_expiry,
|
|
352
|
+
{True: "yes", False: "no"}.get(c.trusted) if c.trusted is not None else None,
|
|
353
|
+
c.protocol,
|
|
354
|
+
]
|
|
355
|
+
for c in report.tls
|
|
356
|
+
],
|
|
357
|
+
),
|
|
358
|
+
_section(
|
|
359
|
+
"Open ports",
|
|
360
|
+
["Port", "Service", "Banner"],
|
|
361
|
+
[[p.port, p.service, p.banner] for p in report.ports],
|
|
362
|
+
),
|
|
363
|
+
_section("Skipped", ["Host", "Reason"], [[s.host, s.reason] for s in report.skipped]),
|
|
364
|
+
_section("Notes", ["Note"], [[n] for n in report.notes]),
|
|
365
|
+
]
|
|
366
|
+
if not any(
|
|
367
|
+
(report.dns and report.dns.records, report.subdomains, report.http, report.tls,
|
|
368
|
+
report.ports, report.errors)
|
|
369
|
+
):
|
|
370
|
+
parts.append("<p class='meta'>No results were collected.</p>")
|
|
371
|
+
return "<section>" + "".join(parts) + "</section>"
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def to_html(reports: Reports) -> str:
|
|
375
|
+
"""Render a self-contained HTML report with no external assets.
|
|
376
|
+
|
|
377
|
+
A Content-Security-Policy meta tag forbids scripts and remote resources as
|
|
378
|
+
defence in depth on top of escaping every value.
|
|
379
|
+
"""
|
|
380
|
+
items = _as_list(reports)
|
|
381
|
+
title = ", ".join(r.target for r in items) or "no targets"
|
|
382
|
+
body = "<hr>".join(_html_report(r) for r in items)
|
|
383
|
+
return f"""<!doctype html>
|
|
384
|
+
<html lang="en"><head><meta charset="utf-8">
|
|
385
|
+
<meta http-equiv="Content-Security-Policy" content="{HTML_CSP}">
|
|
386
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
387
|
+
<title>webrecon — {_esc(title)}</title>
|
|
388
|
+
<style>
|
|
389
|
+
:root {{ color-scheme: light dark; }}
|
|
390
|
+
body {{ font: 15px/1.5 system-ui, sans-serif; margin: 0 auto; padding: 2rem 1rem;
|
|
391
|
+
max-width: 60rem; }}
|
|
392
|
+
h1 {{ margin-bottom: .25rem; }}
|
|
393
|
+
.meta {{ color: #888; margin-bottom: 1rem; }}
|
|
394
|
+
.warn {{ color: #b35c00; }}
|
|
395
|
+
.ok {{ color: #2f9e44; }}
|
|
396
|
+
h2 {{ margin-top: 2rem; font-size: 1.1rem; }}
|
|
397
|
+
h2.bad {{ color: #c92a2a; }}
|
|
398
|
+
.wrap {{ overflow-x: auto; }}
|
|
399
|
+
table {{ border-collapse: collapse; width: 100%; font-size: .9rem; }}
|
|
400
|
+
th, td {{ text-align: left; padding: .4rem .6rem;
|
|
401
|
+
border-bottom: 1px solid rgba(128,128,128,.3); vertical-align: top; }}
|
|
402
|
+
th {{ font-weight: 600; white-space: nowrap; }}
|
|
403
|
+
td {{ word-break: break-word; }}
|
|
404
|
+
hr {{ margin: 3rem 0; border: 0; border-top: 1px solid rgba(128,128,128,.4); }}
|
|
405
|
+
</style></head>
|
|
406
|
+
<body>
|
|
407
|
+
{body}
|
|
408
|
+
</body></html>"""
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Security-header and cookie-flag analysis of an HTTP response."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from collections.abc import Iterable, Mapping
|
|
7
|
+
|
|
8
|
+
HSTS_MIN_SECONDS = 180 * 24 * 3600
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def cookie_attributes(raw: str) -> tuple[str, set[str], dict[str, str]]:
|
|
12
|
+
"""Split a Set-Cookie value into (name, flag attributes, valued attributes)."""
|
|
13
|
+
pair, *attrs = raw.split(";")
|
|
14
|
+
name = pair.split("=", 1)[0].strip()
|
|
15
|
+
flags: set[str] = set()
|
|
16
|
+
values: dict[str, str] = {}
|
|
17
|
+
for attr in attrs:
|
|
18
|
+
key, sep, value = attr.strip().partition("=")
|
|
19
|
+
key = key.strip().lower()
|
|
20
|
+
if not key:
|
|
21
|
+
continue
|
|
22
|
+
if sep:
|
|
23
|
+
values[key] = value.strip().lower()
|
|
24
|
+
else:
|
|
25
|
+
flags.add(key)
|
|
26
|
+
return name, flags, values
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def analyse(headers: Mapping[str, str], cookies: Iterable[str], https: bool) -> list[str]:
|
|
30
|
+
"""Return human-readable findings for missing or weak protections."""
|
|
31
|
+
h = {k.lower(): v for k, v in headers.items()}
|
|
32
|
+
findings: list[str] = []
|
|
33
|
+
|
|
34
|
+
if https:
|
|
35
|
+
hsts = h.get("strict-transport-security")
|
|
36
|
+
if not hsts:
|
|
37
|
+
findings.append("missing Strict-Transport-Security")
|
|
38
|
+
else:
|
|
39
|
+
match = re.search(r"max-age\s*=\s*\"?(\d+)", hsts, re.IGNORECASE)
|
|
40
|
+
if not match or int(match.group(1)) < HSTS_MIN_SECONDS:
|
|
41
|
+
findings.append("Strict-Transport-Security max-age below 180 days")
|
|
42
|
+
|
|
43
|
+
csp = h.get("content-security-policy")
|
|
44
|
+
if not csp:
|
|
45
|
+
findings.append("missing Content-Security-Policy")
|
|
46
|
+
if h.get("x-content-type-options", "").strip().lower() != "nosniff":
|
|
47
|
+
findings.append("missing X-Content-Type-Options: nosniff")
|
|
48
|
+
if "x-frame-options" not in h and not (csp and "frame-ancestors" in csp.lower()):
|
|
49
|
+
findings.append("missing X-Frame-Options or CSP frame-ancestors (clickjacking)")
|
|
50
|
+
if "referrer-policy" not in h:
|
|
51
|
+
findings.append("missing Referrer-Policy")
|
|
52
|
+
|
|
53
|
+
for raw in cookies:
|
|
54
|
+
name, flags, values = cookie_attributes(raw)
|
|
55
|
+
if not name:
|
|
56
|
+
continue
|
|
57
|
+
missing = []
|
|
58
|
+
if https and "secure" not in flags:
|
|
59
|
+
missing.append("Secure")
|
|
60
|
+
if "httponly" not in flags:
|
|
61
|
+
missing.append("HttpOnly")
|
|
62
|
+
if "samesite" not in values:
|
|
63
|
+
missing.append("SameSite")
|
|
64
|
+
elif values["samesite"] == "none" and "secure" not in flags:
|
|
65
|
+
missing.append("Secure (required by SameSite=None)")
|
|
66
|
+
if missing:
|
|
67
|
+
findings.append(f"cookie {name} without {', '.join(missing)}")
|
|
68
|
+
|
|
69
|
+
return findings
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Signature-based technology fingerprinting from HTTP responses.
|
|
2
|
+
|
|
3
|
+
Deliberately conservative: a signature only fires on evidence the server
|
|
4
|
+
actually sent (headers, cookies, body markers), never on guesswork. Patterns
|
|
5
|
+
are anchored to markup or header structure so that prose mentioning a product
|
|
6
|
+
name does not count as evidence.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
from collections.abc import Iterable, Mapping
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class Signature:
|
|
18
|
+
name: str
|
|
19
|
+
header: tuple[str, str] | None = None # (header name, regex over its value)
|
|
20
|
+
cookie: str | None = None # cookie name, matched case-insensitively
|
|
21
|
+
cookie_value: str | None = None # optional regex the cookie value must match
|
|
22
|
+
body: str | None = None # regex over the response body
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
# itsdangerous-signed Flask session: payload.timestamp.signature, base64url,
|
|
26
|
+
# with a leading "." when the payload is compressed.
|
|
27
|
+
_FLASK_SESSION = r"^\.?[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$"
|
|
28
|
+
|
|
29
|
+
SIGNATURES: tuple[Signature, ...] = (
|
|
30
|
+
Signature("nginx", header=("server", r"\bnginx\b")),
|
|
31
|
+
Signature("Apache Tomcat", header=("server", r"apache-coyote|\btomcat\b")),
|
|
32
|
+
Signature("Apache", header=("server", r"^apache(?!-coyote)(?:/|\s|$)")),
|
|
33
|
+
Signature("Microsoft IIS", header=("server", r"^microsoft-iis")),
|
|
34
|
+
Signature("Caddy", header=("server", r"^caddy\b")),
|
|
35
|
+
Signature("LiteSpeed", header=("server", r"^litespeed")),
|
|
36
|
+
Signature("Cloudflare", header=("server", r"^cloudflare")),
|
|
37
|
+
Signature("Cloudflare", header=("cf-ray", r".")),
|
|
38
|
+
Signature("Amazon CloudFront", header=("x-amz-cf-id", r".")),
|
|
39
|
+
Signature("Fastly", header=("x-served-by", r"\bcache-")),
|
|
40
|
+
Signature("Varnish", header=("x-varnish", r".")),
|
|
41
|
+
Signature("PHP", header=("x-powered-by", r"\bphp\b")),
|
|
42
|
+
Signature("PHP", cookie="PHPSESSID"),
|
|
43
|
+
Signature("ASP.NET", header=("x-powered-by", r"asp\.net")),
|
|
44
|
+
Signature("ASP.NET", header=("x-aspnet-version", r".")),
|
|
45
|
+
Signature("ASP.NET", cookie="ASP.NET_SessionId"),
|
|
46
|
+
Signature("Express", header=("x-powered-by", r"^express\b")),
|
|
47
|
+
Signature("Next.js", header=("x-powered-by", r"next\.js")),
|
|
48
|
+
Signature("Next.js", body=r"""["'(]/_next/static/"""),
|
|
49
|
+
Signature("Nuxt", body=r"window\.__NUXT__|<div id=[\"']__nuxt[\"']"),
|
|
50
|
+
Signature("React", body=r"\bdata-reactroot\b|__REACT_DEVTOOLS"),
|
|
51
|
+
Signature("Vue.js", body=r"\bdata-v-app\b|__VUE__"),
|
|
52
|
+
Signature("Angular", body=r"\bng-version=[\"']"),
|
|
53
|
+
Signature("WordPress", body=r"""["'(][^"'()]*/wp-(?:content|includes)/"""),
|
|
54
|
+
Signature("WordPress", header=("link", r"/wp-json/")),
|
|
55
|
+
Signature(
|
|
56
|
+
"Joomla",
|
|
57
|
+
body=r"<meta[^>]+name=[\"']generator[\"'][^>]+content=[\"']Joomla|"
|
|
58
|
+
r"[\"'][^\"']*/media/(?:jui|system/js)/",
|
|
59
|
+
),
|
|
60
|
+
Signature("Drupal", header=("x-generator", r"^drupal")),
|
|
61
|
+
Signature("Drupal", header=("x-drupal-cache", r".")),
|
|
62
|
+
Signature("Drupal", body=r"""["'][^"']*/sites/default/files/"""),
|
|
63
|
+
Signature("Shopify", header=("x-shopid", r".")),
|
|
64
|
+
Signature("Django", cookie="csrftoken"),
|
|
65
|
+
Signature("Django", body=r"name=[\"']csrfmiddlewaretoken[\"']"),
|
|
66
|
+
Signature("Laravel", cookie="laravel_session"),
|
|
67
|
+
Signature("Flask", cookie="session", cookie_value=_FLASK_SESSION),
|
|
68
|
+
Signature("Ruby on Rails", cookie="_rails_session"),
|
|
69
|
+
Signature("jQuery", body=r"jquery[.-][\d.]+(?:\.min)?\.js"),
|
|
70
|
+
Signature("Bootstrap", body=r"bootstrap[.-][\d.]+(?:\.min)?\.(?:js|css)"),
|
|
71
|
+
Signature("Google Analytics", body=r"google-analytics\.com/|gtag\(\s*['\"]config['\"]"),
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def parse_cookies(set_cookie_values: Iterable[str]) -> dict[str, str]:
|
|
76
|
+
"""Map cookie name (lowercased) to value, one Set-Cookie header per item.
|
|
77
|
+
|
|
78
|
+
Only the part before the first ``;`` is the name/value pair; attributes
|
|
79
|
+
such as ``Expires=Wed, 21 Oct 2026`` are never mistaken for cookies.
|
|
80
|
+
"""
|
|
81
|
+
jar: dict[str, str] = {}
|
|
82
|
+
for raw in set_cookie_values:
|
|
83
|
+
pair = raw.split(";", 1)[0]
|
|
84
|
+
name, sep, value = pair.partition("=")
|
|
85
|
+
name = name.strip()
|
|
86
|
+
if sep and name:
|
|
87
|
+
jar[name.lower()] = value.strip().strip('"')
|
|
88
|
+
return jar
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def detect(
|
|
92
|
+
headers: Mapping[str, str], body: str = "", cookies: Iterable[str] | None = None
|
|
93
|
+
) -> list[str]:
|
|
94
|
+
"""Return the technologies evidenced by these headers, cookies and body, sorted.
|
|
95
|
+
|
|
96
|
+
``cookies`` is the list of raw Set-Cookie header values (one per header,
|
|
97
|
+
as ``httpx.Headers.get_list`` returns them). When omitted, a ``set-cookie``
|
|
98
|
+
entry in ``headers`` is split on newlines.
|
|
99
|
+
"""
|
|
100
|
+
lowered = {k.lower(): v for k, v in headers.items()}
|
|
101
|
+
if cookies is None:
|
|
102
|
+
cookies = [c for c in lowered.get("set-cookie", "").split("\n") if c.strip()]
|
|
103
|
+
jar = parse_cookies(cookies)
|
|
104
|
+
found: set[str] = set()
|
|
105
|
+
|
|
106
|
+
for sig in SIGNATURES:
|
|
107
|
+
if sig.name in found:
|
|
108
|
+
continue
|
|
109
|
+
if sig.header:
|
|
110
|
+
key, pattern = sig.header
|
|
111
|
+
value = lowered.get(key)
|
|
112
|
+
if value and re.search(pattern, value, re.IGNORECASE):
|
|
113
|
+
found.add(sig.name)
|
|
114
|
+
continue
|
|
115
|
+
if sig.cookie:
|
|
116
|
+
value = jar.get(sig.cookie.lower())
|
|
117
|
+
if value is not None and (
|
|
118
|
+
sig.cookie_value is None or re.search(sig.cookie_value, value)
|
|
119
|
+
):
|
|
120
|
+
found.add(sig.name)
|
|
121
|
+
continue
|
|
122
|
+
if sig.body and body and re.search(sig.body, body, re.IGNORECASE):
|
|
123
|
+
found.add(sig.name)
|
|
124
|
+
|
|
125
|
+
return sorted(found)
|