huntkit 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- huntkit/__init__.py +4 -0
- huntkit/__main__.py +8 -0
- huntkit/ai/__init__.py +42 -0
- huntkit/ai/advisor.py +133 -0
- huntkit/ai/providers.py +200 -0
- huntkit/app/__init__.py +13 -0
- huntkit/app/config_cmd.py +65 -0
- huntkit/app/context.py +50 -0
- huntkit/app/main.py +1166 -0
- huntkit/app/plugins_cmd.py +79 -0
- huntkit/app/workspace_cmd.py +62 -0
- huntkit/core/__init__.py +6 -0
- huntkit/core/cache.py +67 -0
- huntkit/core/config.py +272 -0
- huntkit/core/logger.py +66 -0
- huntkit/core/pacing.py +71 -0
- huntkit/core/runner.py +117 -0
- huntkit/core/state.py +154 -0
- huntkit/core/workspace.py +145 -0
- huntkit/enrich/__init__.py +39 -0
- huntkit/enrich/asn.py +139 -0
- huntkit/enrich/engine.py +167 -0
- huntkit/enrich/favicon.py +91 -0
- huntkit/enrich/mmh3.py +69 -0
- huntkit/fingerprint/__init__.py +26 -0
- huntkit/fingerprint/engine.py +83 -0
- huntkit/fingerprint/rules.py +170 -0
- huntkit/history/__init__.py +38 -0
- huntkit/history/diff.py +95 -0
- huntkit/history/store.py +133 -0
- huntkit/intel/__init__.py +48 -0
- huntkit/intel/engine.py +233 -0
- huntkit/intel/signals.py +826 -0
- huntkit/knowledge/__init__.py +27 -0
- huntkit/knowledge/playbooks.py +452 -0
- huntkit/monitor/__init__.py +24 -0
- huntkit/monitor/runner.py +111 -0
- huntkit/notify/__init__.py +16 -0
- huntkit/notify/webhook.py +73 -0
- huntkit/pipeline.py +302 -0
- huntkit/plugins/__init__.py +36 -0
- huntkit/plugins/amass.py +25 -0
- huntkit/plugins/arjun.py +18 -0
- huntkit/plugins/assetfinder.py +18 -0
- huntkit/plugins/base.py +220 -0
- huntkit/plugins/chaos.py +20 -0
- huntkit/plugins/dalfox.py +24 -0
- huntkit/plugins/dnsx.py +19 -0
- huntkit/plugins/ffuf.py +34 -0
- huntkit/plugins/findomain.py +18 -0
- huntkit/plugins/gau.py +18 -0
- huntkit/plugins/hakrawler.py +19 -0
- huntkit/plugins/httpx.py +30 -0
- huntkit/plugins/katana.py +23 -0
- huntkit/plugins/naabu.py +22 -0
- huntkit/plugins/nmap.py +39 -0
- huntkit/plugins/nuclei.py +27 -0
- huntkit/plugins/registry.py +117 -0
- huntkit/plugins/subfinder.py +21 -0
- huntkit/plugins/waybackurls.py +29 -0
- huntkit/plugins/whatweb.py +19 -0
- huntkit/py.typed +0 -0
- huntkit/report/__init__.py +103 -0
- huntkit/report/builder.py +145 -0
- huntkit/report/render.py +323 -0
- huntkit/scanner/__init__.py +28 -0
- huntkit/scanner/runner.py +239 -0
- huntkit/secrets/__init__.py +34 -0
- huntkit/secrets/patterns.py +172 -0
- huntkit/secrets/scanner.py +133 -0
- huntkit/shots/__init__.py +29 -0
- huntkit/shots/runner.py +186 -0
- huntkit/tui/__init__.py +13 -0
- huntkit/tui/dashboard.py +160 -0
- huntkit/utils/__init__.py +6 -0
- huntkit/utils/filesystem.py +109 -0
- huntkit/utils/process.py +104 -0
- huntkit/utils/severity.py +49 -0
- huntkit/utils/terminal.py +88 -0
- huntkit/utils/validators.py +99 -0
- huntkit-0.3.0.dist-info/METADATA +340 -0
- huntkit-0.3.0.dist-info/RECORD +86 -0
- huntkit-0.3.0.dist-info/WHEEL +5 -0
- huntkit-0.3.0.dist-info/entry_points.txt +2 -0
- huntkit-0.3.0.dist-info/licenses/LICENSE +21 -0
- huntkit-0.3.0.dist-info/top_level.txt +1 -0
huntkit/enrich/engine.py
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""Enrichment engine — favicon hashes and ASN origins over the recon surface.
|
|
2
|
+
|
|
3
|
+
The recon pipeline gathers live URLs and resolved hosts; this layer adds the
|
|
4
|
+
two *asset-correlation* signals that turn that surface into leads elsewhere:
|
|
5
|
+
|
|
6
|
+
- a **favicon hash** per origin (the Shodan pivot), and
|
|
7
|
+
- the **ASN / BGP prefix** behind each host (the network to sweep next).
|
|
8
|
+
|
|
9
|
+
It mirrors the scanner's shape — targets are read from the workspace's own,
|
|
10
|
+
already scope-filtered result files *and* re-checked against scope here, the
|
|
11
|
+
work runs on a thread pool, and results are merged and persisted once on the
|
|
12
|
+
main thread. Both the HTTP fetch and the ASN resolve/lookup are injected, so
|
|
13
|
+
the engine stays testable without the network.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
import time
|
|
20
|
+
import urllib.request
|
|
21
|
+
from collections.abc import Callable
|
|
22
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
23
|
+
from dataclasses import dataclass, field
|
|
24
|
+
from typing import Optional
|
|
25
|
+
from urllib.parse import urlparse
|
|
26
|
+
|
|
27
|
+
from ..core.workspace import Workspace
|
|
28
|
+
from .asn import AsnResult, LookupFn, ResolveFn, asn_for, default_lookup, default_resolve
|
|
29
|
+
from .favicon import FaviconResult, FetchFn, hash_favicon
|
|
30
|
+
|
|
31
|
+
# URL sources (already scope-filtered by recon; re-checked here anyway).
|
|
32
|
+
URL_FILES = ["recon/live.txt", "urls/urls.txt"]
|
|
33
|
+
HOST_FILES = ["recon/live.txt", "recon/resolved.txt", "urls/urls.txt"]
|
|
34
|
+
|
|
35
|
+
ENRICH_JSON = "scans/enrich.json"
|
|
36
|
+
ENRICH_STATE = "enrich"
|
|
37
|
+
|
|
38
|
+
_UA = "HuntKit/enrich (+authorised recon)"
|
|
39
|
+
_HTTP_TIMEOUT = 10
|
|
40
|
+
|
|
41
|
+
EventFn = Callable[[dict], None]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass
|
|
45
|
+
class EnrichReport:
|
|
46
|
+
program: str
|
|
47
|
+
favicons: list[FaviconResult] = field(default_factory=list)
|
|
48
|
+
asns: list[AsnResult] = field(default_factory=list)
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def favicon_hits(self) -> int:
|
|
52
|
+
return sum(1 for f in self.favicons if f.ok)
|
|
53
|
+
|
|
54
|
+
@property
|
|
55
|
+
def asn_hits(self) -> int:
|
|
56
|
+
return sum(1 for a in self.asns if a.ok)
|
|
57
|
+
|
|
58
|
+
@property
|
|
59
|
+
def unique_asns(self) -> list[str]:
|
|
60
|
+
seen = {a.asn for a in self.asns if a.ok}
|
|
61
|
+
return sorted(seen, key=lambda s: int(s))
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def default_fetch(url: str) -> bytes:
|
|
65
|
+
"""Fetch a URL's body over HTTP, raising on any non-200 or transport error."""
|
|
66
|
+
# scheme guard first: never let a file:// or other scheme reach urlopen
|
|
67
|
+
if urlparse(url).scheme not in ("http", "https"):
|
|
68
|
+
raise ValueError("only http(s) favicons are fetched")
|
|
69
|
+
req = urllib.request.Request(url, headers={"User-Agent": _UA}) # noqa: S310 — http(s) only
|
|
70
|
+
with urllib.request.urlopen(req, timeout=_HTTP_TIMEOUT) as resp: # noqa: S310
|
|
71
|
+
if resp.status != 200:
|
|
72
|
+
raise OSError(f"HTTP {resp.status}")
|
|
73
|
+
return resp.read()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _host_of(url: str) -> Optional[str]:
|
|
77
|
+
parsed = urlparse(url if "://" in url else f"//{url}", scheme="http")
|
|
78
|
+
return parsed.hostname
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def enrich_origins(ws: Workspace) -> list[str]:
|
|
82
|
+
"""One representative URL per in-scope origin (scheme + netloc), deduped."""
|
|
83
|
+
seen: dict[str, str] = {}
|
|
84
|
+
for rel in URL_FILES:
|
|
85
|
+
for url in ws.read_lines(rel):
|
|
86
|
+
parsed = urlparse(url if "://" in url else f"http://{url}")
|
|
87
|
+
host = parsed.hostname
|
|
88
|
+
if not host or not ws.in_scope(host):
|
|
89
|
+
continue
|
|
90
|
+
origin = f"{parsed.scheme}://{parsed.netloc}"
|
|
91
|
+
seen.setdefault(origin, origin)
|
|
92
|
+
return sorted(seen)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def enrich_hosts(ws: Workspace) -> list[str]:
|
|
96
|
+
"""Unique in-scope hostnames across the recon result files."""
|
|
97
|
+
seen: dict[str, None] = {}
|
|
98
|
+
for rel in HOST_FILES:
|
|
99
|
+
for line in ws.read_lines(rel):
|
|
100
|
+
host = _host_of(line)
|
|
101
|
+
if host and ws.in_scope(host):
|
|
102
|
+
seen[host] = None
|
|
103
|
+
return sorted(seen)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def run_enrich(
|
|
107
|
+
ws: Workspace,
|
|
108
|
+
*,
|
|
109
|
+
do_favicon: bool = True,
|
|
110
|
+
do_asn: bool = True,
|
|
111
|
+
threads: int = 10,
|
|
112
|
+
fetch: FetchFn = default_fetch,
|
|
113
|
+
resolve: ResolveFn = default_resolve,
|
|
114
|
+
lookup: LookupFn = default_lookup,
|
|
115
|
+
on_event: Optional[EventFn] = None,
|
|
116
|
+
) -> EnrichReport:
|
|
117
|
+
"""Run the requested enrichers over the workspace and persist the report."""
|
|
118
|
+
emit: EventFn = on_event or (lambda _e: None)
|
|
119
|
+
report = EnrichReport(program=ws.program)
|
|
120
|
+
ws.state.start(ENRICH_STATE)
|
|
121
|
+
|
|
122
|
+
if do_favicon:
|
|
123
|
+
origins = enrich_origins(ws)
|
|
124
|
+
emit({"event": "favicon_start", "targets": len(origins)})
|
|
125
|
+
report.favicons = _run_pool(origins, lambda o: hash_favicon(o, fetch), threads)
|
|
126
|
+
emit({"event": "favicon_done", "hits": report.favicon_hits})
|
|
127
|
+
|
|
128
|
+
if do_asn:
|
|
129
|
+
hosts = enrich_hosts(ws)
|
|
130
|
+
emit({"event": "asn_start", "targets": len(hosts)})
|
|
131
|
+
report.asns = _run_pool(
|
|
132
|
+
hosts, lambda h: asn_for(h, resolve=resolve, lookup=lookup), threads
|
|
133
|
+
)
|
|
134
|
+
emit({"event": "asn_done", "hits": report.asn_hits})
|
|
135
|
+
|
|
136
|
+
_persist(ws, report)
|
|
137
|
+
ws.state.done(ENRICH_STATE, favicons=report.favicon_hits, asns=report.asn_hits)
|
|
138
|
+
return report
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _run_pool(items: list[str], work: Callable[[str], object], threads: int) -> list:
|
|
142
|
+
"""Map ``work`` over ``items`` on a thread pool, order-preserving.
|
|
143
|
+
|
|
144
|
+
A worker that raises is isolated: the batch keeps its other results.
|
|
145
|
+
"""
|
|
146
|
+
if not items:
|
|
147
|
+
return []
|
|
148
|
+
out: dict[int, object] = {}
|
|
149
|
+
with ThreadPoolExecutor(max_workers=max(1, min(threads, len(items)))) as pool:
|
|
150
|
+
futures = {pool.submit(work, item): i for i, item in enumerate(items)}
|
|
151
|
+
for fut in as_completed(futures):
|
|
152
|
+
idx = futures[fut]
|
|
153
|
+
try:
|
|
154
|
+
out[idx] = fut.result()
|
|
155
|
+
except Exception: # noqa: BLE001 — a crashing worker must not sink the pool
|
|
156
|
+
out[idx] = None
|
|
157
|
+
return [out[i] for i in range(len(items)) if out.get(i) is not None]
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _persist(ws: Workspace, report: EnrichReport) -> None:
|
|
161
|
+
payload = {
|
|
162
|
+
"program": report.program,
|
|
163
|
+
"generated": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
164
|
+
"favicons": [f.to_dict() for f in report.favicons],
|
|
165
|
+
"asns": [a.to_dict() for a in report.asns],
|
|
166
|
+
}
|
|
167
|
+
ws.path(ENRICH_JSON).write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""Favicon hashing — the Shodan pivot from one icon to every host serving it.
|
|
2
|
+
|
|
3
|
+
A favicon is a strong asset-correlation signal: load balancers, dev/staging
|
|
4
|
+
mirrors, and forgotten origins behind a CDN very often ship the *same* icon as
|
|
5
|
+
the flagship site. Hashing it (``mmh3`` over the base64 of the raw bytes, the
|
|
6
|
+
Shodan convention) gives a single integer you can paste into
|
|
7
|
+
``http.favicon.hash:<h>`` on Shodan/Censys to surface the rest of the estate.
|
|
8
|
+
|
|
9
|
+
The fetch is a plain HTTP GET to an in-scope target — the same kind of request
|
|
10
|
+
recon already makes — and is injectable so the engine (and the tests) can swap
|
|
11
|
+
in a stubbed fetcher instead of hitting the network.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import base64
|
|
17
|
+
from collections.abc import Callable
|
|
18
|
+
from dataclasses import dataclass
|
|
19
|
+
from typing import Optional
|
|
20
|
+
from urllib.parse import urljoin, urlparse
|
|
21
|
+
|
|
22
|
+
from . import mmh3
|
|
23
|
+
|
|
24
|
+
# Fetcher contract: given an absolute URL, return the response body bytes, or
|
|
25
|
+
# raise on any failure (timeout, non-200, connection error). The engine wraps
|
|
26
|
+
# the default urllib fetcher so callers can inject a stub in tests.
|
|
27
|
+
FetchFn = Callable[[str], bytes]
|
|
28
|
+
|
|
29
|
+
_MAX_BYTES = 1024 * 1024 # a favicon over 1 MiB is not a favicon
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class FaviconResult:
|
|
34
|
+
"""One host's favicon hash and the search pivot it unlocks."""
|
|
35
|
+
|
|
36
|
+
url: str
|
|
37
|
+
hash: Optional[int] = None
|
|
38
|
+
size: int = 0
|
|
39
|
+
error: str = ""
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def ok(self) -> bool:
|
|
43
|
+
return self.hash is not None
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def shodan(self) -> str:
|
|
47
|
+
return f"http.favicon.hash:{self.hash}" if self.ok else ""
|
|
48
|
+
|
|
49
|
+
def to_dict(self) -> dict:
|
|
50
|
+
return {
|
|
51
|
+
"url": self.url,
|
|
52
|
+
"hash": self.hash,
|
|
53
|
+
"size": self.size,
|
|
54
|
+
"shodan": self.shodan,
|
|
55
|
+
"error": self.error,
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def favicon_url(base: str) -> str:
|
|
60
|
+
"""The ``/favicon.ico`` URL for a site, given any URL on that origin."""
|
|
61
|
+
parsed = urlparse(base if "://" in base else f"http://{base}")
|
|
62
|
+
origin = f"{parsed.scheme}://{parsed.netloc}"
|
|
63
|
+
return urljoin(origin + "/", "favicon.ico")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def favicon_hash(data: bytes) -> int:
|
|
67
|
+
"""The Shodan-convention favicon hash: mmh3 over the base64 of the bytes.
|
|
68
|
+
|
|
69
|
+
``base64.encodebytes`` is used (not ``b64encode``) because it inserts a
|
|
70
|
+
newline every 76 chars and a trailing newline — the exact byte stream
|
|
71
|
+
Shodan hashes, so the result matches ``http.favicon.hash:`` queries.
|
|
72
|
+
"""
|
|
73
|
+
return mmh3.hash_bytes(base64.encodebytes(data))
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def hash_favicon(base_url: str, fetch: FetchFn) -> FaviconResult:
|
|
77
|
+
"""Fetch the favicon for ``base_url`` and hash it, never raising.
|
|
78
|
+
|
|
79
|
+
A fetch failure (missing icon, timeout, oversize) is recorded on the
|
|
80
|
+
result rather than propagated, so one dead host cannot sink a batch.
|
|
81
|
+
"""
|
|
82
|
+
url = favicon_url(base_url)
|
|
83
|
+
try:
|
|
84
|
+
data = fetch(url)
|
|
85
|
+
except Exception as exc: # noqa: BLE001 — a bad fetch is data, not a crash
|
|
86
|
+
return FaviconResult(url=url, error=str(exc) or type(exc).__name__)
|
|
87
|
+
if not data:
|
|
88
|
+
return FaviconResult(url=url, error="empty response")
|
|
89
|
+
if len(data) > _MAX_BYTES:
|
|
90
|
+
return FaviconResult(url=url, size=len(data), error="too large")
|
|
91
|
+
return FaviconResult(url=url, hash=favicon_hash(data), size=len(data))
|
huntkit/enrich/mmh3.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""MurmurHash3 (x86, 32-bit) — a self-contained, pure-Python port.
|
|
2
|
+
|
|
3
|
+
Shodan indexes a site's favicon by ``mmh3.hash(base64(favicon_bytes))``, so a
|
|
4
|
+
favicon hash is the pivot that turns one asset into *every* host serving the
|
|
5
|
+
same icon. The reference hasher ships as a C extension (``mmh3``); HuntKit
|
|
6
|
+
keeps its runtime dependencies to three pure-Python packages, so rather than
|
|
7
|
+
pull a compiled wheel onto a hunter's box we port the ~40-line algorithm here.
|
|
8
|
+
|
|
9
|
+
This is a faithful port of Austin Appleby's ``MurmurHash3_x86_32`` (public
|
|
10
|
+
domain). It returns a *signed* 32-bit int by default, matching the ``mmh3``
|
|
11
|
+
package — which is the exact value Shodan's ``http.favicon.hash:`` expects.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
_C1 = 0xCC9E2D51
|
|
17
|
+
_C2 = 0x1B873593
|
|
18
|
+
_MASK = 0xFFFFFFFF
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _rotl32(x: int, r: int) -> int:
|
|
22
|
+
return ((x << r) | (x >> (32 - r))) & _MASK
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def hash_bytes(data: bytes, seed: int = 0, *, signed: bool = True) -> int:
|
|
26
|
+
"""MurmurHash3_x86_32 of ``data``.
|
|
27
|
+
|
|
28
|
+
``signed`` (the default) folds the result into the signed 32-bit range,
|
|
29
|
+
so it equals ``mmh3.hash(data, seed)`` from the C package.
|
|
30
|
+
"""
|
|
31
|
+
length = len(data)
|
|
32
|
+
nblocks = length // 4
|
|
33
|
+
h1 = seed & _MASK
|
|
34
|
+
|
|
35
|
+
# body — four bytes at a time, little-endian
|
|
36
|
+
for i in range(nblocks):
|
|
37
|
+
k1 = int.from_bytes(data[i * 4 : i * 4 + 4], "little")
|
|
38
|
+
k1 = (k1 * _C1) & _MASK
|
|
39
|
+
k1 = _rotl32(k1, 15)
|
|
40
|
+
k1 = (k1 * _C2) & _MASK
|
|
41
|
+
h1 ^= k1
|
|
42
|
+
h1 = _rotl32(h1, 13)
|
|
43
|
+
h1 = (h1 * 5 + 0xE6546B64) & _MASK
|
|
44
|
+
|
|
45
|
+
# tail — the trailing 0..3 bytes
|
|
46
|
+
tail = data[nblocks * 4 :]
|
|
47
|
+
k1 = 0
|
|
48
|
+
if len(tail) >= 3:
|
|
49
|
+
k1 ^= tail[2] << 16
|
|
50
|
+
if len(tail) >= 2:
|
|
51
|
+
k1 ^= tail[1] << 8
|
|
52
|
+
if len(tail) >= 1:
|
|
53
|
+
k1 ^= tail[0]
|
|
54
|
+
k1 = (k1 * _C1) & _MASK
|
|
55
|
+
k1 = _rotl32(k1, 15)
|
|
56
|
+
k1 = (k1 * _C2) & _MASK
|
|
57
|
+
h1 ^= k1
|
|
58
|
+
|
|
59
|
+
# finalisation mix (fmix32)
|
|
60
|
+
h1 ^= length
|
|
61
|
+
h1 ^= h1 >> 16
|
|
62
|
+
h1 = (h1 * 0x85EBCA6B) & _MASK
|
|
63
|
+
h1 ^= h1 >> 13
|
|
64
|
+
h1 = (h1 * 0xC2B2AE35) & _MASK
|
|
65
|
+
h1 ^= h1 >> 16
|
|
66
|
+
|
|
67
|
+
if signed and h1 & 0x80000000:
|
|
68
|
+
return h1 - 0x100000000
|
|
69
|
+
return h1
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Fingerprint layer — recognise CDN/WAF/cloud/tech from HTTP headers.
|
|
2
|
+
|
|
3
|
+
A pure feature layer (depends only on ``utils``): it classifies response
|
|
4
|
+
headers the operator already captured against a declarative catalog. Offline;
|
|
5
|
+
never touches a target.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from .engine import (
|
|
11
|
+
FingerprintMatch,
|
|
12
|
+
fingerprint_headers,
|
|
13
|
+
has_waf,
|
|
14
|
+
parse_headers,
|
|
15
|
+
)
|
|
16
|
+
from .rules import FINGERPRINT_RULES, KINDS, FingerprintRule
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"FINGERPRINT_RULES",
|
|
20
|
+
"FingerprintMatch",
|
|
21
|
+
"FingerprintRule",
|
|
22
|
+
"KINDS",
|
|
23
|
+
"fingerprint_headers",
|
|
24
|
+
"has_waf",
|
|
25
|
+
"parse_headers",
|
|
26
|
+
]
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""Fingerprint engine — classify HTTP response headers against the catalog.
|
|
2
|
+
|
|
3
|
+
Accepts either a parsed header mapping or a raw header block (as printed by
|
|
4
|
+
``curl -sI`` or saved from a response), and returns the vendors it recognises,
|
|
5
|
+
WAF first. Pure: it only reads headers the operator already captured.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
|
|
12
|
+
from ..utils.severity import Severity
|
|
13
|
+
from .rules import FINGERPRINT_RULES, KINDS
|
|
14
|
+
|
|
15
|
+
_KIND_ORDER = {k: i for i, k in enumerate(KINDS)}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class FingerprintMatch:
|
|
20
|
+
id: str
|
|
21
|
+
name: str
|
|
22
|
+
kind: str
|
|
23
|
+
severity: Severity
|
|
24
|
+
evidence: str # the header that matched, e.g. "cf-ray: 7a1b…"
|
|
25
|
+
|
|
26
|
+
def to_dict(self) -> dict:
|
|
27
|
+
return {
|
|
28
|
+
"id": self.id,
|
|
29
|
+
"name": self.name,
|
|
30
|
+
"kind": self.kind,
|
|
31
|
+
"severity": self.severity.label,
|
|
32
|
+
"evidence": self.evidence,
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def parse_headers(text: str) -> dict[str, str]:
|
|
37
|
+
"""Parse a raw header block into a lowercased name->value map.
|
|
38
|
+
|
|
39
|
+
The HTTP status line is skipped and repeated headers (notably Set-Cookie)
|
|
40
|
+
are joined with '; ' so a single substring check still sees them all.
|
|
41
|
+
"""
|
|
42
|
+
headers: dict[str, str] = {}
|
|
43
|
+
for line in text.splitlines():
|
|
44
|
+
if ":" not in line:
|
|
45
|
+
continue
|
|
46
|
+
name, _, value = line.partition(":")
|
|
47
|
+
name = name.strip().lower()
|
|
48
|
+
value = value.strip()
|
|
49
|
+
if not name or name.startswith("http/"):
|
|
50
|
+
continue
|
|
51
|
+
headers[name] = f"{headers[name]}; {value}" if name in headers else value
|
|
52
|
+
return headers
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _evidence(rule, headers: dict[str, str]) -> str:
|
|
56
|
+
for hname, vsub in rule.headers:
|
|
57
|
+
val = headers.get(hname)
|
|
58
|
+
if val is None:
|
|
59
|
+
continue
|
|
60
|
+
if vsub == "" or vsub in val.lower():
|
|
61
|
+
shown = val if len(val) <= 60 else val[:57] + "…"
|
|
62
|
+
return f"{hname}: {shown}"
|
|
63
|
+
return ""
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def fingerprint_headers(headers: str | dict) -> list[FingerprintMatch]:
|
|
67
|
+
"""Return matched vendors for the given headers, WAF first then by kind."""
|
|
68
|
+
hmap = (
|
|
69
|
+
parse_headers(headers)
|
|
70
|
+
if isinstance(headers, str)
|
|
71
|
+
else {str(k).lower(): str(v) for k, v in headers.items()}
|
|
72
|
+
)
|
|
73
|
+
out: list[FingerprintMatch] = []
|
|
74
|
+
for rule in FINGERPRINT_RULES:
|
|
75
|
+
ev = _evidence(rule, hmap)
|
|
76
|
+
if ev:
|
|
77
|
+
out.append(FingerprintMatch(rule.id, rule.name, rule.kind, rule.severity, ev))
|
|
78
|
+
out.sort(key=lambda m: (-int(m.severity), _KIND_ORDER.get(m.kind, 9), m.name))
|
|
79
|
+
return out
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def has_waf(matches: list[FingerprintMatch]) -> bool:
|
|
83
|
+
return any(m.kind == "waf" for m in matches)
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
"""CDN / WAF / cloud / tech fingerprint catalog.
|
|
2
|
+
|
|
3
|
+
Each rule maps one vendor to the HTTP response headers (and Set-Cookie values)
|
|
4
|
+
that give it away. A rule matches when ANY of its header signals hit — a header
|
|
5
|
+
is present and, if a value substring is given, contains it. Plain data, so
|
|
6
|
+
adding a vendor is one row (same ethos as the signal and secret catalogs).
|
|
7
|
+
|
|
8
|
+
Pure and offline: it classifies response headers the operator already captured
|
|
9
|
+
(``curl -sI``, a saved response). WAF matches are flagged MEDIUM because a WAF
|
|
10
|
+
changes how the rest of the engagement should be run; everything else is INFO.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
|
|
17
|
+
from ..utils.severity import Severity
|
|
18
|
+
|
|
19
|
+
# kind values, in the order they should surface.
|
|
20
|
+
KINDS = ("waf", "cdn", "cloud", "server", "tech")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class FingerprintRule:
|
|
25
|
+
id: str
|
|
26
|
+
name: str
|
|
27
|
+
kind: str # one of KINDS
|
|
28
|
+
headers: tuple[tuple[str, str], ...] # (header-name-lower, value-substr-lower); "" = presence
|
|
29
|
+
severity: Severity = Severity.INFO
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
_WAF = Severity.MEDIUM
|
|
33
|
+
_INFO = Severity.INFO
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
FINGERPRINT_RULES: tuple[FingerprintRule, ...] = (
|
|
37
|
+
# ---- WAF / security proxies (flagged) --------------------------------
|
|
38
|
+
FingerprintRule(
|
|
39
|
+
"imperva",
|
|
40
|
+
"Imperva / Incapsula WAF",
|
|
41
|
+
"waf",
|
|
42
|
+
(
|
|
43
|
+
("x-iinfo", ""),
|
|
44
|
+
("x-cdn", "incapsula"),
|
|
45
|
+
("set-cookie", "incap_ses"),
|
|
46
|
+
("set-cookie", "visid_incap"),
|
|
47
|
+
),
|
|
48
|
+
_WAF,
|
|
49
|
+
),
|
|
50
|
+
FingerprintRule(
|
|
51
|
+
"sucuri",
|
|
52
|
+
"Sucuri CloudProxy WAF",
|
|
53
|
+
"waf",
|
|
54
|
+
(("x-sucuri-id", ""), ("x-sucuri-cache", ""), ("server", "sucuri/cloudproxy")),
|
|
55
|
+
_WAF,
|
|
56
|
+
),
|
|
57
|
+
FingerprintRule(
|
|
58
|
+
"f5-bigip",
|
|
59
|
+
"F5 BIG-IP / ASM",
|
|
60
|
+
"waf",
|
|
61
|
+
(("set-cookie", "bigipserver"), ("server", "big-ip"), ("x-waf-event-info", "")),
|
|
62
|
+
_WAF,
|
|
63
|
+
),
|
|
64
|
+
FingerprintRule(
|
|
65
|
+
"citrix-netscaler",
|
|
66
|
+
"Citrix NetScaler",
|
|
67
|
+
"waf",
|
|
68
|
+
(("set-cookie", "ns_af"), ("set-cookie", "citrix_ns_id"), ("via", "ns-cache")),
|
|
69
|
+
_WAF,
|
|
70
|
+
),
|
|
71
|
+
FingerprintRule(
|
|
72
|
+
"barracuda",
|
|
73
|
+
"Barracuda WAF",
|
|
74
|
+
"waf",
|
|
75
|
+
(("set-cookie", "barra_counter_session"), ("server", "barracuda")),
|
|
76
|
+
_WAF,
|
|
77
|
+
),
|
|
78
|
+
FingerprintRule("fortiweb", "Fortinet FortiWeb", "waf", (("set-cookie", "fortiwafsid"),), _WAF),
|
|
79
|
+
FingerprintRule(
|
|
80
|
+
"modsecurity",
|
|
81
|
+
"ModSecurity",
|
|
82
|
+
"waf",
|
|
83
|
+
(("server", "mod_security"), ("server", "modsecurity")),
|
|
84
|
+
_WAF,
|
|
85
|
+
),
|
|
86
|
+
FingerprintRule("aws-waf", "AWS WAF", "waf", (("x-amzn-waf-action", ""),), _WAF),
|
|
87
|
+
# ---- CDN -------------------------------------------------------------
|
|
88
|
+
FingerprintRule(
|
|
89
|
+
"cloudflare",
|
|
90
|
+
"Cloudflare",
|
|
91
|
+
"cdn",
|
|
92
|
+
(("server", "cloudflare"), ("cf-ray", ""), ("cf-cache-status", "")),
|
|
93
|
+
_INFO,
|
|
94
|
+
),
|
|
95
|
+
FingerprintRule(
|
|
96
|
+
"fastly",
|
|
97
|
+
"Fastly",
|
|
98
|
+
"cdn",
|
|
99
|
+
(("x-served-by", "cache"), ("x-fastly-request-id", ""), ("via", "fastly")),
|
|
100
|
+
_INFO,
|
|
101
|
+
),
|
|
102
|
+
FingerprintRule(
|
|
103
|
+
"cloudfront",
|
|
104
|
+
"Amazon CloudFront",
|
|
105
|
+
"cdn",
|
|
106
|
+
(("via", "cloudfront"), ("x-amz-cf-id", ""), ("x-amz-cf-pop", "")),
|
|
107
|
+
_INFO,
|
|
108
|
+
),
|
|
109
|
+
FingerprintRule(
|
|
110
|
+
"akamai",
|
|
111
|
+
"Akamai",
|
|
112
|
+
"cdn",
|
|
113
|
+
(("server", "akamaighost"), ("x-akamai-transformed", ""), ("x-akamai-request-id", "")),
|
|
114
|
+
_INFO,
|
|
115
|
+
),
|
|
116
|
+
FingerprintRule("keycdn", "KeyCDN", "cdn", (("server", "keycdn-engine"),), _INFO),
|
|
117
|
+
FingerprintRule("bunnycdn", "BunnyCDN", "cdn", (("server", "bunnycdn"),), _INFO),
|
|
118
|
+
# ---- cloud / hosting -------------------------------------------------
|
|
119
|
+
FingerprintRule(
|
|
120
|
+
"aws-s3", "Amazon S3", "cloud", (("server", "amazons3"), ("x-amz-bucket-region", "")), _INFO
|
|
121
|
+
),
|
|
122
|
+
FingerprintRule(
|
|
123
|
+
"aws", "Amazon Web Services", "cloud", (("x-amz-request-id", ""), ("x-amz-id-2", "")), _INFO
|
|
124
|
+
),
|
|
125
|
+
FingerprintRule(
|
|
126
|
+
"gcp",
|
|
127
|
+
"Google Cloud",
|
|
128
|
+
"cloud",
|
|
129
|
+
(("server", "gws"), ("server", "gse"), ("x-goog-generation", "")),
|
|
130
|
+
_INFO,
|
|
131
|
+
),
|
|
132
|
+
FingerprintRule(
|
|
133
|
+
"azure",
|
|
134
|
+
"Microsoft Azure",
|
|
135
|
+
"cloud",
|
|
136
|
+
(("x-azure-ref", ""), ("x-ms-request-id", ""), ("server", "microsoft-azure")),
|
|
137
|
+
_INFO,
|
|
138
|
+
),
|
|
139
|
+
FingerprintRule("heroku", "Heroku", "cloud", (("via", "vegur"), ("server", "cowboy")), _INFO),
|
|
140
|
+
FingerprintRule(
|
|
141
|
+
"vercel", "Vercel", "cloud", (("x-vercel-id", ""), ("server", "vercel")), _INFO
|
|
142
|
+
),
|
|
143
|
+
FingerprintRule(
|
|
144
|
+
"netlify", "Netlify", "cloud", (("x-nf-request-id", ""), ("server", "netlify")), _INFO
|
|
145
|
+
),
|
|
146
|
+
FingerprintRule("github-pages", "GitHub Pages", "cloud", (("server", "github.com"),), _INFO),
|
|
147
|
+
FingerprintRule(
|
|
148
|
+
"fly-io", "Fly.io", "cloud", (("fly-request-id", ""), ("server", "fly")), _INFO
|
|
149
|
+
),
|
|
150
|
+
FingerprintRule("render", "Render", "cloud", (("x-render-origin-server", ""),), _INFO),
|
|
151
|
+
# ---- server / framework ---------------------------------------------
|
|
152
|
+
FingerprintRule("nginx", "nginx", "server", (("server", "nginx"),), _INFO),
|
|
153
|
+
FingerprintRule("apache", "Apache httpd", "server", (("server", "apache"),), _INFO),
|
|
154
|
+
FingerprintRule("iis", "Microsoft IIS", "server", (("server", "microsoft-iis"),), _INFO),
|
|
155
|
+
FingerprintRule("litespeed", "LiteSpeed", "server", (("server", "litespeed"),), _INFO),
|
|
156
|
+
FingerprintRule("openresty", "OpenResty", "server", (("server", "openresty"),), _INFO),
|
|
157
|
+
FingerprintRule("express", "Node.js / Express", "tech", (("x-powered-by", "express"),), _INFO),
|
|
158
|
+
FingerprintRule("php", "PHP", "tech", (("x-powered-by", "php"),), _INFO),
|
|
159
|
+
FingerprintRule(
|
|
160
|
+
"aspnet", "ASP.NET", "tech", (("x-powered-by", "asp.net"), ("x-aspnet-version", "")), _INFO
|
|
161
|
+
),
|
|
162
|
+
FingerprintRule(
|
|
163
|
+
"nextjs", "Next.js", "tech", (("x-powered-by", "next.js"), ("x-nextjs-cache", "")), _INFO
|
|
164
|
+
),
|
|
165
|
+
FingerprintRule(
|
|
166
|
+
"wordpress", "WordPress", "tech", (("x-pingback", ""), ("link", "wp-json")), _INFO
|
|
167
|
+
),
|
|
168
|
+
FingerprintRule("laravel", "Laravel", "tech", (("set-cookie", "laravel_session"),), _INFO),
|
|
169
|
+
FingerprintRule("django", "Django", "tech", (("set-cookie", "csrftoken"),), _INFO),
|
|
170
|
+
)
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""History layer — snapshot a program's results and diff across runs.
|
|
2
|
+
|
|
3
|
+
Bug bounty is the same scope re-swept over time; this layer freezes the
|
|
4
|
+
workspace's result sets into timestamped snapshots (``history/``) and reports
|
|
5
|
+
the delta between two points — new subdomains, freshly-live URLs, and above
|
|
6
|
+
all new findings. Depends only on ``core``; pure and offline.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from .diff import CategoryDiff, DiffReport, current_vs, diff_snapshots
|
|
12
|
+
from .store import (
|
|
13
|
+
CATEGORIES,
|
|
14
|
+
HISTORY_DIR,
|
|
15
|
+
Snapshot,
|
|
16
|
+
capture,
|
|
17
|
+
find,
|
|
18
|
+
latest,
|
|
19
|
+
list_snapshots,
|
|
20
|
+
load,
|
|
21
|
+
save,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"CATEGORIES",
|
|
26
|
+
"HISTORY_DIR",
|
|
27
|
+
"CategoryDiff",
|
|
28
|
+
"DiffReport",
|
|
29
|
+
"Snapshot",
|
|
30
|
+
"capture",
|
|
31
|
+
"current_vs",
|
|
32
|
+
"diff_snapshots",
|
|
33
|
+
"find",
|
|
34
|
+
"latest",
|
|
35
|
+
"list_snapshots",
|
|
36
|
+
"load",
|
|
37
|
+
"save",
|
|
38
|
+
]
|