stackscan 2.2.2__tar.gz → 2.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {stackscan-2.2.2 → stackscan-2.5.0}/PKG-INFO +4 -3
- {stackscan-2.2.2 → stackscan-2.5.0}/README.md +3 -2
- {stackscan-2.2.2 → stackscan-2.5.0}/scripts/build_cve_db.py +12 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/__init__.py +1 -1
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/analyzers/__init__.py +2 -1
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/analyzers/cve.py +64 -12
- stackscan-2.5.0/src/stackscan/analyzers/generic.py +319 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/analyzers/infra.py +57 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/analyzers/osdetect.py +3 -1
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/analyzers/services.py +5 -15
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/analyzers/tech.py +89 -6
- stackscan-2.5.0/src/stackscan/analyzers/vibe.py +248 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/cli.py +202 -90
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/core/core.py +14 -2
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/data/subdomains.txt +32 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/export.py +42 -9
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/net/__init__.py +4 -1
- stackscan-2.5.0/src/stackscan/net/dns.py +247 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/net/fingerprint.py +7 -1
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/net/ipinfo.py +13 -6
- stackscan-2.5.0/src/stackscan/net/resolver.py +58 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/net/subdomains.py +171 -15
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/net/tld.py +56 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/net/tls.py +31 -19
- stackscan-2.5.0/src/stackscan/net/whois.py +139 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/render.py +220 -82
- stackscan-2.5.0/src/stackscan/scan.py +1092 -0
- stackscan-2.5.0/src/stackscan/scanners/__init__.py +0 -0
- stackscan-2.5.0/src/stackscan/scanners/isp_blocked.py +133 -0
- stackscan-2.5.0/src/stackscan/scanners/secrets.py +107 -0
- stackscan-2.5.0/src/stackscan/scanners/takeover.py +215 -0
- stackscan-2.5.0/src/stackscan/theme.py +63 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/types/__init__.py +6 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/types/output.py +78 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/tests/test_brute.py +6 -4
- stackscan-2.5.0/tests/test_cve.py +254 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/tests/test_extra.py +5 -5
- {stackscan-2.2.2 → stackscan-2.5.0}/tests/test_infra.py +24 -1
- {stackscan-2.2.2 → stackscan-2.5.0}/tests/test_ports.py +3 -1
- {stackscan-2.2.2 → stackscan-2.5.0}/tests/test_scan.py +29 -1
- stackscan-2.5.0/tests/test_scanners.py +97 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/tests/test_services.py +18 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/tests/test_subdomains.py +23 -0
- stackscan-2.5.0/tests/test_tech.py +246 -0
- stackscan-2.5.0/tests/test_theme.py +26 -0
- stackscan-2.5.0/tests/test_vhost.py +133 -0
- stackscan-2.5.0/tests/test_vibe.py +38 -0
- stackscan-2.5.0/tests/test_whois.py +89 -0
- stackscan-2.2.2/src/stackscan/net/dns.py +0 -168
- stackscan-2.2.2/src/stackscan/net/resolver.py +0 -54
- stackscan-2.2.2/src/stackscan/scan.py +0 -560
- stackscan-2.2.2/src/stackscan/theme.py +0 -23
- stackscan-2.2.2/tests/test_cve.py +0 -120
- stackscan-2.2.2/tests/test_tech.py +0 -71
- {stackscan-2.2.2 → stackscan-2.5.0}/.gitignore +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/LICENSE +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/pyproject.toml +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/__main__.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/analyzers/creds.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/analyzers/exposure.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/analyzers/security.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/analyzers/social.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/config/__init__.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/config/sigdb_loader.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/config/sources.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/core/__init__.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/data/builtin.sigdb +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/data/cve.json.gz +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/data/reekeer-logo.png +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/net/geo.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/net/ports.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/types/models.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/utils/__init__.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/utils/paths.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/src/stackscan/utils/urls.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/tests/test_builtin_sigdb.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/tests/test_cli_helpers.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/tests/test_creds.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/tests/test_dns_records.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/tests/test_export.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/tests/test_osdetect.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/tests/test_social.py +0 -0
- {stackscan-2.2.2 → stackscan-2.5.0}/tests/test_sources.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: stackscan
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.5.0
|
|
4
4
|
Summary: Full web stack, infrastructure, DNS, port, subdomain and CVE analyzer.
|
|
5
5
|
Project-URL: Source, https://github.com/reekeer/stackscan
|
|
6
6
|
Project-URL: Issues, https://github.com/reekeer/stackscan/issues
|
|
@@ -117,7 +117,7 @@ stackscan --cve-online example.com
|
|
|
117
117
|
|
|
118
118
|
# Speed / coverage knobs
|
|
119
119
|
stackscan --full --workers 500 --subdomain-limit 10000 example.com
|
|
120
|
-
stackscan --full --site-limit
|
|
120
|
+
stackscan --full --site-limit 20 --workers 200 example.com
|
|
121
121
|
|
|
122
122
|
# JSON (full detail, includes timing) or a compact table
|
|
123
123
|
stackscan --json example.com
|
|
@@ -169,7 +169,8 @@ Below are performance benchmarks conducted on two production targets using diffe
|
|
|
169
169
|
| `--ports` / `--no-nmap` | off | Active port scan (nmap, else Python) / force the Python scanner. |
|
|
170
170
|
| `--subdomains` | off | Enumerate subdomains (AXFR + wordlist + TLS SANs). |
|
|
171
171
|
| `--subdomain-limit` | `5000` | Max ranked labels to resolve (`0` = full list). |
|
|
172
|
-
| `--site-limit` | `
|
|
172
|
+
| `--site-limit` | `50` | Max derived sites to analyze from discovered open ports (`0` = unlimited). |
|
|
173
|
+
| `--cve-min-confidence` | `50` | Hide CVE matches with confidence below N (`0` shows all). |
|
|
173
174
|
| `--default-creds` | off | Bounded default-credential / open-device check (prompts before brute-forcing). |
|
|
174
175
|
| `--full-auto` | off | Auto-accept every brute prompt on discovered devices (enables default-cred checks). |
|
|
175
176
|
| `--cred-limit` | `50` | Max default-credential pairs per device (`0` = full SecLists list). |
|
|
@@ -79,7 +79,7 @@ stackscan --cve-online example.com
|
|
|
79
79
|
|
|
80
80
|
# Speed / coverage knobs
|
|
81
81
|
stackscan --full --workers 500 --subdomain-limit 10000 example.com
|
|
82
|
-
stackscan --full --site-limit
|
|
82
|
+
stackscan --full --site-limit 20 --workers 200 example.com
|
|
83
83
|
|
|
84
84
|
# JSON (full detail, includes timing) or a compact table
|
|
85
85
|
stackscan --json example.com
|
|
@@ -131,7 +131,8 @@ Below are performance benchmarks conducted on two production targets using diffe
|
|
|
131
131
|
| `--ports` / `--no-nmap` | off | Active port scan (nmap, else Python) / force the Python scanner. |
|
|
132
132
|
| `--subdomains` | off | Enumerate subdomains (AXFR + wordlist + TLS SANs). |
|
|
133
133
|
| `--subdomain-limit` | `5000` | Max ranked labels to resolve (`0` = full list). |
|
|
134
|
-
| `--site-limit` | `
|
|
134
|
+
| `--site-limit` | `50` | Max derived sites to analyze from discovered open ports (`0` = unlimited). |
|
|
135
|
+
| `--cve-min-confidence` | `50` | Hide CVE matches with confidence below N (`0` shows all). |
|
|
135
136
|
| `--default-creds` | off | Bounded default-credential / open-device check (prompts before brute-forcing). |
|
|
136
137
|
| `--full-auto` | off | Auto-accept every brute prompt on discovered devices (enables default-cred checks). |
|
|
137
138
|
| `--cred-limit` | `50` | Max default-credential pairs per device (`0` = full SecLists list). |
|
|
@@ -138,6 +138,16 @@ def ranges_for(cve: dict[str, Any], needle: str) -> list[dict[str, str]]:
|
|
|
138
138
|
return out
|
|
139
139
|
|
|
140
140
|
|
|
141
|
+
def _skip_entry(cve: dict[str, Any]) -> bool:
|
|
142
|
+
status = str(cve.get("vulnStatus", "")).upper()
|
|
143
|
+
if status in {"REJECTED", "REPLACED", "DEPRECATED"}:
|
|
144
|
+
return True
|
|
145
|
+
descriptions = cve.get("descriptions") or []
|
|
146
|
+
if not english_summary(descriptions):
|
|
147
|
+
return True
|
|
148
|
+
return False
|
|
149
|
+
|
|
150
|
+
|
|
141
151
|
def collect(product: str, vendor_product: str, throttle: float) -> list[dict[str, Any]]:
|
|
142
152
|
needle = ":" + vendor_product + ":"
|
|
143
153
|
entries: list[dict[str, Any]] = []
|
|
@@ -149,6 +159,8 @@ def collect(product: str, vendor_product: str, throttle: float) -> list[dict[str
|
|
|
149
159
|
vulns = payload.get("vulnerabilities") or []
|
|
150
160
|
for wrapper in vulns:
|
|
151
161
|
cve = wrapper["cve"]
|
|
162
|
+
if _skip_entry(cve):
|
|
163
|
+
continue
|
|
152
164
|
ranges = ranges_for(cve, needle)
|
|
153
165
|
if not ranges:
|
|
154
166
|
continue
|
|
@@ -7,7 +7,7 @@ from stackscan.analyzers.cve import (
|
|
|
7
7
|
software_from_ports,
|
|
8
8
|
)
|
|
9
9
|
from stackscan.analyzers.exposure import ExposureProbe, analyze_exposure
|
|
10
|
-
from stackscan.analyzers.infra import analyze_infra
|
|
10
|
+
from stackscan.analyzers.infra import analyze_infra, summarize_edge
|
|
11
11
|
from stackscan.analyzers.osdetect import detect_os
|
|
12
12
|
from stackscan.analyzers.security import analyze_security_headers
|
|
13
13
|
from stackscan.analyzers.services import classify_services, port_category
|
|
@@ -19,6 +19,7 @@ __all__ = [
|
|
|
19
19
|
"TechAnalyzer",
|
|
20
20
|
"analyze_exposure",
|
|
21
21
|
"analyze_infra",
|
|
22
|
+
"summarize_edge",
|
|
22
23
|
"analyze_security_headers",
|
|
23
24
|
"brute_devices",
|
|
24
25
|
"detect_devices",
|
|
@@ -9,12 +9,13 @@ from functools import lru_cache
|
|
|
9
9
|
from importlib import resources
|
|
10
10
|
from typing import Any, cast
|
|
11
11
|
|
|
12
|
+
from stackscan.analyzers.generic import SERVER_NAMES, extract_generic_software, is_commit_hash
|
|
12
13
|
from stackscan.types import CveMatch, Headers, PortScan, Software
|
|
13
14
|
|
|
14
15
|
CveEntry = dict[str, Any]
|
|
15
16
|
_BACKPORT_DISTRO_RE = re.compile(
|
|
16
17
|
r"(0?ubuntu0?[._]\d[\d.]+)"
|
|
17
|
-
r"|\b(ubuntu|debian|centos|rhel|red\s*hat|fedora|amzn|amazon|rocky|almalinux|alpine|raspbian|suse|opensuse)\b"
|
|
18
|
+
r"|\b(ubuntu|debian|centos|rhel|red\s*hat|fedora|amzn|amazon|rocky|almalinux|alpine|raspbian|suse|opensuse|oracle|sles)\b"
|
|
18
19
|
r"|(\+deb\d+u?\d*)"
|
|
19
20
|
r"|(-\d+ubuntu\d+(?![\d._]))"
|
|
20
21
|
r"|(el\d+)"
|
|
@@ -99,7 +100,10 @@ _JQUERY_RE = re.compile("jquery[-/]?v?(\\d+\\.\\d+(?:\\.\\d+)?)", re.IGNORECASE)
|
|
|
99
100
|
_GENERATOR_RE = re.compile(
|
|
100
101
|
"<meta[^>]+name=[\\\"']generator[\\\"'][^>]+content=[\\\"']([^\\\"']+)[\\\"']", re.IGNORECASE
|
|
101
102
|
)
|
|
102
|
-
_SSH_RE = re.compile("openssh[
|
|
103
|
+
_SSH_RE = re.compile(r"openssh[\s_/:-](\d+\.\d+(?:p\d+)?)", re.IGNORECASE)
|
|
104
|
+
_CORE_COMMIT_RE = re.compile(
|
|
105
|
+
r"([A-Za-z][A-Za-z0-9\s-]*?)\s+Core\s+\(([a-f0-9]{4,})\)", re.IGNORECASE
|
|
106
|
+
)
|
|
103
107
|
|
|
104
108
|
|
|
105
109
|
@lru_cache(maxsize=1)
|
|
@@ -177,8 +181,26 @@ def extract_software(headers: Headers, body: str, location: str = "") -> list[So
|
|
|
177
181
|
server = headers.get("server")
|
|
178
182
|
if server:
|
|
179
183
|
server_os = _distro_tag(server)
|
|
180
|
-
|
|
184
|
+
server_items = _tokens(server, "header:server", location, os=server_os)
|
|
185
|
+
for item in server_items:
|
|
181
186
|
add(item)
|
|
187
|
+
# If the Server banner names a known product without a version (e.g.
|
|
188
|
+
# "nginx" or "Apache"), still record it so it shows up in the software
|
|
189
|
+
# table even when no CVE can be matched yet.
|
|
190
|
+
if not server_items:
|
|
191
|
+
server_lower = server.lower()
|
|
192
|
+
for name in sorted(SERVER_NAMES, key=len, reverse=True):
|
|
193
|
+
if name in server_lower:
|
|
194
|
+
add(
|
|
195
|
+
Software(
|
|
196
|
+
name=name.lower().replace(" ", ""),
|
|
197
|
+
version=None,
|
|
198
|
+
source="header:server",
|
|
199
|
+
location=location,
|
|
200
|
+
os=server_os,
|
|
201
|
+
)
|
|
202
|
+
)
|
|
203
|
+
break
|
|
182
204
|
powered = headers.get("x-powered-by")
|
|
183
205
|
if powered:
|
|
184
206
|
for item in _tokens(powered, "header:x-powered-by", location):
|
|
@@ -199,6 +221,22 @@ def extract_software(headers: Headers, body: str, location: str = "") -> list[So
|
|
|
199
221
|
location=location,
|
|
200
222
|
)
|
|
201
223
|
)
|
|
224
|
+
core_match = _CORE_COMMIT_RE.search(body)
|
|
225
|
+
if core_match:
|
|
226
|
+
product = core_match.group(1).strip()
|
|
227
|
+
commit = core_match.group(2).lower()
|
|
228
|
+
if product:
|
|
229
|
+
add(
|
|
230
|
+
Software(
|
|
231
|
+
name=product.lower().replace(" ", ""),
|
|
232
|
+
version=commit,
|
|
233
|
+
source="body:core-commit",
|
|
234
|
+
location=location,
|
|
235
|
+
)
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
for item in extract_generic_software(body, location=location):
|
|
239
|
+
add(item)
|
|
202
240
|
return software
|
|
203
241
|
|
|
204
242
|
|
|
@@ -211,6 +249,7 @@ def software_from_ports(scan: PortScan | None) -> list[Software]:
|
|
|
211
249
|
if not blob:
|
|
212
250
|
continue
|
|
213
251
|
location = f"{port.host}:{port.port}" if port.host else f":{port.port}"
|
|
252
|
+
port_os = port.os or _distro_tag(blob)
|
|
214
253
|
ssh = _SSH_RE.search(blob)
|
|
215
254
|
if ssh:
|
|
216
255
|
out.append(
|
|
@@ -219,7 +258,7 @@ def software_from_ports(scan: PortScan | None) -> list[Software]:
|
|
|
219
258
|
version=ssh.group(1),
|
|
220
259
|
source="port-banner",
|
|
221
260
|
location=location,
|
|
222
|
-
os=
|
|
261
|
+
os=port_os,
|
|
223
262
|
)
|
|
224
263
|
)
|
|
225
264
|
continue
|
|
@@ -230,7 +269,7 @@ def software_from_ports(scan: PortScan | None) -> list[Software]:
|
|
|
230
269
|
version=port.version,
|
|
231
270
|
source="port-banner",
|
|
232
271
|
location=location,
|
|
233
|
-
os=
|
|
272
|
+
os=port_os,
|
|
234
273
|
)
|
|
235
274
|
)
|
|
236
275
|
return out
|
|
@@ -240,7 +279,11 @@ _AUTHORITATIVE = {"header:server", "header:x-powered-by", "port-banner"}
|
|
|
240
279
|
_SEVERITY_RANK = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3}
|
|
241
280
|
|
|
242
281
|
|
|
243
|
-
def _confidence(version: str, rng: dict[str, str], source: str) -> int:
|
|
282
|
+
def _confidence(version: str, rng: dict[str, str], source: str, backported: bool = False) -> int:
|
|
283
|
+
if backported:
|
|
284
|
+
if "start_incl" in rng or "start_excl" in rng:
|
|
285
|
+
return 35
|
|
286
|
+
return 20
|
|
244
287
|
comps = len([c for c in re.split("[.\\-_]", version) if c[:1].isdigit()])
|
|
245
288
|
bounded = ("start_incl" in rng or "start_excl" in rng) and (
|
|
246
289
|
"end_incl" in rng or "end_excl" in rng
|
|
@@ -278,6 +321,8 @@ def _match_entries(
|
|
|
278
321
|
version = item.version
|
|
279
322
|
if not version:
|
|
280
323
|
return
|
|
324
|
+
if is_commit_hash(version):
|
|
325
|
+
return
|
|
281
326
|
backported = bool(item.os)
|
|
282
327
|
caveat = "distro backport likely — patchlevel not in banner" if backported else ""
|
|
283
328
|
for entry in entries:
|
|
@@ -286,7 +331,7 @@ def _match_entries(
|
|
|
286
331
|
if hit_rng is None:
|
|
287
332
|
continue
|
|
288
333
|
cve_id = str(entry["id"])
|
|
289
|
-
confidence =
|
|
334
|
+
confidence = _confidence(version, hit_rng, item.source, backported=backported)
|
|
290
335
|
record = agg.get(cve_id)
|
|
291
336
|
if record is None:
|
|
292
337
|
record = _CveAgg(
|
|
@@ -309,7 +354,8 @@ def _match_entries(
|
|
|
309
354
|
if backported:
|
|
310
355
|
record.unconfirmed = True
|
|
311
356
|
record.caveat = caveat
|
|
312
|
-
|
|
357
|
+
if confidence < record.confidence:
|
|
358
|
+
record.confidence = confidence
|
|
313
359
|
elif confidence > record.confidence:
|
|
314
360
|
record.confidence = confidence
|
|
315
361
|
record.version = version
|
|
@@ -363,7 +409,7 @@ def merge_cve_matches(offline: list[CveMatch], online: list[CveMatch]) -> list[C
|
|
|
363
409
|
return _sort_matches(list(by_id.values()))
|
|
364
410
|
|
|
365
411
|
|
|
366
|
-
def match_cves(software: list[Software]) -> list[CveMatch]:
|
|
412
|
+
def match_cves(software: list[Software], *, min_confidence: int = 0) -> list[CveMatch]:
|
|
367
413
|
db = load_cve_db()
|
|
368
414
|
agg: dict[str, _CveAgg] = {}
|
|
369
415
|
for item in software:
|
|
@@ -373,7 +419,10 @@ def match_cves(software: list[Software]) -> list[CveMatch]:
|
|
|
373
419
|
entries = db.get(product_key)
|
|
374
420
|
if entries:
|
|
375
421
|
_match_entries(item, product_key, entries, agg)
|
|
376
|
-
|
|
422
|
+
matches = _agg_to_matches(agg)
|
|
423
|
+
if min_confidence > 0:
|
|
424
|
+
matches = [m for m in matches if m.confidence >= min_confidence]
|
|
425
|
+
return matches
|
|
377
426
|
|
|
378
427
|
|
|
379
428
|
_NVD_URL = "https://services.nvd.nist.gov/rest/json/cves/2.0"
|
|
@@ -450,7 +499,7 @@ def _nvd_summary(descriptions: list[dict[str, str]]) -> str:
|
|
|
450
499
|
|
|
451
500
|
|
|
452
501
|
async def match_cves_online(
|
|
453
|
-
software: list[Software], *, timeout: float = 25.0, workers: int = 3
|
|
502
|
+
software: list[Software], *, timeout: float = 25.0, workers: int = 3, min_confidence: int = 0
|
|
454
503
|
) -> list[CveMatch]:
|
|
455
504
|
import aiohttp
|
|
456
505
|
|
|
@@ -487,4 +536,7 @@ async def match_cves_online(
|
|
|
487
536
|
for (product_key, item), entries in zip(products.items(), gathered, strict=True):
|
|
488
537
|
if entries:
|
|
489
538
|
_match_entries(item, product_key, entries, agg)
|
|
490
|
-
|
|
539
|
+
matches = _agg_to_matches(agg)
|
|
540
|
+
if min_confidence > 0:
|
|
541
|
+
matches = [m for m in matches if m.confidence >= min_confidence]
|
|
542
|
+
return matches
|
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
from stackscan.types import Software, Technology
|
|
6
|
+
|
|
7
|
+
SERVER_NAMES: frozenset[str] = frozenset(
|
|
8
|
+
{
|
|
9
|
+
"nginx",
|
|
10
|
+
"apache",
|
|
11
|
+
"httpd",
|
|
12
|
+
"lighttpd",
|
|
13
|
+
"litespeed",
|
|
14
|
+
"caddy",
|
|
15
|
+
"openresty",
|
|
16
|
+
"iis",
|
|
17
|
+
"microsoft-iis",
|
|
18
|
+
"cherokee",
|
|
19
|
+
"h2o",
|
|
20
|
+
"boa",
|
|
21
|
+
"thttpd",
|
|
22
|
+
"mini_httpd",
|
|
23
|
+
"rejetto",
|
|
24
|
+
"cowboy",
|
|
25
|
+
"tornado",
|
|
26
|
+
"gunicorn",
|
|
27
|
+
"uwsgi",
|
|
28
|
+
"jetty",
|
|
29
|
+
"tomcat",
|
|
30
|
+
"apache tomcat",
|
|
31
|
+
"websphere",
|
|
32
|
+
"glassfish",
|
|
33
|
+
"play",
|
|
34
|
+
"spray",
|
|
35
|
+
"kestrel",
|
|
36
|
+
"cassini",
|
|
37
|
+
"kangle",
|
|
38
|
+
"resin",
|
|
39
|
+
"weblogic",
|
|
40
|
+
"zope",
|
|
41
|
+
"aolserver",
|
|
42
|
+
"yaws",
|
|
43
|
+
}
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
# Product names we never want to emit as a generic technology/software hit.
|
|
47
|
+
_NOISE_NAMES: frozenset[str] = frozenset(
|
|
48
|
+
{
|
|
49
|
+
"http",
|
|
50
|
+
"https",
|
|
51
|
+
"www",
|
|
52
|
+
"html",
|
|
53
|
+
"css",
|
|
54
|
+
"json",
|
|
55
|
+
"xml",
|
|
56
|
+
"js",
|
|
57
|
+
"png",
|
|
58
|
+
"jpg",
|
|
59
|
+
"jpeg",
|
|
60
|
+
"gif",
|
|
61
|
+
"svg",
|
|
62
|
+
"ico",
|
|
63
|
+
"woff",
|
|
64
|
+
"woff2",
|
|
65
|
+
"ttf",
|
|
66
|
+
"eot",
|
|
67
|
+
"php",
|
|
68
|
+
"asp",
|
|
69
|
+
"aspx",
|
|
70
|
+
"jsp",
|
|
71
|
+
"cgi",
|
|
72
|
+
}
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
_CORE_COMMIT_RE = re.compile(
|
|
76
|
+
r"([A-Za-z][A-Za-z0-9\s_-]{1,40})\s+Core\s+\(([a-f0-9]{4,})\b\)", re.IGNORECASE
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
_POWERED_BY_VERSION_RE = re.compile(
|
|
80
|
+
r"(?:powered\s+by|running\s+on|built\s+with|made\s+with)\s+"
|
|
81
|
+
r"([A-Za-z][A-Za-z0-9_-]*(?:\s+[A-Za-z][A-Za-z0-9_-]*){0,4})"
|
|
82
|
+
r"\s+(?:v\.?|version\s*)?(\d+\.\d+(?:\.\d+)?)",
|
|
83
|
+
re.IGNORECASE,
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
_POWERED_BY_PLAIN_RE = re.compile(
|
|
87
|
+
r"(?:powered\s+by|running\s+on|built\s+with|made\s+with)\s+"
|
|
88
|
+
r"([A-Za-z][A-Za-z0-9_-]*(?:\s+[A-Za-z][A-Za-z0-9_-]*){0,4})"
|
|
89
|
+
r"(?!\s+(?:v\.?|version\s*)?\d)",
|
|
90
|
+
re.IGNORECASE,
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
_SERVER_VERSION_RE = re.compile(
|
|
94
|
+
r"\b("
|
|
95
|
+
+ "|".join(re.escape(name) for name in sorted(SERVER_NAMES, key=len, reverse=True))
|
|
96
|
+
+ r")[/ ]v?(\d+\.\d+(?:\.\d+){0,2})",
|
|
97
|
+
re.IGNORECASE,
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
_COMMIT_AFTER_NAME_RE = re.compile(
|
|
101
|
+
r"\b([A-Za-z][A-Za-z0-9._-]{1,40})\s+\(([a-f0-9]{7,40})\)", re.IGNORECASE
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
_STOPWORDS: frozenset[str] = frozenset(
|
|
105
|
+
{
|
|
106
|
+
"and",
|
|
107
|
+
"or",
|
|
108
|
+
"the",
|
|
109
|
+
"a",
|
|
110
|
+
"an",
|
|
111
|
+
"with",
|
|
112
|
+
"for",
|
|
113
|
+
"to",
|
|
114
|
+
"of",
|
|
115
|
+
"in",
|
|
116
|
+
"on",
|
|
117
|
+
"our",
|
|
118
|
+
"your",
|
|
119
|
+
"my",
|
|
120
|
+
"this",
|
|
121
|
+
"that",
|
|
122
|
+
"is",
|
|
123
|
+
"are",
|
|
124
|
+
"was",
|
|
125
|
+
"were",
|
|
126
|
+
"by",
|
|
127
|
+
"from",
|
|
128
|
+
"using",
|
|
129
|
+
"use",
|
|
130
|
+
"plus",
|
|
131
|
+
"via",
|
|
132
|
+
}
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
_VERSION_TOKEN_RE = re.compile(r"^v?\d[\w.]*$", re.IGNORECASE)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _normalize_name(name: str) -> str:
|
|
140
|
+
return " ".join(name.split()).strip()
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _clean_product_name(name: str) -> str:
|
|
144
|
+
kept: list[str] = []
|
|
145
|
+
for word in _normalize_name(name).split():
|
|
146
|
+
if word.lower() in _STOPWORDS:
|
|
147
|
+
break
|
|
148
|
+
kept.append(word)
|
|
149
|
+
while len(kept) > 1 and _VERSION_TOKEN_RE.match(kept[-1]):
|
|
150
|
+
kept.pop()
|
|
151
|
+
return " ".join(kept)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _software_name(name: str) -> str:
|
|
155
|
+
return _normalize_name(name).lower().replace(" ", "")
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _category(name: str) -> str:
|
|
159
|
+
if _software_name(name) in {n.replace(" ", "") for n in SERVER_NAMES}:
|
|
160
|
+
return "infrastructure"
|
|
161
|
+
return "service"
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _is_noise(name: str) -> bool:
|
|
165
|
+
return _software_name(name) in _NOISE_NAMES
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _is_plausible_name(name: str) -> bool:
|
|
169
|
+
"""Reject single-letter-plus-digit noise (e.g. SVG path commands like M368)."""
|
|
170
|
+
return sum(1 for ch in name if ch.isalpha()) >= 2
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def is_commit_hash(value: str) -> bool:
|
|
174
|
+
"""Return True when value looks like a Git commit hash rather than a version."""
|
|
175
|
+
if len(value) < 7:
|
|
176
|
+
return False
|
|
177
|
+
if value.startswith("v"):
|
|
178
|
+
return False
|
|
179
|
+
return bool(re.fullmatch(r"[a-f0-9]{7,40}", value))
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def extract_generic_tech(body: str) -> list[Technology]:
|
|
183
|
+
hits: list[tuple[str, str, str, str | None]] = []
|
|
184
|
+
seen: set[tuple[str, str | None]] = set()
|
|
185
|
+
|
|
186
|
+
def remember(name: str, evidence: str, version: str | None) -> None:
|
|
187
|
+
name = _clean_product_name(name)
|
|
188
|
+
if len(name) < 2 or _is_noise(name) or not _is_plausible_name(name):
|
|
189
|
+
return
|
|
190
|
+
key = (name.lower(), version)
|
|
191
|
+
if key in seen:
|
|
192
|
+
return
|
|
193
|
+
seen.add(key)
|
|
194
|
+
hits.append((name, evidence, _category(name), version))
|
|
195
|
+
|
|
196
|
+
for match in _SERVER_VERSION_RE.finditer(body):
|
|
197
|
+
name = _normalize_name(match.group(1))
|
|
198
|
+
version = match.group(2)
|
|
199
|
+
remember(name, f"body:{name}/{version}", version)
|
|
200
|
+
|
|
201
|
+
powered_starts: set[int] = set()
|
|
202
|
+
for match in _POWERED_BY_VERSION_RE.finditer(body):
|
|
203
|
+
name = _normalize_name(match.group(1))
|
|
204
|
+
version = match.group(2)
|
|
205
|
+
powered_starts.add(match.start())
|
|
206
|
+
if name and not _is_noise(name):
|
|
207
|
+
remember(name, f"body:powered-by {name} {version}", version)
|
|
208
|
+
|
|
209
|
+
for match in _POWERED_BY_PLAIN_RE.finditer(body):
|
|
210
|
+
if match.start() in powered_starts:
|
|
211
|
+
continue
|
|
212
|
+
name = _normalize_name(match.group(1))
|
|
213
|
+
if name and not _is_noise(name):
|
|
214
|
+
remember(name, f"body:powered-by {name}", None)
|
|
215
|
+
|
|
216
|
+
core_spans: set[tuple[int, int]] = set()
|
|
217
|
+
for match in _CORE_COMMIT_RE.finditer(body):
|
|
218
|
+
name = _normalize_name(match.group(1))
|
|
219
|
+
commit = match.group(2).lower()
|
|
220
|
+
if name and not _is_noise(name):
|
|
221
|
+
remember(name, f"body:{name} Core ({commit})", commit)
|
|
222
|
+
core_spans.add(match.span())
|
|
223
|
+
|
|
224
|
+
def _overlaps_core(span: tuple[int, int]) -> bool:
|
|
225
|
+
start, end = span
|
|
226
|
+
for c_start, c_end in core_spans:
|
|
227
|
+
if start < c_end and end > c_start:
|
|
228
|
+
return True
|
|
229
|
+
return False
|
|
230
|
+
|
|
231
|
+
for match in _COMMIT_AFTER_NAME_RE.finditer(body):
|
|
232
|
+
if _overlaps_core(match.span()):
|
|
233
|
+
continue
|
|
234
|
+
name = _normalize_name(match.group(1))
|
|
235
|
+
commit = match.group(2).lower()
|
|
236
|
+
if name and not _is_noise(name) and is_commit_hash(commit):
|
|
237
|
+
if (name.lower(), commit) not in seen:
|
|
238
|
+
remember(name, f"body:{name} ({commit})", commit)
|
|
239
|
+
|
|
240
|
+
return [
|
|
241
|
+
Technology(
|
|
242
|
+
name=name,
|
|
243
|
+
categories=(category,),
|
|
244
|
+
evidence=(evidence,),
|
|
245
|
+
confidence=70,
|
|
246
|
+
version=version,
|
|
247
|
+
)
|
|
248
|
+
for name, evidence, category, version in hits
|
|
249
|
+
]
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def extract_generic_software(body: str, location: str = "") -> list[Software]:
|
|
253
|
+
out: list[Software] = []
|
|
254
|
+
seen: set[tuple[str, str | None]] = set()
|
|
255
|
+
|
|
256
|
+
def add(name: str, version: str | None, evidence: str) -> None:
|
|
257
|
+
name = _clean_product_name(name)
|
|
258
|
+
if len(name) < 2 or not _is_plausible_name(name):
|
|
259
|
+
return
|
|
260
|
+
sname = _software_name(name)
|
|
261
|
+
if sname in _NOISE_NAMES:
|
|
262
|
+
return
|
|
263
|
+
key = (sname, version)
|
|
264
|
+
if key in seen:
|
|
265
|
+
return
|
|
266
|
+
seen.add(key)
|
|
267
|
+
out.append(
|
|
268
|
+
Software(
|
|
269
|
+
name=sname,
|
|
270
|
+
version=version,
|
|
271
|
+
source=evidence,
|
|
272
|
+
location=location,
|
|
273
|
+
)
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
for match in _SERVER_VERSION_RE.finditer(body):
|
|
277
|
+
name = _normalize_name(match.group(1))
|
|
278
|
+
version = match.group(2)
|
|
279
|
+
add(name, version, f"body:{name}/{version}")
|
|
280
|
+
|
|
281
|
+
powered_starts: set[int] = set()
|
|
282
|
+
for match in _POWERED_BY_VERSION_RE.finditer(body):
|
|
283
|
+
name = _normalize_name(match.group(1))
|
|
284
|
+
version = match.group(2)
|
|
285
|
+
powered_starts.add(match.start())
|
|
286
|
+
if name:
|
|
287
|
+
add(name, version, f"body:powered-by {name} {version}")
|
|
288
|
+
|
|
289
|
+
for match in _POWERED_BY_PLAIN_RE.finditer(body):
|
|
290
|
+
if match.start() in powered_starts:
|
|
291
|
+
continue
|
|
292
|
+
name = _normalize_name(match.group(1))
|
|
293
|
+
if name:
|
|
294
|
+
add(name, None, f"body:powered-by {name}")
|
|
295
|
+
|
|
296
|
+
core_spans: set[tuple[int, int]] = set()
|
|
297
|
+
for match in _CORE_COMMIT_RE.finditer(body):
|
|
298
|
+
name = _normalize_name(match.group(1))
|
|
299
|
+
commit = match.group(2).lower()
|
|
300
|
+
if name:
|
|
301
|
+
add(name, commit, f"body:core-commit {name} ({commit})")
|
|
302
|
+
core_spans.add(match.span())
|
|
303
|
+
|
|
304
|
+
def _overlaps_core(span: tuple[int, int]) -> bool:
|
|
305
|
+
start, end = span
|
|
306
|
+
for c_start, c_end in core_spans:
|
|
307
|
+
if start < c_end and end > c_start:
|
|
308
|
+
return True
|
|
309
|
+
return False
|
|
310
|
+
|
|
311
|
+
for match in _COMMIT_AFTER_NAME_RE.finditer(body):
|
|
312
|
+
if _overlaps_core(match.span()):
|
|
313
|
+
continue
|
|
314
|
+
name = _normalize_name(match.group(1))
|
|
315
|
+
commit = match.group(2).lower()
|
|
316
|
+
if name and is_commit_hash(commit):
|
|
317
|
+
add(name, commit, f"body:commit {name} ({commit})")
|
|
318
|
+
|
|
319
|
+
return out
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
from collections.abc import Iterable
|
|
4
|
+
|
|
3
5
|
from stackscan.types import Headers, InfraInfo
|
|
4
6
|
|
|
5
7
|
_Signature = tuple[str, str, str | None]
|
|
@@ -92,6 +94,61 @@ def _proxy_notes(headers: Headers, host: str) -> list[str]:
|
|
|
92
94
|
return notes
|
|
93
95
|
|
|
94
96
|
|
|
97
|
+
_ROLE_LABEL: dict[str, str] = {"cdn": "CDN", "waf": "WAF", "proxy": "reverse proxy"}
|
|
98
|
+
_ROLE_ORDER: tuple[str, ...] = ("cdn", "waf", "proxy")
|
|
99
|
+
_ORG_SUFFIXES: tuple[str, ...] = (
|
|
100
|
+
", inc.",
|
|
101
|
+
", inc",
|
|
102
|
+
" inc.",
|
|
103
|
+
" inc",
|
|
104
|
+
" llc",
|
|
105
|
+
" ltd",
|
|
106
|
+
" ltd.",
|
|
107
|
+
" gmbh",
|
|
108
|
+
" corporation",
|
|
109
|
+
" technologies",
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _canonical_provider(org: str) -> str:
|
|
114
|
+
name = org.strip()
|
|
115
|
+
low = name.lower()
|
|
116
|
+
for suffix in _ORG_SUFFIXES:
|
|
117
|
+
if low.endswith(suffix):
|
|
118
|
+
name = name[: -len(suffix)].strip()
|
|
119
|
+
low = name.lower()
|
|
120
|
+
return name
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def summarize_edge(infra: InfraInfo, cdn_orgs: Iterable[str] = (), *, sep: str = " → ") -> str:
|
|
124
|
+
roles: dict[str, list[str]] = {}
|
|
125
|
+
order: list[str] = []
|
|
126
|
+
role_names = {"cdn": infra.cdn, "waf": infra.waf, "proxy": infra.proxy}
|
|
127
|
+
for role in _ROLE_ORDER:
|
|
128
|
+
for name in role_names[role]:
|
|
129
|
+
if name not in roles:
|
|
130
|
+
roles[name] = []
|
|
131
|
+
order.append(name)
|
|
132
|
+
if role not in roles[name]:
|
|
133
|
+
roles[name].append(role)
|
|
134
|
+
for org in cdn_orgs:
|
|
135
|
+
name = _canonical_provider(org)
|
|
136
|
+
if not name:
|
|
137
|
+
continue
|
|
138
|
+
if any(name.lower() in known.lower() or known.lower() in name.lower() for known in roles):
|
|
139
|
+
continue
|
|
140
|
+
roles[name] = ["cdn"]
|
|
141
|
+
order.append(name)
|
|
142
|
+
if not order:
|
|
143
|
+
return ""
|
|
144
|
+
order.sort(key=lambda n: 0 if ({"waf", "proxy"} & set(roles[n])) else 1)
|
|
145
|
+
parts: list[str] = []
|
|
146
|
+
for name in order:
|
|
147
|
+
labels = ", ".join(_ROLE_LABEL[r] for r in _ROLE_ORDER if r in roles[name])
|
|
148
|
+
parts.append(f"{name} ({labels})" if labels else name)
|
|
149
|
+
return sep.join(parts)
|
|
150
|
+
|
|
151
|
+
|
|
95
152
|
def analyze_infra(headers: Headers, cookies: tuple[str, ...], host: str) -> InfraInfo:
|
|
96
153
|
cdn = _collect(headers, CDN_SIGNATURES)
|
|
97
154
|
waf = _collect(headers, WAF_SIGNATURES)
|
|
@@ -106,7 +106,9 @@ def detect_os(report: ScanReport) -> list[OsFinding]:
|
|
|
106
106
|
for os, src in pairs:
|
|
107
107
|
by_source.setdefault(src, []).append(os)
|
|
108
108
|
dominant_source = max(by_source, key=lambda s: len(by_source[s]))
|
|
109
|
-
if dominant_source == "port-banner" or dominant_source.startswith(
|
|
109
|
+
if dominant_source == "port-banner" or dominant_source.startswith(
|
|
110
|
+
("header:", "meta:", "script")
|
|
111
|
+
):
|
|
110
112
|
category = "banner"
|
|
111
113
|
elif dominant_source.startswith("port "):
|
|
112
114
|
category = "network"
|