vulpfest 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vulpfest/__init__.py +1 -0
- vulpfest/__main__.py +108 -0
- vulpfest/banner.py +69 -0
- vulpfest/logo.py +19 -0
- vulpfest/tools/gmapscout/cli.py +51 -0
- vulpfest/tools/gmapscout/gmapscout_core.py +202 -0
- vulpfest/tools/leadscout/cli.py +52 -0
- vulpfest/tools/leadscout/leadscout_core.py +72 -0
- vulpfest/tools/scamcheck/checker.py +266 -0
- vulpfest/tools/scamcheck/cli.py +88 -0
- vulpfest/tools/scamcheck/ssrf_guard.py +142 -0
- vulpfest/tools/scanner/auth_scan.py +175 -0
- vulpfest/tools/scanner/cli.py +167 -0
- vulpfest/tools/scanner/deep_web.py +353 -0
- vulpfest/tools/scanner/dns_email.py +202 -0
- vulpfest/tools/scanner/manual_review.py +60 -0
- vulpfest/tools/scanner/repo_scan.py +434 -0
- vulpfest/tools/scanner/scanner.py +886 -0
- vulpfest/tools/scanner/ssrf_guard.py +142 -0
- vulpfest/ui.py +38 -0
- vulpfest-1.0.0.dist-info/METADATA +103 -0
- vulpfest-1.0.0.dist-info/RECORD +25 -0
- vulpfest-1.0.0.dist-info/WHEEL +4 -0
- vulpfest-1.0.0.dist-info/entry_points.txt +2 -0
- vulpfest-1.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""Cyber Web Scanner from the command line.
|
|
2
|
+
|
|
3
|
+
Examples:
|
|
4
|
+
python cli.py example.com # surface + deep web checks
|
|
5
|
+
python cli.py example.com --surface-only # quick surface checks only
|
|
6
|
+
python cli.py --repo . # check a local project folder
|
|
7
|
+
python cli.py example.com --repo . # both, in one report
|
|
8
|
+
python cli.py --repo . --fail-on critical # for CI: exit code 3 on critical findings
|
|
9
|
+
python cli.py example.com --json --out report.json
|
|
10
|
+
"""
|
|
11
|
+
import argparse
|
|
12
|
+
import json
|
|
13
|
+
import re
|
|
14
|
+
import sys
|
|
15
|
+
from datetime import datetime, timezone
|
|
16
|
+
|
|
17
|
+
import json as _json
|
|
18
|
+
import manual_review
|
|
19
|
+
import repo_scan
|
|
20
|
+
import auth_scan
|
|
21
|
+
from scanner import calculate_security_score, run_full_scan
|
|
22
|
+
|
|
23
|
+
LEVEL_ORDER = {"Kritis": 0, "Tinggi": 1, "Menengah": 2, "Rendah": 3}
|
|
24
|
+
LEVEL_PREFIX = re.compile(r"^\s*Risiko\s+(Kritis|Tinggi|Menengah|Rendah)\s*:\s*", re.I)
|
|
25
|
+
FAIL_ON = {"critical": {"critical"}, "high": {"critical", "high"}}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def pick(d, *keys):
|
|
29
|
+
for k in keys:
|
|
30
|
+
if d.get(k):
|
|
31
|
+
return d[k]
|
|
32
|
+
return ""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def split_level(f):
|
|
36
|
+
"""Level may be a separate field or written at the start of the finding ("Risiko Menengah: ...")."""
|
|
37
|
+
text = pick(f, "finding", "text", "temuan")
|
|
38
|
+
level = pick(f, "level", "severity", "tingkat")
|
|
39
|
+
m = LEVEL_PREFIX.match(text)
|
|
40
|
+
if m:
|
|
41
|
+
level = level or m.group(1).capitalize()
|
|
42
|
+
text = text[m.end():]
|
|
43
|
+
return level or "Info", text, f
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def main():
|
|
47
|
+
parser = argparse.ArgumentParser(description="Check your own website and project for common security problems (Vulpfest Lab).")
|
|
48
|
+
parser.add_argument("url", nargs="?", help="website to scan, e.g. example.com or https://example.com")
|
|
49
|
+
parser.add_argument("--json", action="store_true", help="print the full result as JSON")
|
|
50
|
+
parser.add_argument("--out", metavar="FILE", help="also save the full result as JSON to FILE")
|
|
51
|
+
parser.add_argument("--surface-only", action="store_true", help="skip the deep web checks (JavaScript, exposed files, DNS and email)")
|
|
52
|
+
parser.add_argument("--repo", metavar="PATH", help="also check a local project folder (dependencies, secrets, git history, CI/CD, Docker)")
|
|
53
|
+
parser.add_argument("--offline", action="store_true", help="repository check without internet (no OSV.dev or registry lookups)")
|
|
54
|
+
parser.add_argument("--no-history", action="store_true", help="repository check without reading the git history")
|
|
55
|
+
parser.add_argument("--fail-on", choices=sorted(FAIL_ON), help="exit with code 3 when a finding at this level or higher is found")
|
|
56
|
+
parser.add_argument("--senior", action="store_true", help="show finding ID, category, track and confidence")
|
|
57
|
+
parser.add_argument("--checklist", action="store_true", help="print the manual review checklist (Markdown)")
|
|
58
|
+
parser.add_argument("--auth", metavar="FILE", help="authenticated checks (terminal only) on your own site; see auth.example.json")
|
|
59
|
+
parser.add_argument("--yes", action="store_true", help="skip the confirmation prompt for authenticated checks")
|
|
60
|
+
args = parser.parse_args()
|
|
61
|
+
if args.checklist:
|
|
62
|
+
print(manual_review.to_markdown())
|
|
63
|
+
if not args.url and not args.repo:
|
|
64
|
+
return 0
|
|
65
|
+
if not args.url and not args.repo:
|
|
66
|
+
parser.error("give a website, --repo PATH, or both")
|
|
67
|
+
|
|
68
|
+
if args.url:
|
|
69
|
+
print("Reminder: scan only websites you own or are authorized to test.", file=sys.stderr)
|
|
70
|
+
result = run_full_scan(args.url, modes=("surface",) if args.surface_only else ("surface", "deep_web"))
|
|
71
|
+
else:
|
|
72
|
+
result = {"target": None, "valid": True, "scanned_at": datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")}
|
|
73
|
+
|
|
74
|
+
if args.repo:
|
|
75
|
+
print(f"Checking repository {args.repo} ...", file=sys.stderr)
|
|
76
|
+
repo = repo_scan.run(args.repo, online=not args.offline, history=not args.no_history)
|
|
77
|
+
result["repo"] = repo
|
|
78
|
+
if repo.get("error"):
|
|
79
|
+
print(f"Repository: {repo['error']}", file=sys.stderr)
|
|
80
|
+
elif args.url:
|
|
81
|
+
if result.get("valid"):
|
|
82
|
+
result["coverage_modes"] = list(result.get("coverage_modes", ["surface"])) + ["repo"]
|
|
83
|
+
result["score_info"] = calculate_security_score(result)
|
|
84
|
+
else:
|
|
85
|
+
result["target"] = "repo: " + repo["path"]
|
|
86
|
+
result["score_info"] = repo_scan.score(repo)
|
|
87
|
+
if not args.url and repo.get("error"):
|
|
88
|
+
result["valid"] = False
|
|
89
|
+
result["score_info"] = {"score": 0, "grade": "N/A", "findings": [{"finding": repo["error"], "recommendation": "Check the folder path."}]}
|
|
90
|
+
|
|
91
|
+
if args.auth and args.url:
|
|
92
|
+
from urllib.parse import urlparse
|
|
93
|
+
host = urlparse(result.get("evidence", {}).get("final_url") or ("https://" + args.url)).hostname
|
|
94
|
+
try:
|
|
95
|
+
cfg = auth_scan.AuthConfig(_json.load(open(args.auth, encoding="utf-8")))
|
|
96
|
+
except (OSError, ValueError) as e:
|
|
97
|
+
print(f"Could not read {args.auth}: {e}", file=sys.stderr)
|
|
98
|
+
return 2
|
|
99
|
+
if not cfg.ok:
|
|
100
|
+
print("The auth file needs at least one test account.", file=sys.stderr)
|
|
101
|
+
return 2
|
|
102
|
+
token, how = auth_scan.verify_ownership(host)
|
|
103
|
+
ok, _method = auth_scan.confirm_ownership(host, token)
|
|
104
|
+
if not ok:
|
|
105
|
+
print("\nAuthenticated checks need proof that you own this site. Add ONE of these, then run again:", file=sys.stderr)
|
|
106
|
+
print(" - file: " + how["file"], file=sys.stderr)
|
|
107
|
+
print(" - DNS : " + how["dns"], file=sys.stderr)
|
|
108
|
+
return 2
|
|
109
|
+
if not args.yes:
|
|
110
|
+
print("\nAuthenticated checks will send a small number of real requests to " + str(host) + ".", file=sys.stderr)
|
|
111
|
+
if input("Type 'ya' to continue: ").strip().lower() not in ("ya", "yes"):
|
|
112
|
+
print("Cancelled.", file=sys.stderr)
|
|
113
|
+
return 0
|
|
114
|
+
result["auth"] = auth_scan.run(cfg)
|
|
115
|
+
if result.get("valid"):
|
|
116
|
+
result["coverage_modes"] = list(dict.fromkeys(list(result.get("coverage_modes", ["surface"])) + ["auth"]))
|
|
117
|
+
result["score_info"] = calculate_security_score(result)
|
|
118
|
+
elif args.auth:
|
|
119
|
+
print("--auth needs a website argument.", file=sys.stderr)
|
|
120
|
+
return 2
|
|
121
|
+
|
|
122
|
+
if args.out:
|
|
123
|
+
with open(args.out, "w", encoding="utf-8") as f:
|
|
124
|
+
json.dump(result, f, indent=2, ensure_ascii=False)
|
|
125
|
+
info = result.get("score_info", {})
|
|
126
|
+
if args.json:
|
|
127
|
+
print(json.dumps(result, indent=2, ensure_ascii=False))
|
|
128
|
+
else:
|
|
129
|
+
print(f"\nTarget : {result.get('target')}")
|
|
130
|
+
print(f"Scanned: {result.get('scanned_at')}")
|
|
131
|
+
if not result.get("valid"):
|
|
132
|
+
print(f"Status : could not scan reliably - {result.get('invalid_reason', 'see below')}")
|
|
133
|
+
print(f"Grade : {info.get('grade', '-')} (score {info.get('score', '-')}/100)")
|
|
134
|
+
cov = info.get("coverage") or {}
|
|
135
|
+
if cov.get("percent"):
|
|
136
|
+
print(f"Cover : {cov.get('label')} ({cov.get('percent')}%); not checked yet: {', '.join(cov.get('not_checked', [])) or '-'}")
|
|
137
|
+
subs, labels = info.get("subscores") or {}, info.get("category_labels") or {}
|
|
138
|
+
if subs:
|
|
139
|
+
print("Scores : " + " · ".join(f"{labels.get(k, k)} {v if v is not None else '-'}" for k, v in subs.items()))
|
|
140
|
+
if info.get("grade_capped"):
|
|
141
|
+
print("Note : grade capped because of critical findings")
|
|
142
|
+
repo = result.get("repo") or {}
|
|
143
|
+
if repo and not repo.get("error"):
|
|
144
|
+
hist = "not a git repository" if not repo.get("git") else ("partial" if repo.get("history_partial") else "yes")
|
|
145
|
+
print(f"Repo : {repo.get('files_scanned')} files, {repo['deps']['checked']} packages checked"
|
|
146
|
+
f"{' (offline: no OSV/registry)' if not repo.get('online') else ''}, git history: {hist}")
|
|
147
|
+
for note in repo.get("notes", []):
|
|
148
|
+
print(f"Note : {note}")
|
|
149
|
+
print()
|
|
150
|
+
items = sorted((split_level(f) for f in info.get("findings", [])), key=lambda x: LEVEL_ORDER.get(x[0], 9))
|
|
151
|
+
for i, (level, text, f) in enumerate(items, 1):
|
|
152
|
+
print(f"{i:>2}. [{level}] {text}")
|
|
153
|
+
if args.senior and f.get("id"):
|
|
154
|
+
print(f" [{f.get('id')} · {f.get('category', '-')} · track {f.get('track', '-')} · confidence {f.get('confidence', '-')}]")
|
|
155
|
+
rec = pick(f, "recommendation", "rec", "rekomendasi")
|
|
156
|
+
if rec:
|
|
157
|
+
print(f" -> {rec}")
|
|
158
|
+
if args.out:
|
|
159
|
+
print(f"\nFull result saved to {args.out}")
|
|
160
|
+
|
|
161
|
+
if args.fail_on and any(f.get("severity") in FAIL_ON[args.fail_on] for f in info.get("findings", [])):
|
|
162
|
+
return 3
|
|
163
|
+
return 0 if result.get("valid") else 2
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
if __name__ == "__main__":
|
|
167
|
+
sys.exit(main())
|
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
"""Mode B: pemeriksaan web mendalam (tanpa browser headless).
|
|
2
|
+
|
|
3
|
+
Mengunduh file JavaScript milik situs yang dimuat halaman awal lalu memeriksa isinya, serta mencari
|
|
4
|
+
file dan endpoint yang sering tertinggal di produksi. Semua permintaan adalah GET/POST biasa
|
|
5
|
+
bervolume rendah lewat sesi yang sudah dipasangi pengaman SSRF; tidak ada payload eksploitasi.
|
|
6
|
+
Setiap temuan divalidasi dari ISI respons, bukan hanya status HTTP, agar situs yang membalas
|
|
7
|
+
200 untuk semua alamat tidak menghasilkan temuan palsu.
|
|
8
|
+
"""
|
|
9
|
+
import base64
|
|
10
|
+
import json
|
|
11
|
+
import re
|
|
12
|
+
import secrets as _rand
|
|
13
|
+
import time
|
|
14
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
15
|
+
from urllib.parse import urljoin, urlparse
|
|
16
|
+
|
|
17
|
+
import requests
|
|
18
|
+
|
|
19
|
+
MAX_SCRIPTS = 12
|
|
20
|
+
JS_MAX_BYTES = 2 * 1024 * 1024
|
|
21
|
+
INLINE_MAX_CHARS = 1_000_000
|
|
22
|
+
BUDGET_SECONDS = 25
|
|
23
|
+
REQ_TIMEOUT = 6
|
|
24
|
+
|
|
25
|
+
# (jenis, pola, tingkat, keyakinan). critical = kunci rahasia; low = kunci publik yang perlu dibatasi.
|
|
26
|
+
SECRET_PATTERNS = (
|
|
27
|
+
('AWS access key', r'\b(?:AKIA|ASIA)[0-9A-Z]{16}\b', 'critical', 'high'),
|
|
28
|
+
('Stripe secret key', r'\b[sr]k_live_[0-9A-Za-z]{20,}\b', 'critical', 'high'),
|
|
29
|
+
('OpenAI API key', r'\bsk-(?:proj-)?[A-Za-z0-9_-]{40,}\b', 'critical', 'medium'),
|
|
30
|
+
('Anthropic API key', r'\bsk-ant-[A-Za-z0-9_-]{30,}\b', 'critical', 'high'),
|
|
31
|
+
('GitHub token', r'\b(?:gh[pousr]_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{60,})\b', 'critical', 'high'),
|
|
32
|
+
('Slack token', r'\bxox[baprs]-[A-Za-z0-9-]{10,}\b', 'critical', 'high'),
|
|
33
|
+
('SendGrid API key', r'\bSG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}\b', 'critical', 'high'),
|
|
34
|
+
('Mailgun API key', r'\bkey-[0-9a-f]{32}\b', 'critical', 'medium'),
|
|
35
|
+
('Midtrans server key', r'\b(?:SB-)?Mid-server-[A-Za-z0-9_-]{20,}\b', 'critical', 'high'),
|
|
36
|
+
('Xendit secret key', r'\bxnd_(?:production|development)_[A-Za-z0-9]{40,}\b', 'critical', 'high'),
|
|
37
|
+
('Private key', r'-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----', 'critical', 'high'),
|
|
38
|
+
('Google API key', r'\bAIza[0-9A-Za-z_-]{35}\b', 'low', 'medium'),
|
|
39
|
+
)
|
|
40
|
+
JWT_RE = re.compile(r'\beyJ[A-Za-z0-9_-]{10,}\.(eyJ[A-Za-z0-9_-]{10,})\.[A-Za-z0-9_-]{10,}')
|
|
41
|
+
|
|
42
|
+
# (nama, pola versi, batas aman / 'eol' / None=aturan khusus, penjelasan)
|
|
43
|
+
LIB_RULES = (
|
|
44
|
+
('jQuery UI', r'jQuery UI - v(\d+\.\d+\.\d+)', (1, 13, 0),
|
|
45
|
+
'XSS pada beberapa widget (CVE-2021-41182 dan sejenisnya); perbarui ke 1.13 atau lebih baru'),
|
|
46
|
+
('jQuery', r'jQuery (?:JavaScript Library )?v(\d+\.\d+\.\d+)', (3, 5, 0),
|
|
47
|
+
'XSS saat memproses HTML (CVE-2020-11022, CVE-2020-11023); perbarui ke 3.5.0 atau lebih baru'),
|
|
48
|
+
('Bootstrap', r'Bootstrap v(\d+\.\d+\.\d+)', None,
|
|
49
|
+
'XSS pada tooltip dan popover (CVE-2019-8331); perbarui ke 3.4.1, 4.3.1, atau lebih baru'),
|
|
50
|
+
('AngularJS', r'AngularJS v(1\.\d+\.\d+)', 'eol',
|
|
51
|
+
'AngularJS 1.x tidak lagi didukung sejak Januari 2022 dan tidak menerima perbaikan keamanan'),
|
|
52
|
+
('Moment.js', r'//! version : (\d+\.\d+\.\d+)', (2, 29, 4),
|
|
53
|
+
'ReDoS saat mem-parsing tanggal (CVE-2022-31129); perbarui ke 2.29.4 atau lebih baru'),
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
SOURCEMAP_RE = re.compile(r'//[#@]\s*sourceMappingURL=([^\s\'"]+)')
|
|
57
|
+
# Layanan yang memang tidak mendukung SRI (skrip dinamis), dikecualikan agar laporan tidak berisik.
|
|
58
|
+
SRI_EXEMPT = ('googletagmanager.com', 'google-analytics.com', 'gstatic.com', 'google.com', 'recaptcha.net',
|
|
59
|
+
'challenges.cloudflare.com', 'js.stripe.com', 'vercel-scripts.com', 'vercel-insights.com',
|
|
60
|
+
'paypal.com', 'paypalobjects.com', 'midtrans.com', 'facebook.net', 'clarity.ms', 'hotjar.com')
|
|
61
|
+
|
|
62
|
+
BACKUP_NAMES = ('backup.zip', 'backup.tar.gz', 'backup.sql', 'database.sql', 'db.sql', 'dump.sql',
|
|
63
|
+
'site.zip', 'www.zip', 'public_html.zip', 'html.zip')
|
|
64
|
+
CONFIG_FILES = {
|
|
65
|
+
'/.env.local': (r'^[A-Z0-9_]+\s*=', 'File environment (.env.local)'),
|
|
66
|
+
'/.env.production': (r'^[A-Z0-9_]+\s*=', 'File environment (.env.production)'),
|
|
67
|
+
'/.env.backup': (r'^[A-Z0-9_]+\s*=', 'File environment (.env.backup)'),
|
|
68
|
+
'/.git-credentials': (r'https?://[^\s:/]+:[^\s@/]+@', 'Kredensial git'),
|
|
69
|
+
'/.npmrc': (r'_authToken\s*=', 'Token npm'),
|
|
70
|
+
'/wp-config.php.bak': (r'DB_PASSWORD', 'Backup konfigurasi WordPress'),
|
|
71
|
+
}
|
|
72
|
+
# jalur: (pola isi, jenis, boleh HTML)
|
|
73
|
+
OPS_ENDPOINTS = {
|
|
74
|
+
'/phpinfo.php': (r'phpinfo\(\)', 'Halaman phpinfo', True),
|
|
75
|
+
'/info.php': (r'phpinfo\(\)', 'Halaman phpinfo', True),
|
|
76
|
+
'/actuator/env': (r'"propertySources"', 'Spring Boot Actuator /env', False),
|
|
77
|
+
'/server-status': (r'Apache Server Status for', 'Apache server-status', True),
|
|
78
|
+
'/debug/pprof/': (r'Types of profiles available', 'Go pprof', True),
|
|
79
|
+
'/metrics': (r'^# (?:HELP|TYPE) \w+', 'Metrik Prometheus', False),
|
|
80
|
+
}
|
|
81
|
+
API_DOC_PATHS = ('/openapi.json', '/swagger.json', '/v3/api-docs', '/api-docs', '/swagger-ui.html',
|
|
82
|
+
'/swagger-ui/', '/docs', '/redoc')
|
|
83
|
+
GRAPHQL_PATHS = ('/graphql', '/api/graphql')
|
|
84
|
+
DEBUG_MARKERS = ('traceback (most recent call last)', 'werkzeug debugger', 'debug = true</code>',
|
|
85
|
+
'whoops! there was an error', 'symfony\\component\\errorhandler', 'stack trace:</',
|
|
86
|
+
"server error in '/' application", 'at org.springframework', 'nextjs__container_errors')
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class _OutOfTime(Exception):
|
|
90
|
+
pass
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class _Ctx:
|
|
94
|
+
def __init__(self, session, verify, budget):
|
|
95
|
+
self.s, self.verify = session, verify
|
|
96
|
+
self.deadline = time.monotonic() + budget
|
|
97
|
+
self.partial = False
|
|
98
|
+
|
|
99
|
+
def _timeout(self):
|
|
100
|
+
left = self.deadline - time.monotonic()
|
|
101
|
+
if left <= 1:
|
|
102
|
+
self.partial = True
|
|
103
|
+
raise _OutOfTime()
|
|
104
|
+
return min(REQ_TIMEOUT, left)
|
|
105
|
+
|
|
106
|
+
def first_bytes(self, url, n=8192, any_status=False):
|
|
107
|
+
r = self.s.get(url, timeout=self._timeout(), verify=self.verify, allow_redirects=False, stream=True)
|
|
108
|
+
try:
|
|
109
|
+
data = r.raw.read(n, decode_content=True) if (any_status or r.status_code == 200) else b''
|
|
110
|
+
finally:
|
|
111
|
+
r.close()
|
|
112
|
+
return r, data
|
|
113
|
+
|
|
114
|
+
def post_json(self, url, payload):
|
|
115
|
+
return self.s.post(url, json=payload, timeout=self._timeout(), verify=self.verify, allow_redirects=False)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def mask(value):
|
|
119
|
+
v = value.strip()
|
|
120
|
+
return v[:6] + '…' + v[-4:] if len(v) > 14 else v[:3] + '…'
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _short(url):
|
|
124
|
+
p = urlparse(url)
|
|
125
|
+
path = p.path or '/'
|
|
126
|
+
return path if len(path) <= 80 else '…' + path[-77:]
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _ver(s):
|
|
130
|
+
return tuple(int(x) for x in s.split('.')[:3])
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _jwt_payload(segment):
|
|
134
|
+
try:
|
|
135
|
+
raw = base64.urlsafe_b64decode(segment + '=' * (-len(segment) % 4))
|
|
136
|
+
data = json.loads(raw)
|
|
137
|
+
return data if isinstance(data, dict) else None
|
|
138
|
+
except Exception:
|
|
139
|
+
return None
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def find_secrets(text, where):
|
|
143
|
+
found = []
|
|
144
|
+
for kind, pattern, level, conf in SECRET_PATTERNS:
|
|
145
|
+
for m in re.finditer(pattern, text):
|
|
146
|
+
found.append({'type': kind, 'file': where, 'masked': mask(m.group(0)), 'severity': level, 'confidence': conf})
|
|
147
|
+
for m in JWT_RE.finditer(text):
|
|
148
|
+
payload = _jwt_payload(m.group(1))
|
|
149
|
+
if payload and payload.get('role') == 'service_role':
|
|
150
|
+
found.append({'type': 'Supabase service_role key', 'file': where, 'masked': mask(m.group(0)),
|
|
151
|
+
'severity': 'critical', 'confidence': 'high'})
|
|
152
|
+
return found
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _bootstrap_vulnerable(v):
|
|
156
|
+
return v < (3, 4, 1) or (4, 0, 0) <= v < (4, 3, 1)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def find_libraries(text):
|
|
160
|
+
out = []
|
|
161
|
+
for name, pattern, rule, issue in LIB_RULES:
|
|
162
|
+
m = re.search(pattern, text)
|
|
163
|
+
if not m:
|
|
164
|
+
continue
|
|
165
|
+
v = _ver(m.group(1))
|
|
166
|
+
vulnerable = (rule == 'eol') or (rule is None and _bootstrap_vulnerable(v)) or (isinstance(rule, tuple) and v < rule)
|
|
167
|
+
if vulnerable:
|
|
168
|
+
out.append({'name': name, 'version': m.group(1), 'issue': issue})
|
|
169
|
+
return out
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _debug_kind(head, ctype):
|
|
173
|
+
text = head.decode('utf-8', 'ignore').lower()
|
|
174
|
+
return next((m for m in DEBUG_MARKERS if m in text), None)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _backup_kind(head, ctype):
|
|
178
|
+
if not head or 'html' in ctype.lower():
|
|
179
|
+
return None
|
|
180
|
+
if head.startswith(b'PK\x03\x04'):
|
|
181
|
+
return 'Arsip ZIP'
|
|
182
|
+
if head.startswith(b'\x1f\x8b'):
|
|
183
|
+
return 'Arsip GZIP/TAR.GZ'
|
|
184
|
+
if head.startswith(b'Rar!'):
|
|
185
|
+
return 'Arsip RAR'
|
|
186
|
+
if re.search(r'(-- MySQL dump|PostgreSQL database dump|CREATE TABLE|INSERT INTO)', head.decode('utf-8', 'ignore'), re.I):
|
|
187
|
+
return 'Dump database SQL'
|
|
188
|
+
return None
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _text_rule(pattern, kind, allow_html=False):
|
|
192
|
+
def check(head, ctype):
|
|
193
|
+
text = head.decode('utf-8', 'ignore')
|
|
194
|
+
if not text:
|
|
195
|
+
return None
|
|
196
|
+
if not allow_html and ('html' in ctype.lower() or text.lstrip().startswith('<')):
|
|
197
|
+
return None
|
|
198
|
+
return kind if re.search(pattern, text, re.I | re.M) else None
|
|
199
|
+
return check
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _api_doc_kind(head, ctype):
|
|
203
|
+
text = head.decode('utf-8', 'ignore')
|
|
204
|
+
if re.search(r'"(?:openapi|swagger)"\s*:\s*"\d', text):
|
|
205
|
+
return 'Spesifikasi OpenAPI/Swagger'
|
|
206
|
+
if re.search(r'swagger-ui|<redoc|redoc\.standalone', text, re.I):
|
|
207
|
+
return 'Swagger UI / ReDoc'
|
|
208
|
+
return None
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _same_site(host, page_host):
|
|
212
|
+
def root(h):
|
|
213
|
+
return '.'.join(h.split('.')[-2:])
|
|
214
|
+
return host == page_host or root(host) == root(page_host)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _collect_scripts(soup, page_url, page_host):
|
|
218
|
+
scripts, no_sri = [], set()
|
|
219
|
+
for tag in soup.find_all('script'):
|
|
220
|
+
src = (tag.get('src') or '').strip()
|
|
221
|
+
if not src:
|
|
222
|
+
continue
|
|
223
|
+
full = urljoin(page_url, src)
|
|
224
|
+
p = urlparse(full)
|
|
225
|
+
host = (p.hostname or '').lower()
|
|
226
|
+
if p.scheme not in ('http', 'https') or not host:
|
|
227
|
+
continue
|
|
228
|
+
if _same_site(host, page_host):
|
|
229
|
+
if full not in scripts:
|
|
230
|
+
scripts.append(full)
|
|
231
|
+
elif not tag.get('integrity') and not any(host == d or host.endswith('.' + d) for d in SRI_EXEMPT):
|
|
232
|
+
no_sri.add(host)
|
|
233
|
+
return scripts, no_sri
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _check_sourcemap(ctx, map_url):
|
|
237
|
+
r, data = ctx.first_bytes(map_url, 2048)
|
|
238
|
+
text = data.decode('utf-8', 'ignore').lstrip()
|
|
239
|
+
return r.status_code == 200 and text.startswith('{') and ('"mappings"' in text or '"sources"' in text)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _analyze_script(ctx, url, guess_map):
|
|
243
|
+
r = ctx.s.get(url, timeout=ctx._timeout(), verify=ctx.verify, stream=True)
|
|
244
|
+
try:
|
|
245
|
+
if r.status_code != 200:
|
|
246
|
+
return None
|
|
247
|
+
data = r.raw.read(JS_MAX_BYTES, decode_content=True)
|
|
248
|
+
finally:
|
|
249
|
+
r.close()
|
|
250
|
+
text = data.decode('utf-8', 'ignore')
|
|
251
|
+
res = {'secrets': find_secrets(text, _short(url)), 'libraries': find_libraries(text), 'sourcemaps': []}
|
|
252
|
+
candidates = []
|
|
253
|
+
header = r.headers.get('SourceMap') or r.headers.get('X-SourceMap')
|
|
254
|
+
if header:
|
|
255
|
+
candidates.append(urljoin(url, header))
|
|
256
|
+
m = SOURCEMAP_RE.search(text[-4000:])
|
|
257
|
+
if m and not m.group(1).startswith('data:'):
|
|
258
|
+
candidates.append(urljoin(url, m.group(1)))
|
|
259
|
+
if not candidates and guess_map:
|
|
260
|
+
candidates.append(url.split('?')[0] + '.map')
|
|
261
|
+
if candidates and _check_sourcemap(ctx, candidates[0]):
|
|
262
|
+
res['sourcemaps'].append(_short(candidates[0]))
|
|
263
|
+
return res
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _probes(host):
|
|
267
|
+
label = host.split('.')[-2] if host.count('.') >= 1 else host
|
|
268
|
+
items = []
|
|
269
|
+
for name in dict.fromkeys(BACKUP_NAMES + (f'{host}.zip', f'{label}.zip', f'{label}.sql')):
|
|
270
|
+
items.append(('backups', '/' + name, _backup_kind, False))
|
|
271
|
+
for path, (pattern, kind) in CONFIG_FILES.items():
|
|
272
|
+
items.append(('config_files', path, _text_rule(pattern, kind), False))
|
|
273
|
+
for path, (pattern, kind, allow_html) in OPS_ENDPOINTS.items():
|
|
274
|
+
items.append(('ops', path, _text_rule(pattern, kind, allow_html), False))
|
|
275
|
+
for path in API_DOC_PATHS:
|
|
276
|
+
items.append(('api_docs', path, _api_doc_kind, False))
|
|
277
|
+
items.append(('debug', '/__cws_probe_' + _rand.token_hex(4), _debug_kind, True))
|
|
278
|
+
return items
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _run_probe(ctx, base, item):
|
|
282
|
+
category, path, validator, any_status = item
|
|
283
|
+
r, head = ctx.first_bytes(base + path, 8192, any_status)
|
|
284
|
+
kind = validator(head, r.headers.get('Content-Type', ''))
|
|
285
|
+
return category, ({'path': path if category != 'debug' else '(halaman error)', 'kind': kind} if kind else None)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _probe_graphql(ctx, base, path):
|
|
289
|
+
r = ctx.post_json(base + path, {'query': 'query{__schema{queryType{name}}}'})
|
|
290
|
+
if r.status_code != 200 or 'json' not in r.headers.get('Content-Type', '').lower():
|
|
291
|
+
return None
|
|
292
|
+
try:
|
|
293
|
+
data = r.json()
|
|
294
|
+
except ValueError:
|
|
295
|
+
return None
|
|
296
|
+
if isinstance(data, dict) and isinstance(data.get('data'), dict) and data['data'].get('__schema'):
|
|
297
|
+
return {'path': path, 'kind': 'Introspeksi GraphQL aktif'}
|
|
298
|
+
return None
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def run(session, base, resp, soup, verify, budget=BUDGET_SECONDS):
|
|
302
|
+
ctx = _Ctx(session, verify, budget)
|
|
303
|
+
page_host = (urlparse(resp.url).hostname or '').lower()
|
|
304
|
+
out = {'js_checked': 0, 'js_total': 0, 'secrets': [], 'sourcemaps': [], 'libraries': [], 'backups': [],
|
|
305
|
+
'config_files': [], 'debug': [], 'ops': [], 'api_docs': [], 'graphql': [], 'sri_missing': [],
|
|
306
|
+
'partial': False, 'notes': []}
|
|
307
|
+
scripts, no_sri = _collect_scripts(soup, resp.url, page_host)
|
|
308
|
+
out['js_total'] = len(scripts)
|
|
309
|
+
out['sri_missing'] = sorted(no_sri)
|
|
310
|
+
inline = '\n'.join(t.get_text() or '' for t in soup.find_all('script') if not t.get('src'))[:INLINE_MAX_CHARS]
|
|
311
|
+
out['secrets'].extend(find_secrets(inline, 'HTML (skrip inline)'))
|
|
312
|
+
out['libraries'].extend(find_libraries(inline))
|
|
313
|
+
|
|
314
|
+
jobs = []
|
|
315
|
+
with ThreadPoolExecutor(max_workers=4) as ex: # sopan terhadap firewall situs
|
|
316
|
+
for i, url in enumerate(scripts[:MAX_SCRIPTS]):
|
|
317
|
+
jobs.append(('js', ex.submit(_analyze_script, ctx, url, i < 2)))
|
|
318
|
+
for item in _probes(page_host):
|
|
319
|
+
jobs.append(('probe', ex.submit(_run_probe, ctx, base, item)))
|
|
320
|
+
for path in GRAPHQL_PATHS:
|
|
321
|
+
jobs.append(('graphql', ex.submit(_probe_graphql, ctx, base, path)))
|
|
322
|
+
for kind, fut in jobs:
|
|
323
|
+
try:
|
|
324
|
+
res = fut.result()
|
|
325
|
+
except (_OutOfTime, requests.exceptions.RequestException):
|
|
326
|
+
continue
|
|
327
|
+
except Exception as e:
|
|
328
|
+
out['notes'].append(f'{e.__class__.__name__}: {str(e)[:120]}')
|
|
329
|
+
continue
|
|
330
|
+
if not res:
|
|
331
|
+
continue
|
|
332
|
+
if kind == 'js':
|
|
333
|
+
out['js_checked'] += 1
|
|
334
|
+
for key in ('secrets', 'libraries', 'sourcemaps'):
|
|
335
|
+
out[key].extend(res[key])
|
|
336
|
+
elif kind == 'probe':
|
|
337
|
+
category, entry = res
|
|
338
|
+
if entry:
|
|
339
|
+
out[category].append(entry)
|
|
340
|
+
else:
|
|
341
|
+
out['graphql'].append(res)
|
|
342
|
+
|
|
343
|
+
seen, uniq = set(), []
|
|
344
|
+
for x in out['secrets']:
|
|
345
|
+
key = (x['type'], x['masked'])
|
|
346
|
+
if key not in seen:
|
|
347
|
+
seen.add(key)
|
|
348
|
+
uniq.append(x)
|
|
349
|
+
out['secrets'] = uniq
|
|
350
|
+
out['libraries'] = list({x['name']: x for x in out['libraries']}.values())
|
|
351
|
+
out['debug'] = out['debug'][:1]
|
|
352
|
+
out['partial'] = ctx.partial
|
|
353
|
+
return out
|