vulpfest 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,167 @@
1
+ """Cyber Web Scanner from the command line.
2
+
3
+ Examples:
4
+ python cli.py example.com # surface + deep web checks
5
+ python cli.py example.com --surface-only # quick surface checks only
6
+ python cli.py --repo . # check a local project folder
7
+ python cli.py example.com --repo . # both, in one report
8
+ python cli.py --repo . --fail-on critical # for CI: exit code 3 on critical findings
9
+ python cli.py example.com --json --out report.json
10
+ """
11
+ import argparse
12
+ import json
13
+ import re
14
+ import sys
15
+ from datetime import datetime, timezone
16
+
17
+ import json as _json
18
+ import manual_review
19
+ import repo_scan
20
+ import auth_scan
21
+ from scanner import calculate_security_score, run_full_scan
22
+
23
+ LEVEL_ORDER = {"Kritis": 0, "Tinggi": 1, "Menengah": 2, "Rendah": 3}
24
+ LEVEL_PREFIX = re.compile(r"^\s*Risiko\s+(Kritis|Tinggi|Menengah|Rendah)\s*:\s*", re.I)
25
+ FAIL_ON = {"critical": {"critical"}, "high": {"critical", "high"}}
26
+
27
+
28
+ def pick(d, *keys):
29
+ for k in keys:
30
+ if d.get(k):
31
+ return d[k]
32
+ return ""
33
+
34
+
35
+ def split_level(f):
36
+ """Level may be a separate field or written at the start of the finding ("Risiko Menengah: ...")."""
37
+ text = pick(f, "finding", "text", "temuan")
38
+ level = pick(f, "level", "severity", "tingkat")
39
+ m = LEVEL_PREFIX.match(text)
40
+ if m:
41
+ level = level or m.group(1).capitalize()
42
+ text = text[m.end():]
43
+ return level or "Info", text, f
44
+
45
+
46
+ def main():
47
+ parser = argparse.ArgumentParser(description="Check your own website and project for common security problems (Vulpfest Lab).")
48
+ parser.add_argument("url", nargs="?", help="website to scan, e.g. example.com or https://example.com")
49
+ parser.add_argument("--json", action="store_true", help="print the full result as JSON")
50
+ parser.add_argument("--out", metavar="FILE", help="also save the full result as JSON to FILE")
51
+ parser.add_argument("--surface-only", action="store_true", help="skip the deep web checks (JavaScript, exposed files, DNS and email)")
52
+ parser.add_argument("--repo", metavar="PATH", help="also check a local project folder (dependencies, secrets, git history, CI/CD, Docker)")
53
+ parser.add_argument("--offline", action="store_true", help="repository check without internet (no OSV.dev or registry lookups)")
54
+ parser.add_argument("--no-history", action="store_true", help="repository check without reading the git history")
55
+ parser.add_argument("--fail-on", choices=sorted(FAIL_ON), help="exit with code 3 when a finding at this level or higher is found")
56
+ parser.add_argument("--senior", action="store_true", help="show finding ID, category, track and confidence")
57
+ parser.add_argument("--checklist", action="store_true", help="print the manual review checklist (Markdown)")
58
+ parser.add_argument("--auth", metavar="FILE", help="authenticated checks (terminal only) on your own site; see auth.example.json")
59
+ parser.add_argument("--yes", action="store_true", help="skip the confirmation prompt for authenticated checks")
60
+ args = parser.parse_args()
61
+ if args.checklist:
62
+ print(manual_review.to_markdown())
63
+ if not args.url and not args.repo:
64
+ return 0
65
+ if not args.url and not args.repo:
66
+ parser.error("give a website, --repo PATH, or both")
67
+
68
+ if args.url:
69
+ print("Reminder: scan only websites you own or are authorized to test.", file=sys.stderr)
70
+ result = run_full_scan(args.url, modes=("surface",) if args.surface_only else ("surface", "deep_web"))
71
+ else:
72
+ result = {"target": None, "valid": True, "scanned_at": datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")}
73
+
74
+ if args.repo:
75
+ print(f"Checking repository {args.repo} ...", file=sys.stderr)
76
+ repo = repo_scan.run(args.repo, online=not args.offline, history=not args.no_history)
77
+ result["repo"] = repo
78
+ if repo.get("error"):
79
+ print(f"Repository: {repo['error']}", file=sys.stderr)
80
+ elif args.url:
81
+ if result.get("valid"):
82
+ result["coverage_modes"] = list(result.get("coverage_modes", ["surface"])) + ["repo"]
83
+ result["score_info"] = calculate_security_score(result)
84
+ else:
85
+ result["target"] = "repo: " + repo["path"]
86
+ result["score_info"] = repo_scan.score(repo)
87
+ if not args.url and repo.get("error"):
88
+ result["valid"] = False
89
+ result["score_info"] = {"score": 0, "grade": "N/A", "findings": [{"finding": repo["error"], "recommendation": "Check the folder path."}]}
90
+
91
+ if args.auth and args.url:
92
+ from urllib.parse import urlparse
93
+ host = urlparse(result.get("evidence", {}).get("final_url") or ("https://" + args.url)).hostname
94
+ try:
95
+ cfg = auth_scan.AuthConfig(_json.load(open(args.auth, encoding="utf-8")))
96
+ except (OSError, ValueError) as e:
97
+ print(f"Could not read {args.auth}: {e}", file=sys.stderr)
98
+ return 2
99
+ if not cfg.ok:
100
+ print("The auth file needs at least one test account.", file=sys.stderr)
101
+ return 2
102
+ token, how = auth_scan.verify_ownership(host)
103
+ ok, _method = auth_scan.confirm_ownership(host, token)
104
+ if not ok:
105
+ print("\nAuthenticated checks need proof that you own this site. Add ONE of these, then run again:", file=sys.stderr)
106
+ print(" - file: " + how["file"], file=sys.stderr)
107
+ print(" - DNS : " + how["dns"], file=sys.stderr)
108
+ return 2
109
+ if not args.yes:
110
+ print("\nAuthenticated checks will send a small number of real requests to " + str(host) + ".", file=sys.stderr)
111
+ if input("Type 'ya' to continue: ").strip().lower() not in ("ya", "yes"):
112
+ print("Cancelled.", file=sys.stderr)
113
+ return 0
114
+ result["auth"] = auth_scan.run(cfg)
115
+ if result.get("valid"):
116
+ result["coverage_modes"] = list(dict.fromkeys(list(result.get("coverage_modes", ["surface"])) + ["auth"]))
117
+ result["score_info"] = calculate_security_score(result)
118
+ elif args.auth:
119
+ print("--auth needs a website argument.", file=sys.stderr)
120
+ return 2
121
+
122
+ if args.out:
123
+ with open(args.out, "w", encoding="utf-8") as f:
124
+ json.dump(result, f, indent=2, ensure_ascii=False)
125
+ info = result.get("score_info", {})
126
+ if args.json:
127
+ print(json.dumps(result, indent=2, ensure_ascii=False))
128
+ else:
129
+ print(f"\nTarget : {result.get('target')}")
130
+ print(f"Scanned: {result.get('scanned_at')}")
131
+ if not result.get("valid"):
132
+ print(f"Status : could not scan reliably - {result.get('invalid_reason', 'see below')}")
133
+ print(f"Grade : {info.get('grade', '-')} (score {info.get('score', '-')}/100)")
134
+ cov = info.get("coverage") or {}
135
+ if cov.get("percent"):
136
+ print(f"Cover : {cov.get('label')} ({cov.get('percent')}%); not checked yet: {', '.join(cov.get('not_checked', [])) or '-'}")
137
+ subs, labels = info.get("subscores") or {}, info.get("category_labels") or {}
138
+ if subs:
139
+ print("Scores : " + " · ".join(f"{labels.get(k, k)} {v if v is not None else '-'}" for k, v in subs.items()))
140
+ if info.get("grade_capped"):
141
+ print("Note : grade capped because of critical findings")
142
+ repo = result.get("repo") or {}
143
+ if repo and not repo.get("error"):
144
+ hist = "not a git repository" if not repo.get("git") else ("partial" if repo.get("history_partial") else "yes")
145
+ print(f"Repo : {repo.get('files_scanned')} files, {repo['deps']['checked']} packages checked"
146
+ f"{' (offline: no OSV/registry)' if not repo.get('online') else ''}, git history: {hist}")
147
+ for note in repo.get("notes", []):
148
+ print(f"Note : {note}")
149
+ print()
150
+ items = sorted((split_level(f) for f in info.get("findings", [])), key=lambda x: LEVEL_ORDER.get(x[0], 9))
151
+ for i, (level, text, f) in enumerate(items, 1):
152
+ print(f"{i:>2}. [{level}] {text}")
153
+ if args.senior and f.get("id"):
154
+ print(f" [{f.get('id')} · {f.get('category', '-')} · track {f.get('track', '-')} · confidence {f.get('confidence', '-')}]")
155
+ rec = pick(f, "recommendation", "rec", "rekomendasi")
156
+ if rec:
157
+ print(f" -> {rec}")
158
+ if args.out:
159
+ print(f"\nFull result saved to {args.out}")
160
+
161
+ if args.fail_on and any(f.get("severity") in FAIL_ON[args.fail_on] for f in info.get("findings", [])):
162
+ return 3
163
+ return 0 if result.get("valid") else 2
164
+
165
+
166
+ if __name__ == "__main__":
167
+ sys.exit(main())
@@ -0,0 +1,353 @@
1
+ """Mode B: pemeriksaan web mendalam (tanpa browser headless).
2
+
3
+ Mengunduh file JavaScript milik situs yang dimuat halaman awal lalu memeriksa isinya, serta mencari
4
+ file dan endpoint yang sering tertinggal di produksi. Semua permintaan adalah GET/POST biasa
5
+ bervolume rendah lewat sesi yang sudah dipasangi pengaman SSRF; tidak ada payload eksploitasi.
6
+ Setiap temuan divalidasi dari ISI respons, bukan hanya status HTTP, agar situs yang membalas
7
+ 200 untuk semua alamat tidak menghasilkan temuan palsu.
8
+ """
9
+ import base64
10
+ import json
11
+ import re
12
+ import secrets as _rand
13
+ import time
14
+ from concurrent.futures import ThreadPoolExecutor
15
+ from urllib.parse import urljoin, urlparse
16
+
17
+ import requests
18
+
19
+ MAX_SCRIPTS = 12
20
+ JS_MAX_BYTES = 2 * 1024 * 1024
21
+ INLINE_MAX_CHARS = 1_000_000
22
+ BUDGET_SECONDS = 25
23
+ REQ_TIMEOUT = 6
24
+
25
+ # (jenis, pola, tingkat, keyakinan). critical = kunci rahasia; low = kunci publik yang perlu dibatasi.
26
+ SECRET_PATTERNS = (
27
+ ('AWS access key', r'\b(?:AKIA|ASIA)[0-9A-Z]{16}\b', 'critical', 'high'),
28
+ ('Stripe secret key', r'\b[sr]k_live_[0-9A-Za-z]{20,}\b', 'critical', 'high'),
29
+ ('OpenAI API key', r'\bsk-(?:proj-)?[A-Za-z0-9_-]{40,}\b', 'critical', 'medium'),
30
+ ('Anthropic API key', r'\bsk-ant-[A-Za-z0-9_-]{30,}\b', 'critical', 'high'),
31
+ ('GitHub token', r'\b(?:gh[pousr]_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{60,})\b', 'critical', 'high'),
32
+ ('Slack token', r'\bxox[baprs]-[A-Za-z0-9-]{10,}\b', 'critical', 'high'),
33
+ ('SendGrid API key', r'\bSG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}\b', 'critical', 'high'),
34
+ ('Mailgun API key', r'\bkey-[0-9a-f]{32}\b', 'critical', 'medium'),
35
+ ('Midtrans server key', r'\b(?:SB-)?Mid-server-[A-Za-z0-9_-]{20,}\b', 'critical', 'high'),
36
+ ('Xendit secret key', r'\bxnd_(?:production|development)_[A-Za-z0-9]{40,}\b', 'critical', 'high'),
37
+ ('Private key', r'-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----', 'critical', 'high'),
38
+ ('Google API key', r'\bAIza[0-9A-Za-z_-]{35}\b', 'low', 'medium'),
39
+ )
40
+ JWT_RE = re.compile(r'\beyJ[A-Za-z0-9_-]{10,}\.(eyJ[A-Za-z0-9_-]{10,})\.[A-Za-z0-9_-]{10,}')
41
+
42
+ # (nama, pola versi, batas aman / 'eol' / None=aturan khusus, penjelasan)
43
+ LIB_RULES = (
44
+ ('jQuery UI', r'jQuery UI - v(\d+\.\d+\.\d+)', (1, 13, 0),
45
+ 'XSS pada beberapa widget (CVE-2021-41182 dan sejenisnya); perbarui ke 1.13 atau lebih baru'),
46
+ ('jQuery', r'jQuery (?:JavaScript Library )?v(\d+\.\d+\.\d+)', (3, 5, 0),
47
+ 'XSS saat memproses HTML (CVE-2020-11022, CVE-2020-11023); perbarui ke 3.5.0 atau lebih baru'),
48
+ ('Bootstrap', r'Bootstrap v(\d+\.\d+\.\d+)', None,
49
+ 'XSS pada tooltip dan popover (CVE-2019-8331); perbarui ke 3.4.1, 4.3.1, atau lebih baru'),
50
+ ('AngularJS', r'AngularJS v(1\.\d+\.\d+)', 'eol',
51
+ 'AngularJS 1.x tidak lagi didukung sejak Januari 2022 dan tidak menerima perbaikan keamanan'),
52
+ ('Moment.js', r'//! version : (\d+\.\d+\.\d+)', (2, 29, 4),
53
+ 'ReDoS saat mem-parsing tanggal (CVE-2022-31129); perbarui ke 2.29.4 atau lebih baru'),
54
+ )
55
+
56
+ SOURCEMAP_RE = re.compile(r'//[#@]\s*sourceMappingURL=([^\s\'"]+)')
57
+ # Layanan yang memang tidak mendukung SRI (skrip dinamis), dikecualikan agar laporan tidak berisik.
58
+ SRI_EXEMPT = ('googletagmanager.com', 'google-analytics.com', 'gstatic.com', 'google.com', 'recaptcha.net',
59
+ 'challenges.cloudflare.com', 'js.stripe.com', 'vercel-scripts.com', 'vercel-insights.com',
60
+ 'paypal.com', 'paypalobjects.com', 'midtrans.com', 'facebook.net', 'clarity.ms', 'hotjar.com')
61
+
62
+ BACKUP_NAMES = ('backup.zip', 'backup.tar.gz', 'backup.sql', 'database.sql', 'db.sql', 'dump.sql',
63
+ 'site.zip', 'www.zip', 'public_html.zip', 'html.zip')
64
+ CONFIG_FILES = {
65
+ '/.env.local': (r'^[A-Z0-9_]+\s*=', 'File environment (.env.local)'),
66
+ '/.env.production': (r'^[A-Z0-9_]+\s*=', 'File environment (.env.production)'),
67
+ '/.env.backup': (r'^[A-Z0-9_]+\s*=', 'File environment (.env.backup)'),
68
+ '/.git-credentials': (r'https?://[^\s:/]+:[^\s@/]+@', 'Kredensial git'),
69
+ '/.npmrc': (r'_authToken\s*=', 'Token npm'),
70
+ '/wp-config.php.bak': (r'DB_PASSWORD', 'Backup konfigurasi WordPress'),
71
+ }
72
+ # jalur: (pola isi, jenis, boleh HTML)
73
+ OPS_ENDPOINTS = {
74
+ '/phpinfo.php': (r'phpinfo\(\)', 'Halaman phpinfo', True),
75
+ '/info.php': (r'phpinfo\(\)', 'Halaman phpinfo', True),
76
+ '/actuator/env': (r'"propertySources"', 'Spring Boot Actuator /env', False),
77
+ '/server-status': (r'Apache Server Status for', 'Apache server-status', True),
78
+ '/debug/pprof/': (r'Types of profiles available', 'Go pprof', True),
79
+ '/metrics': (r'^# (?:HELP|TYPE) \w+', 'Metrik Prometheus', False),
80
+ }
81
+ API_DOC_PATHS = ('/openapi.json', '/swagger.json', '/v3/api-docs', '/api-docs', '/swagger-ui.html',
82
+ '/swagger-ui/', '/docs', '/redoc')
83
+ GRAPHQL_PATHS = ('/graphql', '/api/graphql')
84
+ DEBUG_MARKERS = ('traceback (most recent call last)', 'werkzeug debugger', 'debug = true</code>',
85
+ 'whoops! there was an error', 'symfony\\component\\errorhandler', 'stack trace:</',
86
+ "server error in '/' application", 'at org.springframework', 'nextjs__container_errors')
87
+
88
+
89
+ class _OutOfTime(Exception):
90
+ pass
91
+
92
+
93
+ class _Ctx:
94
+ def __init__(self, session, verify, budget):
95
+ self.s, self.verify = session, verify
96
+ self.deadline = time.monotonic() + budget
97
+ self.partial = False
98
+
99
+ def _timeout(self):
100
+ left = self.deadline - time.monotonic()
101
+ if left <= 1:
102
+ self.partial = True
103
+ raise _OutOfTime()
104
+ return min(REQ_TIMEOUT, left)
105
+
106
+ def first_bytes(self, url, n=8192, any_status=False):
107
+ r = self.s.get(url, timeout=self._timeout(), verify=self.verify, allow_redirects=False, stream=True)
108
+ try:
109
+ data = r.raw.read(n, decode_content=True) if (any_status or r.status_code == 200) else b''
110
+ finally:
111
+ r.close()
112
+ return r, data
113
+
114
+ def post_json(self, url, payload):
115
+ return self.s.post(url, json=payload, timeout=self._timeout(), verify=self.verify, allow_redirects=False)
116
+
117
+
118
+ def mask(value):
119
+ v = value.strip()
120
+ return v[:6] + '…' + v[-4:] if len(v) > 14 else v[:3] + '…'
121
+
122
+
123
+ def _short(url):
124
+ p = urlparse(url)
125
+ path = p.path or '/'
126
+ return path if len(path) <= 80 else '…' + path[-77:]
127
+
128
+
129
+ def _ver(s):
130
+ return tuple(int(x) for x in s.split('.')[:3])
131
+
132
+
133
+ def _jwt_payload(segment):
134
+ try:
135
+ raw = base64.urlsafe_b64decode(segment + '=' * (-len(segment) % 4))
136
+ data = json.loads(raw)
137
+ return data if isinstance(data, dict) else None
138
+ except Exception:
139
+ return None
140
+
141
+
142
+ def find_secrets(text, where):
143
+ found = []
144
+ for kind, pattern, level, conf in SECRET_PATTERNS:
145
+ for m in re.finditer(pattern, text):
146
+ found.append({'type': kind, 'file': where, 'masked': mask(m.group(0)), 'severity': level, 'confidence': conf})
147
+ for m in JWT_RE.finditer(text):
148
+ payload = _jwt_payload(m.group(1))
149
+ if payload and payload.get('role') == 'service_role':
150
+ found.append({'type': 'Supabase service_role key', 'file': where, 'masked': mask(m.group(0)),
151
+ 'severity': 'critical', 'confidence': 'high'})
152
+ return found
153
+
154
+
155
+ def _bootstrap_vulnerable(v):
156
+ return v < (3, 4, 1) or (4, 0, 0) <= v < (4, 3, 1)
157
+
158
+
159
+ def find_libraries(text):
160
+ out = []
161
+ for name, pattern, rule, issue in LIB_RULES:
162
+ m = re.search(pattern, text)
163
+ if not m:
164
+ continue
165
+ v = _ver(m.group(1))
166
+ vulnerable = (rule == 'eol') or (rule is None and _bootstrap_vulnerable(v)) or (isinstance(rule, tuple) and v < rule)
167
+ if vulnerable:
168
+ out.append({'name': name, 'version': m.group(1), 'issue': issue})
169
+ return out
170
+
171
+
172
+ def _debug_kind(head, ctype):
173
+ text = head.decode('utf-8', 'ignore').lower()
174
+ return next((m for m in DEBUG_MARKERS if m in text), None)
175
+
176
+
177
+ def _backup_kind(head, ctype):
178
+ if not head or 'html' in ctype.lower():
179
+ return None
180
+ if head.startswith(b'PK\x03\x04'):
181
+ return 'Arsip ZIP'
182
+ if head.startswith(b'\x1f\x8b'):
183
+ return 'Arsip GZIP/TAR.GZ'
184
+ if head.startswith(b'Rar!'):
185
+ return 'Arsip RAR'
186
+ if re.search(r'(-- MySQL dump|PostgreSQL database dump|CREATE TABLE|INSERT INTO)', head.decode('utf-8', 'ignore'), re.I):
187
+ return 'Dump database SQL'
188
+ return None
189
+
190
+
191
+ def _text_rule(pattern, kind, allow_html=False):
192
+ def check(head, ctype):
193
+ text = head.decode('utf-8', 'ignore')
194
+ if not text:
195
+ return None
196
+ if not allow_html and ('html' in ctype.lower() or text.lstrip().startswith('<')):
197
+ return None
198
+ return kind if re.search(pattern, text, re.I | re.M) else None
199
+ return check
200
+
201
+
202
+ def _api_doc_kind(head, ctype):
203
+ text = head.decode('utf-8', 'ignore')
204
+ if re.search(r'"(?:openapi|swagger)"\s*:\s*"\d', text):
205
+ return 'Spesifikasi OpenAPI/Swagger'
206
+ if re.search(r'swagger-ui|<redoc|redoc\.standalone', text, re.I):
207
+ return 'Swagger UI / ReDoc'
208
+ return None
209
+
210
+
211
+ def _same_site(host, page_host):
212
+ def root(h):
213
+ return '.'.join(h.split('.')[-2:])
214
+ return host == page_host or root(host) == root(page_host)
215
+
216
+
217
+ def _collect_scripts(soup, page_url, page_host):
218
+ scripts, no_sri = [], set()
219
+ for tag in soup.find_all('script'):
220
+ src = (tag.get('src') or '').strip()
221
+ if not src:
222
+ continue
223
+ full = urljoin(page_url, src)
224
+ p = urlparse(full)
225
+ host = (p.hostname or '').lower()
226
+ if p.scheme not in ('http', 'https') or not host:
227
+ continue
228
+ if _same_site(host, page_host):
229
+ if full not in scripts:
230
+ scripts.append(full)
231
+ elif not tag.get('integrity') and not any(host == d or host.endswith('.' + d) for d in SRI_EXEMPT):
232
+ no_sri.add(host)
233
+ return scripts, no_sri
234
+
235
+
236
+ def _check_sourcemap(ctx, map_url):
237
+ r, data = ctx.first_bytes(map_url, 2048)
238
+ text = data.decode('utf-8', 'ignore').lstrip()
239
+ return r.status_code == 200 and text.startswith('{') and ('"mappings"' in text or '"sources"' in text)
240
+
241
+
242
+ def _analyze_script(ctx, url, guess_map):
243
+ r = ctx.s.get(url, timeout=ctx._timeout(), verify=ctx.verify, stream=True)
244
+ try:
245
+ if r.status_code != 200:
246
+ return None
247
+ data = r.raw.read(JS_MAX_BYTES, decode_content=True)
248
+ finally:
249
+ r.close()
250
+ text = data.decode('utf-8', 'ignore')
251
+ res = {'secrets': find_secrets(text, _short(url)), 'libraries': find_libraries(text), 'sourcemaps': []}
252
+ candidates = []
253
+ header = r.headers.get('SourceMap') or r.headers.get('X-SourceMap')
254
+ if header:
255
+ candidates.append(urljoin(url, header))
256
+ m = SOURCEMAP_RE.search(text[-4000:])
257
+ if m and not m.group(1).startswith('data:'):
258
+ candidates.append(urljoin(url, m.group(1)))
259
+ if not candidates and guess_map:
260
+ candidates.append(url.split('?')[0] + '.map')
261
+ if candidates and _check_sourcemap(ctx, candidates[0]):
262
+ res['sourcemaps'].append(_short(candidates[0]))
263
+ return res
264
+
265
+
266
+ def _probes(host):
267
+ label = host.split('.')[-2] if host.count('.') >= 1 else host
268
+ items = []
269
+ for name in dict.fromkeys(BACKUP_NAMES + (f'{host}.zip', f'{label}.zip', f'{label}.sql')):
270
+ items.append(('backups', '/' + name, _backup_kind, False))
271
+ for path, (pattern, kind) in CONFIG_FILES.items():
272
+ items.append(('config_files', path, _text_rule(pattern, kind), False))
273
+ for path, (pattern, kind, allow_html) in OPS_ENDPOINTS.items():
274
+ items.append(('ops', path, _text_rule(pattern, kind, allow_html), False))
275
+ for path in API_DOC_PATHS:
276
+ items.append(('api_docs', path, _api_doc_kind, False))
277
+ items.append(('debug', '/__cws_probe_' + _rand.token_hex(4), _debug_kind, True))
278
+ return items
279
+
280
+
281
+ def _run_probe(ctx, base, item):
282
+ category, path, validator, any_status = item
283
+ r, head = ctx.first_bytes(base + path, 8192, any_status)
284
+ kind = validator(head, r.headers.get('Content-Type', ''))
285
+ return category, ({'path': path if category != 'debug' else '(halaman error)', 'kind': kind} if kind else None)
286
+
287
+
288
+ def _probe_graphql(ctx, base, path):
289
+ r = ctx.post_json(base + path, {'query': 'query{__schema{queryType{name}}}'})
290
+ if r.status_code != 200 or 'json' not in r.headers.get('Content-Type', '').lower():
291
+ return None
292
+ try:
293
+ data = r.json()
294
+ except ValueError:
295
+ return None
296
+ if isinstance(data, dict) and isinstance(data.get('data'), dict) and data['data'].get('__schema'):
297
+ return {'path': path, 'kind': 'Introspeksi GraphQL aktif'}
298
+ return None
299
+
300
+
301
+ def run(session, base, resp, soup, verify, budget=BUDGET_SECONDS):
302
+ ctx = _Ctx(session, verify, budget)
303
+ page_host = (urlparse(resp.url).hostname or '').lower()
304
+ out = {'js_checked': 0, 'js_total': 0, 'secrets': [], 'sourcemaps': [], 'libraries': [], 'backups': [],
305
+ 'config_files': [], 'debug': [], 'ops': [], 'api_docs': [], 'graphql': [], 'sri_missing': [],
306
+ 'partial': False, 'notes': []}
307
+ scripts, no_sri = _collect_scripts(soup, resp.url, page_host)
308
+ out['js_total'] = len(scripts)
309
+ out['sri_missing'] = sorted(no_sri)
310
+ inline = '\n'.join(t.get_text() or '' for t in soup.find_all('script') if not t.get('src'))[:INLINE_MAX_CHARS]
311
+ out['secrets'].extend(find_secrets(inline, 'HTML (skrip inline)'))
312
+ out['libraries'].extend(find_libraries(inline))
313
+
314
+ jobs = []
315
+ with ThreadPoolExecutor(max_workers=4) as ex: # sopan terhadap firewall situs
316
+ for i, url in enumerate(scripts[:MAX_SCRIPTS]):
317
+ jobs.append(('js', ex.submit(_analyze_script, ctx, url, i < 2)))
318
+ for item in _probes(page_host):
319
+ jobs.append(('probe', ex.submit(_run_probe, ctx, base, item)))
320
+ for path in GRAPHQL_PATHS:
321
+ jobs.append(('graphql', ex.submit(_probe_graphql, ctx, base, path)))
322
+ for kind, fut in jobs:
323
+ try:
324
+ res = fut.result()
325
+ except (_OutOfTime, requests.exceptions.RequestException):
326
+ continue
327
+ except Exception as e:
328
+ out['notes'].append(f'{e.__class__.__name__}: {str(e)[:120]}')
329
+ continue
330
+ if not res:
331
+ continue
332
+ if kind == 'js':
333
+ out['js_checked'] += 1
334
+ for key in ('secrets', 'libraries', 'sourcemaps'):
335
+ out[key].extend(res[key])
336
+ elif kind == 'probe':
337
+ category, entry = res
338
+ if entry:
339
+ out[category].append(entry)
340
+ else:
341
+ out['graphql'].append(res)
342
+
343
+ seen, uniq = set(), []
344
+ for x in out['secrets']:
345
+ key = (x['type'], x['masked'])
346
+ if key not in seen:
347
+ seen.add(key)
348
+ uniq.append(x)
349
+ out['secrets'] = uniq
350
+ out['libraries'] = list({x['name']: x for x in out['libraries']}.values())
351
+ out['debug'] = out['debug'][:1]
352
+ out['partial'] = ctx.partial
353
+ return out