tech-teardown 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- teardown/__init__.py +5 -0
- teardown/__main__.py +4 -0
- teardown/analyze.py +421 -0
- teardown/cache.py +60 -0
- teardown/cli.py +241 -0
- teardown/collectors/__init__.py +1 -0
- teardown/collectors/api_discovery.py +165 -0
- teardown/collectors/headers.py +169 -0
- teardown/collectors/html_js.py +475 -0
- teardown/data/LICENSE-opentechalyzer +21 -0
- teardown/data/LICENSE.md +6 -0
- teardown/data/__init__.py +3 -0
- teardown/data/curated.json +15 -0
- teardown/data/fingerprints.json +8631 -0
- teardown/diff.py +56 -0
- teardown/enumerate.py +239 -0
- teardown/fingerprints.py +246 -0
- teardown/http_client.py +201 -0
- teardown/mcp_server.py +81 -0
- teardown/models.py +196 -0
- teardown/render.py +299 -0
- teardown/safety.py +85 -0
- teardown/scan_modules.py +142 -0
- tech_teardown-0.1.1.dist-info/METADATA +242 -0
- tech_teardown-0.1.1.dist-info/RECORD +28 -0
- tech_teardown-0.1.1.dist-info/WHEEL +4 -0
- tech_teardown-0.1.1.dist-info/entry_points.txt +3 -0
- tech_teardown-0.1.1.dist-info/licenses/LICENSE +21 -0
teardown/__init__.py
ADDED
teardown/__main__.py
ADDED
teardown/analyze.py
ADDED
|
@@ -0,0 +1,421 @@
|
|
|
1
|
+
"""Deterministic OPSEC analysis + cross-host aggregation.
|
|
2
|
+
|
|
3
|
+
`analyze_host` turns the raw signals a collector already gathered on a
|
|
4
|
+
`HostTeardown` into a prioritized list of `OpsecFinding`s. `build_aggregate`
|
|
5
|
+
rolls the whole scan up into tech-stack / third-party unions and pulls out every
|
|
6
|
+
critical finding. Pure and offline — no network, no mutation of inputs.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
from urllib.parse import urlsplit
|
|
13
|
+
|
|
14
|
+
from .models import (
|
|
15
|
+
Aggregate,
|
|
16
|
+
HostTeardown,
|
|
17
|
+
OpsecFinding,
|
|
18
|
+
Severity,
|
|
19
|
+
)
|
|
20
|
+
from .safety import registrable_domain, source_map_url
|
|
21
|
+
|
|
22
|
+
# A dotted version number ("nginx/1.18.0", "PHP/8.1") or a "/<digit>" suffix.
|
|
23
|
+
_VERSION_RE = re.compile(r"\d+\.\d+|/\s*\d")
|
|
24
|
+
|
|
25
|
+
# Private / internal network markers that should never appear in a public CSP.
|
|
26
|
+
_PRIVATE_IP_RE = re.compile(
|
|
27
|
+
r"\b(?:10\.\d{1,3}\.\d{1,3}\.\d{1,3}"
|
|
28
|
+
r"|192\.168\.\d{1,3}\.\d{1,3}"
|
|
29
|
+
r"|172\.(?:1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3})\b"
|
|
30
|
+
)
|
|
31
|
+
_INTERNAL_MARKERS = (
|
|
32
|
+
"internal",
|
|
33
|
+
"intranet",
|
|
34
|
+
"corp",
|
|
35
|
+
"vpn",
|
|
36
|
+
"staging",
|
|
37
|
+
"preprod",
|
|
38
|
+
"pre-prod",
|
|
39
|
+
".local",
|
|
40
|
+
".svc",
|
|
41
|
+
"cluster.local",
|
|
42
|
+
"localhost",
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
# Names we treat as analytics / tracking when a key is exposed.
|
|
46
|
+
_ANALYTICS_NAMES = {
|
|
47
|
+
"ga",
|
|
48
|
+
"ga4",
|
|
49
|
+
"gtag",
|
|
50
|
+
"gtm",
|
|
51
|
+
"google-analytics",
|
|
52
|
+
"googleanalytics",
|
|
53
|
+
"segment",
|
|
54
|
+
"mixpanel",
|
|
55
|
+
"posthog",
|
|
56
|
+
"amplitude",
|
|
57
|
+
"hotjar",
|
|
58
|
+
"heap",
|
|
59
|
+
"plausible",
|
|
60
|
+
"matomo",
|
|
61
|
+
"fathom",
|
|
62
|
+
}
|
|
63
|
+
_ANALYTICS_CATEGORIES = {"analytics", "tracking", "telemetry"}
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _is_https(host: HostTeardown) -> bool:
|
|
67
|
+
url = (host.http.final_url or host.target or "").strip().lower()
|
|
68
|
+
return url.startswith("https://")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _reveals_version(value: str | None) -> bool:
|
|
72
|
+
return bool(value) and bool(_VERSION_RE.search(value or ""))
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _header_present(host: HostTeardown, sec_key: str, header_name: str) -> bool:
|
|
76
|
+
"""True if a security header is set via either the parsed dict or raw headers."""
|
|
77
|
+
sec = host.http.security_headers or {}
|
|
78
|
+
if sec.get(sec_key):
|
|
79
|
+
return True
|
|
80
|
+
lowered = {k.lower() for k in (host.http.all_headers or {})}
|
|
81
|
+
return header_name.lower() in lowered
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _is_live_secret(type_: str, value_prefix: str) -> bool:
|
|
85
|
+
t = (type_ or "").lower()
|
|
86
|
+
p = (value_prefix or "").lower()
|
|
87
|
+
if "live" in t or "live" in p:
|
|
88
|
+
return True
|
|
89
|
+
# Server-side secret keys are always live-grade exposure.
|
|
90
|
+
return p.startswith(("sk_", "rk_")) or "secret" in t
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def analyze_host(host: HostTeardown) -> list[OpsecFinding]:
|
|
94
|
+
"""Derive OPSEC findings from the already-collected signals on `host`.
|
|
95
|
+
|
|
96
|
+
Defensive: every field may be missing/empty. Returns a fresh list; does not
|
|
97
|
+
mutate `host`.
|
|
98
|
+
"""
|
|
99
|
+
findings: list[OpsecFinding] = []
|
|
100
|
+
target = host.target or (f"https://{host.host}" if host.host else None)
|
|
101
|
+
|
|
102
|
+
api = host.api_surface
|
|
103
|
+
gql = api.graphql
|
|
104
|
+
openapi = api.openapi
|
|
105
|
+
wp = api.wordpress
|
|
106
|
+
|
|
107
|
+
# ---- CRITICAL --------------------------------------------------------
|
|
108
|
+
if gql and gql.introspection_open:
|
|
109
|
+
findings.append(
|
|
110
|
+
OpsecFinding(
|
|
111
|
+
severity=Severity.CRITICAL,
|
|
112
|
+
title="GraphQL introspection enabled",
|
|
113
|
+
detail=(
|
|
114
|
+
"The GraphQL endpoint answers __schema introspection queries, "
|
|
115
|
+
f"exposing the full type system ({len(gql.types)} types) to anyone."
|
|
116
|
+
),
|
|
117
|
+
evidence_url=f"{target}/graphql" if target else None,
|
|
118
|
+
)
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
if wp and wp.users_enumerable:
|
|
122
|
+
names = ", ".join(wp.users[:10]) if wp.users else "author accounts"
|
|
123
|
+
findings.append(
|
|
124
|
+
OpsecFinding(
|
|
125
|
+
severity=Severity.CRITICAL,
|
|
126
|
+
title="WordPress user enumeration open",
|
|
127
|
+
detail=(
|
|
128
|
+
"The WP REST API returns the user list, leaking login names "
|
|
129
|
+
f"({names}) for credential-stuffing / brute-force targeting."
|
|
130
|
+
),
|
|
131
|
+
evidence_url=f"{target}/wp-json/wp/v2/users" if target else None,
|
|
132
|
+
)
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
if openapi and openapi.present:
|
|
136
|
+
findings.append(
|
|
137
|
+
OpsecFinding(
|
|
138
|
+
severity=Severity.CRITICAL,
|
|
139
|
+
title="OpenAPI/Swagger specification publicly exposed",
|
|
140
|
+
detail=(
|
|
141
|
+
"A machine-readable API spec is reachable without auth, "
|
|
142
|
+
f"enumerating {len(openapi.endpoints)} endpoints and their shapes."
|
|
143
|
+
),
|
|
144
|
+
evidence_url=openapi.url or (f"{target}/openapi.json" if target else None),
|
|
145
|
+
)
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
for leak in host.secret_leaks or []:
|
|
149
|
+
if _is_live_secret(leak.type, leak.value_prefix):
|
|
150
|
+
findings.append(
|
|
151
|
+
OpsecFinding(
|
|
152
|
+
severity=Severity.CRITICAL,
|
|
153
|
+
title=f"Live secret exposed: {leak.type}",
|
|
154
|
+
detail=(
|
|
155
|
+
f"A live-scope secret ({leak.type}) is present in client-visible "
|
|
156
|
+
f"content, redacted prefix `{leak.value_prefix}`."
|
|
157
|
+
),
|
|
158
|
+
evidence_url=leak.source_url,
|
|
159
|
+
)
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
for bundle in host.bundles or []:
|
|
163
|
+
# Only first-party source maps are the target's own exposure. A third-party
|
|
164
|
+
# CDN (Sentry, jsdelivr, …) publishing maps for its own SDK is not this
|
|
165
|
+
# host leaking anything, so those are skipped. Source-map disclosure is an
|
|
166
|
+
# information leak, not a credential/data leak -> MEDIUM, not CRITICAL.
|
|
167
|
+
if bundle.has_sourcemap and _is_first_party(bundle.url, host.host):
|
|
168
|
+
findings.append(
|
|
169
|
+
OpsecFinding(
|
|
170
|
+
severity=Severity.MEDIUM,
|
|
171
|
+
title="Source map exposed on first-party bundle",
|
|
172
|
+
detail=(
|
|
173
|
+
f"The first-party bundle {bundle.url} ships a source map, "
|
|
174
|
+
"reconstructing original source (unminified logic, comments, "
|
|
175
|
+
"internal paths). Vendor libraries served first-party leak the "
|
|
176
|
+
"library's source; app bundles leak your own."
|
|
177
|
+
),
|
|
178
|
+
evidence_url=source_map_url(bundle.url),
|
|
179
|
+
)
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
# ---- MEDIUM ----------------------------------------------------------
|
|
183
|
+
version_bits = []
|
|
184
|
+
if _reveals_version(host.http.server):
|
|
185
|
+
version_bits.append(f"Server: `{host.http.server}`")
|
|
186
|
+
if _reveals_version(host.http.powered_by):
|
|
187
|
+
version_bits.append(f"X-Powered-By: `{host.http.powered_by}`")
|
|
188
|
+
if version_bits:
|
|
189
|
+
findings.append(
|
|
190
|
+
OpsecFinding(
|
|
191
|
+
severity=Severity.MEDIUM,
|
|
192
|
+
title="Server / framework version disclosure",
|
|
193
|
+
detail=(
|
|
194
|
+
"Response headers reveal exact software versions, narrowing CVE "
|
|
195
|
+
"lookup for an attacker — " + "; ".join(version_bits) + "."
|
|
196
|
+
),
|
|
197
|
+
evidence_url=target,
|
|
198
|
+
)
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
internal = _internal_hosts_in_csp(host)
|
|
202
|
+
if internal:
|
|
203
|
+
findings.append(
|
|
204
|
+
OpsecFinding(
|
|
205
|
+
severity=Severity.MEDIUM,
|
|
206
|
+
title="CSP leaks internal/infrastructure hostnames",
|
|
207
|
+
detail=(
|
|
208
|
+
"The Content-Security-Policy references non-public hosts, mapping "
|
|
209
|
+
"internal infrastructure: " + ", ".join(internal) + "."
|
|
210
|
+
),
|
|
211
|
+
evidence_url=target,
|
|
212
|
+
)
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
if _is_https(host) and not _header_present(host, "hsts", "strict-transport-security"):
|
|
216
|
+
findings.append(
|
|
217
|
+
OpsecFinding(
|
|
218
|
+
severity=Severity.MEDIUM,
|
|
219
|
+
title="Missing HSTS on HTTPS site",
|
|
220
|
+
detail=(
|
|
221
|
+
"No Strict-Transport-Security header, so browsers will honor a "
|
|
222
|
+
"downgrade to HTTP and are exposed to SSL-stripping on first visit."
|
|
223
|
+
),
|
|
224
|
+
evidence_url=target,
|
|
225
|
+
)
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
# ---- LOW -------------------------------------------------------------
|
|
229
|
+
analytics = [
|
|
230
|
+
tp
|
|
231
|
+
for tp in (host.third_parties or [])
|
|
232
|
+
if tp.key
|
|
233
|
+
and (
|
|
234
|
+
(tp.category or "").lower() in _ANALYTICS_CATEGORIES
|
|
235
|
+
or (tp.name or "").lower() in _ANALYTICS_NAMES
|
|
236
|
+
)
|
|
237
|
+
]
|
|
238
|
+
if analytics:
|
|
239
|
+
listed = ", ".join(f"{tp.name} (`{tp.key}`)" for tp in analytics)
|
|
240
|
+
findings.append(
|
|
241
|
+
OpsecFinding(
|
|
242
|
+
severity=Severity.LOW,
|
|
243
|
+
title="Analytics / tracking keys visible",
|
|
244
|
+
detail=(
|
|
245
|
+
"Client-side analytics keys are exposed (expected for these SDKs, "
|
|
246
|
+
"but worth noting): " + listed + "."
|
|
247
|
+
),
|
|
248
|
+
evidence_url=target,
|
|
249
|
+
)
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
fingerprint = list(host.frameworks or [])
|
|
253
|
+
if host.cms and host.cms.type:
|
|
254
|
+
cms_label = host.cms.type + (f" {host.cms.version}" if host.cms.version else "")
|
|
255
|
+
fingerprint.append(cms_label)
|
|
256
|
+
if fingerprint:
|
|
257
|
+
findings.append(
|
|
258
|
+
OpsecFinding(
|
|
259
|
+
severity=Severity.LOW,
|
|
260
|
+
title="Framework / CMS fingerprintable",
|
|
261
|
+
detail=(
|
|
262
|
+
"The stack is identifiable from public markers: "
|
|
263
|
+
+ ", ".join(fingerprint)
|
|
264
|
+
+ "."
|
|
265
|
+
),
|
|
266
|
+
evidence_url=target,
|
|
267
|
+
)
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
if not _header_present(host, "x_frame_options", "x-frame-options"):
|
|
271
|
+
findings.append(
|
|
272
|
+
OpsecFinding(
|
|
273
|
+
severity=Severity.LOW,
|
|
274
|
+
title="Missing X-Frame-Options",
|
|
275
|
+
detail=(
|
|
276
|
+
"No X-Frame-Options header (and no frame-ancestors CSP seen), "
|
|
277
|
+
"leaving the site open to clickjacking via framing."
|
|
278
|
+
),
|
|
279
|
+
evidence_url=target,
|
|
280
|
+
)
|
|
281
|
+
)
|
|
282
|
+
if not _header_present(host, "x_content_type_options", "x-content-type-options"):
|
|
283
|
+
findings.append(
|
|
284
|
+
OpsecFinding(
|
|
285
|
+
severity=Severity.LOW,
|
|
286
|
+
title="Missing X-Content-Type-Options",
|
|
287
|
+
detail=(
|
|
288
|
+
"No X-Content-Type-Options: nosniff header, allowing MIME-sniffing "
|
|
289
|
+
"of responses."
|
|
290
|
+
),
|
|
291
|
+
evidence_url=target,
|
|
292
|
+
)
|
|
293
|
+
)
|
|
294
|
+
|
|
295
|
+
# ---- INFO ------------------------------------------------------------
|
|
296
|
+
if host.third_parties:
|
|
297
|
+
names = sorted({tp.name for tp in host.third_parties if tp.name})
|
|
298
|
+
if names:
|
|
299
|
+
findings.append(
|
|
300
|
+
OpsecFinding(
|
|
301
|
+
severity=Severity.INFO,
|
|
302
|
+
title="Third-party services enumerated",
|
|
303
|
+
detail=(
|
|
304
|
+
"Public signals enumerate the third parties this host talks to: "
|
|
305
|
+
+ ", ".join(names)
|
|
306
|
+
+ "."
|
|
307
|
+
),
|
|
308
|
+
evidence_url=target,
|
|
309
|
+
)
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
for finding in findings:
|
|
313
|
+
if finding.title == "Framework / CMS fingerprintable" and host.detections:
|
|
314
|
+
finding.confidence = max(item.confidence for item in host.detections)
|
|
315
|
+
finding.evidence = sorted({signal for item in host.detections
|
|
316
|
+
for signal in item.evidence})[:8]
|
|
317
|
+
elif finding.title == "Third-party services enumerated":
|
|
318
|
+
finding.evidence = sorted({tp.evidence or tp.name for tp in host.third_parties})[:8]
|
|
319
|
+
elif finding.evidence_url:
|
|
320
|
+
finding.evidence = [finding.evidence_url]
|
|
321
|
+
return findings
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _registrable(hostname: str) -> str:
|
|
325
|
+
return registrable_domain(hostname or "")
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _is_first_party(bundle_url: str, host_name: str) -> bool:
|
|
329
|
+
"""True if the bundle shares the host's registrable domain (or is a relative URL)."""
|
|
330
|
+
bundle_host = urlsplit(bundle_url).hostname
|
|
331
|
+
if not bundle_host:
|
|
332
|
+
return True # relative/same-origin URL
|
|
333
|
+
return _registrable(bundle_host) == _registrable(host_name)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _internal_hosts_in_csp(host: HostTeardown) -> list[str]:
|
|
337
|
+
"""Return sorted unique CSP sources that look internal/infra (not public)."""
|
|
338
|
+
csp = host.csp
|
|
339
|
+
if not csp or not csp.present:
|
|
340
|
+
return []
|
|
341
|
+
candidates: set[str] = set(csp.third_party_domains or [])
|
|
342
|
+
for sources in (csp.directives or {}).values():
|
|
343
|
+
for src in sources or []:
|
|
344
|
+
s = src.strip().strip("'")
|
|
345
|
+
if s and not s.startswith(("'", "data:", "blob:")):
|
|
346
|
+
candidates.add(s)
|
|
347
|
+
hits: set[str] = set()
|
|
348
|
+
for dom in candidates:
|
|
349
|
+
low = dom.lower()
|
|
350
|
+
if _PRIVATE_IP_RE.search(low) or any(m in low for m in _INTERNAL_MARKERS):
|
|
351
|
+
hits.add(dom)
|
|
352
|
+
return sorted(hits)
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def _detect_hosting(host: HostTeardown) -> list[str]:
|
|
356
|
+
"""Infer hosting/CDN providers from cache headers and the server string."""
|
|
357
|
+
providers: set[str] = set()
|
|
358
|
+
cache_keys = {k.lower() for k in (host.http.cache or {})}
|
|
359
|
+
all_header_keys = {k.lower() for k in (host.http.all_headers or {})}
|
|
360
|
+
keys = cache_keys | all_header_keys
|
|
361
|
+
server = (host.http.server or "").lower()
|
|
362
|
+
powered = (host.http.powered_by or "").lower()
|
|
363
|
+
|
|
364
|
+
markers = {
|
|
365
|
+
"Cloudflare": ("cf_cache_status", "cf-cache-status", "cf-ray"),
|
|
366
|
+
"Vercel": ("x_vercel_cache", "x-vercel-cache", "x_vercel_id", "x-vercel-id"),
|
|
367
|
+
"Netlify": ("x_nf_request_id", "x-nf-request-id"),
|
|
368
|
+
"Fastly": ("x-served-by", "x_served_by", "fastly"),
|
|
369
|
+
"AWS CloudFront": ("x-amz-cf-id", "x_amz_cf_id", "x-amz-cf-pop"),
|
|
370
|
+
}
|
|
371
|
+
for name, hints in markers.items():
|
|
372
|
+
if any(h in keys for h in hints):
|
|
373
|
+
providers.add(name)
|
|
374
|
+
for name in ("cloudflare", "vercel", "netlify", "fastly", "akamai"):
|
|
375
|
+
if name in server or name in powered:
|
|
376
|
+
providers.add(name.capitalize())
|
|
377
|
+
return sorted(providers)
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def build_aggregate(hosts: list[HostTeardown]) -> Aggregate:
|
|
381
|
+
"""Union tech stack + third parties across hosts and collect critical findings."""
|
|
382
|
+
frameworks: set[str] = set()
|
|
383
|
+
detections: set[str] = set()
|
|
384
|
+
cms: set[str] = set()
|
|
385
|
+
servers: set[str] = set()
|
|
386
|
+
hosting: set[str] = set()
|
|
387
|
+
third_parties: set[str] = set()
|
|
388
|
+
criticals: list[OpsecFinding] = []
|
|
389
|
+
|
|
390
|
+
for host in hosts or []:
|
|
391
|
+
detections.update(item.name for item in (host.detections or []) if item.name)
|
|
392
|
+
for fw in host.frameworks or []:
|
|
393
|
+
if fw:
|
|
394
|
+
frameworks.add(fw)
|
|
395
|
+
if host.cms and host.cms.type:
|
|
396
|
+
cms.add(host.cms.type)
|
|
397
|
+
if host.http.server:
|
|
398
|
+
servers.add(host.http.server)
|
|
399
|
+
if host.http.powered_by:
|
|
400
|
+
servers.add(host.http.powered_by)
|
|
401
|
+
for provider in _detect_hosting(host):
|
|
402
|
+
hosting.add(provider)
|
|
403
|
+
for tp in host.third_parties or []:
|
|
404
|
+
if tp.name:
|
|
405
|
+
third_parties.add(tp.name)
|
|
406
|
+
for finding in host.opsec_findings or []:
|
|
407
|
+
if finding.severity == Severity.CRITICAL:
|
|
408
|
+
criticals.append(finding)
|
|
409
|
+
|
|
410
|
+
tech_stack_union = {
|
|
411
|
+
"frameworks": sorted(frameworks),
|
|
412
|
+
"cms": sorted(cms),
|
|
413
|
+
"servers": sorted(servers),
|
|
414
|
+
"hosting": sorted(hosting),
|
|
415
|
+
}
|
|
416
|
+
return Aggregate(
|
|
417
|
+
tech_stack_union=tech_stack_union,
|
|
418
|
+
detections_union=sorted(detections),
|
|
419
|
+
third_parties_union=sorted(third_parties),
|
|
420
|
+
critical_findings=criticals,
|
|
421
|
+
)
|
teardown/cache.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Bounded daily response cache; never writes known secret-bearing bodies."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import datetime as dt
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import re
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from .http_client import FetchResponse
|
|
12
|
+
from .safety import safe_headers, safe_text, safe_url
|
|
13
|
+
|
|
14
|
+
_SENSITIVE_CONTEXT = re.compile(
|
|
15
|
+
r"password|secret|token|credential|session|authorization|auth|"
|
|
16
|
+
r"private[-_ ]?key|api[-_ ]?key|access[-_ ]?key",
|
|
17
|
+
re.I,
|
|
18
|
+
)
|
|
19
|
+
_OPAQUE_VALUE = re.compile(r"(?:=|:)\s*[\"']([A-Za-z0-9_-]{16,})[\"']")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class ResponseCache:
|
|
23
|
+
def __init__(self, directory: str | Path, day: str | None = None) -> None:
|
|
24
|
+
self.directory = Path(directory)
|
|
25
|
+
self.day = day or dt.datetime.now(dt.timezone.utc).date().isoformat()
|
|
26
|
+
|
|
27
|
+
def _path(self, url: str) -> Path:
|
|
28
|
+
key = hashlib.sha256(f"GET\n{url}\n{self.day}".encode()).hexdigest()
|
|
29
|
+
return self.directory / self.day / f"{key}.json"
|
|
30
|
+
|
|
31
|
+
def get(self, url: str) -> FetchResponse | None:
|
|
32
|
+
try:
|
|
33
|
+
data = json.loads(self._path(url).read_text(encoding="utf-8"))
|
|
34
|
+
return FetchResponse(**data)
|
|
35
|
+
except (OSError, ValueError, TypeError):
|
|
36
|
+
return None
|
|
37
|
+
|
|
38
|
+
def put(self, url: str, response: FetchResponse) -> None:
|
|
39
|
+
if response.status is None or response.status >= 400 or response.error:
|
|
40
|
+
return
|
|
41
|
+
body = response.text or ""
|
|
42
|
+
# Free-form bodies can contain credentials in attributes, JSON, or text.
|
|
43
|
+
# If a body even names a sensitive concept, do not persist it.
|
|
44
|
+
if _SENSITIVE_CONTEXT.search(body) or _OPAQUE_VALUE.search(body):
|
|
45
|
+
return
|
|
46
|
+
from .collectors.html_js import detect_secret_leaks
|
|
47
|
+
if detect_secret_leaks(body, response.final_url):
|
|
48
|
+
return
|
|
49
|
+
path = self._path(url)
|
|
50
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
51
|
+
record = {
|
|
52
|
+
"url": safe_url(response.url), "final_url": safe_url(response.final_url),
|
|
53
|
+
"status": response.status, "headers": safe_headers(response.headers),
|
|
54
|
+
"text": safe_text(body), "error": None,
|
|
55
|
+
"status_chain": response.status_chain, "tls": response.tls,
|
|
56
|
+
}
|
|
57
|
+
tmp = path.with_suffix(".tmp")
|
|
58
|
+
tmp.write_text(json.dumps(record), encoding="utf-8")
|
|
59
|
+
tmp.chmod(0o600)
|
|
60
|
+
tmp.replace(path)
|