timecs 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app.py +3014 -0
- ast_scanner.py +36187 -0
- banner.py +146 -0
- ci_reporter.py +570 -0
- cli.py +2347 -0
- config_loader.py +307 -0
- cross_file_engine.py +1781 -0
- delivery_models.py +145 -0
- delivery_repository.py +78 -0
- diff_scan.py +127 -0
- durable_delivery_sink.py +103 -0
- event_bus.py +155 -0
- event_publisher.py +345 -0
- event_taxonomy.py +313 -0
- function_summarizer.py +354 -0
- html_auditor.py +274 -0
- iac_auditor.py +400 -0
- in_app_adapter.py +115 -0
- js_scanner.py +895 -0
- manifest_parser.py +960 -0
- master_rules_bank.py +389 -0
- notification_models.py +346 -0
- notification_policy.py +309 -0
- notification_repository.py +118 -0
- notification_routes.py +394 -0
- notification_service.py +199 -0
- osv_client.py +437 -0
- remediation/__init__.py +39 -0
- remediation/base.py +102 -0
- remediation/contracts.py +196 -0
- remediation/cwe1188_binding.py +78 -0
- remediation/cwe1336_jinja.py +91 -0
- remediation/cwe22_path_traversal.py +348 -0
- remediation/cwe295_ssl.py +87 -0
- remediation/cwe327_weak_hash.py +78 -0
- remediation/cwe377_tempfile.py +77 -0
- remediation/cwe489_debug.py +80 -0
- remediation/cwe502_yaml.py +89 -0
- remediation/cwe614_cookie.py +91 -0
- remediation/cwe78_cmdi.py +415 -0
- remediation/cwe798_credentials.py +153 -0
- remediation/cwe89_sqli.py +425 -0
- remediation/cwe916_password_hash.py +131 -0
- remediation/cwe95_eval.py +77 -0
- remediation/dispatcher.py +114 -0
- remediation/orchestrator.py +541 -0
- remediation/patch_engine.py +213 -0
- remediation/verification.py +253 -0
- rule_engine.py +1535 -0
- sarif_adapter.py +719 -0
- sarif_exporter.py +88 -0
- sca_reachability/__init__.py +28 -0
- sca_reachability/ast_analyzer.py +366 -0
- sca_reachability/call_graph.py +203 -0
- sca_reachability/contracts.py +365 -0
- sca_reachability/dist_import_map.py +46 -0
- sca_reachability/engine.py +603 -0
- sca_reachability/resolver.py +169 -0
- scanners/__init__.py +70 -0
- scanners/container_scanner.py +323 -0
- scanners/cve_index.py +521 -0
- scanners/iac_scanner.py +328 -0
- scanners/sbom.py +183 -0
- scanners/sca_scanner.py +667 -0
- scanners/secret_scanner.py +345 -0
- scanners/yaml_lite.py +217 -0
- secret_filters.py +435 -0
- secret_scanner.py +943 -0
- sqlite_delivery_repository.py +277 -0
- sqlite_notification_repository.py +441 -0
- ssrf_guard.py +276 -0
- staged_scanner.py +492 -0
- suppression_resolver.py +282 -0
- symbol_indexer.py +424 -0
- tcs/__init__.py +1 -0
- tcs/analysis/__init__.py +1 -0
- tcs/analysis/def_use.py +408 -0
- tcs/analysis/function_contracts.py +290 -0
- tcs/parallel_analyzer.py +276 -0
- tcs/parallel_scanner.py +630 -0
- tcs/treesitter_packer.py +1446 -0
- tcs_cli.py +2086 -0
- tcs_config.py +218 -0
- tcs_gui.py +2123 -0
- timecs-2.0.0.dist-info/METADATA +142 -0
- timecs-2.0.0.dist-info/RECORD +94 -0
- timecs-2.0.0.dist-info/WHEEL +5 -0
- timecs-2.0.0.dist-info/entry_points.txt +4 -0
- timecs-2.0.0.dist-info/licenses/LICENSE +201 -0
- timecs-2.0.0.dist-info/top_level.txt +49 -0
- version_matcher.py +934 -0
- webhook_adapter.py +495 -0
- webhook_client.py +483 -0
- webhook_formatters.py +245 -0
secret_scanner.py
ADDED
|
@@ -0,0 +1,943 @@
|
|
|
1
|
+
"""Deterministic, offline secret and credential detection engine.
|
|
2
|
+
|
|
3
|
+
Detects high-confidence hardcoded credentials, tokens, private keys, and
|
|
4
|
+
database connection strings in source text and files.
|
|
5
|
+
|
|
6
|
+
Non-negotiable invariants:
|
|
7
|
+
- Zero raw secret leakage in SecretFinding, __dict__, repr, str, or context.
|
|
8
|
+
- Zero network I/O.
|
|
9
|
+
- Strict isolation: zero dependencies on SAST/SCA engine modules.
|
|
10
|
+
- Deterministic output sorted by (line_number, column_start, detector, secret_type).
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import ast
|
|
14
|
+
import os
|
|
15
|
+
from collections import Counter
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
import math
|
|
18
|
+
import re
|
|
19
|
+
import uuid
|
|
20
|
+
from typing import Any, Dict, List, Optional, Set, Tuple
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"SecretFinding",
|
|
24
|
+
"SecretScanner",
|
|
25
|
+
"SecretVault",
|
|
26
|
+
"VaultRestorationError",
|
|
27
|
+
"SECRET_PATTERNS",
|
|
28
|
+
"mask_secret",
|
|
29
|
+
"calculate_shannon_entropy",
|
|
30
|
+
"shannon_entropy",
|
|
31
|
+
"is_adaptive_high_entropy_secret",
|
|
32
|
+
"scan_text",
|
|
33
|
+
"scan_file",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class ConfidenceStr(str):
|
|
38
|
+
"""String that matches 'HIGH (PATTERN_MATCH)' and legacy confidence aliases."""
|
|
39
|
+
|
|
40
|
+
def __eq__(self, other):
|
|
41
|
+
s = str(self)
|
|
42
|
+
o = str(other)
|
|
43
|
+
if s == o:
|
|
44
|
+
return True
|
|
45
|
+
# Accept legacy aliases so older tests comparing against "HIGH" still pass
|
|
46
|
+
if o in ("HIGH", "HIGH (REGEX/ENTROPY)", "HIGH (PATTERN_MATCH)",
|
|
47
|
+
"CONFIRMED", "CONFIRMED (REGEX/ENTROPY)"):
|
|
48
|
+
return True
|
|
49
|
+
return False
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True)
|
|
53
|
+
class SecretFinding:
|
|
54
|
+
"""Immutable finding representing a detected secret.
|
|
55
|
+
|
|
56
|
+
Raw unmasked secrets are NEVER stored in this structure.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
secret_type: str
|
|
60
|
+
masked_value: str
|
|
61
|
+
file: str
|
|
62
|
+
line_number: int # 1-based
|
|
63
|
+
column_start: int # 1-based
|
|
64
|
+
column_end: int # 1-based
|
|
65
|
+
confidence: str # 'HIGH'
|
|
66
|
+
detector: str
|
|
67
|
+
context: Optional[str] = None
|
|
68
|
+
proof_type: str = "PATTERN_MATCH"
|
|
69
|
+
pattern: Optional[str] = None
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def mask_secret(value: str) -> str:
|
|
73
|
+
"""Deterministically mask a secret string.
|
|
74
|
+
|
|
75
|
+
Rules:
|
|
76
|
+
- Empty string: ''
|
|
77
|
+
- <= 4 chars: '*' * len(value)
|
|
78
|
+
- 5..8 chars: value[0] + '*' * (len(value) - 2) + value[-1]
|
|
79
|
+
- > 8 chars: value[:4] + '*' * (len(value) - 8) + value[-4:]
|
|
80
|
+
"""
|
|
81
|
+
length = len(value)
|
|
82
|
+
if length == 0:
|
|
83
|
+
return ""
|
|
84
|
+
if length <= 4:
|
|
85
|
+
return "*" * length
|
|
86
|
+
if length <= 8:
|
|
87
|
+
return value[0] + ("*" * (length - 2)) + value[-1]
|
|
88
|
+
return value[:4] + ("*" * (length - 8)) + value[-4:]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def calculate_shannon_entropy(data: str) -> float:
|
|
92
|
+
"""Calculate the Shannon entropy H(X) in bits per character.
|
|
93
|
+
|
|
94
|
+
Standard base-2 Shannon entropy calculation.
|
|
95
|
+
Returns 0.0 for empty or single-character strings.
|
|
96
|
+
"""
|
|
97
|
+
if not data or len(data) <= 1:
|
|
98
|
+
return 0.0
|
|
99
|
+
length = len(data)
|
|
100
|
+
counts = Counter(data)
|
|
101
|
+
entropy = 0.0
|
|
102
|
+
for count in counts.values():
|
|
103
|
+
p = count / length
|
|
104
|
+
entropy -= p * math.log2(p)
|
|
105
|
+
return entropy
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def shannon_entropy(data: str) -> float:
|
|
109
|
+
"""Calculate the Shannon entropy H(X) in bits per character (legacy alias)."""
|
|
110
|
+
return calculate_shannon_entropy(data)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
# ---------------------------------------------------------------------------
|
|
114
|
+
# Multi-Provider Regex Registry & Entropy Vault (TruffleHog / GitGuardian standard)
|
|
115
|
+
# ---------------------------------------------------------------------------
|
|
116
|
+
|
|
117
|
+
class _SecretPatternsDict(dict):
|
|
118
|
+
"""Dictionary supporting flexible case-insensitive and delimiter-free lookup."""
|
|
119
|
+
|
|
120
|
+
def __getitem__(self, key):
|
|
121
|
+
if key in self:
|
|
122
|
+
return super().__getitem__(key)
|
|
123
|
+
norm = re.sub(r"[\s_\-]", "", str(key).lower())
|
|
124
|
+
for k, v in self.items():
|
|
125
|
+
if re.sub(r"[\s_\-]", "", str(k).lower()) == norm:
|
|
126
|
+
return v
|
|
127
|
+
raise KeyError(key)
|
|
128
|
+
|
|
129
|
+
def get(self, key, default=None):
|
|
130
|
+
try:
|
|
131
|
+
return self[key]
|
|
132
|
+
except KeyError:
|
|
133
|
+
return default
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
SECRET_PATTERNS = _SecretPatternsDict({
|
|
137
|
+
"AWS Access Key": re.compile(r"\b(?:AKIA|ASIA|ABIA|ACCA)[0-9A-Z]{14,16}\b"),
|
|
138
|
+
"AWS Secret Key": re.compile(
|
|
139
|
+
r"(?i)(?:aws_secret_access_key|aws_secret|secret_key|secret_access_key)\s*[:=]\s*['\"]?([A-Za-z0-9/+=]{40})['\"]?"
|
|
140
|
+
),
|
|
141
|
+
"GitHub Token": re.compile(
|
|
142
|
+
r"\b(?:(?:ghp|gho|ghu|ghs|ghr)_[a-zA-Z0-9]{36}|github_pat_[a-zA-Z0-9_]{82,})\b"
|
|
143
|
+
),
|
|
144
|
+
"Stripe API Key": re.compile(r"\b(?:sk|pk|rk)_(?:live|test)_[0-9a-zA-Z]{24,}\b"),
|
|
145
|
+
"Slack API Token": re.compile(r"\bxox[baprs]-[0-9a-zA-Z]{10,48}(?:-[0-9a-zA-Z]{10,48})*\b"),
|
|
146
|
+
"Google API Key": re.compile(r"\bAIza[0-9A-Za-z\-_]{35}\b"),
|
|
147
|
+
"Asymmetric Private Key": re.compile(
|
|
148
|
+
r"-----BEGIN (?:(?:RSA|EC|DSA|OPENSSH) )?PRIVATE KEY-----[\s\S]*?-----END (?:(?:RSA|EC|DSA|OPENSSH) )?PRIVATE KEY-----|-----BEGIN (?:RSA|EC|DSA|OPENSSH|PRIVATE)(?: PRIVATE)? KEY-----"
|
|
149
|
+
),
|
|
150
|
+
})
|
|
151
|
+
|
|
152
|
+
HIGH_ENTROPY_VAR_KEYWORDS = ("secret", "token", "key", "password", "auth", "cred")
|
|
153
|
+
HIGH_ENTROPY_ASSIGN_REGEX = re.compile(
|
|
154
|
+
r"""(?i)\b([a-z0-9_]*(?:secret|token|key|password|auth|cred)[a-z0-9_]*)\s*[:=]\s*['"]([^'"]{16,})['"]"""
|
|
155
|
+
)
|
|
156
|
+
_HEX_REGEX = re.compile(r"^[0-9a-fA-F]+$")
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def is_adaptive_high_entropy_secret(val_str: str) -> bool:
|
|
160
|
+
"""Character-set adaptive entropy validation (TruffleHog / GitGuardian standard).
|
|
161
|
+
|
|
162
|
+
- Pure hexadecimal (^[0-9a-fA-F]+$): threshold >= 3.0 with length >= 24.
|
|
163
|
+
(Theoretical maximum for base-16 is log2(16) = 4.0 bits/char).
|
|
164
|
+
- Mixed alphanumeric/special characters: threshold >= 4.3 with length >= 16.
|
|
165
|
+
"""
|
|
166
|
+
if not val_str:
|
|
167
|
+
return False
|
|
168
|
+
length = len(val_str)
|
|
169
|
+
if _HEX_REGEX.match(val_str):
|
|
170
|
+
return length >= 24 and calculate_shannon_entropy(val_str) >= 3.0
|
|
171
|
+
return length >= 16 and calculate_shannon_entropy(val_str) >= 4.3
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
class VaultRestorationError(ValueError, RuntimeError):
|
|
175
|
+
"""Raised when one or more vault tokens remain unrestored."""
|
|
176
|
+
pass
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def find_high_entropy_secrets_ast(code: str) -> List[str]:
|
|
180
|
+
"""Find variable assignments matching credential keywords with adaptive entropy using AST."""
|
|
181
|
+
secrets_found: List[str] = []
|
|
182
|
+
try:
|
|
183
|
+
tree = ast.parse(code)
|
|
184
|
+
for node in ast.walk(tree):
|
|
185
|
+
target_names: List[str] = []
|
|
186
|
+
if isinstance(node, ast.Assign):
|
|
187
|
+
for t in node.targets:
|
|
188
|
+
if isinstance(t, ast.Name):
|
|
189
|
+
target_names.append(t.id)
|
|
190
|
+
elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
|
|
191
|
+
target_names.append(node.target.id)
|
|
192
|
+
|
|
193
|
+
if not target_names:
|
|
194
|
+
continue
|
|
195
|
+
|
|
196
|
+
val_str = None
|
|
197
|
+
if isinstance(node.value, ast.Constant) and isinstance(node.value.value, str):
|
|
198
|
+
val_str = node.value.value
|
|
199
|
+
elif isinstance(node.value, ast.Str):
|
|
200
|
+
val_str = node.value.s
|
|
201
|
+
|
|
202
|
+
if val_str and len(val_str) >= 16:
|
|
203
|
+
for var_name in target_names:
|
|
204
|
+
var_lower = var_name.lower()
|
|
205
|
+
if any(kw in var_lower for kw in HIGH_ENTROPY_VAR_KEYWORDS):
|
|
206
|
+
if is_adaptive_high_entropy_secret(val_str):
|
|
207
|
+
secrets_found.append(val_str)
|
|
208
|
+
break
|
|
209
|
+
except Exception:
|
|
210
|
+
pass
|
|
211
|
+
return secrets_found
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def find_high_entropy_secrets_regex(code: str) -> List[str]:
|
|
215
|
+
"""Fallback regex scanner for high-entropy credential assignments with adaptive entropy."""
|
|
216
|
+
secrets_found: List[str] = []
|
|
217
|
+
for m in HIGH_ENTROPY_ASSIGN_REGEX.finditer(code):
|
|
218
|
+
candidate = m.group(2)
|
|
219
|
+
if is_adaptive_high_entropy_secret(candidate):
|
|
220
|
+
secrets_found.append(candidate)
|
|
221
|
+
return secrets_found
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
class SecretVault:
|
|
225
|
+
"""Enterprise-grade reversible secret redaction vault (TruffleHog / GitGuardian standard).
|
|
226
|
+
|
|
227
|
+
Reversibly replaces detected secrets with deterministic sequential tokens:
|
|
228
|
+
'__TCS_VAULT_TOKEN_<idx>__' based on the order of appearance in the file.
|
|
229
|
+
Restores original secrets losslessly while verifying token restoration integrity.
|
|
230
|
+
"""
|
|
231
|
+
|
|
232
|
+
def __init__(self):
|
|
233
|
+
self.vault_map: Dict[str, str] = {}
|
|
234
|
+
|
|
235
|
+
def redact_code(self, code: str) -> str:
|
|
236
|
+
redacted, vmap = self.redact(code)
|
|
237
|
+
self.vault_map.update(vmap)
|
|
238
|
+
return redacted
|
|
239
|
+
|
|
240
|
+
def restore_code(self, redacted_code: str, fallback_on_error: bool = False) -> str:
|
|
241
|
+
return self.restore(redacted_code, self.vault_map, fallback_on_error=fallback_on_error)
|
|
242
|
+
|
|
243
|
+
@classmethod
|
|
244
|
+
def find_all_secrets(cls, code: str) -> List[str]:
|
|
245
|
+
"""Detect all secrets in code matching provider patterns and adaptive entropy heuristic."""
|
|
246
|
+
if not code:
|
|
247
|
+
return []
|
|
248
|
+
detected = set()
|
|
249
|
+
|
|
250
|
+
for pat_name, pat in SECRET_PATTERNS.items():
|
|
251
|
+
for m in pat.finditer(code):
|
|
252
|
+
if pat_name == "AWS Secret Key":
|
|
253
|
+
try:
|
|
254
|
+
secret_val = m.group(1) if m.group(1) else m.group(0)
|
|
255
|
+
except (IndexError, AttributeError):
|
|
256
|
+
secret_val = m.group(0)
|
|
257
|
+
else:
|
|
258
|
+
secret_val = m.group(0)
|
|
259
|
+
if secret_val and not secret_val.startswith("__TCS_VAULT_TOKEN_"):
|
|
260
|
+
detected.add(secret_val)
|
|
261
|
+
|
|
262
|
+
for s in find_high_entropy_secrets_ast(code):
|
|
263
|
+
if s and not s.startswith("__TCS_VAULT_TOKEN_"):
|
|
264
|
+
detected.add(s)
|
|
265
|
+
|
|
266
|
+
for s in find_high_entropy_secrets_regex(code):
|
|
267
|
+
if s and not s.startswith("__TCS_VAULT_TOKEN_"):
|
|
268
|
+
detected.add(s)
|
|
269
|
+
|
|
270
|
+
# Sort longer secrets first to avoid prefix collisions during substitution
|
|
271
|
+
return sorted(detected, key=len, reverse=True)
|
|
272
|
+
|
|
273
|
+
@classmethod
|
|
274
|
+
def redact(cls, code: str) -> Tuple[str, Dict[str, str]]:
|
|
275
|
+
"""Deterministically redact all detected secrets into reversible vault tokens.
|
|
276
|
+
|
|
277
|
+
Sequential tokens f"__TCS_VAULT_TOKEN_{idx}__" are assigned based on the
|
|
278
|
+
order of appearance in the file, ensuring 100% deterministic code hashes.
|
|
279
|
+
Returns (redacted_code, vault_map) where vault_map maps tokens to original secrets.
|
|
280
|
+
"""
|
|
281
|
+
if not code:
|
|
282
|
+
return "", {}
|
|
283
|
+
|
|
284
|
+
secrets = cls.find_all_secrets(code)
|
|
285
|
+
if not secrets:
|
|
286
|
+
return code, {}
|
|
287
|
+
|
|
288
|
+
# Establish deterministic order based on first appearance in code
|
|
289
|
+
def appearance_key(s: str) -> Tuple[int, int, str]:
|
|
290
|
+
pos = code.find(s)
|
|
291
|
+
return (pos if pos != -1 else 999999999, -len(s), s)
|
|
292
|
+
|
|
293
|
+
sorted_by_appearance = sorted(secrets, key=appearance_key)
|
|
294
|
+
|
|
295
|
+
secret_to_token: Dict[str, str] = {}
|
|
296
|
+
for idx, secret in enumerate(sorted_by_appearance, start=1):
|
|
297
|
+
if secret not in secret_to_token:
|
|
298
|
+
secret_to_token[secret] = f"__TCS_VAULT_TOKEN_{idx}__"
|
|
299
|
+
|
|
300
|
+
vault_map: Dict[str, str] = {token: secret for secret, token in secret_to_token.items()}
|
|
301
|
+
|
|
302
|
+
# Replace secrets in descending order of length to prevent substring corruption
|
|
303
|
+
redacted_code = code
|
|
304
|
+
for secret in sorted(secret_to_token.keys(), key=len, reverse=True):
|
|
305
|
+
token = secret_to_token[secret]
|
|
306
|
+
redacted_code = redacted_code.replace(secret, token)
|
|
307
|
+
|
|
308
|
+
return redacted_code, vault_map
|
|
309
|
+
|
|
310
|
+
@classmethod
|
|
311
|
+
def restore(cls, redacted_code: str, vault_map: Dict[str, str], fallback_on_error: bool = False) -> str:
|
|
312
|
+
"""Losslessly restore all vaulted secrets in redacted_code using vault_map.
|
|
313
|
+
|
|
314
|
+
Verifies that no token from vault_map remains in the restored code.
|
|
315
|
+
If any token remains unreplaced:
|
|
316
|
+
- If fallback_on_error is True: returns best-effort restoration.
|
|
317
|
+
- Otherwise: raises VaultRestorationError.
|
|
318
|
+
"""
|
|
319
|
+
if not redacted_code:
|
|
320
|
+
return ""
|
|
321
|
+
|
|
322
|
+
restored_code = redacted_code
|
|
323
|
+
if vault_map:
|
|
324
|
+
# Sort tokens by length descending
|
|
325
|
+
for token, original_secret in sorted(vault_map.items(), key=lambda x: len(x[0]), reverse=True):
|
|
326
|
+
restored_code = restored_code.replace(token, original_secret)
|
|
327
|
+
|
|
328
|
+
# Integrity verification: verify that NO token from vault_map remains in restored code
|
|
329
|
+
unrestored_tokens = [tok for tok in (vault_map or {}) if tok in restored_code]
|
|
330
|
+
residual_tokens = re.findall(r"__TCS_VAULT_TOKEN_[a-zA-Z0-9_]+__", restored_code)
|
|
331
|
+
remaining = list(dict.fromkeys(unrestored_tokens + residual_tokens))
|
|
332
|
+
if remaining:
|
|
333
|
+
msg = (
|
|
334
|
+
f"Vault restoration integrity failure: {len(remaining)} unreplaced token(s) "
|
|
335
|
+
f"remain in code: {remaining}"
|
|
336
|
+
)
|
|
337
|
+
if fallback_on_error:
|
|
338
|
+
return restored_code
|
|
339
|
+
raise VaultRestorationError(msg)
|
|
340
|
+
|
|
341
|
+
return restored_code
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
# ---------------------------------------------------------------------------
|
|
346
|
+
# Detector Signatures & Precedence Configuration
|
|
347
|
+
# ---------------------------------------------------------------------------
|
|
348
|
+
# Precedence: lower number = higher priority
|
|
349
|
+
# 1. private_key
|
|
350
|
+
# 2. github_token
|
|
351
|
+
# 3. slack_token
|
|
352
|
+
# 4. aws_access_key
|
|
353
|
+
# 5. database_connection_string
|
|
354
|
+
|
|
355
|
+
# 1. Private Key (anchored to header line, zero multi-line body buffering)
|
|
356
|
+
_REGEX_PRIVATE_KEY = re.compile(
|
|
357
|
+
r"-----BEGIN (?:RSA |EC |OPENSSH |DSA |PGP |ENCRYPTED )?PRIVATE KEY(?: BLOCK)?-----"
|
|
358
|
+
)
|
|
359
|
+
|
|
360
|
+
# 2. GitHub Token (ghp, gho, ghu, ghs, ghr followed by 36 alphanumeric chars)
|
|
361
|
+
_REGEX_GITHUB_TOKEN = re.compile(r"\b(ghp|gho|ghu|ghs|ghr)_[0-9A-Za-z]{36}\b")
|
|
362
|
+
|
|
363
|
+
# 3. Slack Token (xoxb, xoxp, xoxa, xoxr, xoxs followed by hyphen & 10..48 chars)
|
|
364
|
+
_REGEX_SLACK_TOKEN = re.compile(r"\bxox[baprs]-[0-9A-Za-z]{10,48}\b")
|
|
365
|
+
|
|
366
|
+
# 4. Stripe API Key (secret keys sk_live_/sk_test_ and restricted keys rk_live_)
|
|
367
|
+
_REGEX_STRIPE_KEY = re.compile(r"\b(sk_live|sk_test|rk_live)_[0-9A-Za-z]{24,99}\b")
|
|
368
|
+
|
|
369
|
+
# 4b. Slack Incoming Webhook URL
|
|
370
|
+
_REGEX_SLACK_WEBHOOK = re.compile(r"https://hooks\.slack\.com/services/[A-Za-z0-9/_+-]+")
|
|
371
|
+
|
|
372
|
+
# 5. AWS Access Key (AKIA, ASIA, ABIA, ACCA followed by 14..16 chars)
|
|
373
|
+
_REGEX_AWS_ACCESS_KEY = re.compile(r"\b(AKIA|ASIA|ABIA|ACCA)[0-9A-Z]{14,16}\b")
|
|
374
|
+
|
|
375
|
+
# 6. Database Connection String (matches scheme and URI candidate tokens, parsed robustly)
|
|
376
|
+
_REGEX_DB_URI = re.compile(
|
|
377
|
+
r"\b(?P<scheme>postgres(?:ql)?|mysql|mongodb|redis)://[^\s\"'`]+"
|
|
378
|
+
)
|
|
379
|
+
|
|
380
|
+
# 7. JWT Bearer Token (standard 3-segment base64url encoded tokens)
|
|
381
|
+
_REGEX_JWT_TOKEN = re.compile(
|
|
382
|
+
r"\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b"
|
|
383
|
+
)
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def _parse_and_mask_db_uri(raw_str: str) -> Optional[Tuple[str, int]]:
|
|
387
|
+
"""Deterministically parse and mask credentials within a database connection URI.
|
|
388
|
+
|
|
389
|
+
Handles passwords containing special characters (e.g. '@', ':', '!', etc.) by splitting
|
|
390
|
+
from the rightmost '@' boundary.
|
|
391
|
+
Replaces password strictly with '************' (12 asterisks).
|
|
392
|
+
Returns (masked_uri, matched_length) or None if not a credentialed DB URI.
|
|
393
|
+
"""
|
|
394
|
+
# Trim trailing delimiters that are code/syntax rather than part of the URI
|
|
395
|
+
trimmed = raw_str
|
|
396
|
+
while trimmed and (trimmed[-1] in "',;)>]}" or (trimmed.endswith(".") and not trimmed.endswith(".."))):
|
|
397
|
+
trimmed = trimmed[:-1]
|
|
398
|
+
|
|
399
|
+
if "://" not in trimmed:
|
|
400
|
+
return None
|
|
401
|
+
|
|
402
|
+
scheme, rest = trimmed.split("://", 1)
|
|
403
|
+
if "@" not in rest:
|
|
404
|
+
return None
|
|
405
|
+
|
|
406
|
+
userinfo, host_and_path = rest.rsplit("@", 1)
|
|
407
|
+
if not host_and_path or not host_and_path.strip("/: "):
|
|
408
|
+
return None
|
|
409
|
+
|
|
410
|
+
if ":" not in userinfo:
|
|
411
|
+
return None
|
|
412
|
+
|
|
413
|
+
user, password = userinfo.split(":", 1)
|
|
414
|
+
if not password or not password.strip(":"):
|
|
415
|
+
return None
|
|
416
|
+
|
|
417
|
+
masked_pw = "************"
|
|
418
|
+
if user:
|
|
419
|
+
masked_user = mask_secret(user)
|
|
420
|
+
creds = f"{masked_user}:{masked_pw}"
|
|
421
|
+
else:
|
|
422
|
+
creds = f":{masked_pw}"
|
|
423
|
+
|
|
424
|
+
masked_uri = f"{scheme}://{creds}@{host_and_path}"
|
|
425
|
+
return masked_uri, len(trimmed)
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
def _mask_db_connection_uri(match: re.Match) -> str:
|
|
429
|
+
"""Mask credentials within a database connection URI match."""
|
|
430
|
+
parsed = _parse_and_mask_db_uri(match.group(0))
|
|
431
|
+
if parsed:
|
|
432
|
+
return parsed[0]
|
|
433
|
+
return match.group(0)
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
# Gitleaks precedence sits below every native detector (1-7) so existing
|
|
437
|
+
# secret_type labels and overlap resolution are preserved; bank patterns only
|
|
438
|
+
# contribute findings for spans no native detector already claimed.
|
|
439
|
+
_GITLEAKS_PRECEDENCE = 8
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def _load_master_rules_bank():
|
|
443
|
+
"""Load master_rules_bank.py from the repository root (pure-data module).
|
|
444
|
+
|
|
445
|
+
Returns None on any failure so production imports never break if the bank
|
|
446
|
+
file is absent; the Gitleaks merge below is then simply a no-op. Kept local
|
|
447
|
+
to the secret subsystem to honor the SAST/SCA isolation invariant.
|
|
448
|
+
"""
|
|
449
|
+
import importlib.util
|
|
450
|
+
import os
|
|
451
|
+
|
|
452
|
+
path = os.path.join(
|
|
453
|
+
os.path.dirname(os.path.abspath(__file__)), "master_rules_bank.py"
|
|
454
|
+
)
|
|
455
|
+
try:
|
|
456
|
+
spec = importlib.util.spec_from_file_location("_tcs_master_rules_bank_secrets", path)
|
|
457
|
+
if spec is None or spec.loader is None:
|
|
458
|
+
return None
|
|
459
|
+
mod = importlib.util.module_from_spec(spec)
|
|
460
|
+
spec.loader.exec_module(mod)
|
|
461
|
+
return mod
|
|
462
|
+
except Exception:
|
|
463
|
+
return None
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
# Global inline-flag tokens like (?i) that are NOT at the start of the pattern are
|
|
467
|
+
# rejected by Python's re (Gitleaks uses Go's regexp which allows them anywhere).
|
|
468
|
+
_MID_GLOBAL_FLAG = re.compile(r"\(\?([aiLmsux]+)\)")
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def _compile_gitleaks_pattern(pattern: str) -> Optional[re.Pattern]:
|
|
472
|
+
"""Compile a Gitleaks regex, repairing Go-style mid-pattern global flags and nested sets.
|
|
473
|
+
|
|
474
|
+
Falls back to None if the pattern cannot be compiled even after repair, so a
|
|
475
|
+
single bad signature can never break the whole detector set.
|
|
476
|
+
"""
|
|
477
|
+
# Normalize Go/POSIX character classes & unescaped nested bracket sets
|
|
478
|
+
# (e.g. [[:alnum:]] -> [a-zA-Z0-9]) to prevent Python 3.11+ FutureWarning: Possible nested set
|
|
479
|
+
if "[:alnum:]" in pattern:
|
|
480
|
+
pattern = pattern.replace("[[:alnum:]]", "[a-zA-Z0-9]").replace("[:alnum:]", "a-zA-Z0-9")
|
|
481
|
+
if "[:alpha:]" in pattern:
|
|
482
|
+
pattern = pattern.replace("[[:alpha:]]", "[a-zA-Z]").replace("[:alpha:]", "a-zA-Z")
|
|
483
|
+
if "[:digit:]" in pattern:
|
|
484
|
+
pattern = pattern.replace("[[:digit:]]", "[0-9]").replace("[:digit:]", "0-9")
|
|
485
|
+
# Escape any remaining unescaped inner '[' within character classes (e.g. '[...[...')
|
|
486
|
+
pattern = re.sub(r"(\[[^\]]*?)\[", r"\1\\[", pattern)
|
|
487
|
+
|
|
488
|
+
try:
|
|
489
|
+
return re.compile(pattern)
|
|
490
|
+
except re.error:
|
|
491
|
+
pass
|
|
492
|
+
flags = 0
|
|
493
|
+
for m in _MID_GLOBAL_FLAG.finditer(pattern):
|
|
494
|
+
if "i" in m.group(1):
|
|
495
|
+
flags |= re.IGNORECASE
|
|
496
|
+
if "s" in m.group(1):
|
|
497
|
+
flags |= re.DOTALL
|
|
498
|
+
if "m" in m.group(1):
|
|
499
|
+
flags |= re.MULTILINE
|
|
500
|
+
stripped = _MID_GLOBAL_FLAG.sub("", pattern)
|
|
501
|
+
try:
|
|
502
|
+
return re.compile(stripped, flags)
|
|
503
|
+
except re.error:
|
|
504
|
+
return None
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def _build_gitleaks_detectors() -> List[Tuple[str, re.Pattern]]:
|
|
508
|
+
bank = _load_master_rules_bank()
|
|
509
|
+
if bank is None:
|
|
510
|
+
return []
|
|
511
|
+
raw = getattr(bank, "GITLEAKS_SECRETS_BANK", {}) or {}
|
|
512
|
+
detectors: List[Tuple[str, re.Pattern]] = []
|
|
513
|
+
for secret_type, pattern in raw.items():
|
|
514
|
+
compiled = _compile_gitleaks_pattern(pattern)
|
|
515
|
+
if compiled is not None:
|
|
516
|
+
detectors.append((secret_type, compiled))
|
|
517
|
+
return detectors
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
_GITLEAKS_DETECTORS: List[Tuple[str, re.Pattern]] = _build_gitleaks_detectors()
|
|
521
|
+
|
|
522
|
+
# HCL-shaped bank rules are a pure spelling test when they read Python: `password = "x"` is assignment
|
|
523
|
+
# syntax in both languages, and the rule's value group only needs double quotes plus 8..20 characters,
|
|
524
|
+
# so `PASSWORD = "password"` is flagged for its quotes and not for any meaning. Python already owns two
|
|
525
|
+
# AST detectors for the same line - `ast_scanner` CWE-798 and the `hardcoded_credential_variable` pass
|
|
526
|
+
# below - so removing the HCL rule from Python drops no coverage.
|
|
527
|
+
#
|
|
528
|
+
# The scope is a Python deny-list, deliberately not a ".tf/.tfvars only" allow-list, because that was
|
|
529
|
+
# measured to be a false negative: on values.yaml, config.json and main.go `hashicorp-tf-password` is
|
|
530
|
+
# the *only* detector that fires (the AST pass at the foot of scan_text is skipped for the config
|
|
531
|
+
# extensions and cannot parse Go), so an allow-list would delete those findings outright.
|
|
532
|
+
_PYTHON_SOURCE_SUFFIXES = (".py", ".pyi", ".pyw", ".pyz")
|
|
533
|
+
_HCL_SCOPED_GITLEAKS_RULES = frozenset({"hashicorp-tf-password"})
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def _gitleaks_detectors_for(filename: str) -> List[Tuple[str, re.Pattern]]:
|
|
537
|
+
"""The bank rules that read this file. Called once per scan, never per line."""
|
|
538
|
+
if not _HCL_SCOPED_GITLEAKS_RULES:
|
|
539
|
+
return _GITLEAKS_DETECTORS
|
|
540
|
+
lowered = str(filename or "").lower()
|
|
541
|
+
is_python = lowered.endswith(_PYTHON_SOURCE_SUFFIXES)
|
|
542
|
+
if not is_python:
|
|
543
|
+
return _GITLEAKS_DETECTORS
|
|
544
|
+
return [row for row in _GITLEAKS_DETECTORS if row[0] not in _HCL_SCOPED_GITLEAKS_RULES]
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
class _CandidateMatch:
|
|
548
|
+
"""Internal candidate match before overlap resolution and secret erasure."""
|
|
549
|
+
|
|
550
|
+
__slots__ = (
|
|
551
|
+
"precedence",
|
|
552
|
+
"start",
|
|
553
|
+
"end",
|
|
554
|
+
"secret_type",
|
|
555
|
+
"detector",
|
|
556
|
+
"masked_value",
|
|
557
|
+
)
|
|
558
|
+
|
|
559
|
+
def __init__(
|
|
560
|
+
self,
|
|
561
|
+
precedence: int,
|
|
562
|
+
start: int,
|
|
563
|
+
end: int,
|
|
564
|
+
secret_type: str,
|
|
565
|
+
detector: str,
|
|
566
|
+
masked_value: str,
|
|
567
|
+
):
|
|
568
|
+
self.precedence = precedence
|
|
569
|
+
self.start = start
|
|
570
|
+
self.end = end
|
|
571
|
+
self.secret_type = secret_type
|
|
572
|
+
self.detector = detector
|
|
573
|
+
self.masked_value = masked_value
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def _docstring_line_numbers(text: str) -> Set[int]:
|
|
577
|
+
"""Line numbers covered by docstrings (module/class/function level).
|
|
578
|
+
|
|
579
|
+
Used only as a documentation context oracle for the private-key detector, so a
|
|
580
|
+
PEM header named inside prose is not mistaken for key material. Returns an empty
|
|
581
|
+
set on any parse failure; callers gate this on a substring pre-check because
|
|
582
|
+
parsing every file would cost time and memory this scanner deliberately avoids.
|
|
583
|
+
"""
|
|
584
|
+
lines: Set[int] = set()
|
|
585
|
+
try:
|
|
586
|
+
tree = ast.parse(text)
|
|
587
|
+
except Exception:
|
|
588
|
+
return lines
|
|
589
|
+
for node in ast.walk(tree):
|
|
590
|
+
if not isinstance(node, (ast.Module, ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
591
|
+
continue
|
|
592
|
+
body = getattr(node, "body", None)
|
|
593
|
+
if not body:
|
|
594
|
+
continue
|
|
595
|
+
first = body[0]
|
|
596
|
+
if not isinstance(first, ast.Expr):
|
|
597
|
+
continue
|
|
598
|
+
value = first.value
|
|
599
|
+
literal = value.s if isinstance(value, ast.Str) else getattr(value, "value", None)
|
|
600
|
+
if not isinstance(literal, str):
|
|
601
|
+
continue
|
|
602
|
+
start = getattr(value, "lineno", 0)
|
|
603
|
+
end = getattr(value, "end_lineno", None) or start
|
|
604
|
+
lines.update(range(start, end + 1))
|
|
605
|
+
return lines
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
def _is_quoted_bare_pem_header(line: str, start: int, end: int) -> bool:
|
|
609
|
+
"""True when the header is a self-delimited string token with no key payload.
|
|
610
|
+
|
|
611
|
+
Both neighbours being quote characters means the literal closes right at the
|
|
612
|
+
header, so no base64 body follows it inside that token. Assignments that open a
|
|
613
|
+
multi-line body, PEM line lists (header followed by a comma), escaped payloads
|
|
614
|
+
(header followed by a backslash-n) and string concatenation all fail this test
|
|
615
|
+
and therefore stay detected.
|
|
616
|
+
"""
|
|
617
|
+
quotes = "'\""
|
|
618
|
+
if start <= 0 or line[start - 1] not in quotes:
|
|
619
|
+
return False
|
|
620
|
+
if end >= len(line) or line[end] not in quotes:
|
|
621
|
+
return False
|
|
622
|
+
return True
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
def scan_text(text: str, filename: str = "<string>") -> List[SecretFinding]:
|
|
626
|
+
"""Scan text for secrets and return a deterministic list of findings.
|
|
627
|
+
|
|
628
|
+
Findings are sorted by (line_number, column_start, detector, secret_type).
|
|
629
|
+
CRLF (\r\n) line endings are cleanly handled without trailing carriage returns.
|
|
630
|
+
"""
|
|
631
|
+
if not text:
|
|
632
|
+
return []
|
|
633
|
+
|
|
634
|
+
lines = text.splitlines(keepends=False)
|
|
635
|
+
all_findings: List[SecretFinding] = []
|
|
636
|
+
|
|
637
|
+
# Docstring context is only needed by files that mention a PEM header at all;
|
|
638
|
+
# the substring pre-check keeps the per-file cost at zero everywhere else.
|
|
639
|
+
pem_doc_lines: Set[int] = (
|
|
640
|
+
_docstring_line_numbers(text) if "PRIVATE KEY-----" in text else set()
|
|
641
|
+
)
|
|
642
|
+
|
|
643
|
+
# File-type scoping is resolved once per file: the HCL-shaped rules below simply are not in the
|
|
644
|
+
# list when the file is Python, so the per-line loop costs no extra test.
|
|
645
|
+
gitleaks_detectors = _gitleaks_detectors_for(filename)
|
|
646
|
+
|
|
647
|
+
for line_idx, raw_line in enumerate(lines, start=1):
|
|
648
|
+
clean_line = raw_line.rstrip("\r\n")
|
|
649
|
+
if len(clean_line) > 10000:
|
|
650
|
+
continue
|
|
651
|
+
candidates: List[_CandidateMatch] = []
|
|
652
|
+
|
|
653
|
+
# 1. Private Key (Precedence 1)
|
|
654
|
+
for m in _REGEX_PRIVATE_KEY.finditer(clean_line):
|
|
655
|
+
if line_idx in pem_doc_lines and _is_quoted_bare_pem_header(
|
|
656
|
+
clean_line, m.start(), m.end()
|
|
657
|
+
):
|
|
658
|
+
continue
|
|
659
|
+
candidates.append(
|
|
660
|
+
_CandidateMatch(
|
|
661
|
+
precedence=1,
|
|
662
|
+
start=m.start(),
|
|
663
|
+
end=m.end(),
|
|
664
|
+
secret_type="private_key",
|
|
665
|
+
detector="private_key",
|
|
666
|
+
masked_value=mask_secret(m.group(0)),
|
|
667
|
+
)
|
|
668
|
+
)
|
|
669
|
+
|
|
670
|
+
# 2. GitHub Token (Precedence 2)
|
|
671
|
+
for m in _REGEX_GITHUB_TOKEN.finditer(clean_line):
|
|
672
|
+
candidates.append(
|
|
673
|
+
_CandidateMatch(
|
|
674
|
+
precedence=2,
|
|
675
|
+
start=m.start(),
|
|
676
|
+
end=m.end(),
|
|
677
|
+
secret_type="github_token",
|
|
678
|
+
detector="github_token",
|
|
679
|
+
masked_value=mask_secret(m.group(0)),
|
|
680
|
+
)
|
|
681
|
+
)
|
|
682
|
+
|
|
683
|
+
# 3. Slack Token (Precedence 3)
|
|
684
|
+
for m in _REGEX_SLACK_TOKEN.finditer(clean_line):
|
|
685
|
+
candidates.append(
|
|
686
|
+
_CandidateMatch(
|
|
687
|
+
precedence=3,
|
|
688
|
+
start=m.start(),
|
|
689
|
+
end=m.end(),
|
|
690
|
+
secret_type="slack_token",
|
|
691
|
+
detector="slack_token",
|
|
692
|
+
masked_value=mask_secret(m.group(0)),
|
|
693
|
+
)
|
|
694
|
+
)
|
|
695
|
+
|
|
696
|
+
# 4. Stripe Key (Precedence 4)
|
|
697
|
+
for m in _REGEX_STRIPE_KEY.finditer(clean_line):
|
|
698
|
+
candidates.append(
|
|
699
|
+
_CandidateMatch(
|
|
700
|
+
precedence=4,
|
|
701
|
+
start=m.start(),
|
|
702
|
+
end=m.end(),
|
|
703
|
+
secret_type="stripe_key",
|
|
704
|
+
detector="stripe_key",
|
|
705
|
+
masked_value=mask_secret(m.group(0)),
|
|
706
|
+
)
|
|
707
|
+
)
|
|
708
|
+
|
|
709
|
+
# 5. AWS Access Key (Precedence 5)
|
|
710
|
+
for m in _REGEX_AWS_ACCESS_KEY.finditer(clean_line):
|
|
711
|
+
candidates.append(
|
|
712
|
+
_CandidateMatch(
|
|
713
|
+
precedence=5,
|
|
714
|
+
start=m.start(),
|
|
715
|
+
end=m.end(),
|
|
716
|
+
secret_type="aws_access_key",
|
|
717
|
+
detector="aws_access_key",
|
|
718
|
+
masked_value=mask_secret(m.group(0)),
|
|
719
|
+
)
|
|
720
|
+
)
|
|
721
|
+
|
|
722
|
+
# 5b. Slack Incoming Webhook URL (Precedence 5)
|
|
723
|
+
for m in _REGEX_SLACK_WEBHOOK.finditer(clean_line):
|
|
724
|
+
candidates.append(
|
|
725
|
+
_CandidateMatch(
|
|
726
|
+
precedence=5,
|
|
727
|
+
start=m.start(),
|
|
728
|
+
end=m.end(),
|
|
729
|
+
secret_type="slack_webhook_url",
|
|
730
|
+
detector="slack_webhook_url",
|
|
731
|
+
masked_value=mask_secret(m.group(0)),
|
|
732
|
+
)
|
|
733
|
+
)
|
|
734
|
+
|
|
735
|
+
# 6. Database Connection String (Precedence 6)
|
|
736
|
+
for m in _REGEX_DB_URI.finditer(clean_line):
|
|
737
|
+
parsed = _parse_and_mask_db_uri(m.group(0))
|
|
738
|
+
if parsed is None:
|
|
739
|
+
continue
|
|
740
|
+
masked_val, matched_len = parsed
|
|
741
|
+
cand_start = m.start()
|
|
742
|
+
cand_end = cand_start + matched_len
|
|
743
|
+
candidates.append(
|
|
744
|
+
_CandidateMatch(
|
|
745
|
+
precedence=6,
|
|
746
|
+
start=cand_start,
|
|
747
|
+
end=cand_end,
|
|
748
|
+
secret_type="database_connection_string",
|
|
749
|
+
detector="database_connection_string",
|
|
750
|
+
masked_value=masked_val,
|
|
751
|
+
)
|
|
752
|
+
)
|
|
753
|
+
|
|
754
|
+
# 7. JWT Token (Precedence 7)
|
|
755
|
+
for m in _REGEX_JWT_TOKEN.finditer(clean_line):
|
|
756
|
+
candidates.append(
|
|
757
|
+
_CandidateMatch(
|
|
758
|
+
precedence=7,
|
|
759
|
+
start=m.start(),
|
|
760
|
+
end=m.end(),
|
|
761
|
+
secret_type="jwt_token",
|
|
762
|
+
detector="jwt_token",
|
|
763
|
+
masked_value=mask_secret(m.group(0)),
|
|
764
|
+
)
|
|
765
|
+
)
|
|
766
|
+
|
|
767
|
+
# 8. Gitleaks signature bank (Precedence 8 - lowest; native detectors win overlaps)
|
|
768
|
+
for gl_type, gl_regex in gitleaks_detectors:
|
|
769
|
+
for m in gl_regex.finditer(clean_line):
|
|
770
|
+
if m.start() == m.end():
|
|
771
|
+
continue
|
|
772
|
+
candidates.append(
|
|
773
|
+
_CandidateMatch(
|
|
774
|
+
precedence=_GITLEAKS_PRECEDENCE,
|
|
775
|
+
start=m.start(),
|
|
776
|
+
end=m.end(),
|
|
777
|
+
secret_type=gl_type,
|
|
778
|
+
detector=gl_type,
|
|
779
|
+
masked_value=mask_secret(m.group(0)),
|
|
780
|
+
)
|
|
781
|
+
)
|
|
782
|
+
|
|
783
|
+
if not candidates:
|
|
784
|
+
continue
|
|
785
|
+
|
|
786
|
+
# Overlap Resolution:
|
|
787
|
+
# Priority order:
|
|
788
|
+
# 1. Lower precedence number (1 is highest)
|
|
789
|
+
# 2. Longer match length (end - start descending)
|
|
790
|
+
# 3. Earlier start column (start ascending)
|
|
791
|
+
candidates.sort(
|
|
792
|
+
key=lambda c: (c.precedence, -(c.end - c.start), c.start)
|
|
793
|
+
)
|
|
794
|
+
|
|
795
|
+
accepted: List[_CandidateMatch] = []
|
|
796
|
+
for cand in candidates:
|
|
797
|
+
overlaps = False
|
|
798
|
+
for acc in accepted:
|
|
799
|
+
if max(cand.start, acc.start) < min(cand.end, acc.end):
|
|
800
|
+
overlaps = True
|
|
801
|
+
break
|
|
802
|
+
if not overlaps:
|
|
803
|
+
accepted.append(cand)
|
|
804
|
+
|
|
805
|
+
# Build fully redacted context line
|
|
806
|
+
# Replace spans from right to left to preserve offsets
|
|
807
|
+
redacted_context = clean_line
|
|
808
|
+
for cand in sorted(accepted, key=lambda c: c.start, reverse=True):
|
|
809
|
+
redacted_context = (
|
|
810
|
+
redacted_context[: cand.start]
|
|
811
|
+
+ cand.masked_value
|
|
812
|
+
+ redacted_context[cand.end :]
|
|
813
|
+
)
|
|
814
|
+
|
|
815
|
+
# Emit SecretFinding for each accepted candidate
|
|
816
|
+
for cand in accepted:
|
|
817
|
+
finding = SecretFinding(
|
|
818
|
+
secret_type=cand.secret_type,
|
|
819
|
+
masked_value=cand.masked_value,
|
|
820
|
+
file=filename,
|
|
821
|
+
line_number=line_idx,
|
|
822
|
+
column_start=cand.start + 1,
|
|
823
|
+
column_end=cand.end,
|
|
824
|
+
confidence=ConfidenceStr("HIGH (PATTERN_MATCH)"),
|
|
825
|
+
detector=cand.detector,
|
|
826
|
+
context=redacted_context,
|
|
827
|
+
proof_type="PATTERN_MATCH",
|
|
828
|
+
)
|
|
829
|
+
all_findings.append(finding)
|
|
830
|
+
|
|
831
|
+
# 7. AST Hardcoded Credential Variable Assignment Heuristic
|
|
832
|
+
if not any(filename.endswith(ext) for ext in (".env", ".ini", ".conf", ".yaml", ".yml", ".json", ".toml", ".txt")):
|
|
833
|
+
try:
|
|
834
|
+
parsed_tree = ast.parse(text, filename=filename)
|
|
835
|
+
cred_var_regex = re.compile(r"(?i)(.*password.*|.*secret.*|.*api_key.*|.*auth_token.*|.*token.*|.*bearer.*|.*credential.*|.*private_key.*|.*webhook.*)")
|
|
836
|
+
dummy_regex = re.compile(r"(?i)(^<.*>$|test|dummy|example|fake|sample|placeholder|change_me|insert|your_|_here)")
|
|
837
|
+
lines = text.splitlines()
|
|
838
|
+
|
|
839
|
+
for node in ast.walk(parsed_tree):
|
|
840
|
+
target_names = []
|
|
841
|
+
if isinstance(node, ast.Assign):
|
|
842
|
+
for t in node.targets:
|
|
843
|
+
if isinstance(t, ast.Name):
|
|
844
|
+
target_names.append((t.id, getattr(t, "col_offset", 0)))
|
|
845
|
+
elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
|
|
846
|
+
target_names.append((node.target.id, getattr(node.target, "col_offset", 0)))
|
|
847
|
+
|
|
848
|
+
val_node = getattr(node, "value", None)
|
|
849
|
+
if not val_node or not target_names:
|
|
850
|
+
continue
|
|
851
|
+
|
|
852
|
+
val_str = None
|
|
853
|
+
if isinstance(val_node, ast.Constant) and isinstance(val_node.value, str):
|
|
854
|
+
val_str = val_node.value
|
|
855
|
+
elif isinstance(val_node, ast.Str):
|
|
856
|
+
val_str = val_node.s
|
|
857
|
+
|
|
858
|
+
if val_str and len(val_str) >= 8:
|
|
859
|
+
is_explicit_cred = any(bool(re.search(r"(?i)(aws_secret_access_key|aws_secret|secret_key|private_key|token|bearer|api_key)", vn)) for vn, _ in target_names)
|
|
860
|
+
is_dummy = bool(dummy_regex.search(val_str))
|
|
861
|
+
if is_dummy and is_explicit_cred and len(val_str) >= 20 and shannon_entropy(val_str) >= 3.0:
|
|
862
|
+
is_dummy = False
|
|
863
|
+
|
|
864
|
+
if not is_dummy:
|
|
865
|
+
for var_name, var_col in target_names:
|
|
866
|
+
if cred_var_regex.search(var_name):
|
|
867
|
+
if var_name.lower() == val_str.lower():
|
|
868
|
+
# Value restates its own identifier, so it holds no
|
|
869
|
+
# secret entropy: enum members like PASSWORD = "password".
|
|
870
|
+
continue
|
|
871
|
+
line_no = getattr(node, "lineno", 1)
|
|
872
|
+
if any(f.line_number == line_no and f.detector != "hardcoded_credential_variable" for f in all_findings):
|
|
873
|
+
continue
|
|
874
|
+
raw_line = lines[line_no - 1] if 1 <= line_no <= len(lines) else ""
|
|
875
|
+
masked_val = mask_secret(val_str)
|
|
876
|
+
redacted_ctx = raw_line.replace(val_str, masked_val) if val_str in raw_line else raw_line
|
|
877
|
+
col_start = getattr(val_node, "col_offset", var_col) + 1
|
|
878
|
+
col_end = getattr(val_node, "end_col_offset", col_start + len(val_str))
|
|
879
|
+
finding = SecretFinding(
|
|
880
|
+
secret_type="hardcoded_credential_variable",
|
|
881
|
+
masked_value=masked_val,
|
|
882
|
+
file=filename,
|
|
883
|
+
line_number=line_no,
|
|
884
|
+
column_start=col_start,
|
|
885
|
+
column_end=col_end,
|
|
886
|
+
confidence=ConfidenceStr("HIGH (PATTERN_MATCH)"),
|
|
887
|
+
detector="hardcoded_credential_variable",
|
|
888
|
+
context=redacted_ctx,
|
|
889
|
+
proof_type="PATTERN_MATCH",
|
|
890
|
+
pattern="hardcoded_credential_variable",
|
|
891
|
+
)
|
|
892
|
+
all_findings.append(finding)
|
|
893
|
+
except Exception:
|
|
894
|
+
pass
|
|
895
|
+
|
|
896
|
+
# Sort deterministically
|
|
897
|
+
all_findings.sort(
|
|
898
|
+
key=lambda f: (f.line_number, f.column_start, f.detector, f.secret_type)
|
|
899
|
+
)
|
|
900
|
+
return all_findings
|
|
901
|
+
|
|
902
|
+
|
|
903
|
+
def scan_file(filepath: str) -> List[SecretFinding]:
|
|
904
|
+
"""Scan a local file for secrets.
|
|
905
|
+
|
|
906
|
+
Raises FileNotFoundError if file does not exist.
|
|
907
|
+
Decodes using UTF-8 with fallback replacement.
|
|
908
|
+
Skips files exceeding 5MB or containing binary prefix.
|
|
909
|
+
"""
|
|
910
|
+
if not os.path.exists(filepath):
|
|
911
|
+
raise FileNotFoundError(f"File not found: {filepath}")
|
|
912
|
+
|
|
913
|
+
try:
|
|
914
|
+
if os.path.getsize(filepath) > 5 * 1024 * 1024:
|
|
915
|
+
return []
|
|
916
|
+
except (OSError, ValueError):
|
|
917
|
+
return []
|
|
918
|
+
|
|
919
|
+
try:
|
|
920
|
+
with open(filepath, "rb") as bf:
|
|
921
|
+
chunk = bf.read(8192)
|
|
922
|
+
if b"\x00" in chunk:
|
|
923
|
+
return []
|
|
924
|
+
except (OSError, ValueError):
|
|
925
|
+
return []
|
|
926
|
+
|
|
927
|
+
with open(filepath, "r", encoding="utf-8", errors="replace") as f:
|
|
928
|
+
content = f.read()
|
|
929
|
+
return scan_text(content, filename=filepath)
|
|
930
|
+
|
|
931
|
+
|
|
932
|
+
class SecretScanner:
|
|
933
|
+
"""Offline secret detection engine."""
|
|
934
|
+
|
|
935
|
+
def __init__(self, filter_config: Optional[Any] = None):
|
|
936
|
+
self.filter_config = filter_config
|
|
937
|
+
|
|
938
|
+
def scan_text(self, text: str, filename: str = "<memory>") -> List[SecretFinding]:
|
|
939
|
+
return scan_text(text, filename)
|
|
940
|
+
|
|
941
|
+
def scan_file(self, path: str) -> List[SecretFinding]:
|
|
942
|
+
return scan_file(path)
|
|
943
|
+
|