timecs 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app.py +3014 -0
- ast_scanner.py +36187 -0
- banner.py +146 -0
- ci_reporter.py +570 -0
- cli.py +2347 -0
- config_loader.py +307 -0
- cross_file_engine.py +1781 -0
- delivery_models.py +145 -0
- delivery_repository.py +78 -0
- diff_scan.py +127 -0
- durable_delivery_sink.py +103 -0
- event_bus.py +155 -0
- event_publisher.py +345 -0
- event_taxonomy.py +313 -0
- function_summarizer.py +354 -0
- html_auditor.py +274 -0
- iac_auditor.py +400 -0
- in_app_adapter.py +115 -0
- js_scanner.py +895 -0
- manifest_parser.py +960 -0
- master_rules_bank.py +389 -0
- notification_models.py +346 -0
- notification_policy.py +309 -0
- notification_repository.py +118 -0
- notification_routes.py +394 -0
- notification_service.py +199 -0
- osv_client.py +437 -0
- remediation/__init__.py +39 -0
- remediation/base.py +102 -0
- remediation/contracts.py +196 -0
- remediation/cwe1188_binding.py +78 -0
- remediation/cwe1336_jinja.py +91 -0
- remediation/cwe22_path_traversal.py +348 -0
- remediation/cwe295_ssl.py +87 -0
- remediation/cwe327_weak_hash.py +78 -0
- remediation/cwe377_tempfile.py +77 -0
- remediation/cwe489_debug.py +80 -0
- remediation/cwe502_yaml.py +89 -0
- remediation/cwe614_cookie.py +91 -0
- remediation/cwe78_cmdi.py +415 -0
- remediation/cwe798_credentials.py +153 -0
- remediation/cwe89_sqli.py +425 -0
- remediation/cwe916_password_hash.py +131 -0
- remediation/cwe95_eval.py +77 -0
- remediation/dispatcher.py +114 -0
- remediation/orchestrator.py +541 -0
- remediation/patch_engine.py +213 -0
- remediation/verification.py +253 -0
- rule_engine.py +1535 -0
- sarif_adapter.py +719 -0
- sarif_exporter.py +88 -0
- sca_reachability/__init__.py +28 -0
- sca_reachability/ast_analyzer.py +366 -0
- sca_reachability/call_graph.py +203 -0
- sca_reachability/contracts.py +365 -0
- sca_reachability/dist_import_map.py +46 -0
- sca_reachability/engine.py +603 -0
- sca_reachability/resolver.py +169 -0
- scanners/__init__.py +70 -0
- scanners/container_scanner.py +323 -0
- scanners/cve_index.py +521 -0
- scanners/iac_scanner.py +328 -0
- scanners/sbom.py +183 -0
- scanners/sca_scanner.py +667 -0
- scanners/secret_scanner.py +345 -0
- scanners/yaml_lite.py +217 -0
- secret_filters.py +435 -0
- secret_scanner.py +943 -0
- sqlite_delivery_repository.py +277 -0
- sqlite_notification_repository.py +441 -0
- ssrf_guard.py +276 -0
- staged_scanner.py +492 -0
- suppression_resolver.py +282 -0
- symbol_indexer.py +424 -0
- tcs/__init__.py +1 -0
- tcs/analysis/__init__.py +1 -0
- tcs/analysis/def_use.py +408 -0
- tcs/analysis/function_contracts.py +290 -0
- tcs/parallel_analyzer.py +276 -0
- tcs/parallel_scanner.py +630 -0
- tcs/treesitter_packer.py +1446 -0
- tcs_cli.py +2086 -0
- tcs_config.py +218 -0
- tcs_gui.py +2123 -0
- timecs-2.0.0.dist-info/METADATA +142 -0
- timecs-2.0.0.dist-info/RECORD +94 -0
- timecs-2.0.0.dist-info/WHEEL +5 -0
- timecs-2.0.0.dist-info/entry_points.txt +4 -0
- timecs-2.0.0.dist-info/licenses/LICENSE +201 -0
- timecs-2.0.0.dist-info/top_level.txt +49 -0
- version_matcher.py +934 -0
- webhook_adapter.py +495 -0
- webhook_client.py +483 -0
- webhook_formatters.py +245 -0
|
@@ -0,0 +1,345 @@
|
|
|
1
|
+
"""Hard-coded secrets in configuration, environment and shell documents -- stdlib only.
|
|
2
|
+
|
|
3
|
+
The Python engine already owns hard-coded credentials inside `.py` source. This reader owns the
|
|
4
|
+
documents around it: `.env`, `*.ini`, `*.cfg`, `*.conf`, `*.toml`, `*.json`, `*.yaml`, `*.yml` and
|
|
5
|
+
`*.sh`, where a credential is not an expression the AST can reason about but a literal sitting in
|
|
6
|
+
a config file that ships with the image.
|
|
7
|
+
|
|
8
|
+
Precision is the whole design. Two families are detected:
|
|
9
|
+
|
|
10
|
+
* Vendor shapes -- an AWS key ID, a GitHub token, a Stripe live key, a Slack token or webhook, a
|
|
11
|
+
PEM private-key header, a service URL carrying a password. These are recognisable on their own,
|
|
12
|
+
so no entropy argument is needed: the prefix IS the proof.
|
|
13
|
+
* Generic sensitive assignments -- `API_KEY`, `SECRET_KEY`, `AUTH_TOKEN`, `ACCESS_TOKEN`,
|
|
14
|
+
`CLIENT_SECRET`, `DB_PASSWORD`, `PRIVATE_KEY` (also under a vendor prefix, e.g.
|
|
15
|
+
`STRIPE_SECRET_KEY`). A name alone is not a secret, so the value must additionally be a literal
|
|
16
|
+
of at least 16 characters spanning at least two character classes with Shannon entropy >= 3.3.
|
|
17
|
+
|
|
18
|
+
Everything else is refused, and the refusals are the reason this reader can be loud by default:
|
|
19
|
+
shell interpolation (`${VAR}`, `$VAR`), empty and boolean/null literals, a 40-character hex string
|
|
20
|
+
with no credential name (a git commit SHA), and any value carrying a placeholder word. A template
|
|
21
|
+
or example document is never read at all -- `.env.example` exists to hold fake values.
|
|
22
|
+
|
|
23
|
+
Statistics alone cannot tell a credential from a sentence about a credential, so a generic
|
|
24
|
+
assignment must also pass a lexical test (`_is_literal_credential`): it may carry no space or tab,
|
|
25
|
+
and it may not be wrapped in a macro or reference marker. The bug this closes is Home Assistant's
|
|
26
|
+
`strings.json` localisation catalogues, whose UI labels sit under real sensitive names and clear
|
|
27
|
+
every statistical floor -- `"access_token": "Personal Access Token (PAT)"` measures 3.78 bits per
|
|
28
|
+
character over 27, and `"api_key": "[%key:common::config_flow::data::api_key%]"` (4.14 over 42) is a
|
|
29
|
+
reference to a string stored elsewhere, not the string itself. Both gates apply to the generic
|
|
30
|
+
branch only: a PEM header contains spaces by definition, and a URL password may legitimately carry
|
|
31
|
+
one, so those families keep their own proof and gain no new way to go quiet.
|
|
32
|
+
|
|
33
|
+
The matched secret is never copied into a finding. Reports, SARIF and logs are shared artefacts;
|
|
34
|
+
a scanner that re-publishes the credential it found has moved the leak rather than reported it.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
from __future__ import annotations
|
|
38
|
+
|
|
39
|
+
import math
|
|
40
|
+
import re
|
|
41
|
+
from typing import Any, Dict, Iterator, List, Optional, Tuple
|
|
42
|
+
|
|
43
|
+
__all__ = ["is_secret_path", "audit_secret_files", "CATEGORY"]
|
|
44
|
+
|
|
45
|
+
CATEGORY = "HARDCODED_SECRET_CONFIG"
|
|
46
|
+
|
|
47
|
+
CONFIG_SUFFIXES = (".ini", ".cfg", ".conf", ".toml", ".json", ".yaml", ".yml", ".sh")
|
|
48
|
+
# A lockfile is a resolved dependency graph, not a credential store, and the SCA scanner owns it.
|
|
49
|
+
LOCKFILE_NAMES = frozenset({
|
|
50
|
+
"package-lock.json", "npm-shrinkwrap.json", "yarn.lock", "poetry.lock", "pipfile.lock",
|
|
51
|
+
"cargo.lock", "gemfile.lock", "composer.lock", "pnpm-lock.yaml",
|
|
52
|
+
})
|
|
53
|
+
# These documents exist to carry fake values. Reading them would report the documentation.
|
|
54
|
+
TEMPLATE_SUFFIXES = (".example", ".sample", ".template")
|
|
55
|
+
|
|
56
|
+
# (label, pattern, severity). The pattern matches the credential itself, not the assignment.
|
|
57
|
+
VENDOR_PATTERNS: Tuple[Tuple[str, "re.Pattern[str]", str], ...] = (
|
|
58
|
+
("AWS Access Key ID", re.compile(r"\bAKIA[0-9A-Z]{16}\b"), "HIGH"),
|
|
59
|
+
("GitHub Personal Access Token", re.compile(r"\bghp_[A-Za-z0-9]{36}\b"), "HIGH"),
|
|
60
|
+
("GitHub Fine-Grained Token", re.compile(r"\bgithub_pat_[A-Za-z0-9_]{22,}"), "HIGH"),
|
|
61
|
+
("GitHub OAuth Token", re.compile(r"\bgho_[A-Za-z0-9]{36}\b"), "HIGH"),
|
|
62
|
+
("Stripe Live Secret Key", re.compile(r"\bsk_live_[0-9a-zA-Z]{24,}"), "CRITICAL"),
|
|
63
|
+
("Slack Token", re.compile(r"\bxox[baprs]-[0-9A-Za-z-]{10,}"), "HIGH"),
|
|
64
|
+
("Slack Incoming Webhook",
|
|
65
|
+
re.compile(r"https://hooks\.slack\.com/services/T[A-Z0-9]+/B[A-Z0-9]+/[A-Za-z0-9]+"), "HIGH"),
|
|
66
|
+
("PEM Private Key",
|
|
67
|
+
re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY-----"), "CRITICAL"),
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
SERVICE_URL = re.compile(
|
|
71
|
+
r"(?:postgres|postgresql|mysql|mongodb|redis|amqp)://[^:\s]+:([^@\s]{6,})@[^/\s]+")
|
|
72
|
+
|
|
73
|
+
SENSITIVE_NAMES = ("API_KEY", "SECRET_KEY", "AUTH_TOKEN", "ACCESS_TOKEN",
|
|
74
|
+
"CLIENT_SECRET", "DB_PASSWORD", "PRIVATE_KEY")
|
|
75
|
+
# One assignment shape covers dotenv/ini/shell (`K=v`), YAML (`K: v`), TOML (`K = v`) and
|
|
76
|
+
# JSON (`"K": "v"`): the optional closing quote after the key is what makes JSON fit.
|
|
77
|
+
ASSIGNMENT = re.compile(
|
|
78
|
+
r"""(?P<key>[A-Za-z_][A-Za-z0-9_.\-]*)["']?\s*[:=]\s*"""
|
|
79
|
+
r"""(?P<value>"[^"]*"|'[^']*'|[^\s,;#}\]]+)""")
|
|
80
|
+
|
|
81
|
+
PLACEHOLDER_WORDS = ("example", "changeme", "placeholder", "dummy", "test", "sample",
|
|
82
|
+
"your_", "xxxx", "123456", "password", "secret")
|
|
83
|
+
# Four tokens that say what a value is *for* instead of carrying one. Kept apart from
|
|
84
|
+
# PLACEHOLDER_WORDS so this reader's newer refusal can be measured without moving the older one.
|
|
85
|
+
NON_SECRET_PLACEHOLDER_TOKENS = ("temporary_key", "replace_me", "not_found", "not_a_real_key")
|
|
86
|
+
# `user:user@localhost` proves a developer default rather than a deployed credential: the secret and
|
|
87
|
+
# its own username are the same dictionary word, and the host is this machine, so no deployment can
|
|
88
|
+
# be relying on it. Airflow ships exactly this pair in `default_airflow.cfg`, `default_test.cfg` and
|
|
89
|
+
# the documentation template `config.yml`, all three of which describe a broker URL rather than
|
|
90
|
+
# configure one.
|
|
91
|
+
LOCAL_SERVICE_HOSTS = frozenset({"localhost", "127.0.0.1", "0.0.0.0", "::1"})
|
|
92
|
+
# A document whose own name declares it a test fixture or the shipped default of a product. Only the
|
|
93
|
+
# generic-assignment branch consults this: a vendor token, a PEM block or a service URL proves itself
|
|
94
|
+
# by shape and cannot be silenced by the name of the file it sits in.
|
|
95
|
+
NON_PRODUCTION_CONFIG_SUFFIXES = ("_test.cfg", "_test.ini", "_test.conf")
|
|
96
|
+
NON_PRODUCTION_CONFIG_PREFIXES = ("test_", "default_test.")
|
|
97
|
+
NON_PRODUCTION_CONFIG_EXTENSIONS = (".cfg", ".ini", ".conf", ".yml", ".yaml", ".env", ".properties")
|
|
98
|
+
NULL_LITERALS = frozenset({"null", "none", "true", "false", "~", "nil", "undefined"})
|
|
99
|
+
INTERPOLATION = re.compile(r"\$(?:\{[^}]*\}|[A-Za-z_][A-Za-z0-9_]*)")
|
|
100
|
+
COMMIT_SHA = re.compile(r"[0-9a-fA-F]{40}")
|
|
101
|
+
MIN_GENERIC_LENGTH = 16
|
|
102
|
+
MIN_GENERIC_ENTROPY = 3.3
|
|
103
|
+
# The lexical floor for a *literal* credential, sitting under a sensitive name. Prose says what a
|
|
104
|
+
# credential is; a macro reference says where one lives. Neither is a secret, and no length,
|
|
105
|
+
# character-class or entropy threshold can separate them from one -- only shape can.
|
|
106
|
+
PROSE_CHARACTERS = (" ", "\t")
|
|
107
|
+
# A filesystem path says where a credential is kept, not what it is. `private_key = /etc/salt/pki/
|
|
108
|
+
# minion.pem` and `api_key: ~/credentials.json` clear every statistical floor -- a path is long, spans
|
|
109
|
+
# several character classes and is high-entropy by construction -- while shipping no secret at all; the
|
|
110
|
+
# file it names is the artefact an operator protects and rotates.
|
|
111
|
+
CREDENTIAL_PATH_PREFIXES = ("/", "./", "../", "~/")
|
|
112
|
+
CREDENTIAL_PATH_SUFFIXES = (".pem", ".key", ".crt", ".cer", ".p12", ".pfx", ".pub", ".json",
|
|
113
|
+
".yaml", ".yml", ".conf", ".cfg", ".ini", ".env")
|
|
114
|
+
WINDOWS_DRIVE_PATH = re.compile(r"[A-Za-z]:[\\/].*")
|
|
115
|
+
# The closing marker is optional in every pattern below because ASSIGNMENT's unquoted alternative
|
|
116
|
+
# stops before `]`, `}` and `)`: `api_key: [%key:common::config_flow::data::api_key%]` arrives here
|
|
117
|
+
# as `[%key:common::config_flow::data::api_key%`. A reference is a reference either way.
|
|
118
|
+
REFERENCE_WRAPPERS = (
|
|
119
|
+
# Home Assistant localisation catalogues (and Jinja/Template Toolkit references): the whole
|
|
120
|
+
# value is `[%key:section::name%]`, a pointer into a string table.
|
|
121
|
+
re.compile(r"\[\%.*\%\]?"),
|
|
122
|
+
re.compile(r"\{\{.*\}\}?"), # {{ expression }}
|
|
123
|
+
re.compile(r"\{%.*%\}?"), # {% statement %}
|
|
124
|
+
re.compile(r"\$\{.*\}?"), # ${VAR}
|
|
125
|
+
re.compile(r"\$\(.*\)?"), # $(command)
|
|
126
|
+
re.compile(r"\<.*\>?"), # <value supplied somewhere else>
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _is_literal_credential(value: str) -> bool:
|
|
131
|
+
"""False when the value is prose, a pointer, or a filesystem path rather than a secret itself.
|
|
132
|
+
|
|
133
|
+
Gated on the generic-assignment branch only, by design: `-----BEGIN RSA PRIVATE KEY-----`
|
|
134
|
+
carries spaces and `VENDOR_PATTERNS` / `SERVICE_URL` prove their own shapes, so this rule must
|
|
135
|
+
never be able to silence them. A generic assignment is the one place where the name, not the
|
|
136
|
+
value, did the identifying -- which is exactly where a UI label can impersonate a credential.
|
|
137
|
+
"""
|
|
138
|
+
if not value:
|
|
139
|
+
return False
|
|
140
|
+
if any(char in value for char in PROSE_CHARACTERS):
|
|
141
|
+
return False
|
|
142
|
+
if any(pattern.fullmatch(value) for pattern in REFERENCE_WRAPPERS):
|
|
143
|
+
return False
|
|
144
|
+
if value.startswith(CREDENTIAL_PATH_PREFIXES) or WINDOWS_DRIVE_PATH.fullmatch(value):
|
|
145
|
+
return False
|
|
146
|
+
if value.lower().endswith(CREDENTIAL_PATH_SUFFIXES):
|
|
147
|
+
return False
|
|
148
|
+
return True
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _service_host(authority: str) -> str:
|
|
152
|
+
"""The bare host of an already-split `host` or `host:port`, IPv6 included."""
|
|
153
|
+
host = authority.lower()
|
|
154
|
+
if host.startswith("["):
|
|
155
|
+
return host[1:].split("]", 1)[0]
|
|
156
|
+
head, separator, tail = host.rpartition(":")
|
|
157
|
+
if separator and tail.isdigit():
|
|
158
|
+
return head
|
|
159
|
+
return host
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _service_credential_is_dev_default(url: str) -> bool:
|
|
163
|
+
"""True for `scheme://user:user@localhost/db`: the password is the username, on this machine.
|
|
164
|
+
|
|
165
|
+
`sqla+mysql://airflow:airflow@localhost:3306/airflow` is a worked example of a broker URL, not a
|
|
166
|
+
credential a deployment relies on -- and a deployment that really does run it that way is guarding
|
|
167
|
+
the word `airflow`, which is not a secret worth rotating. `postgres://app:S3cr3tP4ss@db.internal
|
|
168
|
+
/prod` fails both halves, so a production service URL cannot be silenced here.
|
|
169
|
+
"""
|
|
170
|
+
_scheme, separator, rest = url.partition("://")
|
|
171
|
+
if not separator:
|
|
172
|
+
return False
|
|
173
|
+
authority = rest.split("/", 1)[0]
|
|
174
|
+
userinfo, at, host = authority.rpartition("@")
|
|
175
|
+
if not at:
|
|
176
|
+
return False
|
|
177
|
+
user, colon, password = userinfo.partition(":")
|
|
178
|
+
if not colon or user != password:
|
|
179
|
+
return False
|
|
180
|
+
return _service_host(host) in LOCAL_SERVICE_HOSTS
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _is_non_production_document(path: str) -> bool:
|
|
184
|
+
"""True when a config document's own name says it is a test fixture or a shipped default.
|
|
185
|
+
|
|
186
|
+
Scoped to the generic-assignment branch by the caller: those are the hits identified by the key
|
|
187
|
+
name alone, which is exactly where a template's `api_key = REPLACE_ME_NOW` can impersonate a
|
|
188
|
+
credential. A value that proves its own shape is never reached by this rule.
|
|
189
|
+
"""
|
|
190
|
+
name = str(path).replace("\\", "/").rsplit("/", 1)[-1].lower()
|
|
191
|
+
if name.endswith(NON_PRODUCTION_CONFIG_SUFFIXES):
|
|
192
|
+
return True
|
|
193
|
+
if name.startswith(NON_PRODUCTION_CONFIG_PREFIXES):
|
|
194
|
+
return name.endswith(NON_PRODUCTION_CONFIG_EXTENSIONS)
|
|
195
|
+
return False
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def is_secret_path(path: str) -> bool:
|
|
199
|
+
"""True for a config/env/shell document this reader owns, by name alone."""
|
|
200
|
+
name = str(path).replace("\\", "/").rsplit("/", 1)[-1].lower()
|
|
201
|
+
if name in LOCKFILE_NAMES or name.endswith(TEMPLATE_SUFFIXES):
|
|
202
|
+
return False
|
|
203
|
+
if name == ".env" or name.startswith(".env.") or name.endswith(".env"):
|
|
204
|
+
return True
|
|
205
|
+
return name.endswith(CONFIG_SUFFIXES)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def shannon_entropy(text: str) -> float:
|
|
209
|
+
"""Bits per character. A repeated or single-class string cannot reach the generic floor."""
|
|
210
|
+
if not text:
|
|
211
|
+
return 0.0
|
|
212
|
+
counts: Dict[str, int] = {}
|
|
213
|
+
for char in text:
|
|
214
|
+
counts[char] = counts.get(char, 0) + 1
|
|
215
|
+
total = len(text)
|
|
216
|
+
return -sum((count / total) * math.log2(count / total) for count in counts.values())
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _character_classes(text: str) -> int:
|
|
220
|
+
present = 0
|
|
221
|
+
for probe in (str.islower, str.isupper, str.isdigit):
|
|
222
|
+
if any(probe(char) for char in text):
|
|
223
|
+
present += 1
|
|
224
|
+
if any(not char.isalnum() for char in text):
|
|
225
|
+
present += 1
|
|
226
|
+
return present
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _unquote(value: str) -> str:
|
|
230
|
+
text = value.strip()
|
|
231
|
+
if len(text) >= 2 and text[0] == text[-1] and text[0] in "'\"":
|
|
232
|
+
text = text[1:-1]
|
|
233
|
+
return text.strip()
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def is_refused(value: str, named: bool = False) -> bool:
|
|
237
|
+
"""The zero-FP contract: a value that is not a credential, whatever its shape.
|
|
238
|
+
|
|
239
|
+
`named` says a credential name or a password position already identified this value, which is
|
|
240
|
+
what separates a 40-hex secret from a 40-hex git commit SHA.
|
|
241
|
+
"""
|
|
242
|
+
text = _unquote(value)
|
|
243
|
+
if not text:
|
|
244
|
+
return True
|
|
245
|
+
lowered = text.lower()
|
|
246
|
+
if lowered in NULL_LITERALS:
|
|
247
|
+
return True
|
|
248
|
+
if INTERPOLATION.search(text):
|
|
249
|
+
return True
|
|
250
|
+
if any(word in lowered for word in PLACEHOLDER_WORDS):
|
|
251
|
+
return True
|
|
252
|
+
if any(token in lowered for token in NON_SECRET_PLACEHOLDER_TOKENS):
|
|
253
|
+
return True
|
|
254
|
+
if not named and len(text) == 40 and COMMIT_SHA.fullmatch(text):
|
|
255
|
+
return True
|
|
256
|
+
return False
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _sensitive_name(key: str) -> Optional[str]:
|
|
260
|
+
"""The sensitive name a config key carries, exactly or under a vendor prefix."""
|
|
261
|
+
normalized = key.upper().replace("-", "_").replace(".", "_")
|
|
262
|
+
for name in SENSITIVE_NAMES:
|
|
263
|
+
if normalized == name or normalized.endswith("_" + name):
|
|
264
|
+
return name
|
|
265
|
+
return None
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _redact(text: str) -> str:
|
|
269
|
+
"""Enough to locate the credential in the file, never enough to use it."""
|
|
270
|
+
if len(text) <= 8:
|
|
271
|
+
return "*" * len(text)
|
|
272
|
+
return text[:4] + "*" * 8 + text[-2:]
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _iter_hits(source: str) -> Iterator[Tuple[int, str, str, str, str]]:
|
|
276
|
+
"""(lineno, label, severity, redacted, kind) for every credential the line proves."""
|
|
277
|
+
for lineno, line in enumerate(source.splitlines(), start=1):
|
|
278
|
+
if not line.strip():
|
|
279
|
+
continue
|
|
280
|
+
for label, pattern, severity in VENDOR_PATTERNS:
|
|
281
|
+
match = pattern.search(line)
|
|
282
|
+
if match and not is_refused(match.group(0), named=True):
|
|
283
|
+
yield lineno, label, severity, _redact(match.group(0)), "vendor"
|
|
284
|
+
for match in SERVICE_URL.finditer(line):
|
|
285
|
+
if is_refused(match.group(1), named=True):
|
|
286
|
+
continue
|
|
287
|
+
if _service_credential_is_dev_default(match.group(0)):
|
|
288
|
+
continue
|
|
289
|
+
yield (lineno, "Service URL With Embedded Password", "HIGH",
|
|
290
|
+
_redact(match.group(1)), "service_url")
|
|
291
|
+
for match in ASSIGNMENT.finditer(line):
|
|
292
|
+
name = _sensitive_name(match.group("key"))
|
|
293
|
+
if name is None:
|
|
294
|
+
continue
|
|
295
|
+
value = _unquote(match.group("value"))
|
|
296
|
+
if is_refused(value, named=True):
|
|
297
|
+
continue
|
|
298
|
+
if not _is_literal_credential(value):
|
|
299
|
+
continue
|
|
300
|
+
if len(value) < MIN_GENERIC_LENGTH or _character_classes(value) < 2:
|
|
301
|
+
continue
|
|
302
|
+
if shannon_entropy(value) < MIN_GENERIC_ENTROPY:
|
|
303
|
+
continue
|
|
304
|
+
yield (lineno, f"Hard-coded {name}", "HIGH", _redact(value), "generic")
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def audit_secret_files(files: Dict[str, str]) -> List[Dict[str, Any]]:
|
|
308
|
+
"""One finding per (document, line, credential family), in source order."""
|
|
309
|
+
findings: List[Dict[str, Any]] = []
|
|
310
|
+
for file_path in sorted(files):
|
|
311
|
+
# The refusal is enforced here as well as at discovery, so a caller that hands over a
|
|
312
|
+
# lockfile or a `.env.example` cannot make this reader report the documentation.
|
|
313
|
+
if not is_secret_path(file_path):
|
|
314
|
+
continue
|
|
315
|
+
seen = set()
|
|
316
|
+
for lineno, label, severity, redacted, kind in _iter_hits(files[file_path]):
|
|
317
|
+
# A document that names itself a test fixture or the shipped default carries no deployed
|
|
318
|
+
# credential, and the key name is the *only* thing that identified a generic hit -- so
|
|
319
|
+
# `api_key = REPLACE_WITH_YOUR_KEY` in `service_test.cfg` is the one place the document's
|
|
320
|
+
# own name is evidence. Vendor tokens and service URLs prove their shape and never reach
|
|
321
|
+
# this clause.
|
|
322
|
+
if kind == "generic" and _is_non_production_document(file_path):
|
|
323
|
+
continue
|
|
324
|
+
# A line can satisfy two shapes at once -- a service URL whose password is also a
|
|
325
|
+
# sensitive assignment. One credential is one finding, so the first shape wins.
|
|
326
|
+
key = (lineno, label)
|
|
327
|
+
if key in seen:
|
|
328
|
+
continue
|
|
329
|
+
seen.add(key)
|
|
330
|
+
findings.append({
|
|
331
|
+
"file": file_path,
|
|
332
|
+
"line": lineno,
|
|
333
|
+
"cwe": "CWE-798",
|
|
334
|
+
"severity": severity,
|
|
335
|
+
"category": CATEGORY,
|
|
336
|
+
"confidence": "HIGH",
|
|
337
|
+
"secret_type": label,
|
|
338
|
+
"secret_kind": kind,
|
|
339
|
+
"evidence": redacted,
|
|
340
|
+
"message": (f"{label} hard-coded in {file_path}:{lineno} "
|
|
341
|
+
f"(value {redacted}). Move it to a secret manager or an environment "
|
|
342
|
+
f"variable supplied at runtime, and rotate it: a credential committed "
|
|
343
|
+
f"to a config file is already exposed."),
|
|
344
|
+
})
|
|
345
|
+
return sorted(findings, key=lambda item: (item["file"], item["line"], item["secret_type"]))
|
scanners/yaml_lite.py
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
"""A streaming YAML subset reader for security auditing -- stdlib only, no PyYAML.
|
|
2
|
+
|
|
3
|
+
The container and IaC rules need three things from a YAML document: the key path a value sits at,
|
|
4
|
+
the line it sits on, and its literal text. They do not need anchors, tags, merge keys, flow
|
|
5
|
+
collections beyond the empty ``{}``/``[]``, or multi-document streams. This reader implements
|
|
6
|
+
exactly that subset and refuses to guess at the rest: a construct it cannot represent becomes a
|
|
7
|
+
scalar holding the raw text, so a rule reading it sees an unresolvable value and stays quiet
|
|
8
|
+
rather than inventing a structure.
|
|
9
|
+
|
|
10
|
+
Keys are kept as the literal source spelling. PyYAML resolves a bare ``on:`` to the boolean
|
|
11
|
+
``True`` (YAML 1.1), which silently loses every GitHub Actions trigger block; here ``on`` stays
|
|
12
|
+
``on`` because the auditor asks about the document, not about a YAML 1.1 type system.
|
|
13
|
+
|
|
14
|
+
Memory is one line list plus one node per entry -- no whole-file regex sweep, no dependency.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from typing import Any, Iterator, List, Optional, Tuple
|
|
20
|
+
|
|
21
|
+
__all__ = ["Scalar", "Mapping", "Sequence", "parse", "get", "walk", "scalar_text"]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Scalar:
|
|
25
|
+
__slots__ = ("lineno", "text", "end_lineno")
|
|
26
|
+
|
|
27
|
+
def __init__(self, lineno: int, text: str, end_lineno: Optional[int] = None) -> None:
|
|
28
|
+
self.lineno = lineno
|
|
29
|
+
self.text = text
|
|
30
|
+
self.end_lineno = lineno if end_lineno is None else end_lineno
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class Mapping:
|
|
34
|
+
__slots__ = ("lineno", "items")
|
|
35
|
+
|
|
36
|
+
def __init__(self, lineno: int) -> None:
|
|
37
|
+
self.lineno = lineno
|
|
38
|
+
self.items: List[Tuple[str, int, Any]] = []
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class Sequence:
|
|
42
|
+
__slots__ = ("lineno", "items")
|
|
43
|
+
|
|
44
|
+
def __init__(self, lineno: int) -> None:
|
|
45
|
+
self.lineno = lineno
|
|
46
|
+
self.items: List[Tuple[int, Any]] = []
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
_BLOCK_SCALARS = frozenset({"|", "|-", "|+", ">", ">-", ">+"})
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _strip_comment(text: str) -> str:
|
|
53
|
+
"""Drop a trailing ``#`` comment that is outside quotes."""
|
|
54
|
+
quote = ""
|
|
55
|
+
for index, char in enumerate(text):
|
|
56
|
+
if quote:
|
|
57
|
+
if char == quote:
|
|
58
|
+
quote = ""
|
|
59
|
+
elif char in "'\"":
|
|
60
|
+
quote = char
|
|
61
|
+
elif char == "#" and (index == 0 or text[index - 1] in " \t"):
|
|
62
|
+
return text[:index].rstrip()
|
|
63
|
+
return text.rstrip()
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _unquote(text: str) -> str:
|
|
67
|
+
text = text.strip()
|
|
68
|
+
if len(text) >= 2 and text[0] == text[-1] and text[0] in "'\"":
|
|
69
|
+
return text[1:-1]
|
|
70
|
+
return text
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _lines(source: str) -> List[Tuple[int, int, str, str]]:
|
|
74
|
+
"""[(lineno, indent, content, raw)] for every non-blank, non-comment line."""
|
|
75
|
+
out: List[Tuple[int, int, str, str]] = []
|
|
76
|
+
for lineno, raw in enumerate(source.splitlines(), start=1):
|
|
77
|
+
stripped = raw.strip()
|
|
78
|
+
if not stripped or stripped.startswith("#"):
|
|
79
|
+
continue
|
|
80
|
+
if stripped in ("---", "..."):
|
|
81
|
+
continue
|
|
82
|
+
indent = len(raw) - len(raw.lstrip(" "))
|
|
83
|
+
out.append((lineno, indent, _strip_comment(stripped), raw))
|
|
84
|
+
return out
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class _Reader:
|
|
88
|
+
def __init__(self, source: str) -> None:
|
|
89
|
+
self.raw = source.splitlines()
|
|
90
|
+
self.lines = _lines(source)
|
|
91
|
+
self.index = 0
|
|
92
|
+
|
|
93
|
+
def peek(self) -> Optional[Tuple[int, int, str, str]]:
|
|
94
|
+
return self.lines[self.index] if self.index < len(self.lines) else None
|
|
95
|
+
|
|
96
|
+
def block_scalar(self, header_indent: int, style: str) -> Scalar:
|
|
97
|
+
"""Consume the indented body of a ``|``/``>`` scalar, keeping its own line numbers."""
|
|
98
|
+
start = self.index
|
|
99
|
+
body: List[str] = []
|
|
100
|
+
body_indent: Optional[int] = None
|
|
101
|
+
last_lineno = self.lines[start][0] if start < len(self.lines) else 1
|
|
102
|
+
while self.index < len(self.lines):
|
|
103
|
+
lineno, indent, _content, raw = self.lines[self.index]
|
|
104
|
+
if indent <= header_indent:
|
|
105
|
+
break
|
|
106
|
+
if body_indent is None:
|
|
107
|
+
body_indent = indent
|
|
108
|
+
body.append(raw[body_indent:] if len(raw) >= body_indent else raw.strip())
|
|
109
|
+
last_lineno = lineno
|
|
110
|
+
self.index += 1
|
|
111
|
+
if not body:
|
|
112
|
+
return Scalar(last_lineno, "")
|
|
113
|
+
text = "\n".join(body) if style.startswith("|") else " ".join(part.strip() for part in body)
|
|
114
|
+
if style.endswith("-"):
|
|
115
|
+
text = text.rstrip("\n")
|
|
116
|
+
return Scalar(self.lines[start][0], text, last_lineno)
|
|
117
|
+
|
|
118
|
+
def parse_block(self, indent: int) -> Any:
|
|
119
|
+
entry = self.peek()
|
|
120
|
+
if entry is None or entry[1] < indent:
|
|
121
|
+
return Scalar(entry[0] if entry else 1, "")
|
|
122
|
+
if entry[2].startswith("- ") or entry[2] == "-":
|
|
123
|
+
return self.parse_sequence(entry[1])
|
|
124
|
+
return self.parse_mapping(entry[1])
|
|
125
|
+
|
|
126
|
+
def parse_sequence(self, indent: int) -> Sequence:
|
|
127
|
+
node = Sequence(self.peek()[0])
|
|
128
|
+
while True:
|
|
129
|
+
entry = self.peek()
|
|
130
|
+
if entry is None or entry[1] != indent or not (entry[2].startswith("- ") or entry[2] == "-"):
|
|
131
|
+
break
|
|
132
|
+
lineno, _indent, content, _raw = entry
|
|
133
|
+
self.index += 1
|
|
134
|
+
remainder = content[1:].strip()
|
|
135
|
+
if not remainder:
|
|
136
|
+
child = self.parse_block(indent + 1)
|
|
137
|
+
node.items.append((lineno, child))
|
|
138
|
+
continue
|
|
139
|
+
item_indent = indent + (len(content) - len(content[1:].lstrip()))
|
|
140
|
+
if ":" in remainder and not remainder.startswith(("'", '"')):
|
|
141
|
+
key, _, value = remainder.partition(":")
|
|
142
|
+
mapping = Mapping(lineno)
|
|
143
|
+
self.fill_entry(mapping, key.strip(), value.strip(), lineno, item_indent)
|
|
144
|
+
while True:
|
|
145
|
+
nxt = self.peek()
|
|
146
|
+
if nxt is None or nxt[1] != item_indent or nxt[2].startswith("- "):
|
|
147
|
+
break
|
|
148
|
+
self.index += 1
|
|
149
|
+
k, _, v = nxt[2].partition(":")
|
|
150
|
+
self.fill_entry(mapping, k.strip(), v.strip(), nxt[0], item_indent)
|
|
151
|
+
node.items.append((lineno, mapping))
|
|
152
|
+
else:
|
|
153
|
+
node.items.append((lineno, Scalar(lineno, _unquote(remainder))))
|
|
154
|
+
return node
|
|
155
|
+
|
|
156
|
+
def fill_entry(self, mapping: Mapping, key: str, value: str, lineno: int, indent: int) -> None:
|
|
157
|
+
key = _unquote(key)
|
|
158
|
+
if value in _BLOCK_SCALARS:
|
|
159
|
+
mapping.items.append((key, lineno, self.block_scalar(indent, value)))
|
|
160
|
+
return
|
|
161
|
+
if value:
|
|
162
|
+
mapping.items.append((key, lineno, Scalar(lineno, _unquote(value))))
|
|
163
|
+
return
|
|
164
|
+
nxt = self.peek()
|
|
165
|
+
if nxt is not None and nxt[1] > indent:
|
|
166
|
+
mapping.items.append((key, lineno, self.parse_block(nxt[1])))
|
|
167
|
+
elif nxt is not None and nxt[1] == indent and (nxt[2].startswith("- ") or nxt[2] == "-"):
|
|
168
|
+
mapping.items.append((key, lineno, self.parse_sequence(indent)))
|
|
169
|
+
else:
|
|
170
|
+
mapping.items.append((key, lineno, Scalar(lineno, "")))
|
|
171
|
+
|
|
172
|
+
def parse_mapping(self, indent: int) -> Mapping:
|
|
173
|
+
node = Mapping(self.peek()[0])
|
|
174
|
+
while True:
|
|
175
|
+
entry = self.peek()
|
|
176
|
+
if entry is None or entry[1] != indent or entry[2].startswith("- "):
|
|
177
|
+
break
|
|
178
|
+
if ":" not in entry[2]:
|
|
179
|
+
break
|
|
180
|
+
self.index += 1
|
|
181
|
+
key, _, value = entry[2].partition(":")
|
|
182
|
+
self.fill_entry(node, key.strip(), value.strip(), entry[0], indent)
|
|
183
|
+
return node
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def parse(source: str) -> Any:
|
|
187
|
+
"""Parse the first document of ``source`` into Mapping/Sequence/Scalar nodes."""
|
|
188
|
+
reader = _Reader(source)
|
|
189
|
+
if not reader.lines:
|
|
190
|
+
return Scalar(1, "")
|
|
191
|
+
return reader.parse_block(reader.lines[0][1])
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def get(node: Any, key: str) -> Optional[Any]:
|
|
195
|
+
"""The value node under ``key`` in a mapping, or None. Last write wins, as YAML does."""
|
|
196
|
+
if not isinstance(node, Mapping):
|
|
197
|
+
return None
|
|
198
|
+
found = None
|
|
199
|
+
for name, _lineno, value in node.items:
|
|
200
|
+
if name == key:
|
|
201
|
+
found = value
|
|
202
|
+
return found
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def scalar_text(node: Any) -> str:
|
|
206
|
+
return node.text if isinstance(node, Scalar) else ""
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def walk(node: Any, path: Tuple[str, ...] = ()) -> Iterator[Tuple[Tuple[str, ...], int, Any]]:
|
|
210
|
+
"""Yield ``(key_path, lineno, node)`` once per node, depth first, in source order."""
|
|
211
|
+
yield path, getattr(node, "lineno", 1), node
|
|
212
|
+
if isinstance(node, Mapping):
|
|
213
|
+
for key, _lineno, value in node.items:
|
|
214
|
+
yield from walk(value, path + (key,))
|
|
215
|
+
elif isinstance(node, Sequence):
|
|
216
|
+
for _lineno, value in node.items:
|
|
217
|
+
yield from walk(value, path + ("[]",))
|