timecs 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. app.py +3014 -0
  2. ast_scanner.py +36187 -0
  3. banner.py +146 -0
  4. ci_reporter.py +570 -0
  5. cli.py +2347 -0
  6. config_loader.py +307 -0
  7. cross_file_engine.py +1781 -0
  8. delivery_models.py +145 -0
  9. delivery_repository.py +78 -0
  10. diff_scan.py +127 -0
  11. durable_delivery_sink.py +103 -0
  12. event_bus.py +155 -0
  13. event_publisher.py +345 -0
  14. event_taxonomy.py +313 -0
  15. function_summarizer.py +354 -0
  16. html_auditor.py +274 -0
  17. iac_auditor.py +400 -0
  18. in_app_adapter.py +115 -0
  19. js_scanner.py +895 -0
  20. manifest_parser.py +960 -0
  21. master_rules_bank.py +389 -0
  22. notification_models.py +346 -0
  23. notification_policy.py +309 -0
  24. notification_repository.py +118 -0
  25. notification_routes.py +394 -0
  26. notification_service.py +199 -0
  27. osv_client.py +437 -0
  28. remediation/__init__.py +39 -0
  29. remediation/base.py +102 -0
  30. remediation/contracts.py +196 -0
  31. remediation/cwe1188_binding.py +78 -0
  32. remediation/cwe1336_jinja.py +91 -0
  33. remediation/cwe22_path_traversal.py +348 -0
  34. remediation/cwe295_ssl.py +87 -0
  35. remediation/cwe327_weak_hash.py +78 -0
  36. remediation/cwe377_tempfile.py +77 -0
  37. remediation/cwe489_debug.py +80 -0
  38. remediation/cwe502_yaml.py +89 -0
  39. remediation/cwe614_cookie.py +91 -0
  40. remediation/cwe78_cmdi.py +415 -0
  41. remediation/cwe798_credentials.py +153 -0
  42. remediation/cwe89_sqli.py +425 -0
  43. remediation/cwe916_password_hash.py +131 -0
  44. remediation/cwe95_eval.py +77 -0
  45. remediation/dispatcher.py +114 -0
  46. remediation/orchestrator.py +541 -0
  47. remediation/patch_engine.py +213 -0
  48. remediation/verification.py +253 -0
  49. rule_engine.py +1535 -0
  50. sarif_adapter.py +719 -0
  51. sarif_exporter.py +88 -0
  52. sca_reachability/__init__.py +28 -0
  53. sca_reachability/ast_analyzer.py +366 -0
  54. sca_reachability/call_graph.py +203 -0
  55. sca_reachability/contracts.py +365 -0
  56. sca_reachability/dist_import_map.py +46 -0
  57. sca_reachability/engine.py +603 -0
  58. sca_reachability/resolver.py +169 -0
  59. scanners/__init__.py +70 -0
  60. scanners/container_scanner.py +323 -0
  61. scanners/cve_index.py +521 -0
  62. scanners/iac_scanner.py +328 -0
  63. scanners/sbom.py +183 -0
  64. scanners/sca_scanner.py +667 -0
  65. scanners/secret_scanner.py +345 -0
  66. scanners/yaml_lite.py +217 -0
  67. secret_filters.py +435 -0
  68. secret_scanner.py +943 -0
  69. sqlite_delivery_repository.py +277 -0
  70. sqlite_notification_repository.py +441 -0
  71. ssrf_guard.py +276 -0
  72. staged_scanner.py +492 -0
  73. suppression_resolver.py +282 -0
  74. symbol_indexer.py +424 -0
  75. tcs/__init__.py +1 -0
  76. tcs/analysis/__init__.py +1 -0
  77. tcs/analysis/def_use.py +408 -0
  78. tcs/analysis/function_contracts.py +290 -0
  79. tcs/parallel_analyzer.py +276 -0
  80. tcs/parallel_scanner.py +630 -0
  81. tcs/treesitter_packer.py +1446 -0
  82. tcs_cli.py +2086 -0
  83. tcs_config.py +218 -0
  84. tcs_gui.py +2123 -0
  85. timecs-2.0.0.dist-info/METADATA +142 -0
  86. timecs-2.0.0.dist-info/RECORD +94 -0
  87. timecs-2.0.0.dist-info/WHEEL +5 -0
  88. timecs-2.0.0.dist-info/entry_points.txt +4 -0
  89. timecs-2.0.0.dist-info/licenses/LICENSE +201 -0
  90. timecs-2.0.0.dist-info/top_level.txt +49 -0
  91. version_matcher.py +934 -0
  92. webhook_adapter.py +495 -0
  93. webhook_client.py +483 -0
  94. webhook_formatters.py +245 -0
@@ -0,0 +1,345 @@
1
+ """Hard-coded secrets in configuration, environment and shell documents -- stdlib only.
2
+
3
+ The Python engine already owns hard-coded credentials inside `.py` source. This reader owns the
4
+ documents around it: `.env`, `*.ini`, `*.cfg`, `*.conf`, `*.toml`, `*.json`, `*.yaml`, `*.yml` and
5
+ `*.sh`, where a credential is not an expression the AST can reason about but a literal sitting in
6
+ a config file that ships with the image.
7
+
8
+ Precision is the whole design. Two families are detected:
9
+
10
+ * Vendor shapes -- an AWS key ID, a GitHub token, a Stripe live key, a Slack token or webhook, a
11
+ PEM private-key header, a service URL carrying a password. These are recognisable on their own,
12
+ so no entropy argument is needed: the prefix IS the proof.
13
+ * Generic sensitive assignments -- `API_KEY`, `SECRET_KEY`, `AUTH_TOKEN`, `ACCESS_TOKEN`,
14
+ `CLIENT_SECRET`, `DB_PASSWORD`, `PRIVATE_KEY` (also under a vendor prefix, e.g.
15
+ `STRIPE_SECRET_KEY`). A name alone is not a secret, so the value must additionally be a literal
16
+ of at least 16 characters spanning at least two character classes with Shannon entropy >= 3.3.
17
+
18
+ Everything else is refused, and the refusals are the reason this reader can be loud by default:
19
+ shell interpolation (`${VAR}`, `$VAR`), empty and boolean/null literals, a 40-character hex string
20
+ with no credential name (a git commit SHA), and any value carrying a placeholder word. A template
21
+ or example document is never read at all -- `.env.example` exists to hold fake values.
22
+
23
+ Statistics alone cannot tell a credential from a sentence about a credential, so a generic
24
+ assignment must also pass a lexical test (`_is_literal_credential`): it may carry no space or tab,
25
+ and it may not be wrapped in a macro or reference marker. The bug this closes is Home Assistant's
26
+ `strings.json` localisation catalogues, whose UI labels sit under real sensitive names and clear
27
+ every statistical floor -- `"access_token": "Personal Access Token (PAT)"` measures 3.78 bits per
28
+ character over 27, and `"api_key": "[%key:common::config_flow::data::api_key%]"` (4.14 over 42) is a
29
+ reference to a string stored elsewhere, not the string itself. Both gates apply to the generic
30
+ branch only: a PEM header contains spaces by definition, and a URL password may legitimately carry
31
+ one, so those families keep their own proof and gain no new way to go quiet.
32
+
33
+ The matched secret is never copied into a finding. Reports, SARIF and logs are shared artefacts;
34
+ a scanner that re-publishes the credential it found has moved the leak rather than reported it.
35
+ """
36
+
37
+ from __future__ import annotations
38
+
39
+ import math
40
+ import re
41
+ from typing import Any, Dict, Iterator, List, Optional, Tuple
42
+
43
+ __all__ = ["is_secret_path", "audit_secret_files", "CATEGORY"]
44
+
45
+ CATEGORY = "HARDCODED_SECRET_CONFIG"
46
+
47
+ CONFIG_SUFFIXES = (".ini", ".cfg", ".conf", ".toml", ".json", ".yaml", ".yml", ".sh")
48
+ # A lockfile is a resolved dependency graph, not a credential store, and the SCA scanner owns it.
49
+ LOCKFILE_NAMES = frozenset({
50
+ "package-lock.json", "npm-shrinkwrap.json", "yarn.lock", "poetry.lock", "pipfile.lock",
51
+ "cargo.lock", "gemfile.lock", "composer.lock", "pnpm-lock.yaml",
52
+ })
53
+ # These documents exist to carry fake values. Reading them would report the documentation.
54
+ TEMPLATE_SUFFIXES = (".example", ".sample", ".template")
55
+
56
+ # (label, pattern, severity). The pattern matches the credential itself, not the assignment.
57
+ VENDOR_PATTERNS: Tuple[Tuple[str, "re.Pattern[str]", str], ...] = (
58
+ ("AWS Access Key ID", re.compile(r"\bAKIA[0-9A-Z]{16}\b"), "HIGH"),
59
+ ("GitHub Personal Access Token", re.compile(r"\bghp_[A-Za-z0-9]{36}\b"), "HIGH"),
60
+ ("GitHub Fine-Grained Token", re.compile(r"\bgithub_pat_[A-Za-z0-9_]{22,}"), "HIGH"),
61
+ ("GitHub OAuth Token", re.compile(r"\bgho_[A-Za-z0-9]{36}\b"), "HIGH"),
62
+ ("Stripe Live Secret Key", re.compile(r"\bsk_live_[0-9a-zA-Z]{24,}"), "CRITICAL"),
63
+ ("Slack Token", re.compile(r"\bxox[baprs]-[0-9A-Za-z-]{10,}"), "HIGH"),
64
+ ("Slack Incoming Webhook",
65
+ re.compile(r"https://hooks\.slack\.com/services/T[A-Z0-9]+/B[A-Z0-9]+/[A-Za-z0-9]+"), "HIGH"),
66
+ ("PEM Private Key",
67
+ re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY-----"), "CRITICAL"),
68
+ )
69
+
70
+ SERVICE_URL = re.compile(
71
+ r"(?:postgres|postgresql|mysql|mongodb|redis|amqp)://[^:\s]+:([^@\s]{6,})@[^/\s]+")
72
+
73
+ SENSITIVE_NAMES = ("API_KEY", "SECRET_KEY", "AUTH_TOKEN", "ACCESS_TOKEN",
74
+ "CLIENT_SECRET", "DB_PASSWORD", "PRIVATE_KEY")
75
+ # One assignment shape covers dotenv/ini/shell (`K=v`), YAML (`K: v`), TOML (`K = v`) and
76
+ # JSON (`"K": "v"`): the optional closing quote after the key is what makes JSON fit.
77
+ ASSIGNMENT = re.compile(
78
+ r"""(?P<key>[A-Za-z_][A-Za-z0-9_.\-]*)["']?\s*[:=]\s*"""
79
+ r"""(?P<value>"[^"]*"|'[^']*'|[^\s,;#}\]]+)""")
80
+
81
+ PLACEHOLDER_WORDS = ("example", "changeme", "placeholder", "dummy", "test", "sample",
82
+ "your_", "xxxx", "123456", "password", "secret")
83
+ # Four tokens that say what a value is *for* instead of carrying one. Kept apart from
84
+ # PLACEHOLDER_WORDS so this reader's newer refusal can be measured without moving the older one.
85
+ NON_SECRET_PLACEHOLDER_TOKENS = ("temporary_key", "replace_me", "not_found", "not_a_real_key")
86
+ # `user:user@localhost` proves a developer default rather than a deployed credential: the secret and
87
+ # its own username are the same dictionary word, and the host is this machine, so no deployment can
88
+ # be relying on it. Airflow ships exactly this pair in `default_airflow.cfg`, `default_test.cfg` and
89
+ # the documentation template `config.yml`, all three of which describe a broker URL rather than
90
+ # configure one.
91
+ LOCAL_SERVICE_HOSTS = frozenset({"localhost", "127.0.0.1", "0.0.0.0", "::1"})
92
+ # A document whose own name declares it a test fixture or the shipped default of a product. Only the
93
+ # generic-assignment branch consults this: a vendor token, a PEM block or a service URL proves itself
94
+ # by shape and cannot be silenced by the name of the file it sits in.
95
+ NON_PRODUCTION_CONFIG_SUFFIXES = ("_test.cfg", "_test.ini", "_test.conf")
96
+ NON_PRODUCTION_CONFIG_PREFIXES = ("test_", "default_test.")
97
+ NON_PRODUCTION_CONFIG_EXTENSIONS = (".cfg", ".ini", ".conf", ".yml", ".yaml", ".env", ".properties")
98
+ NULL_LITERALS = frozenset({"null", "none", "true", "false", "~", "nil", "undefined"})
99
+ INTERPOLATION = re.compile(r"\$(?:\{[^}]*\}|[A-Za-z_][A-Za-z0-9_]*)")
100
+ COMMIT_SHA = re.compile(r"[0-9a-fA-F]{40}")
101
+ MIN_GENERIC_LENGTH = 16
102
+ MIN_GENERIC_ENTROPY = 3.3
103
+ # The lexical floor for a *literal* credential, sitting under a sensitive name. Prose says what a
104
+ # credential is; a macro reference says where one lives. Neither is a secret, and no length,
105
+ # character-class or entropy threshold can separate them from one -- only shape can.
106
+ PROSE_CHARACTERS = (" ", "\t")
107
+ # A filesystem path says where a credential is kept, not what it is. `private_key = /etc/salt/pki/
108
+ # minion.pem` and `api_key: ~/credentials.json` clear every statistical floor -- a path is long, spans
109
+ # several character classes and is high-entropy by construction -- while shipping no secret at all; the
110
+ # file it names is the artefact an operator protects and rotates.
111
+ CREDENTIAL_PATH_PREFIXES = ("/", "./", "../", "~/")
112
+ CREDENTIAL_PATH_SUFFIXES = (".pem", ".key", ".crt", ".cer", ".p12", ".pfx", ".pub", ".json",
113
+ ".yaml", ".yml", ".conf", ".cfg", ".ini", ".env")
114
+ WINDOWS_DRIVE_PATH = re.compile(r"[A-Za-z]:[\\/].*")
115
+ # The closing marker is optional in every pattern below because ASSIGNMENT's unquoted alternative
116
+ # stops before `]`, `}` and `)`: `api_key: [%key:common::config_flow::data::api_key%]` arrives here
117
+ # as `[%key:common::config_flow::data::api_key%`. A reference is a reference either way.
118
+ REFERENCE_WRAPPERS = (
119
+ # Home Assistant localisation catalogues (and Jinja/Template Toolkit references): the whole
120
+ # value is `[%key:section::name%]`, a pointer into a string table.
121
+ re.compile(r"\[\%.*\%\]?"),
122
+ re.compile(r"\{\{.*\}\}?"), # {{ expression }}
123
+ re.compile(r"\{%.*%\}?"), # {% statement %}
124
+ re.compile(r"\$\{.*\}?"), # ${VAR}
125
+ re.compile(r"\$\(.*\)?"), # $(command)
126
+ re.compile(r"\<.*\>?"), # <value supplied somewhere else>
127
+ )
128
+
129
+
130
+ def _is_literal_credential(value: str) -> bool:
131
+ """False when the value is prose, a pointer, or a filesystem path rather than a secret itself.
132
+
133
+ Gated on the generic-assignment branch only, by design: `-----BEGIN RSA PRIVATE KEY-----`
134
+ carries spaces and `VENDOR_PATTERNS` / `SERVICE_URL` prove their own shapes, so this rule must
135
+ never be able to silence them. A generic assignment is the one place where the name, not the
136
+ value, did the identifying -- which is exactly where a UI label can impersonate a credential.
137
+ """
138
+ if not value:
139
+ return False
140
+ if any(char in value for char in PROSE_CHARACTERS):
141
+ return False
142
+ if any(pattern.fullmatch(value) for pattern in REFERENCE_WRAPPERS):
143
+ return False
144
+ if value.startswith(CREDENTIAL_PATH_PREFIXES) or WINDOWS_DRIVE_PATH.fullmatch(value):
145
+ return False
146
+ if value.lower().endswith(CREDENTIAL_PATH_SUFFIXES):
147
+ return False
148
+ return True
149
+
150
+
151
+ def _service_host(authority: str) -> str:
152
+ """The bare host of an already-split `host` or `host:port`, IPv6 included."""
153
+ host = authority.lower()
154
+ if host.startswith("["):
155
+ return host[1:].split("]", 1)[0]
156
+ head, separator, tail = host.rpartition(":")
157
+ if separator and tail.isdigit():
158
+ return head
159
+ return host
160
+
161
+
162
+ def _service_credential_is_dev_default(url: str) -> bool:
163
+ """True for `scheme://user:user@localhost/db`: the password is the username, on this machine.
164
+
165
+ `sqla+mysql://airflow:airflow@localhost:3306/airflow` is a worked example of a broker URL, not a
166
+ credential a deployment relies on -- and a deployment that really does run it that way is guarding
167
+ the word `airflow`, which is not a secret worth rotating. `postgres://app:S3cr3tP4ss@db.internal
168
+ /prod` fails both halves, so a production service URL cannot be silenced here.
169
+ """
170
+ _scheme, separator, rest = url.partition("://")
171
+ if not separator:
172
+ return False
173
+ authority = rest.split("/", 1)[0]
174
+ userinfo, at, host = authority.rpartition("@")
175
+ if not at:
176
+ return False
177
+ user, colon, password = userinfo.partition(":")
178
+ if not colon or user != password:
179
+ return False
180
+ return _service_host(host) in LOCAL_SERVICE_HOSTS
181
+
182
+
183
+ def _is_non_production_document(path: str) -> bool:
184
+ """True when a config document's own name says it is a test fixture or a shipped default.
185
+
186
+ Scoped to the generic-assignment branch by the caller: those are the hits identified by the key
187
+ name alone, which is exactly where a template's `api_key = REPLACE_ME_NOW` can impersonate a
188
+ credential. A value that proves its own shape is never reached by this rule.
189
+ """
190
+ name = str(path).replace("\\", "/").rsplit("/", 1)[-1].lower()
191
+ if name.endswith(NON_PRODUCTION_CONFIG_SUFFIXES):
192
+ return True
193
+ if name.startswith(NON_PRODUCTION_CONFIG_PREFIXES):
194
+ return name.endswith(NON_PRODUCTION_CONFIG_EXTENSIONS)
195
+ return False
196
+
197
+
198
+ def is_secret_path(path: str) -> bool:
199
+ """True for a config/env/shell document this reader owns, by name alone."""
200
+ name = str(path).replace("\\", "/").rsplit("/", 1)[-1].lower()
201
+ if name in LOCKFILE_NAMES or name.endswith(TEMPLATE_SUFFIXES):
202
+ return False
203
+ if name == ".env" or name.startswith(".env.") or name.endswith(".env"):
204
+ return True
205
+ return name.endswith(CONFIG_SUFFIXES)
206
+
207
+
208
+ def shannon_entropy(text: str) -> float:
209
+ """Bits per character. A repeated or single-class string cannot reach the generic floor."""
210
+ if not text:
211
+ return 0.0
212
+ counts: Dict[str, int] = {}
213
+ for char in text:
214
+ counts[char] = counts.get(char, 0) + 1
215
+ total = len(text)
216
+ return -sum((count / total) * math.log2(count / total) for count in counts.values())
217
+
218
+
219
+ def _character_classes(text: str) -> int:
220
+ present = 0
221
+ for probe in (str.islower, str.isupper, str.isdigit):
222
+ if any(probe(char) for char in text):
223
+ present += 1
224
+ if any(not char.isalnum() for char in text):
225
+ present += 1
226
+ return present
227
+
228
+
229
+ def _unquote(value: str) -> str:
230
+ text = value.strip()
231
+ if len(text) >= 2 and text[0] == text[-1] and text[0] in "'\"":
232
+ text = text[1:-1]
233
+ return text.strip()
234
+
235
+
236
+ def is_refused(value: str, named: bool = False) -> bool:
237
+ """The zero-FP contract: a value that is not a credential, whatever its shape.
238
+
239
+ `named` says a credential name or a password position already identified this value, which is
240
+ what separates a 40-hex secret from a 40-hex git commit SHA.
241
+ """
242
+ text = _unquote(value)
243
+ if not text:
244
+ return True
245
+ lowered = text.lower()
246
+ if lowered in NULL_LITERALS:
247
+ return True
248
+ if INTERPOLATION.search(text):
249
+ return True
250
+ if any(word in lowered for word in PLACEHOLDER_WORDS):
251
+ return True
252
+ if any(token in lowered for token in NON_SECRET_PLACEHOLDER_TOKENS):
253
+ return True
254
+ if not named and len(text) == 40 and COMMIT_SHA.fullmatch(text):
255
+ return True
256
+ return False
257
+
258
+
259
+ def _sensitive_name(key: str) -> Optional[str]:
260
+ """The sensitive name a config key carries, exactly or under a vendor prefix."""
261
+ normalized = key.upper().replace("-", "_").replace(".", "_")
262
+ for name in SENSITIVE_NAMES:
263
+ if normalized == name or normalized.endswith("_" + name):
264
+ return name
265
+ return None
266
+
267
+
268
+ def _redact(text: str) -> str:
269
+ """Enough to locate the credential in the file, never enough to use it."""
270
+ if len(text) <= 8:
271
+ return "*" * len(text)
272
+ return text[:4] + "*" * 8 + text[-2:]
273
+
274
+
275
+ def _iter_hits(source: str) -> Iterator[Tuple[int, str, str, str, str]]:
276
+ """(lineno, label, severity, redacted, kind) for every credential the line proves."""
277
+ for lineno, line in enumerate(source.splitlines(), start=1):
278
+ if not line.strip():
279
+ continue
280
+ for label, pattern, severity in VENDOR_PATTERNS:
281
+ match = pattern.search(line)
282
+ if match and not is_refused(match.group(0), named=True):
283
+ yield lineno, label, severity, _redact(match.group(0)), "vendor"
284
+ for match in SERVICE_URL.finditer(line):
285
+ if is_refused(match.group(1), named=True):
286
+ continue
287
+ if _service_credential_is_dev_default(match.group(0)):
288
+ continue
289
+ yield (lineno, "Service URL With Embedded Password", "HIGH",
290
+ _redact(match.group(1)), "service_url")
291
+ for match in ASSIGNMENT.finditer(line):
292
+ name = _sensitive_name(match.group("key"))
293
+ if name is None:
294
+ continue
295
+ value = _unquote(match.group("value"))
296
+ if is_refused(value, named=True):
297
+ continue
298
+ if not _is_literal_credential(value):
299
+ continue
300
+ if len(value) < MIN_GENERIC_LENGTH or _character_classes(value) < 2:
301
+ continue
302
+ if shannon_entropy(value) < MIN_GENERIC_ENTROPY:
303
+ continue
304
+ yield (lineno, f"Hard-coded {name}", "HIGH", _redact(value), "generic")
305
+
306
+
307
+ def audit_secret_files(files: Dict[str, str]) -> List[Dict[str, Any]]:
308
+ """One finding per (document, line, credential family), in source order."""
309
+ findings: List[Dict[str, Any]] = []
310
+ for file_path in sorted(files):
311
+ # The refusal is enforced here as well as at discovery, so a caller that hands over a
312
+ # lockfile or a `.env.example` cannot make this reader report the documentation.
313
+ if not is_secret_path(file_path):
314
+ continue
315
+ seen = set()
316
+ for lineno, label, severity, redacted, kind in _iter_hits(files[file_path]):
317
+ # A document that names itself a test fixture or the shipped default carries no deployed
318
+ # credential, and the key name is the *only* thing that identified a generic hit -- so
319
+ # `api_key = REPLACE_WITH_YOUR_KEY` in `service_test.cfg` is the one place the document's
320
+ # own name is evidence. Vendor tokens and service URLs prove their shape and never reach
321
+ # this clause.
322
+ if kind == "generic" and _is_non_production_document(file_path):
323
+ continue
324
+ # A line can satisfy two shapes at once -- a service URL whose password is also a
325
+ # sensitive assignment. One credential is one finding, so the first shape wins.
326
+ key = (lineno, label)
327
+ if key in seen:
328
+ continue
329
+ seen.add(key)
330
+ findings.append({
331
+ "file": file_path,
332
+ "line": lineno,
333
+ "cwe": "CWE-798",
334
+ "severity": severity,
335
+ "category": CATEGORY,
336
+ "confidence": "HIGH",
337
+ "secret_type": label,
338
+ "secret_kind": kind,
339
+ "evidence": redacted,
340
+ "message": (f"{label} hard-coded in {file_path}:{lineno} "
341
+ f"(value {redacted}). Move it to a secret manager or an environment "
342
+ f"variable supplied at runtime, and rotate it: a credential committed "
343
+ f"to a config file is already exposed."),
344
+ })
345
+ return sorted(findings, key=lambda item: (item["file"], item["line"], item["secret_type"]))
scanners/yaml_lite.py ADDED
@@ -0,0 +1,217 @@
1
+ """A streaming YAML subset reader for security auditing -- stdlib only, no PyYAML.
2
+
3
+ The container and IaC rules need three things from a YAML document: the key path a value sits at,
4
+ the line it sits on, and its literal text. They do not need anchors, tags, merge keys, flow
5
+ collections beyond the empty ``{}``/``[]``, or multi-document streams. This reader implements
6
+ exactly that subset and refuses to guess at the rest: a construct it cannot represent becomes a
7
+ scalar holding the raw text, so a rule reading it sees an unresolvable value and stays quiet
8
+ rather than inventing a structure.
9
+
10
+ Keys are kept as the literal source spelling. PyYAML resolves a bare ``on:`` to the boolean
11
+ ``True`` (YAML 1.1), which silently loses every GitHub Actions trigger block; here ``on`` stays
12
+ ``on`` because the auditor asks about the document, not about a YAML 1.1 type system.
13
+
14
+ Memory is one line list plus one node per entry -- no whole-file regex sweep, no dependency.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ from typing import Any, Iterator, List, Optional, Tuple
20
+
21
+ __all__ = ["Scalar", "Mapping", "Sequence", "parse", "get", "walk", "scalar_text"]
22
+
23
+
24
+ class Scalar:
25
+ __slots__ = ("lineno", "text", "end_lineno")
26
+
27
+ def __init__(self, lineno: int, text: str, end_lineno: Optional[int] = None) -> None:
28
+ self.lineno = lineno
29
+ self.text = text
30
+ self.end_lineno = lineno if end_lineno is None else end_lineno
31
+
32
+
33
+ class Mapping:
34
+ __slots__ = ("lineno", "items")
35
+
36
+ def __init__(self, lineno: int) -> None:
37
+ self.lineno = lineno
38
+ self.items: List[Tuple[str, int, Any]] = []
39
+
40
+
41
+ class Sequence:
42
+ __slots__ = ("lineno", "items")
43
+
44
+ def __init__(self, lineno: int) -> None:
45
+ self.lineno = lineno
46
+ self.items: List[Tuple[int, Any]] = []
47
+
48
+
49
+ _BLOCK_SCALARS = frozenset({"|", "|-", "|+", ">", ">-", ">+"})
50
+
51
+
52
+ def _strip_comment(text: str) -> str:
53
+ """Drop a trailing ``#`` comment that is outside quotes."""
54
+ quote = ""
55
+ for index, char in enumerate(text):
56
+ if quote:
57
+ if char == quote:
58
+ quote = ""
59
+ elif char in "'\"":
60
+ quote = char
61
+ elif char == "#" and (index == 0 or text[index - 1] in " \t"):
62
+ return text[:index].rstrip()
63
+ return text.rstrip()
64
+
65
+
66
+ def _unquote(text: str) -> str:
67
+ text = text.strip()
68
+ if len(text) >= 2 and text[0] == text[-1] and text[0] in "'\"":
69
+ return text[1:-1]
70
+ return text
71
+
72
+
73
+ def _lines(source: str) -> List[Tuple[int, int, str, str]]:
74
+ """[(lineno, indent, content, raw)] for every non-blank, non-comment line."""
75
+ out: List[Tuple[int, int, str, str]] = []
76
+ for lineno, raw in enumerate(source.splitlines(), start=1):
77
+ stripped = raw.strip()
78
+ if not stripped or stripped.startswith("#"):
79
+ continue
80
+ if stripped in ("---", "..."):
81
+ continue
82
+ indent = len(raw) - len(raw.lstrip(" "))
83
+ out.append((lineno, indent, _strip_comment(stripped), raw))
84
+ return out
85
+
86
+
87
+ class _Reader:
88
+ def __init__(self, source: str) -> None:
89
+ self.raw = source.splitlines()
90
+ self.lines = _lines(source)
91
+ self.index = 0
92
+
93
+ def peek(self) -> Optional[Tuple[int, int, str, str]]:
94
+ return self.lines[self.index] if self.index < len(self.lines) else None
95
+
96
+ def block_scalar(self, header_indent: int, style: str) -> Scalar:
97
+ """Consume the indented body of a ``|``/``>`` scalar, keeping its own line numbers."""
98
+ start = self.index
99
+ body: List[str] = []
100
+ body_indent: Optional[int] = None
101
+ last_lineno = self.lines[start][0] if start < len(self.lines) else 1
102
+ while self.index < len(self.lines):
103
+ lineno, indent, _content, raw = self.lines[self.index]
104
+ if indent <= header_indent:
105
+ break
106
+ if body_indent is None:
107
+ body_indent = indent
108
+ body.append(raw[body_indent:] if len(raw) >= body_indent else raw.strip())
109
+ last_lineno = lineno
110
+ self.index += 1
111
+ if not body:
112
+ return Scalar(last_lineno, "")
113
+ text = "\n".join(body) if style.startswith("|") else " ".join(part.strip() for part in body)
114
+ if style.endswith("-"):
115
+ text = text.rstrip("\n")
116
+ return Scalar(self.lines[start][0], text, last_lineno)
117
+
118
+ def parse_block(self, indent: int) -> Any:
119
+ entry = self.peek()
120
+ if entry is None or entry[1] < indent:
121
+ return Scalar(entry[0] if entry else 1, "")
122
+ if entry[2].startswith("- ") or entry[2] == "-":
123
+ return self.parse_sequence(entry[1])
124
+ return self.parse_mapping(entry[1])
125
+
126
+ def parse_sequence(self, indent: int) -> Sequence:
127
+ node = Sequence(self.peek()[0])
128
+ while True:
129
+ entry = self.peek()
130
+ if entry is None or entry[1] != indent or not (entry[2].startswith("- ") or entry[2] == "-"):
131
+ break
132
+ lineno, _indent, content, _raw = entry
133
+ self.index += 1
134
+ remainder = content[1:].strip()
135
+ if not remainder:
136
+ child = self.parse_block(indent + 1)
137
+ node.items.append((lineno, child))
138
+ continue
139
+ item_indent = indent + (len(content) - len(content[1:].lstrip()))
140
+ if ":" in remainder and not remainder.startswith(("'", '"')):
141
+ key, _, value = remainder.partition(":")
142
+ mapping = Mapping(lineno)
143
+ self.fill_entry(mapping, key.strip(), value.strip(), lineno, item_indent)
144
+ while True:
145
+ nxt = self.peek()
146
+ if nxt is None or nxt[1] != item_indent or nxt[2].startswith("- "):
147
+ break
148
+ self.index += 1
149
+ k, _, v = nxt[2].partition(":")
150
+ self.fill_entry(mapping, k.strip(), v.strip(), nxt[0], item_indent)
151
+ node.items.append((lineno, mapping))
152
+ else:
153
+ node.items.append((lineno, Scalar(lineno, _unquote(remainder))))
154
+ return node
155
+
156
+ def fill_entry(self, mapping: Mapping, key: str, value: str, lineno: int, indent: int) -> None:
157
+ key = _unquote(key)
158
+ if value in _BLOCK_SCALARS:
159
+ mapping.items.append((key, lineno, self.block_scalar(indent, value)))
160
+ return
161
+ if value:
162
+ mapping.items.append((key, lineno, Scalar(lineno, _unquote(value))))
163
+ return
164
+ nxt = self.peek()
165
+ if nxt is not None and nxt[1] > indent:
166
+ mapping.items.append((key, lineno, self.parse_block(nxt[1])))
167
+ elif nxt is not None and nxt[1] == indent and (nxt[2].startswith("- ") or nxt[2] == "-"):
168
+ mapping.items.append((key, lineno, self.parse_sequence(indent)))
169
+ else:
170
+ mapping.items.append((key, lineno, Scalar(lineno, "")))
171
+
172
+ def parse_mapping(self, indent: int) -> Mapping:
173
+ node = Mapping(self.peek()[0])
174
+ while True:
175
+ entry = self.peek()
176
+ if entry is None or entry[1] != indent or entry[2].startswith("- "):
177
+ break
178
+ if ":" not in entry[2]:
179
+ break
180
+ self.index += 1
181
+ key, _, value = entry[2].partition(":")
182
+ self.fill_entry(node, key.strip(), value.strip(), entry[0], indent)
183
+ return node
184
+
185
+
186
+ def parse(source: str) -> Any:
187
+ """Parse the first document of ``source`` into Mapping/Sequence/Scalar nodes."""
188
+ reader = _Reader(source)
189
+ if not reader.lines:
190
+ return Scalar(1, "")
191
+ return reader.parse_block(reader.lines[0][1])
192
+
193
+
194
+ def get(node: Any, key: str) -> Optional[Any]:
195
+ """The value node under ``key`` in a mapping, or None. Last write wins, as YAML does."""
196
+ if not isinstance(node, Mapping):
197
+ return None
198
+ found = None
199
+ for name, _lineno, value in node.items:
200
+ if name == key:
201
+ found = value
202
+ return found
203
+
204
+
205
+ def scalar_text(node: Any) -> str:
206
+ return node.text if isinstance(node, Scalar) else ""
207
+
208
+
209
+ def walk(node: Any, path: Tuple[str, ...] = ()) -> Iterator[Tuple[Tuple[str, ...], int, Any]]:
210
+ """Yield ``(key_path, lineno, node)`` once per node, depth first, in source order."""
211
+ yield path, getattr(node, "lineno", 1), node
212
+ if isinstance(node, Mapping):
213
+ for key, _lineno, value in node.items:
214
+ yield from walk(value, path + (key,))
215
+ elif isinstance(node, Sequence):
216
+ for _lineno, value in node.items:
217
+ yield from walk(value, path + ("[]",))