timecs 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. app.py +3014 -0
  2. ast_scanner.py +36187 -0
  3. banner.py +146 -0
  4. ci_reporter.py +570 -0
  5. cli.py +2347 -0
  6. config_loader.py +307 -0
  7. cross_file_engine.py +1781 -0
  8. delivery_models.py +145 -0
  9. delivery_repository.py +78 -0
  10. diff_scan.py +127 -0
  11. durable_delivery_sink.py +103 -0
  12. event_bus.py +155 -0
  13. event_publisher.py +345 -0
  14. event_taxonomy.py +313 -0
  15. function_summarizer.py +354 -0
  16. html_auditor.py +274 -0
  17. iac_auditor.py +400 -0
  18. in_app_adapter.py +115 -0
  19. js_scanner.py +895 -0
  20. manifest_parser.py +960 -0
  21. master_rules_bank.py +389 -0
  22. notification_models.py +346 -0
  23. notification_policy.py +309 -0
  24. notification_repository.py +118 -0
  25. notification_routes.py +394 -0
  26. notification_service.py +199 -0
  27. osv_client.py +437 -0
  28. remediation/__init__.py +39 -0
  29. remediation/base.py +102 -0
  30. remediation/contracts.py +196 -0
  31. remediation/cwe1188_binding.py +78 -0
  32. remediation/cwe1336_jinja.py +91 -0
  33. remediation/cwe22_path_traversal.py +348 -0
  34. remediation/cwe295_ssl.py +87 -0
  35. remediation/cwe327_weak_hash.py +78 -0
  36. remediation/cwe377_tempfile.py +77 -0
  37. remediation/cwe489_debug.py +80 -0
  38. remediation/cwe502_yaml.py +89 -0
  39. remediation/cwe614_cookie.py +91 -0
  40. remediation/cwe78_cmdi.py +415 -0
  41. remediation/cwe798_credentials.py +153 -0
  42. remediation/cwe89_sqli.py +425 -0
  43. remediation/cwe916_password_hash.py +131 -0
  44. remediation/cwe95_eval.py +77 -0
  45. remediation/dispatcher.py +114 -0
  46. remediation/orchestrator.py +541 -0
  47. remediation/patch_engine.py +213 -0
  48. remediation/verification.py +253 -0
  49. rule_engine.py +1535 -0
  50. sarif_adapter.py +719 -0
  51. sarif_exporter.py +88 -0
  52. sca_reachability/__init__.py +28 -0
  53. sca_reachability/ast_analyzer.py +366 -0
  54. sca_reachability/call_graph.py +203 -0
  55. sca_reachability/contracts.py +365 -0
  56. sca_reachability/dist_import_map.py +46 -0
  57. sca_reachability/engine.py +603 -0
  58. sca_reachability/resolver.py +169 -0
  59. scanners/__init__.py +70 -0
  60. scanners/container_scanner.py +323 -0
  61. scanners/cve_index.py +521 -0
  62. scanners/iac_scanner.py +328 -0
  63. scanners/sbom.py +183 -0
  64. scanners/sca_scanner.py +667 -0
  65. scanners/secret_scanner.py +345 -0
  66. scanners/yaml_lite.py +217 -0
  67. secret_filters.py +435 -0
  68. secret_scanner.py +943 -0
  69. sqlite_delivery_repository.py +277 -0
  70. sqlite_notification_repository.py +441 -0
  71. ssrf_guard.py +276 -0
  72. staged_scanner.py +492 -0
  73. suppression_resolver.py +282 -0
  74. symbol_indexer.py +424 -0
  75. tcs/__init__.py +1 -0
  76. tcs/analysis/__init__.py +1 -0
  77. tcs/analysis/def_use.py +408 -0
  78. tcs/analysis/function_contracts.py +290 -0
  79. tcs/parallel_analyzer.py +276 -0
  80. tcs/parallel_scanner.py +630 -0
  81. tcs/treesitter_packer.py +1446 -0
  82. tcs_cli.py +2086 -0
  83. tcs_config.py +218 -0
  84. tcs_gui.py +2123 -0
  85. timecs-2.0.0.dist-info/METADATA +142 -0
  86. timecs-2.0.0.dist-info/RECORD +94 -0
  87. timecs-2.0.0.dist-info/WHEEL +5 -0
  88. timecs-2.0.0.dist-info/entry_points.txt +4 -0
  89. timecs-2.0.0.dist-info/licenses/LICENSE +201 -0
  90. timecs-2.0.0.dist-info/top_level.txt +49 -0
  91. version_matcher.py +934 -0
  92. webhook_adapter.py +495 -0
  93. webhook_client.py +483 -0
  94. webhook_formatters.py +245 -0
cli.py ADDED
@@ -0,0 +1,2347 @@
1
+ """Standalone command-line interface for the TimeCodeSecurity AST scanner."""
2
+
3
+ import argparse
4
+ import ast
5
+ import fnmatch
6
+ import json
7
+ import os
8
+ from pathlib import Path
9
+ import re
10
+ import shutil
11
+ import signal
12
+ import sys
13
+ import subprocess
14
+ import time
15
+ import traceback
16
+ from typing import Optional
17
+ import banner
18
+ from cross_file_engine import CrossFileTaintEngine
19
+ from ast_scanner import TaintTracker, PARSER_MODES, DEFAULT_PARSER_MODE
20
+ from ci_reporter import sanitize_symbol_display
21
+ from diff_scan import changed_line_intervals, resolve_repo_root
22
+ from html_auditor import audit_templates, is_template_path
23
+ from iac_auditor import audit_iac_files
24
+ from scanners import (
25
+ audit_sca_files,
26
+ audit_secret_files,
27
+ audit_security_files,
28
+ build_sbom,
29
+ collect_components,
30
+ is_container_path,
31
+ is_kubernetes_manifest,
32
+ is_manifest_path,
33
+ is_secret_path,
34
+ is_terraform_path,
35
+ is_workflow_yaml_path,
36
+ tag_reachability,
37
+ )
38
+ from rule_engine import get_rule
39
+ from suppression_resolver import resolve_line_suppressions
40
+ from tcs_config import load_config
41
+ from remediation.patch_engine import RemediationEngine
42
+ from sarif_adapter import bound_sarif_document
43
+ from sarif_exporter import atomic_write_text
44
+
45
+ try:
46
+ from js_scanner import (
47
+ has_js_ts_files,
48
+ js_ts_available,
49
+ run_js_ts_scan_isolated,
50
+ warn_js_ts_unavailable,
51
+ )
52
+ _JS_TS_SCANNER_AVAILABLE = True
53
+ except ImportError:
54
+ _JS_TS_SCANNER_AVAILABLE = False
55
+
56
+ def js_ts_available() -> bool:
57
+ return False
58
+
59
+ def run_js_ts_scan_isolated(target, base_dir=None, skipped_files=None):
60
+ return []
61
+
62
+ def has_js_ts_files(target_path) -> bool:
63
+ return False
64
+
65
+ def warn_js_ts_unavailable() -> None:
66
+ print("[TimeCodeSecurity-JS] WARNING: js_scanner is not importable; JS/TS scanning is disabled.",
67
+ file=sys.stderr)
68
+
69
+
70
+ SARIF_SCHEMA = "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json"
71
+ SEVERITIES = ("CRITICAL", "HIGH", "MEDIUM", "LOW", "UNKNOWN")
72
+ IGNORED_DIRS = {".git", ".venv", "venv", "__pycache__", "node_modules", "build", "dist"}
73
+
74
+ # Directories that hold sample, fixture and harness code rather than shipped product code. A
75
+ # directory scan prunes these before descending; `--include-tests` turns that off, and a path named
76
+ # on the command line is always walked in full. This is a discovery-scope choice, not a suppression:
77
+ # nothing is filtered out of the emission set, so every finding in a scanned file is still reported.
78
+ # The cost is stated plainly because it is real -- `benchmark/gates/workflow_controls_probe.py` names
79
+ # `examples/app.py` and `tests/fixture.py` among the trees a hardcoded secret is classically committed
80
+ # in, and `fixtures/` is where this repository keeps 27 of its own blessed detection cases, so a default
81
+ # directory scan no longer sees a CWE-798 sitting in any of them and needs `--include-tests` to find it.
82
+ # That is why the skip is always announced in the report, why the confidence-floor note and this one are
83
+ # printed even when nothing else is, and why the gates that prove per-rule or per-directory behaviour
84
+ # pass `--include-tests` (or name the directory on the command line) instead of losing their assertions.
85
+ # `debug/` joins the list for the same reason `examples/` and `samples/` are on it: a tree of preview
86
+ # and demo views -- Sentry's `src/sentry/web/frontend/debug/` renders sample mail and component
87
+ # payloads for local development -- whose findings describe developer tooling, not shipped surface.
88
+ # Name the directory or pass `--include-tests` and every row inside it is still reported, which is the
89
+ # pair `benchmark/gates/sentry_tier3_precision_probe.py` locks.
90
+ NON_PRODUCTION_DIRS = frozenset({"tests", "test", "__tests__", "testutils",
91
+ "fixtures", "examples", "example", "samples", "debug"})
92
+ # What the current scan pruned, so the report can say so instead of looking like a clean tree.
93
+ _SCOPE_STATS = {"skipped_dirs": 0, "skipped_files": 0}
94
+
95
+ # Jinja interpolation tokens in a *path*, as opposed to its content: Salt formulae ship files like
96
+ # `templates/{{module_name}}.py`, and that name alone guarantees `ast.parse` raises SyntaxError.
97
+ TEMPLATE_INTERPOLATION_TOKENS = ("{{", "}}")
98
+
99
+ # One flush per file is blocking pipe/disk I/O: a 2,819-file run paid 2,819 of those to say "I am
100
+ # still here". Progress now logs every Nth file, and the in-flight name is kept in
101
+ # `_PARTIAL["current_file"]` so a kill still reports where the scan was stuck.
102
+ PROGRESS_FILE_EVERY = 250
103
+
104
+ # Every finding belongs to exactly one analysis scope: template, container and CI/IaC paths are
105
+ # routed to their auditors, everything else is Python. --scope selects which collectors run.
106
+ SCOPE_ORDER = ("python", "docker", "iac", "sca", "secrets", "html")
107
+ SCOPE_LABELS = {"python": "Python Code", "docker": "Container/Config",
108
+ "iac": "CI/CD & IaC", "sca": "Dependencies (SCA)",
109
+ "secrets": "Config & Env Secrets", "html": "HTML Templates"}
110
+
111
+ # Confidence is a display label, never a detection decision: `--min-confidence` hides rows
112
+ # from the report, the engine still emits every finding it emitted before.
113
+ CONFIDENCE_ORDER = {"LOW": 1, "MEDIUM": 2, "HIGH": 3}
114
+ # Sinks whose finding is a heuristic over an in-process comparison rather than data arriving
115
+ # at a dangerous operation: CWE-208 reports `token == expected`, where both operands are
116
+ # already local values, so the shape alone cannot prove an attacker reaches it.
117
+ LOW_CONFIDENCE_SINK_TYPES = frozenset({"TIMING_ATTACK"})
118
+
119
+
120
+ def _scope_of(path, aux_scopes=None):
121
+ """Scope of one finding's file.
122
+
123
+ ``aux_scopes`` is the discovery map for auxiliary documents. It is the authority for a
124
+ Kubernetes workload manifest, which can sit at any path and is only recognisable from its
125
+ content; every other family is decidable from the path alone.
126
+ """
127
+ key = str(path)
128
+ if aux_scopes and key in aux_scopes:
129
+ return aux_scopes[key]
130
+ lowered = key.lower()
131
+ if is_template_path(lowered):
132
+ return "html"
133
+ if is_container_path(lowered):
134
+ return "docker"
135
+ if is_workflow_yaml_path(lowered) or is_terraform_path(lowered):
136
+ return "iac"
137
+ if is_manifest_path(lowered):
138
+ return "sca"
139
+ if is_secret_path(lowered):
140
+ return "secrets"
141
+ return "python"
142
+
143
+
144
+ def _file_key(path, cwd):
145
+ try:
146
+ return path.resolve().relative_to(cwd).as_posix()
147
+ except ValueError:
148
+ return path.resolve().as_posix()
149
+
150
+
151
+ def _parse_exclude_patterns(raw):
152
+ if not raw:
153
+ return None
154
+ return [p.strip() for p in raw.split(",") if p.strip()] or None
155
+
156
+
157
+ def _is_excluded(path, root, exclude_patterns):
158
+ """True when `path` matches any exclude pattern.
159
+
160
+ Patterns are matched three ways because users write all three: a bare directory
161
+ name (`tests`), a filename glob (`test_*.py`) and a rooted relative glob
162
+ (`contrib/*/tests/*`). Matching only the basename or only the full relative path
163
+ silently keeps nested hits, which is how `--exclude tests` used to survive
164
+ `django/contrib/auth/tests/models.py`.
165
+
166
+ Relative paths are normalised to POSIX before fnmatch, whose separator handling
167
+ is not portable: on Windows `str(Path.relative_to())` yields backslashes, so a
168
+ pattern containing `/` could never match.
169
+ """
170
+ if not exclude_patterns:
171
+ return False
172
+ try:
173
+ rel = path.relative_to(root)
174
+ except ValueError:
175
+ rel = path
176
+ rel_posix = rel.as_posix()
177
+ name = path.name
178
+ parts = rel.parts
179
+ for pattern in exclude_patterns:
180
+ if fnmatch.fnmatchcase(rel_posix, pattern):
181
+ return True
182
+ if fnmatch.fnmatchcase(name, pattern):
183
+ return True
184
+ if any(fnmatch.fnmatchcase(part, pattern) for part in parts):
185
+ return True
186
+ return False
187
+
188
+
189
+ def _is_template_interpolated(path, root):
190
+ """True when a path *component* carries a Jinja interpolation token.
191
+
192
+ Salt formulae ship files named `templates/{{module_name}}.py`: the name is a render-time
193
+ placeholder, so the bytes on disk are pre-render scaffolding and `ast.parse` on them usually
194
+ raises SyntaxError. Only components of the path are inspected, never file content, so a normal
195
+ module that merely uses `{{` inside a string is never mistaken for one.
196
+ """
197
+ try:
198
+ parts = path.relative_to(root).parts
199
+ except ValueError:
200
+ parts = path.parts
201
+ return any(token in part for part in parts for token in TEMPLATE_INTERPOLATION_TOKENS)
202
+
203
+
204
+ def _parses_as_python(source):
205
+ """True when `ast.parse` accepts the text -- the gate that keeps a template file scannable.
206
+
207
+ A Jinja-named path alone is not proof of junk: measured on this repo, `templates/{{x}}.py`
208
+ holding `import os; os.system(p)` parses fine and reports CWE-78. Dropping on the path alone
209
+ would have turned that real finding into a false negative, so the skip requires the content to
210
+ be unparseable as well. One extra parse per interpolated-path file, nothing else pays for it.
211
+ """
212
+ try:
213
+ ast.parse(source)
214
+ return True
215
+ except (SyntaxError, ValueError):
216
+ return False
217
+
218
+
219
+ def _progress(message):
220
+ """Emit one progress line on stderr so a stalled scan names the file it stuck on.
221
+
222
+ stderr keeps stdout parsable (`--format json` consumers read stdout only) and
223
+ flush=True means the line lands even if the process is killed mid-file.
224
+ Callers throttle *how often* this runs (PROGRESS_FILE_EVERY); the in-flight file name
225
+ lives in `_PARTIAL["current_file"]` so the throttled gap stays diagnosable.
226
+ """
227
+ print(f"[progress] {message}", file=sys.stderr, flush=True)
228
+
229
+
230
+ def _ux_enabled(args):
231
+ """True only for a human watching a terminal, never for a data pipe or a log.
232
+
233
+ `--format json` is an absolute veto: that stdout is parsed by every benchmark and CI job, so
234
+ the styled UX stays off even when TIMECS_FORCE_BANNER asks for it. Otherwise a real TTY turns
235
+ it on, and the env var exists so a test can drive the same code path over a pipe.
236
+ """
237
+ if getattr(args, "format", "table") == "json":
238
+ return False
239
+ if os.environ.get("TIMECS_FORCE_BANNER") == "1":
240
+ return True
241
+ try:
242
+ return bool(sys.stderr.isatty())
243
+ except (AttributeError, ValueError):
244
+ return False
245
+
246
+
247
+ class _StageClock:
248
+ """Phase-1 outer instrumentation: disjoint end-to-end stage clocks, stderr only.
249
+
250
+ Each `mark()` closes the stage that was open since the previous mark, so the stage
251
+ seconds are disjoint by construction and sum to TOTAL (modulo the final report call).
252
+ Engine-internal lines (`[PROFILE] Phase n ...`, `[PROFILE] CrossEngine ...`, collector
253
+ profiles) are NESTED inside these stages and must never be added to them. Worker-pool
254
+ time is included in the stage that owns the pool (Parse), not per core.
255
+
256
+ Nothing here feeds a finding, a suppression, the JSON schema, or the exit code;
257
+ `duration_ms` keeps its original semantics (started after file collection). RSS sampling
258
+ uses the already-optional psutil: without it the report says RSS=unavailable.
259
+ """
260
+
261
+ def __init__(self, label):
262
+ try:
263
+ import psutil
264
+ self._rss_fn = psutil.Process().memory_info
265
+ except Exception:
266
+ self._rss_fn = None
267
+ self.label = label
268
+ self.t_start = time.perf_counter()
269
+ self.t_mark = self.t_start
270
+ self.rows = []
271
+ self.peak_rss_mb = 0.0
272
+
273
+ def mark(self, stage):
274
+ now = time.perf_counter()
275
+ rss_mb = 0.0
276
+ if self._rss_fn is not None:
277
+ try:
278
+ rss_mb = self._rss_fn().rss / (1024 ** 2)
279
+ except Exception:
280
+ rss_mb = 0.0
281
+ self.peak_rss_mb = max(self.peak_rss_mb, rss_mb)
282
+ self.rows.append((stage, now - self.t_mark, rss_mb))
283
+ self.t_mark = now
284
+
285
+ def report(self):
286
+ total = time.perf_counter() - self.t_start
287
+ parts = " ".join(f"{name}={secs:.2f}s" for name, secs, _ in self.rows)
288
+ rss_note = (f" peakRSS_parent={self.peak_rss_mb:.0f}MB"
289
+ if self._rss_fn is not None else " RSS=unavailable(psutil missing)")
290
+ print(f"TimeCodeSecurity [PROFILE] Outer({self.label}): {parts} TOTAL={total:.2f}s{rss_note}",
291
+ file=sys.stderr, flush=True)
292
+
293
+
294
+ def _iter_tree_files(root, prune_dirs=None, count_pruned=False):
295
+ """Yield the real files under `root`, never following a symlink.
296
+
297
+ Traversal guard (CWE-59): `Path.rglob` semantics are version dependent — Python
298
+ 3.13+ follows symlinked directories by default, so one link back into an ancestor
299
+ makes the crawl loop. os.walk with followlinks=False plus explicit symlink pruning
300
+ keeps discovery inside the real tree. Ignored directories are pruned before
301
+ descending, so `node_modules` is never walked just to be filtered out again.
302
+
303
+ `prune_dirs` replaces the prune set for one walk; `count_pruned` only ever comes from a requested
304
+ scope narrowing, and records how many directories and Python files it held back so the report can
305
+ name them. Three things keep a file in scope whatever the set says: a file named directly (no
306
+ directory walk happens at all), a directory named as the scan root -- which is why
307
+ `cli.py scan tests/` still walks it -- and `--include-tests`, which passes no set at all.
308
+ """
309
+ skip = IGNORED_DIRS if prune_dirs is None else prune_dirs
310
+ for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
311
+ kept = []
312
+ for name in sorted(dirnames):
313
+ if name in skip:
314
+ if count_pruned and name not in IGNORED_DIRS:
315
+ _SCOPE_STATS["skipped_dirs"] += 1
316
+ _SCOPE_STATS["skipped_files"] += sum(
317
+ 1 for _root, _dirs, _files in os.walk(os.path.join(dirpath, name))
318
+ for _name in _files if _name.lower().endswith(".py"))
319
+ continue
320
+ if os.path.islink(os.path.join(dirpath, name)):
321
+ continue
322
+ kept.append(name)
323
+ dirnames[:] = kept
324
+ for name in sorted(filenames):
325
+ path = Path(dirpath) / name
326
+ if os.path.islink(path):
327
+ continue
328
+ yield path
329
+
330
+
331
+ def _collect_files(scan_path, exclude_patterns=None, config_excludes=None,
332
+ skip_non_production=False):
333
+ """Collect Python files from scan path, optionally excluding by glob patterns.
334
+
335
+ Args:
336
+ scan_path: Path object (file or directory)
337
+ exclude_patterns: List of glob patterns to exclude (e.g., ['tests', 'test_*'])
338
+ config_excludes: Optional callable taking a CWD-relative POSIX path and returning
339
+ True when `.tcs.yaml` excludes it. A file named on the command line is never
340
+ dropped, matching how a tree walk treats `--exclude`.
341
+ skip_non_production: Prune `NON_PRODUCTION_DIRS` from the tree walk only. A path named
342
+ on the command line is scanned in full either way, same contract as `--exclude`.
343
+
344
+ Phase-1 instrumentation only: the enumerate/read split names the cold-tree serial
345
+ read cost that the pre-existing `duration_ms` anchor cannot see (it starts later).
346
+ """
347
+ _t_enum0 = time.perf_counter()
348
+ cwd = Path.cwd().resolve()
349
+ if not scan_path.exists():
350
+ raise FileNotFoundError(f"Path does not exist: {scan_path}")
351
+ is_tree = scan_path.is_dir()
352
+ prune = (IGNORED_DIRS | NON_PRODUCTION_DIRS) if skip_non_production else None
353
+ if scan_path.is_file():
354
+ if scan_path.suffix.lower() != ".py":
355
+ raise ValueError(f"Expected a Python file: {scan_path}")
356
+ paths = [scan_path]
357
+ elif is_tree:
358
+ paths = sorted(
359
+ path for path in _iter_tree_files(scan_path, prune, skip_non_production)
360
+ if path.suffix.lower() == ".py"
361
+ and not any(part in IGNORED_DIRS for part in path.parts)
362
+ and not _is_excluded(path, scan_path, exclude_patterns)
363
+ and not (config_excludes and config_excludes(_file_key(path, cwd)))
364
+ )
365
+ else:
366
+ raise ValueError(f"Path is not a regular file or directory: {scan_path}")
367
+
368
+ if not paths:
369
+ scope_note = non_production_scope_note()
370
+ if scope_note:
371
+ # The tree holds nothing but pruned non-production code: say so rather than letting the
372
+ # caller read "No Python files found" as an empty directory.
373
+ print(scope_note, file=sys.stderr)
374
+ raise ValueError(f"No Python files found: {scan_path}")
375
+
376
+ _t_enum = time.perf_counter()
377
+ files = {}
378
+ skipped = 0
379
+ templated = 0
380
+ total_chars = 0
381
+ total = len(paths)
382
+ _progress(f"discovered {total} python file(s) under {scan_path}")
383
+ for index, path in enumerate(paths, 1):
384
+ _PARTIAL["current_file"] = str(path)
385
+ if index == 1 or index == total or index % PROGRESS_FILE_EVERY == 0:
386
+ _progress(f"reading python ({index}/{total}) {path}")
387
+ try:
388
+ source = path.read_text(encoding="utf-8", errors="replace")
389
+ except (OSError, ValueError, UnicodeDecodeError) as exc:
390
+ # A single unreadable file must not abort the scan of the whole tree.
391
+ skipped += 1
392
+ print(f"Warning: skipping unreadable file {path}: {type(exc).__name__}: {exc}",
393
+ file=sys.stderr)
394
+ continue
395
+ # Only a tree walk drops files; a file named on the command line is always analysed.
396
+ if is_tree and _is_template_interpolated(path, scan_path) and not _parses_as_python(source):
397
+ templated += 1
398
+ continue
399
+ total_chars += len(source)
400
+ files[_file_key(path, cwd)] = source
401
+ _t_reads = time.perf_counter()
402
+ if templated:
403
+ _progress(f"skipped {templated} unparseable Jinja-interpolated file(s) under {scan_path}")
404
+ if skipped:
405
+ print(f"Warning: skipped {skipped} of {len(paths)} Python files; "
406
+ f"{len(files)} scanned.", file=sys.stderr)
407
+ print(f"TimeCodeSecurity [PROFILE] CollectFiles: enumerate={_t_enum - _t_enum0:.2f}s "
408
+ f"(paths={total}), serial_read={_t_reads - _t_enum:.2f}s "
409
+ f"(kept={len(files)}, chars={total_chars}, unreadable={skipped}, jinja_skipped={templated})",
410
+ file=sys.stderr)
411
+ _PARTIAL["current_file"] = None
412
+ return files
413
+
414
+
415
+ AUX_YAML_SUFFIXES = (".yml", ".yaml")
416
+
417
+
418
+ def _collect_auxiliary_files(scan_path, exclude_patterns=None, config_excludes=None,
419
+ skip_non_production=False):
420
+ """Collect non-Python artefacts in one walk.
421
+
422
+ Returns ``(templates, security documents, manifests, secret documents, scope map)``.
423
+ Keys match _collect_files so findings share a single path namespace, and
424
+ `--exclude` applies here too: a template under an excluded directory is just as
425
+ out of scope as a .py file under it. `config_excludes` carries the same contract
426
+ as in _collect_files, as does `skip_non_production` -- a template inside a skipped
427
+ `examples/` tree is out of scope for exactly the same reason.
428
+
429
+ The scope map carries each auxiliary document's key to its scope ("docker" for images and
430
+ compose, "iac" for workflows, Kubernetes and Terraform, "sca" for dependency manifests and
431
+ lockfiles, "secrets" for config and env documents). It exists because a Kubernetes manifest
432
+ is recognised from content, not from its path, so the bucketing decision has to survive the
433
+ walk for `_scope_of` to report it honestly.
434
+ """
435
+ cwd = Path.cwd().resolve()
436
+ if not scan_path.exists():
437
+ raise FileNotFoundError(f"Path does not exist: {scan_path}")
438
+
439
+ prune = (IGNORED_DIRS | NON_PRODUCTION_DIRS) if skip_non_production else None
440
+ if scan_path.is_file():
441
+ paths = [scan_path]
442
+ elif scan_path.is_dir():
443
+ # count_pruned stays off here: the note reports python files held back, and the Python
444
+ # collection below walks the same tree -- counting twice would double the figure.
445
+ paths = sorted(
446
+ path for path in _iter_tree_files(scan_path, prune, False)
447
+ if not any(part in IGNORED_DIRS for part in path.parts)
448
+ and not _is_excluded(path, scan_path, exclude_patterns)
449
+ and not (config_excludes and config_excludes(_file_key(path, cwd)))
450
+ )
451
+ else:
452
+ return {}, {}, {}, {}, {}
453
+
454
+ templates = {}
455
+ iac = {}
456
+ manifests = {}
457
+ secrets = {}
458
+ aux_scopes = {}
459
+ buckets = {}
460
+ skipped = 0
461
+ for path in paths:
462
+ relative = str(path).replace("\\", "/")
463
+ if is_template_path(relative):
464
+ buckets[path] = (templates, "html")
465
+ elif is_container_path(relative):
466
+ buckets[path] = (iac, "docker")
467
+ elif is_workflow_yaml_path(relative) or is_terraform_path(relative):
468
+ buckets[path] = (iac, "iac")
469
+ elif is_manifest_path(relative):
470
+ buckets[path] = (manifests, "sca")
471
+ elif relative.lower().endswith(AUX_YAML_SUFFIXES):
472
+ # Scope undecided until the content is read: a Kubernetes workload manifest belongs
473
+ # to IaC, and only a YAML document that is not one falls through to the secret reader.
474
+ buckets[path] = (iac, None)
475
+ elif is_secret_path(relative):
476
+ buckets[path] = (secrets, "secrets")
477
+
478
+ total = len(buckets)
479
+ _progress(f"discovered {total} template/IaC file(s) under {scan_path}")
480
+ for index, (path, (bucket, scope)) in enumerate(buckets.items(), 1):
481
+ _PARTIAL["current_file"] = str(path)
482
+ if index == 1 or index == total or index % PROGRESS_FILE_EVERY == 0:
483
+ _progress(f"reading auxiliary ({index}/{total}) {path}")
484
+ try:
485
+ source = path.read_text(encoding="utf-8", errors="replace")
486
+ except OSError as exc:
487
+ # Same contract as _collect_files above: one unreadable template or IaC file is a
488
+ # local fault, so it is reported and skipped. Re-raising here used to abort the whole
489
+ # tree scan and exit 2, losing every Python finding in the repository.
490
+ skipped += 1
491
+ print(f"Warning: skipping unreadable auxiliary file {path}: "
492
+ f"{type(exc).__name__}: {exc}", file=sys.stderr)
493
+ continue
494
+ if scope is None:
495
+ if is_kubernetes_manifest(source):
496
+ scope = "iac"
497
+ elif is_secret_path(str(path).replace("\\", "/")):
498
+ bucket, scope = secrets, "secrets"
499
+ else:
500
+ continue
501
+ key = _file_key(path, cwd)
502
+ bucket[key] = source
503
+ if bucket is not templates:
504
+ aux_scopes[key] = scope
505
+ if skipped:
506
+ print(f"Warning: skipped {skipped} of {total} auxiliary files; "
507
+ f"{len(templates) + len(iac) + len(manifests) + len(secrets)} scanned.",
508
+ file=sys.stderr)
509
+ _PARTIAL["current_file"] = None
510
+ return templates, iac, manifests, secrets, aux_scopes
511
+
512
+
513
+ def _imported_roots(tracker):
514
+ """Top-level import roots across the Python files the engine already parsed.
515
+
516
+ Reuses the ``tracker.modules`` trees instead of re-reading or re-parsing anything, so the
517
+ reachability proof never adds a second AST pass to a scan. A tree that is not a stdlib AST
518
+ (the opt-in tree-sitter reader) is parsed from the engine's own source snapshot. The first
519
+ importer wins, which makes the evidence deterministic regardless of module iteration order.
520
+ """
521
+ roots = {}
522
+ if tracker is None:
523
+ return roots
524
+ paths = getattr(tracker, "file_paths", {}) or {}
525
+ sources = getattr(tracker, "files", {}) or {}
526
+ for module, tree in (getattr(tracker, "modules", {}) or {}).items():
527
+ if not isinstance(tree, ast.AST):
528
+ source = sources.get(paths.get(module))
529
+ if not source:
530
+ continue
531
+ try:
532
+ tree = ast.parse(source)
533
+ except (SyntaxError, ValueError):
534
+ continue
535
+ path = str(paths.get(module, module))
536
+ for node in ast.walk(tree):
537
+ if isinstance(node, ast.Import):
538
+ names = [alias.name for alias in node.names]
539
+ elif isinstance(node, ast.ImportFrom):
540
+ # A relative import names this project's own module, not a distribution.
541
+ if node.level or not node.module:
542
+ continue
543
+ names = [node.module]
544
+ else:
545
+ continue
546
+ for name in names:
547
+ root = name.split(".", 1)[0]
548
+ if root and root not in roots:
549
+ roots[root] = (path, node.lineno)
550
+ return roots
551
+
552
+
553
+ def _merge_findings(*finding_groups):
554
+ """Merge auditor findings with AST findings in a deterministic order."""
555
+ seen = set()
556
+ unique = []
557
+ for group in finding_groups:
558
+ for item in group:
559
+ identity = (item["file"], item["line"], item["cwe"], item.get("category"))
560
+ if identity in seen:
561
+ continue
562
+ seen.add(identity)
563
+ unique.append(item)
564
+ return sorted(unique, key=lambda entry: (entry["file"], entry["line"], entry["cwe"]))
565
+
566
+
567
+ # Several CWEs describe one *attribute* of the same defect: a single weak-digest call can
568
+ # be simultaneously a broken algorithm (327/328) and a storage-strength failure (759/916),
569
+ # and one set_cookie() can lack Secure (614), HttpOnly (1004) and SameSite (1275). Reporting
570
+ # every attribute as its own alert multiplies the count without adding an actionable site.
571
+ # An unbounded `f.read()` is the same case from the other direction: the rule engine calls it
572
+ # CWE-400 and the structural batch calls the identical call node CWE-770, so one sink is
573
+ # counted as two findings.
574
+ # The families below collapse to one finding per (file, line); the surviving CWEs are kept
575
+ # in `consolidated_from` so nothing is lost. This is presentation-only: the engine still emits
576
+ # every CWE, which is what the benchmark scores per rule.
577
+ FAMILY_PRIORITY = {
578
+ "CRYPTO_HASH": ("CWE-327", "CWE-916", "CWE-759", "CWE-328"),
579
+ "INSECURE_COOKIE": ("CWE-614", "CWE-1004", "CWE-1275"),
580
+ "RESOURCE_EXHAUSTION": ("CWE-770", "CWE-400"),
581
+ }
582
+ FAMILY_OF_CWE = {
583
+ cwe: family for family, members in FAMILY_PRIORITY.items() for cwe in members
584
+ }
585
+ COOKIE_FLAG_BY_CWE = {"CWE-614": "Secure", "CWE-1004": "HttpOnly", "CWE-1275": "SameSite"}
586
+ SEVERITY_RANK = {"UNKNOWN": 0, "LOW": 1, "MEDIUM": 2, "HIGH": 3, "CRITICAL": 4}
587
+
588
+
589
+ def _family_message(family, primary, members_by_cwe):
590
+ """Alert text naming what the collapsed CWEs actually proved."""
591
+ if family == "INSECURE_COOKIE":
592
+ flags = [COOKIE_FLAG_BY_CWE[cwe] for cwe in FAMILY_PRIORITY[family] if cwe in members_by_cwe]
593
+ return f"{primary}: Insecure Cookie Configuration (Missing {', '.join(flags)})"
594
+ source = members_by_cwe[primary][0]["message"]
595
+ operation = source.split(":", 1)[1].strip() if ":" in source else source
596
+ secondaries = [cwe for cwe in FAMILY_PRIORITY[family] if cwe in members_by_cwe and cwe != primary]
597
+ suffix = f" (consolidated: {', '.join(secondaries)})" if secondaries else ""
598
+ return f"{primary}: {operation}{suffix}"
599
+
600
+
601
+ def _consolidate_family(family, group):
602
+ members_by_cwe = {}
603
+ for finding in group:
604
+ members_by_cwe.setdefault(finding["cwe"], []).append(finding)
605
+ priority = FAMILY_PRIORITY[family]
606
+ primary = next(cwe for cwe in priority if cwe in members_by_cwe)
607
+ representative = members_by_cwe[primary][0]
608
+ consolidated = []
609
+ for cwe in priority:
610
+ consolidated.extend(member for member in members_by_cwe.get(cwe, []) if member is not representative)
611
+ return {
612
+ **representative,
613
+ "cwe": primary,
614
+ "line": min(member["line"] for member in group),
615
+ "severity": max(
616
+ (member["severity"] for member in group),
617
+ key=lambda value: SEVERITY_RANK.get(value, 0),
618
+ ),
619
+ "confidence": max(
620
+ (member.get("confidence", "MEDIUM") for member in group),
621
+ key=lambda value: CONFIDENCE_ORDER.get(value, 0),
622
+ ),
623
+ "message": _family_message(family, primary, members_by_cwe),
624
+ "consolidated_from": [f"{member['cwe']}@{member['line']}" for member in consolidated],
625
+ }
626
+
627
+
628
+ def consolidate_findings(findings):
629
+ """Collapse same-site CWE families into one finding, preserving scan order."""
630
+ buckets = {}
631
+ for finding in findings:
632
+ family = FAMILY_OF_CWE.get(finding["cwe"])
633
+ if family:
634
+ buckets.setdefault((family, finding["file"], finding["line"]), []).append(finding)
635
+
636
+ emitted = set()
637
+ consolidated = []
638
+ for finding in findings:
639
+ family = FAMILY_OF_CWE.get(finding["cwe"])
640
+ if not family:
641
+ consolidated.append(finding)
642
+ continue
643
+ key = (family, finding["file"], finding["line"])
644
+ if key in emitted:
645
+ continue
646
+ emitted.add(key)
647
+ consolidated.append(_consolidate_family(family, buckets[key]))
648
+ return sorted(consolidated, key=lambda item: (item["file"], item["line"], item["cwe"]))
649
+
650
+
651
+ def _findings_for(tracker, edges):
652
+ sinks = {sink.id: sink for sink in tracker.sinks}
653
+ by_identity = {}
654
+ for edge in edges:
655
+ sink = sinks.get(edge.target_id)
656
+ if sink is None:
657
+ continue
658
+ cwe = sink.metadata.get("cwe")
659
+ if not cwe and edge.proof_graph is not None:
660
+ cwe = edge.proof_graph.cwe
661
+ cwe = cwe or "UNKNOWN_CWE"
662
+ rule = get_rule(cwe)
663
+ confidence_label = "CONFIRMED" if edge.kind == "CONFIRMED_DATA_FLOW" else "POTENTIAL"
664
+ severity = rule.get_severity(confidence_label).upper() if rule else "HIGH"
665
+ # HIGH follows the engine's own CONFIRMED label. Phase collectors record a sink with a
666
+ # synthetic source id (`OS_COMMAND_EXECUTION`, `WEAK_HASH_ALGORITHM`) and still say
667
+ # CONFIRMED, so this over-claims rather than under-claims: a `--min-confidence high`
668
+ # run may show a pattern row, but it can never hide a traced flow behind a filter.
669
+ sink_type = sink.metadata.get("sink_type") or sink.metadata.get("category")
670
+ if sink_type in LOW_CONFIDENCE_SINK_TYPES:
671
+ confidence = "LOW"
672
+ elif edge.kind == "CONFIRMED_DATA_FLOW":
673
+ confidence = "HIGH"
674
+ else:
675
+ confidence = "MEDIUM"
676
+ location = sink.location
677
+ file_path = location.file.replace("\\", "/")
678
+ finding = {
679
+ "file": file_path,
680
+ "line": location.line_start,
681
+ "cwe": cwe,
682
+ "severity": severity,
683
+ "confidence": confidence,
684
+ "category": sink.metadata.get("category") or (rule.category if rule else "Security"),
685
+ "message": sink.metadata.get("message") or \
686
+ f"{cwe}: {sink.metadata.get('operation') or sanitize_symbol_display(sink.symbol)}",
687
+ }
688
+ # Triage-only shape label (CWE-611 today). It rides beside the message, never inside it,
689
+ # so the canonical keys the baselines compare cannot move because of it.
690
+ xml_input = sink.metadata.get("xml_input")
691
+ if xml_input:
692
+ finding["xml_input"] = xml_input
693
+ identity = (finding["file"], finding["line"], finding["cwe"])
694
+ existing = by_identity.get(identity)
695
+ if existing is None:
696
+ by_identity[identity] = finding
697
+ elif CONFIDENCE_ORDER[confidence] > CONFIDENCE_ORDER[existing["confidence"]]:
698
+ # One site can be reached by several edges; the loudest label wins so a
699
+ # confidence filter can never hide a confirmed flow behind a weaker edge.
700
+ existing["confidence"] = confidence
701
+ return sorted(by_identity.values(),
702
+ key=lambda item: (item["file"], item["line"], item["cwe"]))
703
+
704
+
705
+ def _parse_line_ranges(spec):
706
+ """Parse a --lines value like '4-5' or '4-5,10' into inclusive (start, end) ranges.
707
+
708
+ Returns None when the spec is empty/absent, meaning a full-file scan.
709
+ Raises ValueError on malformed input so the CLI can report it cleanly.
710
+ """
711
+ if not spec:
712
+ return None
713
+ ranges = []
714
+ for part in str(spec).split(","):
715
+ part = part.strip()
716
+ if not part:
717
+ continue
718
+ bounds = part.split("-", 1)
719
+ try:
720
+ start = int(bounds[0].strip())
721
+ end = int(bounds[1].strip()) if len(bounds) > 1 else start
722
+ except ValueError as exc:
723
+ raise ValueError(f"Invalid --lines range {part!r}; expected START[-END]") from exc
724
+ if start < 1 or end < start:
725
+ raise ValueError(f"Invalid --lines range {part!r}; lines must be positive and ordered")
726
+ ranges.append((start, end))
727
+ return ranges or None
728
+
729
+
730
+ def _in_line_ranges(line, ranges):
731
+ return ranges is None or any(start <= line <= end for start, end in ranges)
732
+
733
+
734
+ def _scope_findings_to_lines(findings, ranges):
735
+ """Keep findings whose sink line intersects the requested ranges.
736
+
737
+ Findings without a usable line number are kept: dropping them would turn an
738
+ incremental scan into a false-negative source.
739
+ """
740
+ if ranges is None:
741
+ return findings
742
+ scoped = []
743
+ for finding in findings:
744
+ line = finding.get("line")
745
+ if not isinstance(line, int):
746
+ scoped.append(finding)
747
+ continue
748
+ if _in_line_ranges(line, ranges):
749
+ scoped.append(finding)
750
+ return scoped
751
+
752
+
753
+ def _apply_inline_suppressions(findings, sources):
754
+ """Drop findings named by a `# tcs: ignore[CWE-XXX]` comment on the finding's own line.
755
+
756
+ `sources` maps every scanned path to its text (Python, templates and IaC alike).
757
+ A finding whose file is absent from that map is kept: an unreadable source can
758
+ never suppress anything.
759
+ """
760
+ if not findings or not sources:
761
+ return findings
762
+ resolve_line_suppressions(findings, sources)
763
+ return [finding for finding in findings if not finding.get("suppressed")]
764
+
765
+
766
+ def _scope_findings_to_diff(findings, intervals, repo_root):
767
+ """Keep findings reported on added/modified lines of the working tree versus a ref.
768
+
769
+ `intervals` is keyed by repo-relative paths; findings carry CWD-relative keys, so both
770
+ sides are mapped through `_file_key`. A finding without a usable line number in a
771
+ changed file is kept, mirroring `_scope_findings_to_lines`.
772
+ """
773
+ cwd = Path.cwd().resolve()
774
+ changed = {}
775
+ for rel_path, spans in intervals.items():
776
+ try:
777
+ changed[_file_key((repo_root / rel_path).resolve(), cwd)] = spans
778
+ except OSError:
779
+ continue
780
+ kept = []
781
+ for finding in findings:
782
+ spans = changed.get(finding.get("file"))
783
+ if spans is None:
784
+ continue
785
+ line = finding.get("line")
786
+ if not isinstance(line, int) or any(start <= line <= end for start, end in spans):
787
+ kept.append(finding)
788
+ return kept
789
+
790
+
791
+ def _filter_by_confidence(findings, minimum):
792
+ """Hide findings below `minimum` from the report; detection itself is untouched."""
793
+ if not minimum:
794
+ return findings
795
+ floor = CONFIDENCE_ORDER[minimum.upper()]
796
+ return [
797
+ finding for finding in findings
798
+ if CONFIDENCE_ORDER.get(str(finding.get("confidence") or "MEDIUM").upper(), 2) >= floor
799
+ ]
800
+
801
+
802
+ # The floor a plain `cli.py scan` table starts at when nobody asked for one. It is a reading order for
803
+ # a person, not a detection decision, so it is applied to the table only and always announced.
804
+ TABLE_DEFAULT_CONFIDENCE = "high"
805
+
806
+
807
+ def _split_by_table_default(findings):
808
+ """Return (rows a default table shows, rows it hides) at `TABLE_DEFAULT_CONFIDENCE`."""
809
+ floor = CONFIDENCE_ORDER[TABLE_DEFAULT_CONFIDENCE.upper()]
810
+ visible, hidden = [], []
811
+ for finding in findings:
812
+ label = str(finding.get("confidence") or "MEDIUM").upper()
813
+ (visible if CONFIDENCE_ORDER.get(label, 2) >= floor else hidden).append(finding)
814
+ return visible, hidden
815
+
816
+
817
+ def non_production_scope_note() -> str:
818
+ """Name the scope a directory walk narrowed by default.
819
+
820
+ Held-back files are invisible to the report, which is the one way a scope choice can be mistaken
821
+ for a clean tree -- and a hardcoded secret in `tests/fixture.py` or `examples/app.py` is exactly
822
+ the finding that lives there. So the count is always stated when the walk pruned anything, and the
823
+ way back is named. A path given on the command line is always walked in full, which is why this
824
+ note never appears for `scan tests/`.
825
+ """
826
+ dirs = _SCOPE_STATS["skipped_dirs"]
827
+ if not dirs:
828
+ return ""
829
+ files = _SCOPE_STATS["skipped_files"]
830
+ return (f"Note: Held back {files} python file(s) in {dirs} non-production "
831
+ f"director{'y' if dirs == 1 else 'ies'} (tests/, testutils/, fixtures/, "
832
+ f"examples/, samples/, debug/); pass --include-tests to scan them.")
833
+
834
+
835
+ def table_default_note(hidden) -> str:
836
+ """One line naming what the default table floor hid, so a short table is never read as a clean scan.
837
+
838
+ A CRITICAL-severity row can sit under the floor when its flow was matched structurally rather than
839
+ traced, so the count of hidden criticals is called out separately: that is the case where a reader
840
+ would otherwise conclude the scan had nothing dangerous in it.
841
+ """
842
+ if not hidden:
843
+ return ""
844
+ counts = {}
845
+ for finding in hidden:
846
+ label = str(finding.get("confidence") or "MEDIUM").upper()
847
+ counts[label] = counts.get(label, 0) + 1
848
+ parts = [f"{counts[label]} {label}" for label in ("HIGH", "MEDIUM", "LOW") if counts.get(label)]
849
+ critical = sum(1 for finding in hidden if finding.get("severity") == "CRITICAL")
850
+ tail = ", all CRITICAL severity" if critical == len(hidden) else ""
851
+ if 0 < critical < len(hidden):
852
+ tail = f", {critical} of them CRITICAL severity"
853
+ noun = "finding" if len(hidden) == 1 else "findings"
854
+ return (f"Note: {' and '.join(parts)} {noun} hidden by the default table filter{tail} "
855
+ "(pass --min-confidence all to view all findings).")
856
+
857
+
858
+ def _apply_config_filters(findings, config, path_excludes, scan_root=None, aux_scopes=None):
859
+ """Apply root `.tcs.yaml` to findings: path exclusions and per-rule path overrides.
860
+
861
+ `path_excludes` is False when the user named a single file on the command line --
862
+ then only the rule-level overrides apply, because an explicitly named file is
863
+ always analysed (same contract as `--exclude`).
864
+
865
+ `scan_root` is the path the user named, as seen from the working directory: collected
866
+ keys carry it as a prefix, and `TcsConfig` needs it to honour an anchored pattern such
867
+ as `docs/**` when the scan is aimed at a subdirectory instead of the repository root.
868
+
869
+ `aux_scopes` is the container/CI-IaC discovery map. Per-rule overrides are a *Python* noise
870
+ policy -- the shipped CWE-78 globs name CI scripts and example trees, where shell-by-design
871
+ is not injection -- and `**/.github/**` over a workflow YAML would delete the GHA injection
872
+ rule at its only source. Those documents answer to their own structural falsifiers instead.
873
+ A global `exclude_paths` entry still never-reads them, and templates are unaffected.
874
+ """
875
+ if config is None:
876
+ return findings
877
+ security_docs = aux_scopes or {}
878
+ kept = []
879
+ for finding in findings:
880
+ path_key = str(finding.get("file", ""))
881
+ if path_excludes and config.excludes_path(path_key, scan_root):
882
+ continue
883
+ if path_key not in security_docs and config.rule_excluded(
884
+ str(finding.get("cwe", "")), path_key, scan_root):
885
+ continue
886
+ kept.append(finding)
887
+ return kept
888
+
889
+
890
+ def _statement_span(source_code, line_number):
891
+ """Return the (start, end) line span of the smallest statement covering line_number.
892
+
893
+ Falls back to (line_number, line_number) when the source is unparseable or no
894
+ statement covers the requested line.
895
+ """
896
+ try:
897
+ tree = ast.parse(source_code)
898
+ except (SyntaxError, ValueError, RecursionError):
899
+ return line_number, line_number
900
+
901
+ candidates = [
902
+ node
903
+ for node in ast.walk(tree)
904
+ if isinstance(node, ast.stmt)
905
+ and getattr(node, "lineno", 0)
906
+ and node.lineno <= line_number <= (node.end_lineno or node.lineno)
907
+ ]
908
+ if not candidates:
909
+ return line_number, line_number
910
+
911
+ stmt = min(candidates, key=lambda n: (n.end_lineno or n.lineno) - n.lineno)
912
+ return stmt.lineno, (stmt.end_lineno or stmt.lineno)
913
+
914
+
915
+ def _generate_autofix(finding, source_code, file_path, engine=None):
916
+ """
917
+ Generate an autofix object for a finding by running the RemediationEngine.
918
+
919
+ Returns a dict with autofix schema or None if no deterministic fix is available.
920
+
921
+ Autofix schema:
922
+ {
923
+ "type": "ast_patch",
924
+ "description": str,
925
+ "replacement_text": str,
926
+ "range": {
927
+ "start_line": int,
928
+ "start_col": int,
929
+ "end_line": int,
930
+ "end_col": int
931
+ },
932
+ "diff": str (optional),
933
+ "verification_passed": bool
934
+ }
935
+ """
936
+ if engine is None:
937
+ engine = RemediationEngine()
938
+
939
+ # Map CLI finding to remediation engine format
940
+ remediation_finding = {
941
+ "id": f"TimeCodeSecurity-{finding['cwe'].replace('-', '')}-{finding['line']}",
942
+ "cwe": finding["cwe"],
943
+ "line_number": finding["line"],
944
+ "file": finding["file"],
945
+ "code_snippet": "", # Will be extracted from source
946
+ }
947
+
948
+ try:
949
+ record = engine.remediate(remediation_finding, source_code, file_path)
950
+ except Exception:
951
+ # If remediation fails entirely, return None
952
+ return None
953
+
954
+ # Only attach autofix for successful patches
955
+ if record.patch_status.value != "SUCCESS":
956
+ return None
957
+
958
+ # The autofix replaces the whole original statement, so derive the range from
959
+ # the statement span in the UNPATCHED source. The snippet's own line count is
960
+ # not used: a multi-line replacement still overwrites a single-line statement.
961
+ stmt_start, stmt_end = _statement_span(source_code, record.line_number)
962
+ start_line = stmt_start
963
+ end_line = stmt_end
964
+
965
+ source_lines = source_code.split("\n")
966
+ if 0 < end_line <= len(source_lines):
967
+ end_col = len(source_lines[end_line - 1].rstrip("\r"))
968
+ else:
969
+ end_col = 0
970
+
971
+ autofix = {
972
+ "type": "ast_patch",
973
+ "description": f"Apply {record.remediation_rule.value} transformation for {record.cwe}",
974
+ "replacement_text": record.patched_code_snippet,
975
+ "range": {
976
+ "start_line": start_line,
977
+ "start_col": 0,
978
+ "end_line": end_line,
979
+ "end_col": end_col
980
+ },
981
+ "diff": record.unified_diff,
982
+ "verification_passed": record.verification_passed
983
+ }
984
+
985
+ return autofix
986
+
987
+
988
+ def _attach_autofixes(findings, source_cache, autofix_engine=None):
989
+ """
990
+ Attach autofix objects to findings where deterministic fixes are available.
991
+
992
+ Args:
993
+ findings: List of finding dicts
994
+ source_cache: Dict mapping file paths to source code strings
995
+ autofix_engine: Optional RemediationEngine instance
996
+
997
+ Returns:
998
+ List of findings with autofix objects attached where applicable
999
+ """
1000
+ if autofix_engine is None:
1001
+ autofix_engine = RemediationEngine()
1002
+
1003
+ enhanced_findings = []
1004
+ for finding in findings:
1005
+ file_path = finding["file"]
1006
+ source_code = source_cache.get(file_path)
1007
+
1008
+ if source_code:
1009
+ autofix = _generate_autofix(finding, source_code, file_path, autofix_engine)
1010
+ if autofix:
1011
+ finding_copy = finding.copy()
1012
+ finding_copy["autofix"] = autofix
1013
+ enhanced_findings.append(finding_copy)
1014
+ else:
1015
+ enhanced_findings.append(finding)
1016
+ else:
1017
+ enhanced_findings.append(finding)
1018
+
1019
+ return enhanced_findings
1020
+
1021
+
1022
+ def _ascii_table(findings):
1023
+ headers = ("#", "File", "Line", "CWE", "Severity", "Category")
1024
+ max_widths = (3, 15, 4, 8, 8, 22)
1025
+ rows = [
1026
+ (str(index), item["file"], str(item["line"]), item["cwe"], item["severity"], item["category"].replace("_", " "))
1027
+ for index, item in enumerate(findings, 1)
1028
+ ]
1029
+ widths = [
1030
+ min(max_widths[index], max([len(headers[index])] + [len(row[index]) for row in rows]))
1031
+ for index in range(len(headers))
1032
+ ]
1033
+
1034
+ def fit(value, width, keep_end=False):
1035
+ if len(value) <= width:
1036
+ return value.ljust(width)
1037
+ if keep_end:
1038
+ return "..." + value[-(width - 3):]
1039
+ return value[:width - 3] + "..."
1040
+
1041
+ border = "+" + "+".join("-" * (width + 2) for width in widths) + "+"
1042
+ lines = [border, "| " + " | ".join(headers[i].ljust(widths[i]) for i in range(len(headers))) + " |", border]
1043
+ for row in rows:
1044
+ cells = [fit(row[index], widths[index], keep_end=index == 1) for index in range(len(headers))]
1045
+ lines.append("| " + " | ".join(cells) + " |")
1046
+ lines.append(border)
1047
+ return "\n".join(lines)
1048
+
1049
+
1050
+ def _sarif_tags(cwe, related):
1051
+ """Tag list for one rule/result: its own CWE plus any secondary CWEs it stands for.
1052
+
1053
+ A deduplicated, order-preserving list, so the same set of secondary ids always renders the
1054
+ same bytes and a CWE that already appears as the primary id is never repeated.
1055
+ """
1056
+ tags = ["security", str(cwe).lower()]
1057
+ for related_cwe in related or ():
1058
+ tag = str(related_cwe).lower()
1059
+ if tag not in tags:
1060
+ tags.append(tag)
1061
+ return tags
1062
+
1063
+
1064
+ def _sarif_rule(cwe, category, severity, related=()):
1065
+ """Build one driver rule carrying the same required members the SARIF adapter emits."""
1066
+ digits = re.search(r"\d+", str(cwe))
1067
+ rule_meta = get_rule(cwe)
1068
+ short = str(category).replace("_", " ")
1069
+ return {
1070
+ "id": cwe,
1071
+ "name": str(cwe).replace("-", "_"),
1072
+ "shortDescription": {"text": short},
1073
+ "fullDescription": {"text": rule_meta.name if rule_meta else short},
1074
+ "helpUri": (
1075
+ f"https://cwe.mitre.org/data/definitions/{digits.group()}.html"
1076
+ if digits else "https://cwe.mitre.org/data/definitions/index.html"
1077
+ ),
1078
+ "defaultConfiguration": {
1079
+ "level": "error" if severity in ("CRITICAL", "HIGH") else "warning"
1080
+ },
1081
+ "properties": {"tags": _sarif_tags(cwe, related)},
1082
+ }
1083
+
1084
+
1085
+ def _sarif_uri(path, cwd):
1086
+ """Normalize a filesystem path into a valid SARIF artifactLocation URI."""
1087
+ resolved = Path(str(path).replace("\\", "/")).resolve()
1088
+ try:
1089
+ return resolved.relative_to(cwd).as_posix()
1090
+ except ValueError:
1091
+ return resolved.as_uri()
1092
+
1093
+
1094
+ def _sarif_code_flows(flow_trace, cwd):
1095
+ """Wrap a cross-file trace in SARIF codeFlows/threadFlows for GHAS and VS Code viewers."""
1096
+ locations = []
1097
+ for step, hop in enumerate(flow_trace, 1):
1098
+ locations.append({
1099
+ "location": {
1100
+ "message": {"text": f"{hop['role']} : {hop['label']}"},
1101
+ "physicalLocation": {
1102
+ "artifactLocation": {"uri": _sarif_uri(hop["file"], cwd)},
1103
+ "region": {"startLine": max(1, hop["line"]), "startColumn": 1},
1104
+ },
1105
+ },
1106
+ "executionOrder": step,
1107
+ "importance": "essential",
1108
+ })
1109
+ return [{
1110
+ "message": {"text": "Cross-file taint execution path"},
1111
+ "threadFlows": [{"locations": locations}],
1112
+ }]
1113
+
1114
+
1115
+ def _sarif_document(findings, cwd, execution_successful=True):
1116
+ """Build the SARIF 2.1.0 run for `findings`.
1117
+
1118
+ `execution_successful` mirrors reality: the crash and timeout-flush callers pass False, because
1119
+ their JSON siblings already report `status: incomplete` / `status: partial`, and both already
1120
+ exit non-zero -- claiming a successful run in the artefact would contradict them.
1121
+ """
1122
+ rules = []
1123
+ rule_indexes = {}
1124
+ # Rules are keyed by CWE alone, so the secondary ids of *every* finding carrying that CWE
1125
+ # are unioned up front. Reading only the first-seen finding would drop a later finding's
1126
+ # related CWEs and leave the rule under-indexed.
1127
+ related_by_cwe = {}
1128
+ for finding in findings:
1129
+ for related_cwe in finding.get("related_cwes") or ():
1130
+ collected = related_by_cwe.setdefault(finding["cwe"], [])
1131
+ if related_cwe not in collected:
1132
+ collected.append(related_cwe)
1133
+ for finding in findings:
1134
+ cwe = finding["cwe"]
1135
+ if cwe in rule_indexes:
1136
+ continue
1137
+ rule_indexes[cwe] = len(rules)
1138
+ rules.append(_sarif_rule(cwe, finding["category"], finding["severity"],
1139
+ related_by_cwe.get(cwe)))
1140
+
1141
+ level_by_severity = {"CRITICAL": "error", "HIGH": "error", "MEDIUM": "warning", "LOW": "note", "UNKNOWN": "note"}
1142
+ results = []
1143
+ for finding in findings:
1144
+ result = {
1145
+ "ruleId": finding["cwe"],
1146
+ "ruleIndex": rule_indexes[finding["cwe"]],
1147
+ "level": level_by_severity.get(finding["severity"], "warning"),
1148
+ "message": {"text": finding["message"]},
1149
+ "locations": [{
1150
+ "physicalLocation": {
1151
+ "artifactLocation": {"uri": _sarif_uri(finding["file"], cwd)},
1152
+ "region": {"startLine": max(1, finding["line"]), "startColumn": 1},
1153
+ }
1154
+ }],
1155
+ }
1156
+ # Mirrors the rule tag list so a reader that only walks results still sees the
1157
+ # secondary CWE. Emitted only when a finding actually carries one, which keeps the
1158
+ # bytes of every existing finding identical.
1159
+ properties = {}
1160
+ related = finding.get("related_cwes")
1161
+ if related:
1162
+ properties["tags"] = _sarif_tags(finding["cwe"], related)
1163
+ # CWE-611 triage hint: which syntactic shape the parser argument had. It exists only
1164
+ # here, never in the message, so SARIF text consumers keep reading the same sentence.
1165
+ xml_input = finding.get("xml_input")
1166
+ if xml_input:
1167
+ properties["xml_input"] = xml_input
1168
+ if properties:
1169
+ result["properties"] = properties
1170
+ flow_trace = finding.get("flow_trace")
1171
+ if flow_trace:
1172
+ result["codeFlows"] = _sarif_code_flows(flow_trace, cwd)
1173
+ results.append(result)
1174
+
1175
+ return {
1176
+ "$schema": SARIF_SCHEMA,
1177
+ "version": "2.1.0",
1178
+ "runs": [{
1179
+ "tool": {"driver": {"name": "TimeCodeSecurity", "rules": rules}},
1180
+ "results": results,
1181
+ "invocations": [{"executionSuccessful": bool(execution_successful)}],
1182
+ }],
1183
+ }
1184
+
1185
+
1186
+ def _scope_breakdown(findings, aux_scopes=None):
1187
+ counts = {scope: 0 for scope in SCOPE_ORDER}
1188
+ for finding in findings:
1189
+ counts[_scope_of(finding["file"], aux_scopes)] += 1
1190
+ return counts
1191
+
1192
+
1193
+ def _print_summary(file_count, duration_ms, findings, file_counts=None,
1194
+ discovered_files=None, fast_path_skipped_files=0,
1195
+ unparseable_files=0, parse_governor=None, aux_scopes=None):
1196
+ counts = {severity: 0 for severity in SEVERITIES}
1197
+ scope_counts = _scope_breakdown(findings, aux_scopes)
1198
+ for finding in findings:
1199
+ severity = finding["severity"]
1200
+ counts[severity if severity in counts else "UNKNOWN"] += 1
1201
+ if discovered_files is not None:
1202
+ print(f"\nTotal discovered files: {discovered_files}")
1203
+ print(f"Total scanned files: {file_count}")
1204
+ if file_counts:
1205
+ print("Files scanned by scope: " + ", ".join(
1206
+ f"{SCOPE_LABELS[scope]}: {file_counts[scope]}" for scope in SCOPE_ORDER
1207
+ ))
1208
+ skipped = discovered_files - file_count if discovered_files is not None else 0
1209
+ if skipped:
1210
+ print(f"Files not scanned: {skipped} "
1211
+ f"({fast_path_skipped_files} fast-path skipped, "
1212
+ f"{unparseable_files} unparseable)")
1213
+ if parse_governor:
1214
+ workers = parse_governor.get("workers_final", parse_governor.get("workers_used"))
1215
+ print(f"Parser: {parse_governor.get('parse_mode', 'unknown')} mode, "
1216
+ f"{workers} worker(s), "
1217
+ f"{parse_governor.get('governor_events', 0)} governor reduction(s)")
1218
+ if parse_governor.get("execution_mode") == "serial_fallback":
1219
+ # A degraded run must announce itself on stdout: the reason, and the fact that
1220
+ # coverage did not shrink. Silent serial re-parsing would look like a normal scan.
1221
+ print(f"[!] EXECUTION MODE: serial_fallback - the parallel parse pool failed "
1222
+ f"({parse_governor.get('parallel_fallback_reason')}). All "
1223
+ f"{file_count} scanned file(s) were re-parsed serially; coverage unchanged, "
1224
+ f"scan slower.")
1225
+ print(f"Scan duration: {duration_ms:.2f} ms")
1226
+ print("Findings by severity: " + ", ".join(f"{severity.title()}: {counts[severity]}" for severity in SEVERITIES))
1227
+ print("Findings by scope: " + ", ".join(
1228
+ f"{SCOPE_LABELS[scope]}: {scope_counts[scope]}" for scope in SCOPE_ORDER
1229
+ ))
1230
+
1231
+
1232
+ def _cross_trace_report(findings, cwd):
1233
+ """Render the multi-hop attack path for every cross-file finding, or '' if none."""
1234
+ traced = [
1235
+ (index, finding)
1236
+ for index, finding in enumerate(findings, 1)
1237
+ if finding.get("category") == "cross_file" and finding.get("flow_trace")
1238
+ ]
1239
+ if not traced:
1240
+ return ""
1241
+
1242
+ rule = "-" * 80
1243
+ lines = ["", rule, " TIMECODESECURITY - CROSS-FILE EXPLOIT CHAINS", rule]
1244
+ for index, finding in traced:
1245
+ rule_meta = get_rule(finding["cwe"])
1246
+ label = (rule_meta.category if rule_meta else finding.get("category", "")).replace("_", " ").upper()
1247
+ lines.append(f"[TRACE] Finding #{index}: {finding['cwe']} ({label})")
1248
+ for step, hop in enumerate(finding["flow_trace"], 1):
1249
+ location = _file_key(Path(hop["file"]), cwd)
1250
+ lines.append(f" {step}. {hop['role'].ljust(7)}: {location}:{hop['line']} ({hop['label']})")
1251
+ lines.append(rule)
1252
+ return "\n".join(lines)
1253
+
1254
+
1255
+
1256
+ class CrossFinding(dict):
1257
+ """Hybrid finding: acts as dict and object simultaneously."""
1258
+ def __init__(self, cwe, file_path, lineno, message, severity="HIGH", flow_trace=None,
1259
+ confidence="HIGH"):
1260
+ super().__init__(
1261
+ cwe=cwe,
1262
+ rule_id=cwe,
1263
+ severity="HIGH",
1264
+ confidence=confidence,
1265
+ file_path=file_path,
1266
+ file=file_path,
1267
+ path=file_path,
1268
+ lineno=lineno,
1269
+ line=lineno,
1270
+ message=message,
1271
+ desc=message,
1272
+ description=message,
1273
+ category="cross_file",
1274
+ flow_trace=flow_trace or [],
1275
+ )
1276
+ self.category = "cross_file"
1277
+ self.confidence = confidence
1278
+ self.flow_trace = flow_trace or []
1279
+ self.cwe = cwe
1280
+ self.rule_id = cwe
1281
+ self.severity = "HIGH"
1282
+ self.file_path = file_path
1283
+ self.file = file_path
1284
+ self.path = file_path
1285
+ self.lineno = lineno
1286
+ self.line = lineno
1287
+ self.message = message
1288
+ self.desc = message
1289
+ self.description = message
1290
+
1291
+ def _cross_scan_reuse_trees(tracker):
1292
+ """The scanner's already-parsed ASTs, offered to the cross-file indexer for a second use.
1293
+
1294
+ Reuse preconditions, each one checked before this dict is handed over:
1295
+ - parser mode is not `treesitter`: the engine reparses with the built-in `ast` module and
1296
+ a treesitter-produced tree is not what `GlobalSymbolIndex._index_file` walks;
1297
+ - files the scanner skipped (unparseable), recovered (block-pass substituted) or rescued
1298
+ are excluded - the engine's own disk parse would build a *different* tree for them,
1299
+ and the baseline behaviour for those files is the fresh parse, not the scanner's one;
1300
+ - the indexer still re-reads every path and reuses its tree only when the on-disk source
1301
+ equals the scanner's snapshot byte-for-byte, so a file changed mid-scan falls back.
1302
+
1303
+ Returns None when nothing qualifies; the engine then behaves exactly as before.
1304
+ """
1305
+ if tracker is None or not getattr(tracker, "modules", None):
1306
+ return None
1307
+ if getattr(tracker, "execution_mode", None) == "treesitter":
1308
+ return None
1309
+ cwd = Path.cwd()
1310
+ excluded = (set(tracker.skipped_files) | set(tracker.recovered_files)
1311
+ | set(tracker.rescued_files))
1312
+ reuse = {}
1313
+ for mod, tree in tracker.modules.items():
1314
+ key = tracker.file_paths.get(mod)
1315
+ if key is None or key in excluded or tree is None:
1316
+ continue
1317
+ try:
1318
+ source = tracker.files[key]
1319
+ reuse[str((cwd / key).resolve())] = (source, tree)
1320
+ except (KeyError, OSError):
1321
+ continue
1322
+ return reuse or None
1323
+
1324
+
1325
+ def _run_cross_scan(target_path, reuse_trees=None):
1326
+ """Executes CrossFileTaintEngine. Returns (findings, engine).
1327
+
1328
+ An engine failure is fatal rather than empty: swallowing the exception made a broken
1329
+ cross-file analysis indistinguishable from a clean scan, silently dropping both the
1330
+ cross-file findings and the sanitizer suppression that depends on the engine.
1331
+ """
1332
+ try:
1333
+ engine = CrossFileTaintEngine(target_path, reuse_trees=reuse_trees)
1334
+ raw_findings = engine.run()
1335
+ except Exception as exc:
1336
+ print(f"[FATAL] CrossFileTaintEngine crashed: {exc}", file=sys.stderr)
1337
+ traceback.print_exc()
1338
+ raise SystemExit(1)
1339
+
1340
+ results = []
1341
+ for cf in raw_findings:
1342
+ msg = f"Cross-file leak: {cf.caller_func} flows into {cf.callee_func} ({cf.callee_sink_name}) at {cf.callee_file}:{cf.callee_sink_lineno}"
1343
+ results.append(CrossFinding(
1344
+ cwe=cf.cwe,
1345
+ file_path=cf.caller_file,
1346
+ lineno=cf.caller_lineno,
1347
+ message=msg,
1348
+ severity="HIGH",
1349
+ flow_trace=[
1350
+ {"role": hop.role, "file": hop.file_path, "line": hop.lineno, "label": hop.label}
1351
+ for hop in getattr(cf, "flow_trace", None) or []
1352
+ ],
1353
+ ))
1354
+ return results, engine
1355
+
1356
+
1357
+ def _suppress_cross_file_sanitized(findings, engine):
1358
+ """Drop single-file findings whose tainted arguments a cross-file sanitizer neutralized.
1359
+
1360
+ The AST scanner has no visibility into callee contracts, so `cmd = format_command(x)`
1361
+ followed by `os.system(cmd)` is reported as injectable even when the helper escapes its
1362
+ argument. Cross-file findings are left untouched: that engine already honours contracts.
1363
+
1364
+ A failure here is fatal, not ignored: returning the findings unchanged would silently
1365
+ re-introduce every sanitizer false positive and still report a successful scan.
1366
+ """
1367
+ try:
1368
+ sanitized_sites = {
1369
+ (_normalize_compare_path(file_path), lineno)
1370
+ for file_path, lineno in engine.sanitized_sink_locations()
1371
+ }
1372
+ except Exception as exc:
1373
+ print(f"[FATAL] CrossFileTaintEngine sanitizer analysis crashed: {exc}", file=sys.stderr)
1374
+ traceback.print_exc()
1375
+ raise SystemExit(1)
1376
+ if not sanitized_sites:
1377
+ return findings
1378
+
1379
+ kept = []
1380
+ for finding in findings:
1381
+ if finding.get("category") == "cross_file":
1382
+ kept.append(finding)
1383
+ continue
1384
+ identity = (_normalize_compare_path(finding["file"]), finding["line"])
1385
+ if identity in sanitized_sites:
1386
+ continue
1387
+ kept.append(finding)
1388
+ return kept
1389
+
1390
+ def _js_ts_findings(path):
1391
+ """Map the isolated tree-sitter JS/TS findings onto this module's finding shape."""
1392
+ mapped = []
1393
+ for finding in run_js_ts_scan_isolated(path, base_dir=Path.cwd().resolve()):
1394
+ mapped.append({
1395
+ "file": str(finding.get("file", "")).replace("\\", "/"),
1396
+ "line": finding.get("line_number") or 1,
1397
+ "cwe": finding.get("cwe") or "UNKNOWN_CWE",
1398
+ "severity": str(finding.get("severity") or "HIGH").upper(),
1399
+ "confidence": "MEDIUM",
1400
+ "category": finding.get("category") or "Security",
1401
+ "message": finding.get("message") or "",
1402
+ })
1403
+ return sorted(mapped, key=lambda item: (item["file"], item["line"], item["cwe"]))
1404
+
1405
+
1406
+ def _scan_failure(args, exc):
1407
+ """A crashed scan still owes CI parsable artefacts, then reports exit 2.
1408
+
1409
+ Without this the process dies on a traceback: stdout JSON stays empty (0-byte file
1410
+ after redirection) and the SARIF target never exists, which GitHub's uploader
1411
+ reports as an unrelated failure.
1412
+ """
1413
+ message = f"{type(exc).__name__}: {exc}"
1414
+ print(f"[FATAL] TimeCodeSecurity scan aborted: {message}", file=sys.stderr)
1415
+ if getattr(args, "format", "") == "json":
1416
+ print(json.dumps({
1417
+ "scope": getattr(args, "scope", "all"),
1418
+ "status": "incomplete",
1419
+ "error": message,
1420
+ "total_discovered_files": 0,
1421
+ "scanned_files": 0,
1422
+ "fast_path_skipped_files": 0,
1423
+ "unparseable_files": 0,
1424
+ "findings": [],
1425
+ }, indent=2))
1426
+ if getattr(args, "sarif", None):
1427
+ try:
1428
+ document = bound_sarif_document(
1429
+ _sarif_document([], Path.cwd().resolve(), execution_successful=False))
1430
+ atomic_write_text(Path(args.sarif), json.dumps(document, indent=2) + "\n")
1431
+ except OSError as write_exc:
1432
+ print(f"Error writing fallback SARIF file: {write_exc}", file=sys.stderr)
1433
+ return 2
1434
+
1435
+
1436
+ # A runner that kills the process at a step timeout leaves no SARIF at all, and the referee can only
1437
+ # report that as "0 findings" - indistinguishable from a clean scan. The handlers below turn that
1438
+ # truncation into a real (partial) report: whatever the scan already holds gets written to the
1439
+ # requested --sarif / --format json targets. Two limits are stated plainly: findings the engine has
1440
+ # not produced yet cannot be flushed, and GitHub's own step timeout does not reliably signal child
1441
+ # processes, so a caller must bound the run with `timeout --signal=TERM` to actually reach this path.
1442
+ _PARTIAL = {
1443
+ "args": None,
1444
+ "findings": [],
1445
+ "tracker": None,
1446
+ "files": None,
1447
+ "written": False,
1448
+ "current_file": None,
1449
+ "t0": None,
1450
+ }
1451
+
1452
+
1453
+ def _partial_findings():
1454
+ """Findings as far as the scan has got: stage list first, live tracker index second."""
1455
+ findings = list(_PARTIAL["findings"])
1456
+ if findings:
1457
+ return findings
1458
+ tracker = _PARTIAL["tracker"]
1459
+ if tracker is None:
1460
+ return []
1461
+ try:
1462
+ return _findings_for(tracker, list(tracker.edges))
1463
+ except Exception as exc:
1464
+ # A half-built index must not hide the artefact: report an empty partial run instead.
1465
+ print(f"[warning] Partial index unreadable: {type(exc).__name__}: {exc}",
1466
+ file=sys.stderr, flush=True)
1467
+ return []
1468
+
1469
+
1470
+ def _print_termination_telemetry():
1471
+ """Say where the scan stopped, on stderr, before the partial payload is emitted.
1472
+
1473
+ A partial SARIF/JSON file only carries the findings that existed at the moment of the
1474
+ kill; it cannot say how far the engine had got. Without this line a timeout at file 3 of
1475
+ 9,000 and a timeout inside the final cascade look identical to whoever reads the artefact.
1476
+ Everything here is observational: it writes to stderr and never touches the JSON shape or
1477
+ the SARIF document.
1478
+ """
1479
+ tracker = _PARTIAL["tracker"]
1480
+ point = None
1481
+ if tracker is not None:
1482
+ try:
1483
+ point = tracker.termination_checkpoint()
1484
+ except Exception as exc:
1485
+ print(f"[telemetry] Checkpoint unavailable: {type(exc).__name__}: {exc}",
1486
+ file=sys.stderr, flush=True)
1487
+ if point is None:
1488
+ print("[telemetry] Active phase: pre-analysis (discovery/parse), analyze() not started",
1489
+ file=sys.stderr, flush=True)
1490
+ else:
1491
+ print(f"[telemetry] Active phase: {point['phase']}", file=sys.stderr, flush=True)
1492
+ profile = point.get("collector_profile_s") or {}
1493
+ if profile:
1494
+ ordered = sorted(profile.items(), key=lambda kv: (-kv[1], kv[0]))
1495
+ print("[telemetry] Collector timings so far (s): "
1496
+ + " ".join(f"{name}={secs:.2f}" for name, secs in ordered),
1497
+ file=sys.stderr, flush=True)
1498
+ t0 = _PARTIAL.get("t0")
1499
+ if t0 is not None:
1500
+ line = f"[telemetry] Elapsed since scan start: {time.perf_counter() - t0:.2f}s"
1501
+ if point is not None and point.get("analyze_elapsed_s") is not None:
1502
+ line += f" (analyze(): {point['analyze_elapsed_s']:.2f}s)"
1503
+ print(line, file=sys.stderr, flush=True)
1504
+
1505
+
1506
+ def _flush_partial(reason):
1507
+ """Serialize the current scan state into the requested artefacts, once."""
1508
+ args = _PARTIAL["args"]
1509
+ if args is None or _PARTIAL["written"]:
1510
+ return []
1511
+ _PARTIAL["written"] = True
1512
+ # Telemetry first: stdout carries the JSON payload, so the checkpoint has to be on stderr
1513
+ # and flushed before it, or a consumer piping stdout never sees it in order.
1514
+ _print_termination_telemetry()
1515
+ findings = _partial_findings()
1516
+ files = _PARTIAL["files"] or {}
1517
+ if getattr(args, "format", "") == "json":
1518
+ print(json.dumps({
1519
+ "scope": getattr(args, "scope", "all"),
1520
+ "status": "partial",
1521
+ "terminated_by": reason,
1522
+ "total_discovered_files": len(files),
1523
+ "scanned_files": len(files),
1524
+ "findings": findings,
1525
+ }, indent=2), flush=True)
1526
+ if getattr(args, "sarif", None):
1527
+ try:
1528
+ document = bound_sarif_document(
1529
+ _sarif_document(findings, Path.cwd().resolve(), execution_successful=False))
1530
+ atomic_write_text(Path(args.sarif), json.dumps(document, indent=2) + "\n")
1531
+ except OSError as exc:
1532
+ print(f"[warning] Partial SARIF write failed: {exc}", file=sys.stderr, flush=True)
1533
+ return findings
1534
+ print("[warning] Caught termination signal; flushed partial findings to SARIF",
1535
+ file=sys.stderr, flush=True)
1536
+ return findings
1537
+
1538
+
1539
+ def _install_partial_flush(args):
1540
+ """Arm SIGTERM/SIGINT for this scan run, then let the caller work normally."""
1541
+ _PARTIAL.update({"args": args, "findings": [], "tracker": None, "files": None,
1542
+ "written": False, "current_file": None, "t0": time.perf_counter()})
1543
+
1544
+ def _handle(signum, _frame):
1545
+ reason = signal.Signals(signum).name
1546
+ # Progress lines are throttled to one per PROGRESS_FILE_EVERY files, so the file the scan
1547
+ # is actually inside comes from _PARTIAL["current_file"] rather than the last log line.
1548
+ current = _PARTIAL["current_file"]
1549
+ where = f" while reading {current}" if current else " mid-scan"
1550
+ print(f"[warning] Caught {reason}{where}; flushing what is known so far",
1551
+ file=sys.stderr, flush=True)
1552
+ _flush_partial(reason)
1553
+ # 128 + signum is the shell convention for "died of that signal": 143 SIGTERM, 130 SIGINT.
1554
+ raise SystemExit(128 + signum)
1555
+
1556
+ for sig in (signal.SIGTERM, signal.SIGINT):
1557
+ try:
1558
+ signal.signal(sig, _handle)
1559
+ except (AttributeError, OSError, ValueError):
1560
+ # No such signal on this platform, or not the main thread.
1561
+ pass
1562
+
1563
+
1564
+ def _parse_parser_arg(parser_str: Optional[str]) -> str:
1565
+ """Normalise --parser into one of PARSER_MODES. Anything unexpected is the default, never a crash.
1566
+
1567
+ argparse restricts the choices for a real command line; this exists for callers that build an args
1568
+ namespace themselves, where `getattr(args, "parser", None)` comes back None.
1569
+ """
1570
+ if isinstance(parser_str, str) and parser_str.strip().lower() in PARSER_MODES:
1571
+ return parser_str.strip().lower()
1572
+ return DEFAULT_PARSER_MODE
1573
+
1574
+
1575
+ def _parse_workers_arg(workers_str: Optional[str]) -> int:
1576
+ """Parse --workers argument into integer worker count.
1577
+
1578
+ Args:
1579
+ workers_str: String value from CLI ('1', '2', '4', '8', 'auto', or None)
1580
+
1581
+ Returns:
1582
+ Integer worker count (defaults to os.cpu_count() for full multi-core utilization)
1583
+ """
1584
+ if workers_str is None:
1585
+ workers_str = "auto"
1586
+
1587
+ workers_str = workers_str.strip().lower()
1588
+
1589
+ if workers_str == "auto":
1590
+ try:
1591
+ import os
1592
+ cpu_count = os.cpu_count() or 4
1593
+ # Use all available CPU cores (dynamic multi-core concurrency)
1594
+ return max(1, cpu_count)
1595
+ except Exception:
1596
+ return 4
1597
+
1598
+ try:
1599
+ n = int(workers_str)
1600
+ if n < 1:
1601
+ print("Warning: --workers must be >= 1, using 1", file=sys.stderr)
1602
+ return 1
1603
+ if n > 64:
1604
+ print(f"Warning: --workers={n} is excessive, capping at 32", file=sys.stderr)
1605
+ return 32
1606
+ return n
1607
+ except ValueError:
1608
+ print(f"Warning: invalid --workers value '{workers_str}', using auto-detect", file=sys.stderr)
1609
+ try:
1610
+ import os
1611
+ return os.cpu_count() or 4
1612
+ except Exception:
1613
+ return 4
1614
+
1615
+
1616
+ def _maybe_warn_js_ts(args):
1617
+ """Warn about a missing JS/TS grammar only when JS/TS was actually part of this run.
1618
+
1619
+ Pure-Python scans used to inherit an import-time warning from js_scanner that the user could
1620
+ do nothing with. The grammar is already imported fine in most runs, so that case costs one
1621
+ boolean; only a broken install pays the (early-exit) tree walk that decides whether the
1622
+ target holds any JS/TS file.
1623
+ """
1624
+ if _JS_TS_SCANNER_AVAILABLE and js_ts_available():
1625
+ return
1626
+ if not getattr(args, "include_js", False) and not has_js_ts_files(args.path):
1627
+ return
1628
+ warn_js_ts_unavailable()
1629
+
1630
+
1631
+ def _scan(args):
1632
+ scope = args.scope
1633
+ # Phase-1 outer clock: anchored at the earliest scan entry point, i.e. BEFORE
1634
+ # `_maybe_warn_js_ts`, auxiliary collection and `_collect_files`. The pre-existing
1635
+ # `started`/`duration_ms` anchor (after file collection) is left untouched so the
1636
+ # JSON schema keeps its documented meaning; Outer(...) TOTAL is the wider boundary.
1637
+ _clock = _StageClock("scan")
1638
+ _install_partial_flush(args)
1639
+ _maybe_warn_js_ts(args)
1640
+
1641
+ # Parse workers argument and pass to TaintTracker
1642
+ workers = _parse_workers_arg(getattr(args, "workers", None))
1643
+ ux = _ux_enabled(args)
1644
+ if ux:
1645
+ banner.print_banner(args.path, scope=scope, workers=workers, stream=sys.stderr)
1646
+
1647
+ try:
1648
+ line_ranges = _parse_line_ranges(getattr(args, "lines", None))
1649
+ exclude_patterns = _parse_exclude_patterns(getattr(args, "exclude", None))
1650
+ # Root configuration is optional and read from the working directory only, so a
1651
+ # checkout without `.tcs.yaml` keeps the byte-identical default scan.
1652
+ config = load_config()
1653
+ # Collected keys are spelled relative to the working directory, so the root is
1654
+ # resolved once here and reused by every `.tcs.yaml` match below.
1655
+ _config_root = _file_key(args.path, Path.cwd().resolve())
1656
+ config_excludes = (
1657
+ (lambda key: config.excludes_path(key, _config_root))
1658
+ if config is not None else None
1659
+ )
1660
+ tree_scan = args.path.is_dir()
1661
+ # Pruning non-production trees is the default for a directory walk; `--include-tests` opts
1662
+ # back in, and `--skip-non-production` is accepted but no longer changes anything. Only a
1663
+ # walk prunes: a file or directory named on the command line is analysed in full.
1664
+ skip_non_production = tree_scan and not bool(getattr(args, "include_tests", False))
1665
+ _SCOPE_STATS["skipped_dirs"] = 0
1666
+ _SCOPE_STATS["skipped_files"] = 0
1667
+ templates, iac, manifests, secrets, aux_scopes = _collect_auxiliary_files(
1668
+ args.path, exclude_patterns, config_excludes, skip_non_production)
1669
+ if scope not in ("all", "html"):
1670
+ templates = {}
1671
+ if scope not in ("all", "sca"):
1672
+ manifests = {}
1673
+ if scope not in ("all", "secrets"):
1674
+ secrets = {}
1675
+ if scope in ("docker", "iac"):
1676
+ iac = {key: source for key, source in iac.items() if aux_scopes.get(key) == scope}
1677
+ elif scope != "all":
1678
+ iac = {}
1679
+ aux_scopes = {key: value for key, value in aux_scopes.items()
1680
+ if scope == "all" or value == scope}
1681
+ _clock.mark("AuxCollect")
1682
+ tracker = None
1683
+ files = {}
1684
+ if scope in ("all", "python"):
1685
+ try:
1686
+ files = _collect_files(args.path, exclude_patterns=exclude_patterns,
1687
+ config_excludes=config_excludes,
1688
+ skip_non_production=skip_non_production)
1689
+ except ValueError:
1690
+ if not templates and not iac and not manifests and not secrets:
1691
+ raise
1692
+ _clock.mark("CollectPy")
1693
+ if ux:
1694
+ auxiliary = len(templates) + len(iac) + len(manifests) + len(secrets)
1695
+ banner.print_phase(1, 4, "discovery", stream=sys.stderr)
1696
+ banner.print_step(1, "files discovered",
1697
+ f"{len(files)} python, {auxiliary} auxiliary", stream=sys.stderr)
1698
+
1699
+ # TimeCodeSecurity: Capture pre-scan memory stats
1700
+ try:
1701
+ import psutil
1702
+ mem_before = psutil.virtual_memory()
1703
+ available_ram_before_gb = mem_before.available / (1024 ** 3)
1704
+ rss_before_mb = psutil.Process().memory_info().rss / (1024 ** 2)
1705
+ except ImportError:
1706
+ available_ram_before_gb = 0
1707
+ rss_before_mb = 0
1708
+
1709
+ if ux:
1710
+ banner.print_phase(2, 4, "analysis", stream=sys.stderr)
1711
+ for worker_id in range(workers if files else 0):
1712
+ banner.print_worker(worker_id, "ready", stream=sys.stderr)
1713
+ started = time.perf_counter()
1714
+ if files:
1715
+ _PARTIAL["files"] = files
1716
+ _parser_mode = _parse_parser_arg(getattr(args, "parser", None))
1717
+ # Default mode keeps the progress line byte-for-byte as it was; a chosen mode names itself,
1718
+ # because "with 8 worker(s)" is not what `treesitter` is doing (it runs single-process).
1719
+ _progress(f"parsing {len(files)} python file(s) with {workers} worker(s)"
1720
+ + ("" if _parser_mode == DEFAULT_PARSER_MODE else f", parser={_parser_mode}"))
1721
+ tracker = TaintTracker(
1722
+ files=files,
1723
+ max_workers=workers,
1724
+ parser_mode=_parser_mode,
1725
+ audit_all=getattr(args, "audit_all", False),
1726
+ )
1727
+ _PARTIAL["tracker"] = tracker
1728
+ _clock.mark("Parse+Init")
1729
+ for fpath, reason in sorted(tracker.skipped_files.items()):
1730
+ print(f"Warning: skipping unparseable file {fpath}: {reason}", file=sys.stderr)
1731
+ _, _, edges = tracker.analyze()
1732
+ _clock.mark("Analyze")
1733
+ ast_findings = _findings_for(tracker, edges)
1734
+ _clock.mark("ASTFindings")
1735
+ _progress(f"parsed + traced, {len(ast_findings)} AST finding(s) so far")
1736
+ else:
1737
+ ast_findings = []
1738
+ _clock.mark("Parse+Init")
1739
+ _clock.mark("Analyze")
1740
+ _clock.mark("ASTFindings")
1741
+ if ux:
1742
+ banner.print_step(2, "ast findings", f"{len(ast_findings)} row(s)", stream=sys.stderr)
1743
+ aux_file_counts = {"docker": 0, "iac": 0, "sca": 0, "secrets": 0}
1744
+ for aux_scope in aux_scopes.values():
1745
+ aux_file_counts[aux_scope] += 1
1746
+ file_counts = {
1747
+ "python": len(tracker.modules) if tracker else 0,
1748
+ "docker": aux_file_counts["docker"],
1749
+ "iac": aux_file_counts["iac"],
1750
+ "sca": aux_file_counts["sca"],
1751
+ "secrets": aux_file_counts["secrets"],
1752
+ "html": len(templates),
1753
+ }
1754
+ scanned_files = sum(file_counts.values())
1755
+ parse_stats = getattr(tracker, "parse_stats", None) or {}
1756
+ python_discovered = parse_stats.get("discovered", len(files))
1757
+ fast_path_skipped_files = parse_stats.get("fast_path_skipped", 0)
1758
+ unparseable_files = parse_stats.get(
1759
+ "unparseable", len(tracker.skipped_files) if tracker else 0)
1760
+ scope_breakdown = {
1761
+ "python": {
1762
+ "discovered": python_discovered,
1763
+ "scanned": file_counts["python"],
1764
+ "fast_path_skipped": fast_path_skipped_files,
1765
+ "unparseable": unparseable_files,
1766
+ },
1767
+ "docker": {
1768
+ "discovered": aux_file_counts["docker"],
1769
+ "scanned": file_counts["docker"],
1770
+ },
1771
+ "iac": {
1772
+ "discovered": aux_file_counts["iac"],
1773
+ "scanned": file_counts["iac"],
1774
+ },
1775
+ "sca": {
1776
+ "discovered": aux_file_counts["sca"],
1777
+ "scanned": file_counts["sca"],
1778
+ },
1779
+ "secrets": {
1780
+ "discovered": aux_file_counts["secrets"],
1781
+ "scanned": file_counts["secrets"],
1782
+ },
1783
+ "html": {
1784
+ "discovered": len(templates),
1785
+ "scanned": file_counts["html"],
1786
+ },
1787
+ }
1788
+ total_discovered_files = sum(bucket["discovered"] for bucket in scope_breakdown.values())
1789
+ parse_governor = {
1790
+ key: parse_stats[key]
1791
+ for key in ("parse_mode", "workers_used", "workers_initial",
1792
+ "workers_final", "governor_events",
1793
+ "execution_mode", "parallel_fallback_reason",
1794
+ # Phase 3: which reader was allowed, and how many modules it actually produced
1795
+ # (a shadow run can be checked against those two numbers instead of a log).
1796
+ "parser_mode", "parsed_by_packer")
1797
+ if key in parse_stats
1798
+ }
1799
+ # The engine attribute is the authority; parse_stats carries it only when a tracker ran.
1800
+ parse_governor.setdefault("execution_mode", getattr(tracker, "execution_mode", "not_started"))
1801
+ parse_governor.setdefault("parallel_fallback_reason",
1802
+ getattr(tracker, "parallel_fallback_reason", None))
1803
+ # The legacy auditor keeps owning every CWE-1357 row and the Dockerfile CWE-250/CWE-269
1804
+ # pair, byte-identical. The modular scanners add the container and CI/IaC rule families on
1805
+ # top and are handed the legacy rows so one root-user defect is never booked twice.
1806
+ legacy_iac = audit_iac_files(iac, audit_all=getattr(args, "audit_all", False))
1807
+ sca_findings = audit_sca_files(manifests)
1808
+ if sca_findings:
1809
+ # The import walk is paid for only when a vulnerable pin exists, so a Python-only tree
1810
+ # -- every latency gate's workload -- does no extra work at all.
1811
+ tag_reachability(sca_findings, _imported_roots(tracker))
1812
+ findings = _merge_findings(
1813
+ ast_findings,
1814
+ audit_templates(templates),
1815
+ legacy_iac,
1816
+ audit_security_files(iac, legacy_findings=legacy_iac),
1817
+ sca_findings,
1818
+ audit_secret_files(secrets),
1819
+ )
1820
+ findings = consolidate_findings(findings)
1821
+ # Template/IaC auditors report a structural match and no data-flow provenance, which
1822
+ # is exactly MEDIUM; AST findings arrive with their own label.
1823
+ for finding in findings:
1824
+ finding.setdefault("confidence", "MEDIUM")
1825
+ _clock.mark("Merge+Consolidate")
1826
+ _PARTIAL["findings"] = findings
1827
+ if scope in ("all", "python"):
1828
+ cross_findings, cross_engine = _run_cross_scan(
1829
+ args.path, reuse_trees=_cross_scan_reuse_trees(tracker))
1830
+ _clock.mark("CrossScan")
1831
+ findings = _suppress_cross_file_sanitized(findings, cross_engine)
1832
+ _clock.mark("Suppression")
1833
+ findings.extend(cross_findings)
1834
+ _PARTIAL["findings"] = findings
1835
+ else:
1836
+ _clock.mark("CrossScan")
1837
+ _clock.mark("Suppression")
1838
+ if ux:
1839
+ banner.print_phase(3, 4, "correlation", stream=sys.stderr)
1840
+ banner.print_step(3, "cross-file + suppression", f"{len(findings)} row(s)",
1841
+ stream=sys.stderr)
1842
+
1843
+ # Incremental scan: keep only diagnostics intersecting the modified lines.
1844
+ # Applied before autofix generation so patching work is scoped too.
1845
+ findings = _scope_findings_to_lines(findings, line_ranges)
1846
+
1847
+ # Opt-in tree-sitter JS/TS pass. Off by default: the grammar is a native
1848
+ # extension whose access violation takes the whole process down.
1849
+ if getattr(args, "include_js", False) and _JS_TS_SCANNER_AVAILABLE and js_ts_available():
1850
+ findings = _merge_findings(findings, _js_ts_findings(args.path))
1851
+
1852
+ # Root `.tcs.yaml`: path exclusions and per-rule path overrides from the working
1853
+ # directory. Applied after every producer so cross-file and JS rows obey it too.
1854
+ findings = _apply_config_filters(findings, config, path_excludes=tree_scan,
1855
+ scan_root=_config_root, aux_scopes=aux_scopes)
1856
+ findings = _apply_inline_suppressions(
1857
+ findings, {**files, **templates, **iac, **manifests, **secrets})
1858
+
1859
+ # --diff narrows *reporting* to added/modified lines. The analysis above already
1860
+ # saw the whole tree, so cross-file context is never truncated.
1861
+ diff_ref = getattr(args, "diff", None)
1862
+ if diff_ref:
1863
+ _diff_repo_root = resolve_repo_root()
1864
+ findings = _scope_findings_to_diff(
1865
+ findings, changed_line_intervals(_diff_repo_root, diff_ref), _diff_repo_root)
1866
+
1867
+ # Display-only confidence filter; the engine's emission set is already final.
1868
+ requested_minimum = getattr(args, "min_confidence", None)
1869
+ confidence_minimum = None if requested_minimum in (None, "all") else requested_minimum
1870
+ findings = _filter_by_confidence(findings, confidence_minimum)
1871
+ # The default high floor belongs to the table alone, and only when nobody named a level: an
1872
+ # explicit `all` is a request to see everything, in the table too. `findings` stays the
1873
+ # complete emission set either way, so JSON, SARIF, the autofix pass and the exit code never
1874
+ # see a floor nobody asked for, and `confidence_filter` keeps reporting null for a machine run.
1875
+ hidden_rows = []
1876
+ display_rows = findings
1877
+ if requested_minimum is None and getattr(args, "format", "table") == "table":
1878
+ display_rows, hidden_rows = _split_by_table_default(findings)
1879
+ _PARTIAL["findings"] = findings
1880
+
1881
+ # Generate autofix suggestions if requested (JSON only)
1882
+ if args.with_autofix and args.format == "json" and files:
1883
+ # Build source cache from scanned files
1884
+ source_cache = {}
1885
+ for file_path in files:
1886
+ try:
1887
+ with open(file_path, 'r', encoding='utf-8') as f:
1888
+ source_cache[file_path.replace("\\", "/")] = f.read()
1889
+ except (OSError, UnicodeDecodeError):
1890
+ pass
1891
+
1892
+ # Attach autofixes where deterministic fixes are available
1893
+ autofix_engine = RemediationEngine()
1894
+ findings = _attach_autofixes(findings, source_cache, autofix_engine)
1895
+ _clock.mark("LineScope+JS+Autofix")
1896
+
1897
+ duration_ms = (time.perf_counter() - started) * 1000
1898
+
1899
+ # TimeCodeSecurity: Capture post-scan memory stats
1900
+ try:
1901
+ import psutil
1902
+ mem_after = psutil.virtual_memory()
1903
+ available_ram_after_gb = mem_after.available / (1024 ** 3)
1904
+ rss_after_mb = psutil.Process().memory_info().rss / (1024 ** 2)
1905
+ peak_rss_mb = rss_after_mb
1906
+ available_ram_gb = min(available_ram_before_gb, available_ram_after_gb)
1907
+ except ImportError:
1908
+ peak_rss_mb = 0
1909
+ available_ram_gb = 0
1910
+ except (OSError, ValueError) as exc:
1911
+ return _scan_failure(args, exc)
1912
+
1913
+ if ux:
1914
+ banner.print_phase(4, 4, "reporting", stream=sys.stderr)
1915
+ if args.format == "json":
1916
+ print(json.dumps({
1917
+ "scope": scope,
1918
+ "line_filter": [list(r) for r in line_ranges] if line_ranges else None,
1919
+ "diff_ref": diff_ref,
1920
+ "confidence_filter": confidence_minimum,
1921
+ "total_discovered_files": total_discovered_files,
1922
+ "scanned_files": scanned_files,
1923
+ "scanned_files_by_scope": file_counts,
1924
+ "fast_path_skipped_files": fast_path_skipped_files,
1925
+ "unparseable_files": unparseable_files,
1926
+ "scope_breakdown": scope_breakdown,
1927
+ "parse_governor": parse_governor,
1928
+ "execution_mode": parse_governor.get("execution_mode", "not_started"),
1929
+ "parallel_fallback_reason": parse_governor.get("parallel_fallback_reason"),
1930
+ "duration_ms": round(duration_ms, 2),
1931
+ "findings_by_scope": _scope_breakdown(findings, aux_scopes),
1932
+ "findings": findings,
1933
+ }, indent=2))
1934
+ # A scope note belongs on stderr for a machine run: the payload and its keys stay exactly
1935
+ # as they are, so nothing that parses this output changes behaviour.
1936
+ scope_note = non_production_scope_note()
1937
+ if scope_note:
1938
+ print(scope_note, file=sys.stderr)
1939
+ else:
1940
+ if display_rows:
1941
+ print(_ascii_table(display_rows))
1942
+ elif hidden_rows:
1943
+ print(f"[+] No findings at the default {TABLE_DEFAULT_CONFIDENCE.upper()} confidence "
1944
+ f"floor; {len(hidden_rows)} row(s) are held back, listed in the note below.")
1945
+ else:
1946
+ if _SCOPE_STATS["skipped_files"]:
1947
+ # Nothing was reported, but the walk did not read everything: "Clean scan!" over a
1948
+ # tree whose tests/ and examples/ were pruned would be the most misleading line
1949
+ # this tool can print.
1950
+ print(f"[+] No findings in the {scanned_files} file(s) that were read; "
1951
+ f"{_SCOPE_STATS['skipped_files']} non-production file(s) were held back "
1952
+ f"-- see the note below.")
1953
+ else:
1954
+ print("[+] No vulnerabilities found. Clean scan!")
1955
+ note = table_default_note(hidden_rows)
1956
+ if note:
1957
+ print(note)
1958
+ scope_note = non_production_scope_note()
1959
+ if scope_note:
1960
+ print(scope_note)
1961
+ _print_summary(scanned_files, duration_ms, display_rows, file_counts,
1962
+ discovered_files=total_discovered_files,
1963
+ fast_path_skipped_files=fast_path_skipped_files,
1964
+ unparseable_files=unparseable_files,
1965
+ parse_governor=parse_governor, aux_scopes=aux_scopes)
1966
+ traces = _cross_trace_report(findings, Path.cwd().resolve())
1967
+ if traces:
1968
+ print(traces)
1969
+ _clock.mark("StdoutOutput")
1970
+
1971
+ if args.sarif:
1972
+ try:
1973
+ document = bound_sarif_document(_sarif_document(findings, Path.cwd().resolve()))
1974
+ sarif_path = Path(args.sarif)
1975
+ atomic_write_text(sarif_path, json.dumps(document, indent=2) + "\n")
1976
+ sarif_size_mb = sarif_path.stat().st_size / (1024 ** 2) if sarif_path.exists() else 0
1977
+ except OSError as exc:
1978
+ print(f"Error writing SARIF file: {exc}", file=sys.stderr)
1979
+ return 2
1980
+ else:
1981
+ sarif_size_mb = 0
1982
+ _clock.mark("SARIFDump")
1983
+
1984
+ sbom_path_text = getattr(args, "sbom", None)
1985
+ if sbom_path_text:
1986
+ try:
1987
+ document = build_sbom(collect_components(manifests), findings,
1988
+ root_name=Path(args.path).resolve().name or "target")
1989
+ sbom_path = Path(sbom_path_text)
1990
+ atomic_write_text(sbom_path, json.dumps(document, indent=2) + "\n")
1991
+ except OSError as exc:
1992
+ print(f"Error writing SBOM file: {exc}", file=sys.stderr)
1993
+ return 2
1994
+ _clock.mark("SBOMDump")
1995
+
1996
+ # TimeCodeSecurity: Print performance metrics to stderr
1997
+ if files and tracker:
1998
+ try:
1999
+ from tcs.parallel_scanner import HARD_SAFETY_THRESHOLD_MB
2000
+
2001
+ print("\n" + "="*70, file=sys.stderr)
2002
+ print("TimeCodeSecurity - Performance Metrics", file=sys.stderr)
2003
+ print("="*70, file=sys.stderr)
2004
+ print(f"Wall-Clock Scan Time: {duration_ms/1000:.2f}s ({duration_ms:.0f}ms)", file=sys.stderr)
2005
+ print(f"Parser Mode: {parse_governor.get('parse_mode', 'unknown')}"
2006
+ f" (execution: {parse_governor.get('execution_mode', 'unknown')})", file=sys.stderr)
2007
+ print(f"Worker Count Utilized: {parse_governor.get('workers_final', parse_governor.get('workers_used', 'n/a'))}"
2008
+ f" (governor start: {parse_governor.get('workers_initial', parse_governor.get('workers_used', 'n/a'))}, "
2009
+ f"reductions: {parse_governor.get('governor_events', 0)})", file=sys.stderr)
2010
+ print(f"Governor RAM Floor: {HARD_SAFETY_THRESHOLD_MB} MB", file=sys.stderr)
2011
+ print(f"Peak Memory RSS: {peak_rss_mb:.1f} MB", file=sys.stderr)
2012
+ print(f"Available RAM (min): {available_ram_gb:.2f} GB", file=sys.stderr)
2013
+ print(f"Total Findings: {len(findings)}", file=sys.stderr)
2014
+ if args.sarif:
2015
+ print(f"SARIF File Size: {sarif_size_mb:.2f} MB {'✓' if sarif_size_mb < 10 else '⚠ EXCEEDS 10 MB'}", file=sys.stderr)
2016
+ print(f"Files Discovered: {total_discovered_files} "
2017
+ f"(python: {python_discovered}, docker: {aux_file_counts['docker']}, "
2018
+ f"iac: {aux_file_counts['iac']}, sca: {aux_file_counts['sca']}, "
2019
+ f"secrets: {aux_file_counts['secrets']}, "
2020
+ f"html: {len(templates)})", file=sys.stderr)
2021
+ print(f"Files Scanned: {scanned_files} "
2022
+ f"(python: {file_counts['python']}, docker: {file_counts['docker']}, "
2023
+ f"iac: {file_counts['iac']}, sca: {file_counts['sca']}, "
2024
+ f"secrets: {file_counts['secrets']}, "
2025
+ f"html: {file_counts['html']})", file=sys.stderr)
2026
+ print(f"Zero-Risk Fast-Path Skip: {fast_path_skipped_files} skipped", file=sys.stderr)
2027
+ print(f"Unparseable Files: {unparseable_files} skipped", file=sys.stderr)
2028
+ print("="*70, file=sys.stderr)
2029
+ except ImportError:
2030
+ pass # Parallel scanner not available, skip metrics
2031
+
2032
+ _clock.mark("MetricsBlock")
2033
+ # Outer TOTAL spans the whole scan command: entry (before auxiliary/python collection)
2034
+ # to just before exit-code computation. Stages before `started` (AuxCollect, CollectPy)
2035
+ # are inside TOTAL but NOT inside duration_ms - that is the CLI-vs-UI boundary gap.
2036
+ _clock.report()
2037
+ if ux:
2038
+ banner.print_completion(duration_ms / 1000, len(findings),
2039
+ sarif_path=str(args.sarif) if args.sarif else None,
2040
+ sbom_path=sbom_path_text or None, stream=sys.stderr)
2041
+
2042
+ # Exit-code parity with `tcs_cli.py`: a scan that finds something exits 1, so CI can gate
2043
+ # on either entry point. 2 stays reserved for an internal error (see the crash handler).
2044
+ # `--fail-on-critical` narrows what blocks: only Critical/High findings count.
2045
+ if args.fail_on_critical:
2046
+ blocking = [item for item in findings if item["severity"] in ("CRITICAL", "HIGH")]
2047
+ else:
2048
+ blocking = findings
2049
+ return 1 if blocking else 0
2050
+
2051
+
2052
+ def _normalize_compare_path(value):
2053
+ return os.path.normcase(str(Path(value).resolve()))
2054
+
2055
+
2056
+ def _normalize_cwes(value):
2057
+ if isinstance(value, dict):
2058
+ value = value.get("id", "")
2059
+ values = value if isinstance(value, list) else [value]
2060
+ cwes = set()
2061
+ for item in values:
2062
+ if isinstance(item, dict):
2063
+ item = item.get("id", "")
2064
+ text = str(item or "").upper()
2065
+ cwes.update(f"CWE-{number}" for number in re.findall(r"\bCWE[-_ ]?(\d+)\b", text))
2066
+ if text.isdigit():
2067
+ cwes.add(f"CWE-{text}")
2068
+ return sorted(cwes) if cwes else ["UNKNOWN_CWE"]
2069
+
2070
+
2071
+ def _competitor_findings(tool, data):
2072
+ findings = []
2073
+ if tool == "semgrep":
2074
+ for item in data.get("results", []):
2075
+ metadata = item.get("extra", {}).get("metadata", {})
2076
+ cwes = _normalize_cwes(metadata.get("cwe"))
2077
+ if cwes == ["UNKNOWN_CWE"]:
2078
+ cwes = _normalize_cwes(item.get("check_id", ""))
2079
+ path = item.get("path", "")
2080
+ line = item.get("start", {}).get("line", 0)
2081
+ rule = item.get("check_id", "")
2082
+ message = item.get("extra", {}).get("message", "")
2083
+ for cwe in cwes:
2084
+ findings.append({"file": _normalize_compare_path(path), "line": line, "cwe": cwe, "rule": rule, "message": message})
2085
+ else:
2086
+ for item in data.get("results", []):
2087
+ cwes = _normalize_cwes(item.get("issue_cwe"))
2088
+ path = item.get("filename", "")
2089
+ line = item.get("line_number", 0)
2090
+ rule = item.get("test_id", "")
2091
+ message = item.get("issue_text", "")
2092
+ for cwe in cwes:
2093
+ findings.append({"file": _normalize_compare_path(path), "line": line, "cwe": cwe, "rule": rule, "message": message})
2094
+ return findings
2095
+
2096
+
2097
+ def _run_competitor(tool, scan_path):
2098
+ executable = shutil.which(tool)
2099
+ if executable is None:
2100
+ print(f"Error: {tool} is not installed or not available on PATH.", file=sys.stderr)
2101
+ return None, None, 127
2102
+ if tool == "semgrep":
2103
+ command = [executable, "scan", "--json", "--config", "auto", str(scan_path)]
2104
+ else:
2105
+ command = [executable, "-r", "-f", "json", str(scan_path)]
2106
+
2107
+ started = time.perf_counter()
2108
+ try:
2109
+ completed = subprocess.run(command, capture_output=True, text=True, encoding="utf-8", errors="replace", check=False)
2110
+ except OSError as exc:
2111
+ print(f"Error: unable to run {tool}: {exc}", file=sys.stderr)
2112
+ return None, None, 2
2113
+ duration_ms = (time.perf_counter() - started) * 1000
2114
+ try:
2115
+ data = json.loads(completed.stdout)
2116
+ except (json.JSONDecodeError, TypeError):
2117
+ detail = completed.stderr.strip() or completed.stdout.strip() or f"process exited with status {completed.returncode}"
2118
+ print(f"Error: {tool} did not return valid JSON: {detail}", file=sys.stderr)
2119
+ return None, None, 2
2120
+ return _competitor_findings(tool, data), duration_ms, 0
2121
+
2122
+
2123
+ def _compare_findings(tcs_findings, competitor_findings, tolerance):
2124
+ matched = []
2125
+ matched_tcs = set()
2126
+ competitor_only = []
2127
+ for competitor in competitor_findings:
2128
+ candidates = [
2129
+ (index, finding) for index, finding in enumerate(tcs_findings)
2130
+ if finding["_normalized_file"] == competitor["file"]
2131
+ and finding["cwe"] == competitor["cwe"]
2132
+ and abs(finding["line"] - competitor["line"]) <= tolerance
2133
+ ]
2134
+ if candidates:
2135
+ tcs_index, finding = min(candidates, key=lambda pair: abs(pair[1]["line"] - competitor["line"]))
2136
+ matched_tcs.add(tcs_index)
2137
+ matched.append({"cwe": competitor["cwe"], "file": finding["file"], "tcs_line": finding["line"], "competitor_line": competitor["line"], "rule": competitor["rule"]})
2138
+ else:
2139
+ competitor_only.append(competitor)
2140
+
2141
+ tcs_only = [finding for index, finding in enumerate(tcs_findings) if index not in matched_tcs]
2142
+ return matched, tcs_only, competitor_only
2143
+
2144
+
2145
+ def _cwe_breakdown(matched, tcs_only, competitor_only):
2146
+ cwes = sorted({item["cwe"] for item in matched + tcs_only + competitor_only})
2147
+ return [
2148
+ {
2149
+ "cwe": cwe,
2150
+ "both": sum(item["cwe"] == cwe for item in matched),
2151
+ "tcs_only": sum(item["cwe"] == cwe for item in tcs_only),
2152
+ "competitor_only": sum(item["cwe"] == cwe for item in competitor_only),
2153
+ }
2154
+ for cwe in cwes
2155
+ ]
2156
+
2157
+
2158
+ def _print_compare_scoreboard(tool, tcs_ms, competitor_ms, matched, tcs_only, competitor_only, breakdown):
2159
+ summary_rows = [
2160
+ ("Execution speed", f"TimeCodeSecurity {tcs_ms:.2f} ms vs {tool.title()} {competitor_ms:.2f} ms"),
2161
+ ("Matched Findings (both)", str(len(matched))),
2162
+ ("TimeCodeSecurity Exclusive Findings", str(len(tcs_only))),
2163
+ (f"{tool.title()} Only Findings", str(len(competitor_only))),
2164
+ ]
2165
+ summary_headers = ("Metric", "Result")
2166
+ summary_widths = [max(len(summary_headers[index]), *(len(row[index]) for row in summary_rows)) for index in range(len(summary_headers))]
2167
+ summary_border = "+" + "+".join("-" * (width + 2) for width in summary_widths) + "+"
2168
+ print("Comparison Scoreboard")
2169
+ print(summary_border)
2170
+ print("| " + " | ".join(summary_headers[index].ljust(summary_widths[index]) for index in range(len(summary_headers))) + " |")
2171
+ print(summary_border)
2172
+ for row in summary_rows:
2173
+ print("| " + " | ".join(row[index].ljust(summary_widths[index]) for index in range(len(summary_headers))) + " |")
2174
+ print(summary_border)
2175
+ rows = [(item["cwe"], str(item["both"]), str(item["tcs_only"]), str(item["competitor_only"])) for item in breakdown]
2176
+ headers = ("CWE", "Both", "TimeCodeSecurity only", f"{tool.title()} only")
2177
+ widths = [max(len(headers[index]), *(len(row[index]) for row in rows)) for index in range(len(headers))]
2178
+ border = "+" + "+".join("-" * (width + 2) for width in widths) + "+"
2179
+ print("\nCWE Breakdown")
2180
+ print(border)
2181
+ print("| " + " | ".join(headers[index].ljust(widths[index]) for index in range(len(headers))) + " |")
2182
+ print(border)
2183
+ for row in rows:
2184
+ print("| " + " | ".join(row[index].ljust(widths[index]) for index in range(len(headers))) + " |")
2185
+ print(border)
2186
+
2187
+
2188
+ def _compare(args):
2189
+ try:
2190
+ exclude_patterns = _parse_exclude_patterns(getattr(args, "exclude", None))
2191
+ templates, iac, manifests, secrets, _aux_scopes = _collect_auxiliary_files(
2192
+ args.path, exclude_patterns)
2193
+
2194
+ try:
2195
+ files = _collect_files(args.path, exclude_patterns=exclude_patterns)
2196
+ except ValueError:
2197
+ if not templates and not iac and not manifests and not secrets:
2198
+ raise
2199
+ files = {}
2200
+ started = time.perf_counter()
2201
+ if files:
2202
+ tracker = TaintTracker(files=files)
2203
+ _, _, edges = tracker.analyze()
2204
+ ast_findings = _findings_for(tracker, edges)
2205
+ else:
2206
+ ast_findings = []
2207
+ tracker = None
2208
+ legacy_iac = audit_iac_files(iac)
2209
+ sca_findings = audit_sca_files(manifests)
2210
+ if sca_findings:
2211
+ tag_reachability(sca_findings, _imported_roots(tracker))
2212
+ tcs_findings = _merge_findings(ast_findings, audit_templates(templates), legacy_iac,
2213
+ audit_security_files(iac, legacy_findings=legacy_iac),
2214
+ sca_findings, audit_secret_files(secrets))
2215
+ tcs_ms = (time.perf_counter() - started) * 1000
2216
+ except (OSError, ValueError) as exc:
2217
+ print(f"Error: {exc}", file=sys.stderr)
2218
+ return 2
2219
+
2220
+ for finding in tcs_findings:
2221
+ finding["_normalized_file"] = _normalize_compare_path(finding["file"])
2222
+ competitor_findings, competitor_ms, status = _run_competitor(args.vs, args.path)
2223
+ if status:
2224
+ return status
2225
+
2226
+ matched, tcs_only, competitor_only = _compare_findings(tcs_findings, competitor_findings, args.tolerance)
2227
+ breakdown = _cwe_breakdown(matched, tcs_only, competitor_only)
2228
+ _print_compare_scoreboard(args.vs, tcs_ms, competitor_ms, matched, tcs_only, competitor_only, breakdown)
2229
+
2230
+ report = {
2231
+ "competitor": args.vs,
2232
+ "tolerance": args.tolerance,
2233
+ "execution_ms": {"tcs": round(tcs_ms, 2), "competitor": round(competitor_ms, 2)},
2234
+ "counts": {"both": len(matched), "tcs_only": len(tcs_only), "competitor_only": len(competitor_only)},
2235
+ "cwe_breakdown": breakdown,
2236
+ "matched": matched,
2237
+ "tcs_exclusive": [{key: value for key, value in item.items() if not key.startswith("_")} for item in tcs_only],
2238
+ "competitor_only_findings": competitor_only,
2239
+ }
2240
+ if args.output_json:
2241
+ try:
2242
+ atomic_write_text(Path(args.output_json), json.dumps(report, indent=2) + "\n")
2243
+ except OSError as exc:
2244
+ print(f"Error writing comparison report: {exc}", file=sys.stderr)
2245
+ return 2
2246
+ return 0
2247
+
2248
+
2249
+ def main(argv=None):
2250
+ parser = argparse.ArgumentParser(prog="cli.py", description="Standalone TimeCodeSecurity scanner")
2251
+ commands = parser.add_subparsers(dest="command", required=True)
2252
+ scan_parser = commands.add_parser("scan", help="Scan a Python file or directory")
2253
+ scan_parser.add_argument("path", type=Path, help="Python file or directory to scan")
2254
+ scan_parser.add_argument("--sarif", metavar="OUTPUT_PATH", help="Write findings as SARIF 2.1.0 JSON")
2255
+ scan_parser.add_argument("--sbom", metavar="OUTPUT_PATH",
2256
+ help="Write the dependency inventory and its advisories as a "
2257
+ "CycloneDX 1.5 JSON SBOM (stdout output is unchanged)")
2258
+ scan_parser.add_argument("--fail-on-critical", action="store_true", help="Only exit 1 on a Critical or High finding (default: any finding exits 1, matching tcs_cli.py)")
2259
+ scan_parser.add_argument("--format", choices=("table", "json"), default="table", help="Output format (default: table)")
2260
+ scan_parser.add_argument(
2261
+ "--scope", choices=("all", "python", "docker", "iac", "sca", "secrets", "html"),
2262
+ default="all",
2263
+ help="Limit analysis to Python code, container images/compose, CI-CD & IaC "
2264
+ "(workflows, Kubernetes, Terraform), dependency manifests/lockfiles (SCA), "
2265
+ "config/env documents carrying credentials, "
2266
+ "or HTML templates (default: all)",
2267
+ )
2268
+ scan_parser.add_argument(
2269
+ "--with-autofix", action="store_true", default=False,
2270
+ help="Include deterministic autofix suggestions in JSON output (only for supported CWEs)",
2271
+ )
2272
+ scan_parser.add_argument(
2273
+ "--lines", metavar="START[-END][,START[-END]...]", default=None,
2274
+ help="Incremental scan: restrict findings to the given 1-indexed line ranges, e.g. --lines 4-5",
2275
+ )
2276
+ scan_parser.add_argument(
2277
+ "--diff", nargs="?", const="HEAD", default=None, metavar="REF",
2278
+ help="Report only findings on lines added or modified in the working tree versus REF "
2279
+ "(default REF: HEAD). The whole tree is still analysed, so cross-file context is "
2280
+ "never truncated. Requires a git repository",
2281
+ )
2282
+ scan_parser.add_argument(
2283
+ "--min-confidence", choices=("all", "low", "medium", "high"), default=None,
2284
+ help="Only show findings at or above this confidence level: high = an explicit "
2285
+ "source->sink flow, medium = pattern/structural match, low = heuristic comparison. "
2286
+ "`all` shows every finding. Default: the human-readable table starts at high, every "
2287
+ "machine format starts at all. Detection is unchanged",
2288
+ )
2289
+ scan_parser.add_argument(
2290
+ "--include-js", dest="include_js", action="store_true", default=False,
2291
+ help="Also run the tree-sitter JS/TS scanner (CWE-79/CWE-95 in .js/.jsx/.ts/.tsx). "
2292
+ "Off by default so the analysis stays scoped to Python",
2293
+ )
2294
+ scan_parser.add_argument(
2295
+ "--workers", type=str, default="auto",
2296
+ help="Number of parallel worker processes for scanning. Use integer (1-64) or 'auto' (default: auto = all available CPU cores via os.cpu_count())",
2297
+ )
2298
+ scan_parser.add_argument(
2299
+ "--exclude", type=str, default=None,
2300
+ help="Comma-separated glob patterns to exclude from scanning (e.g., tests,test_*,*_test.py,migrations)",
2301
+ )
2302
+ scan_parser.add_argument(
2303
+ "--include-tests", action="store_true",
2304
+ help="Scan tests/, test/, __tests__/, examples/, example/ and samples/ too. Directory walks "
2305
+ "prune those trees before descending by default, which is where fixtures and samples "
2306
+ "live -- and also where a hardcoded secret is classically committed, so use this flag "
2307
+ "for a full-coverage sweep. A file or directory named on the command line is always "
2308
+ "scanned in full, with or without it.",
2309
+ )
2310
+ scan_parser.add_argument(
2311
+ "--skip-non-production", action="store_true",
2312
+ help="Accepted for compatibility with scripts written before skipping became the default; "
2313
+ "it now asks for what already happens and changes nothing.",
2314
+ )
2315
+ scan_parser.add_argument(
2316
+ "--parser", choices=PARSER_MODES, default=DEFAULT_PARSER_MODE,
2317
+ help="Which reader builds the ASTs. 'hybrid' (default) = native ast.parse, with the "
2318
+ "tree-sitter rescue shield only on a file that raises SyntaxError; 'treesitter' = the "
2319
+ "tolerant tree-sitter CST reader for every file, single-process, CPython still catches "
2320
+ "what it declines; 'ast' = native ast.parse and the block-level sweep, tree-sitter "
2321
+ "never invoked",
2322
+ )
2323
+ scan_parser.add_argument(
2324
+ "--audit-all", action="store_true", default=False,
2325
+ help="Emit speculative POTENTIAL findings and non-actionable IaC notes",
2326
+ )
2327
+ compare_parser = commands.add_parser("compare", help="Compare TimeCodeSecurity findings with Semgrep or Bandit")
2328
+ compare_parser.add_argument("path", type=Path, help="Python file or directory to compare")
2329
+ compare_parser.add_argument("--vs", choices=("semgrep", "bandit"), default="semgrep", help="Competitor scanner (default: semgrep)")
2330
+ compare_parser.add_argument("--tolerance", type=int, default=5, help="Maximum line distance for a match (default: 5)")
2331
+ compare_parser.add_argument("--output-json", metavar="REPORT_PATH", help="Write detailed comparison report as JSON")
2332
+ args = parser.parse_args(argv)
2333
+ if args.command == "scan":
2334
+ try:
2335
+ return _scan(args)
2336
+ except Exception as exc:
2337
+ # Anything unhandled is an internal error: emit the fallback artefacts and
2338
+ # exit 2 instead of dying on a traceback with an empty report.
2339
+ traceback.print_exc()
2340
+ return _scan_failure(args, exc)
2341
+ if args.tolerance < 0:
2342
+ parser.error("--tolerance must be zero or greater")
2343
+ return _compare(args)
2344
+
2345
+
2346
+ if __name__ == "__main__":
2347
+ sys.exit(main())