timecs 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app.py +3014 -0
- ast_scanner.py +36187 -0
- banner.py +146 -0
- ci_reporter.py +570 -0
- cli.py +2347 -0
- config_loader.py +307 -0
- cross_file_engine.py +1781 -0
- delivery_models.py +145 -0
- delivery_repository.py +78 -0
- diff_scan.py +127 -0
- durable_delivery_sink.py +103 -0
- event_bus.py +155 -0
- event_publisher.py +345 -0
- event_taxonomy.py +313 -0
- function_summarizer.py +354 -0
- html_auditor.py +274 -0
- iac_auditor.py +400 -0
- in_app_adapter.py +115 -0
- js_scanner.py +895 -0
- manifest_parser.py +960 -0
- master_rules_bank.py +389 -0
- notification_models.py +346 -0
- notification_policy.py +309 -0
- notification_repository.py +118 -0
- notification_routes.py +394 -0
- notification_service.py +199 -0
- osv_client.py +437 -0
- remediation/__init__.py +39 -0
- remediation/base.py +102 -0
- remediation/contracts.py +196 -0
- remediation/cwe1188_binding.py +78 -0
- remediation/cwe1336_jinja.py +91 -0
- remediation/cwe22_path_traversal.py +348 -0
- remediation/cwe295_ssl.py +87 -0
- remediation/cwe327_weak_hash.py +78 -0
- remediation/cwe377_tempfile.py +77 -0
- remediation/cwe489_debug.py +80 -0
- remediation/cwe502_yaml.py +89 -0
- remediation/cwe614_cookie.py +91 -0
- remediation/cwe78_cmdi.py +415 -0
- remediation/cwe798_credentials.py +153 -0
- remediation/cwe89_sqli.py +425 -0
- remediation/cwe916_password_hash.py +131 -0
- remediation/cwe95_eval.py +77 -0
- remediation/dispatcher.py +114 -0
- remediation/orchestrator.py +541 -0
- remediation/patch_engine.py +213 -0
- remediation/verification.py +253 -0
- rule_engine.py +1535 -0
- sarif_adapter.py +719 -0
- sarif_exporter.py +88 -0
- sca_reachability/__init__.py +28 -0
- sca_reachability/ast_analyzer.py +366 -0
- sca_reachability/call_graph.py +203 -0
- sca_reachability/contracts.py +365 -0
- sca_reachability/dist_import_map.py +46 -0
- sca_reachability/engine.py +603 -0
- sca_reachability/resolver.py +169 -0
- scanners/__init__.py +70 -0
- scanners/container_scanner.py +323 -0
- scanners/cve_index.py +521 -0
- scanners/iac_scanner.py +328 -0
- scanners/sbom.py +183 -0
- scanners/sca_scanner.py +667 -0
- scanners/secret_scanner.py +345 -0
- scanners/yaml_lite.py +217 -0
- secret_filters.py +435 -0
- secret_scanner.py +943 -0
- sqlite_delivery_repository.py +277 -0
- sqlite_notification_repository.py +441 -0
- ssrf_guard.py +276 -0
- staged_scanner.py +492 -0
- suppression_resolver.py +282 -0
- symbol_indexer.py +424 -0
- tcs/__init__.py +1 -0
- tcs/analysis/__init__.py +1 -0
- tcs/analysis/def_use.py +408 -0
- tcs/analysis/function_contracts.py +290 -0
- tcs/parallel_analyzer.py +276 -0
- tcs/parallel_scanner.py +630 -0
- tcs/treesitter_packer.py +1446 -0
- tcs_cli.py +2086 -0
- tcs_config.py +218 -0
- tcs_gui.py +2123 -0
- timecs-2.0.0.dist-info/METADATA +142 -0
- timecs-2.0.0.dist-info/RECORD +94 -0
- timecs-2.0.0.dist-info/WHEEL +5 -0
- timecs-2.0.0.dist-info/entry_points.txt +4 -0
- timecs-2.0.0.dist-info/licenses/LICENSE +201 -0
- timecs-2.0.0.dist-info/top_level.txt +49 -0
- version_matcher.py +934 -0
- webhook_adapter.py +495 -0
- webhook_client.py +483 -0
- webhook_formatters.py +245 -0
cli.py
ADDED
|
@@ -0,0 +1,2347 @@
|
|
|
1
|
+
"""Standalone command-line interface for the TimeCodeSecurity AST scanner."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import ast
|
|
5
|
+
import fnmatch
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
import re
|
|
10
|
+
import shutil
|
|
11
|
+
import signal
|
|
12
|
+
import sys
|
|
13
|
+
import subprocess
|
|
14
|
+
import time
|
|
15
|
+
import traceback
|
|
16
|
+
from typing import Optional
|
|
17
|
+
import banner
|
|
18
|
+
from cross_file_engine import CrossFileTaintEngine
|
|
19
|
+
from ast_scanner import TaintTracker, PARSER_MODES, DEFAULT_PARSER_MODE
|
|
20
|
+
from ci_reporter import sanitize_symbol_display
|
|
21
|
+
from diff_scan import changed_line_intervals, resolve_repo_root
|
|
22
|
+
from html_auditor import audit_templates, is_template_path
|
|
23
|
+
from iac_auditor import audit_iac_files
|
|
24
|
+
from scanners import (
|
|
25
|
+
audit_sca_files,
|
|
26
|
+
audit_secret_files,
|
|
27
|
+
audit_security_files,
|
|
28
|
+
build_sbom,
|
|
29
|
+
collect_components,
|
|
30
|
+
is_container_path,
|
|
31
|
+
is_kubernetes_manifest,
|
|
32
|
+
is_manifest_path,
|
|
33
|
+
is_secret_path,
|
|
34
|
+
is_terraform_path,
|
|
35
|
+
is_workflow_yaml_path,
|
|
36
|
+
tag_reachability,
|
|
37
|
+
)
|
|
38
|
+
from rule_engine import get_rule
|
|
39
|
+
from suppression_resolver import resolve_line_suppressions
|
|
40
|
+
from tcs_config import load_config
|
|
41
|
+
from remediation.patch_engine import RemediationEngine
|
|
42
|
+
from sarif_adapter import bound_sarif_document
|
|
43
|
+
from sarif_exporter import atomic_write_text
|
|
44
|
+
|
|
45
|
+
try:
|
|
46
|
+
from js_scanner import (
|
|
47
|
+
has_js_ts_files,
|
|
48
|
+
js_ts_available,
|
|
49
|
+
run_js_ts_scan_isolated,
|
|
50
|
+
warn_js_ts_unavailable,
|
|
51
|
+
)
|
|
52
|
+
_JS_TS_SCANNER_AVAILABLE = True
|
|
53
|
+
except ImportError:
|
|
54
|
+
_JS_TS_SCANNER_AVAILABLE = False
|
|
55
|
+
|
|
56
|
+
def js_ts_available() -> bool:
|
|
57
|
+
return False
|
|
58
|
+
|
|
59
|
+
def run_js_ts_scan_isolated(target, base_dir=None, skipped_files=None):
|
|
60
|
+
return []
|
|
61
|
+
|
|
62
|
+
def has_js_ts_files(target_path) -> bool:
|
|
63
|
+
return False
|
|
64
|
+
|
|
65
|
+
def warn_js_ts_unavailable() -> None:
|
|
66
|
+
print("[TimeCodeSecurity-JS] WARNING: js_scanner is not importable; JS/TS scanning is disabled.",
|
|
67
|
+
file=sys.stderr)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
SARIF_SCHEMA = "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json"
|
|
71
|
+
SEVERITIES = ("CRITICAL", "HIGH", "MEDIUM", "LOW", "UNKNOWN")
|
|
72
|
+
IGNORED_DIRS = {".git", ".venv", "venv", "__pycache__", "node_modules", "build", "dist"}
|
|
73
|
+
|
|
74
|
+
# Directories that hold sample, fixture and harness code rather than shipped product code. A
|
|
75
|
+
# directory scan prunes these before descending; `--include-tests` turns that off, and a path named
|
|
76
|
+
# on the command line is always walked in full. This is a discovery-scope choice, not a suppression:
|
|
77
|
+
# nothing is filtered out of the emission set, so every finding in a scanned file is still reported.
|
|
78
|
+
# The cost is stated plainly because it is real -- `benchmark/gates/workflow_controls_probe.py` names
|
|
79
|
+
# `examples/app.py` and `tests/fixture.py` among the trees a hardcoded secret is classically committed
|
|
80
|
+
# in, and `fixtures/` is where this repository keeps 27 of its own blessed detection cases, so a default
|
|
81
|
+
# directory scan no longer sees a CWE-798 sitting in any of them and needs `--include-tests` to find it.
|
|
82
|
+
# That is why the skip is always announced in the report, why the confidence-floor note and this one are
|
|
83
|
+
# printed even when nothing else is, and why the gates that prove per-rule or per-directory behaviour
|
|
84
|
+
# pass `--include-tests` (or name the directory on the command line) instead of losing their assertions.
|
|
85
|
+
# `debug/` joins the list for the same reason `examples/` and `samples/` are on it: a tree of preview
|
|
86
|
+
# and demo views -- Sentry's `src/sentry/web/frontend/debug/` renders sample mail and component
|
|
87
|
+
# payloads for local development -- whose findings describe developer tooling, not shipped surface.
|
|
88
|
+
# Name the directory or pass `--include-tests` and every row inside it is still reported, which is the
|
|
89
|
+
# pair `benchmark/gates/sentry_tier3_precision_probe.py` locks.
|
|
90
|
+
NON_PRODUCTION_DIRS = frozenset({"tests", "test", "__tests__", "testutils",
|
|
91
|
+
"fixtures", "examples", "example", "samples", "debug"})
|
|
92
|
+
# What the current scan pruned, so the report can say so instead of looking like a clean tree.
|
|
93
|
+
_SCOPE_STATS = {"skipped_dirs": 0, "skipped_files": 0}
|
|
94
|
+
|
|
95
|
+
# Jinja interpolation tokens in a *path*, as opposed to its content: Salt formulae ship files like
|
|
96
|
+
# `templates/{{module_name}}.py`, and that name alone guarantees `ast.parse` raises SyntaxError.
|
|
97
|
+
TEMPLATE_INTERPOLATION_TOKENS = ("{{", "}}")
|
|
98
|
+
|
|
99
|
+
# One flush per file is blocking pipe/disk I/O: a 2,819-file run paid 2,819 of those to say "I am
|
|
100
|
+
# still here". Progress now logs every Nth file, and the in-flight name is kept in
|
|
101
|
+
# `_PARTIAL["current_file"]` so a kill still reports where the scan was stuck.
|
|
102
|
+
PROGRESS_FILE_EVERY = 250
|
|
103
|
+
|
|
104
|
+
# Every finding belongs to exactly one analysis scope: template, container and CI/IaC paths are
|
|
105
|
+
# routed to their auditors, everything else is Python. --scope selects which collectors run.
|
|
106
|
+
SCOPE_ORDER = ("python", "docker", "iac", "sca", "secrets", "html")
|
|
107
|
+
SCOPE_LABELS = {"python": "Python Code", "docker": "Container/Config",
|
|
108
|
+
"iac": "CI/CD & IaC", "sca": "Dependencies (SCA)",
|
|
109
|
+
"secrets": "Config & Env Secrets", "html": "HTML Templates"}
|
|
110
|
+
|
|
111
|
+
# Confidence is a display label, never a detection decision: `--min-confidence` hides rows
|
|
112
|
+
# from the report, the engine still emits every finding it emitted before.
|
|
113
|
+
CONFIDENCE_ORDER = {"LOW": 1, "MEDIUM": 2, "HIGH": 3}
|
|
114
|
+
# Sinks whose finding is a heuristic over an in-process comparison rather than data arriving
|
|
115
|
+
# at a dangerous operation: CWE-208 reports `token == expected`, where both operands are
|
|
116
|
+
# already local values, so the shape alone cannot prove an attacker reaches it.
|
|
117
|
+
LOW_CONFIDENCE_SINK_TYPES = frozenset({"TIMING_ATTACK"})
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _scope_of(path, aux_scopes=None):
|
|
121
|
+
"""Scope of one finding's file.
|
|
122
|
+
|
|
123
|
+
``aux_scopes`` is the discovery map for auxiliary documents. It is the authority for a
|
|
124
|
+
Kubernetes workload manifest, which can sit at any path and is only recognisable from its
|
|
125
|
+
content; every other family is decidable from the path alone.
|
|
126
|
+
"""
|
|
127
|
+
key = str(path)
|
|
128
|
+
if aux_scopes and key in aux_scopes:
|
|
129
|
+
return aux_scopes[key]
|
|
130
|
+
lowered = key.lower()
|
|
131
|
+
if is_template_path(lowered):
|
|
132
|
+
return "html"
|
|
133
|
+
if is_container_path(lowered):
|
|
134
|
+
return "docker"
|
|
135
|
+
if is_workflow_yaml_path(lowered) or is_terraform_path(lowered):
|
|
136
|
+
return "iac"
|
|
137
|
+
if is_manifest_path(lowered):
|
|
138
|
+
return "sca"
|
|
139
|
+
if is_secret_path(lowered):
|
|
140
|
+
return "secrets"
|
|
141
|
+
return "python"
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _file_key(path, cwd):
|
|
145
|
+
try:
|
|
146
|
+
return path.resolve().relative_to(cwd).as_posix()
|
|
147
|
+
except ValueError:
|
|
148
|
+
return path.resolve().as_posix()
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _parse_exclude_patterns(raw):
|
|
152
|
+
if not raw:
|
|
153
|
+
return None
|
|
154
|
+
return [p.strip() for p in raw.split(",") if p.strip()] or None
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _is_excluded(path, root, exclude_patterns):
|
|
158
|
+
"""True when `path` matches any exclude pattern.
|
|
159
|
+
|
|
160
|
+
Patterns are matched three ways because users write all three: a bare directory
|
|
161
|
+
name (`tests`), a filename glob (`test_*.py`) and a rooted relative glob
|
|
162
|
+
(`contrib/*/tests/*`). Matching only the basename or only the full relative path
|
|
163
|
+
silently keeps nested hits, which is how `--exclude tests` used to survive
|
|
164
|
+
`django/contrib/auth/tests/models.py`.
|
|
165
|
+
|
|
166
|
+
Relative paths are normalised to POSIX before fnmatch, whose separator handling
|
|
167
|
+
is not portable: on Windows `str(Path.relative_to())` yields backslashes, so a
|
|
168
|
+
pattern containing `/` could never match.
|
|
169
|
+
"""
|
|
170
|
+
if not exclude_patterns:
|
|
171
|
+
return False
|
|
172
|
+
try:
|
|
173
|
+
rel = path.relative_to(root)
|
|
174
|
+
except ValueError:
|
|
175
|
+
rel = path
|
|
176
|
+
rel_posix = rel.as_posix()
|
|
177
|
+
name = path.name
|
|
178
|
+
parts = rel.parts
|
|
179
|
+
for pattern in exclude_patterns:
|
|
180
|
+
if fnmatch.fnmatchcase(rel_posix, pattern):
|
|
181
|
+
return True
|
|
182
|
+
if fnmatch.fnmatchcase(name, pattern):
|
|
183
|
+
return True
|
|
184
|
+
if any(fnmatch.fnmatchcase(part, pattern) for part in parts):
|
|
185
|
+
return True
|
|
186
|
+
return False
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _is_template_interpolated(path, root):
|
|
190
|
+
"""True when a path *component* carries a Jinja interpolation token.
|
|
191
|
+
|
|
192
|
+
Salt formulae ship files named `templates/{{module_name}}.py`: the name is a render-time
|
|
193
|
+
placeholder, so the bytes on disk are pre-render scaffolding and `ast.parse` on them usually
|
|
194
|
+
raises SyntaxError. Only components of the path are inspected, never file content, so a normal
|
|
195
|
+
module that merely uses `{{` inside a string is never mistaken for one.
|
|
196
|
+
"""
|
|
197
|
+
try:
|
|
198
|
+
parts = path.relative_to(root).parts
|
|
199
|
+
except ValueError:
|
|
200
|
+
parts = path.parts
|
|
201
|
+
return any(token in part for part in parts for token in TEMPLATE_INTERPOLATION_TOKENS)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _parses_as_python(source):
|
|
205
|
+
"""True when `ast.parse` accepts the text -- the gate that keeps a template file scannable.
|
|
206
|
+
|
|
207
|
+
A Jinja-named path alone is not proof of junk: measured on this repo, `templates/{{x}}.py`
|
|
208
|
+
holding `import os; os.system(p)` parses fine and reports CWE-78. Dropping on the path alone
|
|
209
|
+
would have turned that real finding into a false negative, so the skip requires the content to
|
|
210
|
+
be unparseable as well. One extra parse per interpolated-path file, nothing else pays for it.
|
|
211
|
+
"""
|
|
212
|
+
try:
|
|
213
|
+
ast.parse(source)
|
|
214
|
+
return True
|
|
215
|
+
except (SyntaxError, ValueError):
|
|
216
|
+
return False
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _progress(message):
|
|
220
|
+
"""Emit one progress line on stderr so a stalled scan names the file it stuck on.
|
|
221
|
+
|
|
222
|
+
stderr keeps stdout parsable (`--format json` consumers read stdout only) and
|
|
223
|
+
flush=True means the line lands even if the process is killed mid-file.
|
|
224
|
+
Callers throttle *how often* this runs (PROGRESS_FILE_EVERY); the in-flight file name
|
|
225
|
+
lives in `_PARTIAL["current_file"]` so the throttled gap stays diagnosable.
|
|
226
|
+
"""
|
|
227
|
+
print(f"[progress] {message}", file=sys.stderr, flush=True)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _ux_enabled(args):
|
|
231
|
+
"""True only for a human watching a terminal, never for a data pipe or a log.
|
|
232
|
+
|
|
233
|
+
`--format json` is an absolute veto: that stdout is parsed by every benchmark and CI job, so
|
|
234
|
+
the styled UX stays off even when TIMECS_FORCE_BANNER asks for it. Otherwise a real TTY turns
|
|
235
|
+
it on, and the env var exists so a test can drive the same code path over a pipe.
|
|
236
|
+
"""
|
|
237
|
+
if getattr(args, "format", "table") == "json":
|
|
238
|
+
return False
|
|
239
|
+
if os.environ.get("TIMECS_FORCE_BANNER") == "1":
|
|
240
|
+
return True
|
|
241
|
+
try:
|
|
242
|
+
return bool(sys.stderr.isatty())
|
|
243
|
+
except (AttributeError, ValueError):
|
|
244
|
+
return False
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
class _StageClock:
|
|
248
|
+
"""Phase-1 outer instrumentation: disjoint end-to-end stage clocks, stderr only.
|
|
249
|
+
|
|
250
|
+
Each `mark()` closes the stage that was open since the previous mark, so the stage
|
|
251
|
+
seconds are disjoint by construction and sum to TOTAL (modulo the final report call).
|
|
252
|
+
Engine-internal lines (`[PROFILE] Phase n ...`, `[PROFILE] CrossEngine ...`, collector
|
|
253
|
+
profiles) are NESTED inside these stages and must never be added to them. Worker-pool
|
|
254
|
+
time is included in the stage that owns the pool (Parse), not per core.
|
|
255
|
+
|
|
256
|
+
Nothing here feeds a finding, a suppression, the JSON schema, or the exit code;
|
|
257
|
+
`duration_ms` keeps its original semantics (started after file collection). RSS sampling
|
|
258
|
+
uses the already-optional psutil: without it the report says RSS=unavailable.
|
|
259
|
+
"""
|
|
260
|
+
|
|
261
|
+
def __init__(self, label):
|
|
262
|
+
try:
|
|
263
|
+
import psutil
|
|
264
|
+
self._rss_fn = psutil.Process().memory_info
|
|
265
|
+
except Exception:
|
|
266
|
+
self._rss_fn = None
|
|
267
|
+
self.label = label
|
|
268
|
+
self.t_start = time.perf_counter()
|
|
269
|
+
self.t_mark = self.t_start
|
|
270
|
+
self.rows = []
|
|
271
|
+
self.peak_rss_mb = 0.0
|
|
272
|
+
|
|
273
|
+
def mark(self, stage):
|
|
274
|
+
now = time.perf_counter()
|
|
275
|
+
rss_mb = 0.0
|
|
276
|
+
if self._rss_fn is not None:
|
|
277
|
+
try:
|
|
278
|
+
rss_mb = self._rss_fn().rss / (1024 ** 2)
|
|
279
|
+
except Exception:
|
|
280
|
+
rss_mb = 0.0
|
|
281
|
+
self.peak_rss_mb = max(self.peak_rss_mb, rss_mb)
|
|
282
|
+
self.rows.append((stage, now - self.t_mark, rss_mb))
|
|
283
|
+
self.t_mark = now
|
|
284
|
+
|
|
285
|
+
def report(self):
|
|
286
|
+
total = time.perf_counter() - self.t_start
|
|
287
|
+
parts = " ".join(f"{name}={secs:.2f}s" for name, secs, _ in self.rows)
|
|
288
|
+
rss_note = (f" peakRSS_parent={self.peak_rss_mb:.0f}MB"
|
|
289
|
+
if self._rss_fn is not None else " RSS=unavailable(psutil missing)")
|
|
290
|
+
print(f"TimeCodeSecurity [PROFILE] Outer({self.label}): {parts} TOTAL={total:.2f}s{rss_note}",
|
|
291
|
+
file=sys.stderr, flush=True)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _iter_tree_files(root, prune_dirs=None, count_pruned=False):
|
|
295
|
+
"""Yield the real files under `root`, never following a symlink.
|
|
296
|
+
|
|
297
|
+
Traversal guard (CWE-59): `Path.rglob` semantics are version dependent — Python
|
|
298
|
+
3.13+ follows symlinked directories by default, so one link back into an ancestor
|
|
299
|
+
makes the crawl loop. os.walk with followlinks=False plus explicit symlink pruning
|
|
300
|
+
keeps discovery inside the real tree. Ignored directories are pruned before
|
|
301
|
+
descending, so `node_modules` is never walked just to be filtered out again.
|
|
302
|
+
|
|
303
|
+
`prune_dirs` replaces the prune set for one walk; `count_pruned` only ever comes from a requested
|
|
304
|
+
scope narrowing, and records how many directories and Python files it held back so the report can
|
|
305
|
+
name them. Three things keep a file in scope whatever the set says: a file named directly (no
|
|
306
|
+
directory walk happens at all), a directory named as the scan root -- which is why
|
|
307
|
+
`cli.py scan tests/` still walks it -- and `--include-tests`, which passes no set at all.
|
|
308
|
+
"""
|
|
309
|
+
skip = IGNORED_DIRS if prune_dirs is None else prune_dirs
|
|
310
|
+
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
|
|
311
|
+
kept = []
|
|
312
|
+
for name in sorted(dirnames):
|
|
313
|
+
if name in skip:
|
|
314
|
+
if count_pruned and name not in IGNORED_DIRS:
|
|
315
|
+
_SCOPE_STATS["skipped_dirs"] += 1
|
|
316
|
+
_SCOPE_STATS["skipped_files"] += sum(
|
|
317
|
+
1 for _root, _dirs, _files in os.walk(os.path.join(dirpath, name))
|
|
318
|
+
for _name in _files if _name.lower().endswith(".py"))
|
|
319
|
+
continue
|
|
320
|
+
if os.path.islink(os.path.join(dirpath, name)):
|
|
321
|
+
continue
|
|
322
|
+
kept.append(name)
|
|
323
|
+
dirnames[:] = kept
|
|
324
|
+
for name in sorted(filenames):
|
|
325
|
+
path = Path(dirpath) / name
|
|
326
|
+
if os.path.islink(path):
|
|
327
|
+
continue
|
|
328
|
+
yield path
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def _collect_files(scan_path, exclude_patterns=None, config_excludes=None,
|
|
332
|
+
skip_non_production=False):
|
|
333
|
+
"""Collect Python files from scan path, optionally excluding by glob patterns.
|
|
334
|
+
|
|
335
|
+
Args:
|
|
336
|
+
scan_path: Path object (file or directory)
|
|
337
|
+
exclude_patterns: List of glob patterns to exclude (e.g., ['tests', 'test_*'])
|
|
338
|
+
config_excludes: Optional callable taking a CWD-relative POSIX path and returning
|
|
339
|
+
True when `.tcs.yaml` excludes it. A file named on the command line is never
|
|
340
|
+
dropped, matching how a tree walk treats `--exclude`.
|
|
341
|
+
skip_non_production: Prune `NON_PRODUCTION_DIRS` from the tree walk only. A path named
|
|
342
|
+
on the command line is scanned in full either way, same contract as `--exclude`.
|
|
343
|
+
|
|
344
|
+
Phase-1 instrumentation only: the enumerate/read split names the cold-tree serial
|
|
345
|
+
read cost that the pre-existing `duration_ms` anchor cannot see (it starts later).
|
|
346
|
+
"""
|
|
347
|
+
_t_enum0 = time.perf_counter()
|
|
348
|
+
cwd = Path.cwd().resolve()
|
|
349
|
+
if not scan_path.exists():
|
|
350
|
+
raise FileNotFoundError(f"Path does not exist: {scan_path}")
|
|
351
|
+
is_tree = scan_path.is_dir()
|
|
352
|
+
prune = (IGNORED_DIRS | NON_PRODUCTION_DIRS) if skip_non_production else None
|
|
353
|
+
if scan_path.is_file():
|
|
354
|
+
if scan_path.suffix.lower() != ".py":
|
|
355
|
+
raise ValueError(f"Expected a Python file: {scan_path}")
|
|
356
|
+
paths = [scan_path]
|
|
357
|
+
elif is_tree:
|
|
358
|
+
paths = sorted(
|
|
359
|
+
path for path in _iter_tree_files(scan_path, prune, skip_non_production)
|
|
360
|
+
if path.suffix.lower() == ".py"
|
|
361
|
+
and not any(part in IGNORED_DIRS for part in path.parts)
|
|
362
|
+
and not _is_excluded(path, scan_path, exclude_patterns)
|
|
363
|
+
and not (config_excludes and config_excludes(_file_key(path, cwd)))
|
|
364
|
+
)
|
|
365
|
+
else:
|
|
366
|
+
raise ValueError(f"Path is not a regular file or directory: {scan_path}")
|
|
367
|
+
|
|
368
|
+
if not paths:
|
|
369
|
+
scope_note = non_production_scope_note()
|
|
370
|
+
if scope_note:
|
|
371
|
+
# The tree holds nothing but pruned non-production code: say so rather than letting the
|
|
372
|
+
# caller read "No Python files found" as an empty directory.
|
|
373
|
+
print(scope_note, file=sys.stderr)
|
|
374
|
+
raise ValueError(f"No Python files found: {scan_path}")
|
|
375
|
+
|
|
376
|
+
_t_enum = time.perf_counter()
|
|
377
|
+
files = {}
|
|
378
|
+
skipped = 0
|
|
379
|
+
templated = 0
|
|
380
|
+
total_chars = 0
|
|
381
|
+
total = len(paths)
|
|
382
|
+
_progress(f"discovered {total} python file(s) under {scan_path}")
|
|
383
|
+
for index, path in enumerate(paths, 1):
|
|
384
|
+
_PARTIAL["current_file"] = str(path)
|
|
385
|
+
if index == 1 or index == total or index % PROGRESS_FILE_EVERY == 0:
|
|
386
|
+
_progress(f"reading python ({index}/{total}) {path}")
|
|
387
|
+
try:
|
|
388
|
+
source = path.read_text(encoding="utf-8", errors="replace")
|
|
389
|
+
except (OSError, ValueError, UnicodeDecodeError) as exc:
|
|
390
|
+
# A single unreadable file must not abort the scan of the whole tree.
|
|
391
|
+
skipped += 1
|
|
392
|
+
print(f"Warning: skipping unreadable file {path}: {type(exc).__name__}: {exc}",
|
|
393
|
+
file=sys.stderr)
|
|
394
|
+
continue
|
|
395
|
+
# Only a tree walk drops files; a file named on the command line is always analysed.
|
|
396
|
+
if is_tree and _is_template_interpolated(path, scan_path) and not _parses_as_python(source):
|
|
397
|
+
templated += 1
|
|
398
|
+
continue
|
|
399
|
+
total_chars += len(source)
|
|
400
|
+
files[_file_key(path, cwd)] = source
|
|
401
|
+
_t_reads = time.perf_counter()
|
|
402
|
+
if templated:
|
|
403
|
+
_progress(f"skipped {templated} unparseable Jinja-interpolated file(s) under {scan_path}")
|
|
404
|
+
if skipped:
|
|
405
|
+
print(f"Warning: skipped {skipped} of {len(paths)} Python files; "
|
|
406
|
+
f"{len(files)} scanned.", file=sys.stderr)
|
|
407
|
+
print(f"TimeCodeSecurity [PROFILE] CollectFiles: enumerate={_t_enum - _t_enum0:.2f}s "
|
|
408
|
+
f"(paths={total}), serial_read={_t_reads - _t_enum:.2f}s "
|
|
409
|
+
f"(kept={len(files)}, chars={total_chars}, unreadable={skipped}, jinja_skipped={templated})",
|
|
410
|
+
file=sys.stderr)
|
|
411
|
+
_PARTIAL["current_file"] = None
|
|
412
|
+
return files
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
AUX_YAML_SUFFIXES = (".yml", ".yaml")
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def _collect_auxiliary_files(scan_path, exclude_patterns=None, config_excludes=None,
|
|
419
|
+
skip_non_production=False):
|
|
420
|
+
"""Collect non-Python artefacts in one walk.
|
|
421
|
+
|
|
422
|
+
Returns ``(templates, security documents, manifests, secret documents, scope map)``.
|
|
423
|
+
Keys match _collect_files so findings share a single path namespace, and
|
|
424
|
+
`--exclude` applies here too: a template under an excluded directory is just as
|
|
425
|
+
out of scope as a .py file under it. `config_excludes` carries the same contract
|
|
426
|
+
as in _collect_files, as does `skip_non_production` -- a template inside a skipped
|
|
427
|
+
`examples/` tree is out of scope for exactly the same reason.
|
|
428
|
+
|
|
429
|
+
The scope map carries each auxiliary document's key to its scope ("docker" for images and
|
|
430
|
+
compose, "iac" for workflows, Kubernetes and Terraform, "sca" for dependency manifests and
|
|
431
|
+
lockfiles, "secrets" for config and env documents). It exists because a Kubernetes manifest
|
|
432
|
+
is recognised from content, not from its path, so the bucketing decision has to survive the
|
|
433
|
+
walk for `_scope_of` to report it honestly.
|
|
434
|
+
"""
|
|
435
|
+
cwd = Path.cwd().resolve()
|
|
436
|
+
if not scan_path.exists():
|
|
437
|
+
raise FileNotFoundError(f"Path does not exist: {scan_path}")
|
|
438
|
+
|
|
439
|
+
prune = (IGNORED_DIRS | NON_PRODUCTION_DIRS) if skip_non_production else None
|
|
440
|
+
if scan_path.is_file():
|
|
441
|
+
paths = [scan_path]
|
|
442
|
+
elif scan_path.is_dir():
|
|
443
|
+
# count_pruned stays off here: the note reports python files held back, and the Python
|
|
444
|
+
# collection below walks the same tree -- counting twice would double the figure.
|
|
445
|
+
paths = sorted(
|
|
446
|
+
path for path in _iter_tree_files(scan_path, prune, False)
|
|
447
|
+
if not any(part in IGNORED_DIRS for part in path.parts)
|
|
448
|
+
and not _is_excluded(path, scan_path, exclude_patterns)
|
|
449
|
+
and not (config_excludes and config_excludes(_file_key(path, cwd)))
|
|
450
|
+
)
|
|
451
|
+
else:
|
|
452
|
+
return {}, {}, {}, {}, {}
|
|
453
|
+
|
|
454
|
+
templates = {}
|
|
455
|
+
iac = {}
|
|
456
|
+
manifests = {}
|
|
457
|
+
secrets = {}
|
|
458
|
+
aux_scopes = {}
|
|
459
|
+
buckets = {}
|
|
460
|
+
skipped = 0
|
|
461
|
+
for path in paths:
|
|
462
|
+
relative = str(path).replace("\\", "/")
|
|
463
|
+
if is_template_path(relative):
|
|
464
|
+
buckets[path] = (templates, "html")
|
|
465
|
+
elif is_container_path(relative):
|
|
466
|
+
buckets[path] = (iac, "docker")
|
|
467
|
+
elif is_workflow_yaml_path(relative) or is_terraform_path(relative):
|
|
468
|
+
buckets[path] = (iac, "iac")
|
|
469
|
+
elif is_manifest_path(relative):
|
|
470
|
+
buckets[path] = (manifests, "sca")
|
|
471
|
+
elif relative.lower().endswith(AUX_YAML_SUFFIXES):
|
|
472
|
+
# Scope undecided until the content is read: a Kubernetes workload manifest belongs
|
|
473
|
+
# to IaC, and only a YAML document that is not one falls through to the secret reader.
|
|
474
|
+
buckets[path] = (iac, None)
|
|
475
|
+
elif is_secret_path(relative):
|
|
476
|
+
buckets[path] = (secrets, "secrets")
|
|
477
|
+
|
|
478
|
+
total = len(buckets)
|
|
479
|
+
_progress(f"discovered {total} template/IaC file(s) under {scan_path}")
|
|
480
|
+
for index, (path, (bucket, scope)) in enumerate(buckets.items(), 1):
|
|
481
|
+
_PARTIAL["current_file"] = str(path)
|
|
482
|
+
if index == 1 or index == total or index % PROGRESS_FILE_EVERY == 0:
|
|
483
|
+
_progress(f"reading auxiliary ({index}/{total}) {path}")
|
|
484
|
+
try:
|
|
485
|
+
source = path.read_text(encoding="utf-8", errors="replace")
|
|
486
|
+
except OSError as exc:
|
|
487
|
+
# Same contract as _collect_files above: one unreadable template or IaC file is a
|
|
488
|
+
# local fault, so it is reported and skipped. Re-raising here used to abort the whole
|
|
489
|
+
# tree scan and exit 2, losing every Python finding in the repository.
|
|
490
|
+
skipped += 1
|
|
491
|
+
print(f"Warning: skipping unreadable auxiliary file {path}: "
|
|
492
|
+
f"{type(exc).__name__}: {exc}", file=sys.stderr)
|
|
493
|
+
continue
|
|
494
|
+
if scope is None:
|
|
495
|
+
if is_kubernetes_manifest(source):
|
|
496
|
+
scope = "iac"
|
|
497
|
+
elif is_secret_path(str(path).replace("\\", "/")):
|
|
498
|
+
bucket, scope = secrets, "secrets"
|
|
499
|
+
else:
|
|
500
|
+
continue
|
|
501
|
+
key = _file_key(path, cwd)
|
|
502
|
+
bucket[key] = source
|
|
503
|
+
if bucket is not templates:
|
|
504
|
+
aux_scopes[key] = scope
|
|
505
|
+
if skipped:
|
|
506
|
+
print(f"Warning: skipped {skipped} of {total} auxiliary files; "
|
|
507
|
+
f"{len(templates) + len(iac) + len(manifests) + len(secrets)} scanned.",
|
|
508
|
+
file=sys.stderr)
|
|
509
|
+
_PARTIAL["current_file"] = None
|
|
510
|
+
return templates, iac, manifests, secrets, aux_scopes
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def _imported_roots(tracker):
|
|
514
|
+
"""Top-level import roots across the Python files the engine already parsed.
|
|
515
|
+
|
|
516
|
+
Reuses the ``tracker.modules`` trees instead of re-reading or re-parsing anything, so the
|
|
517
|
+
reachability proof never adds a second AST pass to a scan. A tree that is not a stdlib AST
|
|
518
|
+
(the opt-in tree-sitter reader) is parsed from the engine's own source snapshot. The first
|
|
519
|
+
importer wins, which makes the evidence deterministic regardless of module iteration order.
|
|
520
|
+
"""
|
|
521
|
+
roots = {}
|
|
522
|
+
if tracker is None:
|
|
523
|
+
return roots
|
|
524
|
+
paths = getattr(tracker, "file_paths", {}) or {}
|
|
525
|
+
sources = getattr(tracker, "files", {}) or {}
|
|
526
|
+
for module, tree in (getattr(tracker, "modules", {}) or {}).items():
|
|
527
|
+
if not isinstance(tree, ast.AST):
|
|
528
|
+
source = sources.get(paths.get(module))
|
|
529
|
+
if not source:
|
|
530
|
+
continue
|
|
531
|
+
try:
|
|
532
|
+
tree = ast.parse(source)
|
|
533
|
+
except (SyntaxError, ValueError):
|
|
534
|
+
continue
|
|
535
|
+
path = str(paths.get(module, module))
|
|
536
|
+
for node in ast.walk(tree):
|
|
537
|
+
if isinstance(node, ast.Import):
|
|
538
|
+
names = [alias.name for alias in node.names]
|
|
539
|
+
elif isinstance(node, ast.ImportFrom):
|
|
540
|
+
# A relative import names this project's own module, not a distribution.
|
|
541
|
+
if node.level or not node.module:
|
|
542
|
+
continue
|
|
543
|
+
names = [node.module]
|
|
544
|
+
else:
|
|
545
|
+
continue
|
|
546
|
+
for name in names:
|
|
547
|
+
root = name.split(".", 1)[0]
|
|
548
|
+
if root and root not in roots:
|
|
549
|
+
roots[root] = (path, node.lineno)
|
|
550
|
+
return roots
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
def _merge_findings(*finding_groups):
|
|
554
|
+
"""Merge auditor findings with AST findings in a deterministic order."""
|
|
555
|
+
seen = set()
|
|
556
|
+
unique = []
|
|
557
|
+
for group in finding_groups:
|
|
558
|
+
for item in group:
|
|
559
|
+
identity = (item["file"], item["line"], item["cwe"], item.get("category"))
|
|
560
|
+
if identity in seen:
|
|
561
|
+
continue
|
|
562
|
+
seen.add(identity)
|
|
563
|
+
unique.append(item)
|
|
564
|
+
return sorted(unique, key=lambda entry: (entry["file"], entry["line"], entry["cwe"]))
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
# Several CWEs describe one *attribute* of the same defect: a single weak-digest call can
|
|
568
|
+
# be simultaneously a broken algorithm (327/328) and a storage-strength failure (759/916),
|
|
569
|
+
# and one set_cookie() can lack Secure (614), HttpOnly (1004) and SameSite (1275). Reporting
|
|
570
|
+
# every attribute as its own alert multiplies the count without adding an actionable site.
|
|
571
|
+
# An unbounded `f.read()` is the same case from the other direction: the rule engine calls it
|
|
572
|
+
# CWE-400 and the structural batch calls the identical call node CWE-770, so one sink is
|
|
573
|
+
# counted as two findings.
|
|
574
|
+
# The families below collapse to one finding per (file, line); the surviving CWEs are kept
|
|
575
|
+
# in `consolidated_from` so nothing is lost. This is presentation-only: the engine still emits
|
|
576
|
+
# every CWE, which is what the benchmark scores per rule.
|
|
577
|
+
FAMILY_PRIORITY = {
|
|
578
|
+
"CRYPTO_HASH": ("CWE-327", "CWE-916", "CWE-759", "CWE-328"),
|
|
579
|
+
"INSECURE_COOKIE": ("CWE-614", "CWE-1004", "CWE-1275"),
|
|
580
|
+
"RESOURCE_EXHAUSTION": ("CWE-770", "CWE-400"),
|
|
581
|
+
}
|
|
582
|
+
FAMILY_OF_CWE = {
|
|
583
|
+
cwe: family for family, members in FAMILY_PRIORITY.items() for cwe in members
|
|
584
|
+
}
|
|
585
|
+
COOKIE_FLAG_BY_CWE = {"CWE-614": "Secure", "CWE-1004": "HttpOnly", "CWE-1275": "SameSite"}
|
|
586
|
+
SEVERITY_RANK = {"UNKNOWN": 0, "LOW": 1, "MEDIUM": 2, "HIGH": 3, "CRITICAL": 4}
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
def _family_message(family, primary, members_by_cwe):
|
|
590
|
+
"""Alert text naming what the collapsed CWEs actually proved."""
|
|
591
|
+
if family == "INSECURE_COOKIE":
|
|
592
|
+
flags = [COOKIE_FLAG_BY_CWE[cwe] for cwe in FAMILY_PRIORITY[family] if cwe in members_by_cwe]
|
|
593
|
+
return f"{primary}: Insecure Cookie Configuration (Missing {', '.join(flags)})"
|
|
594
|
+
source = members_by_cwe[primary][0]["message"]
|
|
595
|
+
operation = source.split(":", 1)[1].strip() if ":" in source else source
|
|
596
|
+
secondaries = [cwe for cwe in FAMILY_PRIORITY[family] if cwe in members_by_cwe and cwe != primary]
|
|
597
|
+
suffix = f" (consolidated: {', '.join(secondaries)})" if secondaries else ""
|
|
598
|
+
return f"{primary}: {operation}{suffix}"
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
def _consolidate_family(family, group):
|
|
602
|
+
members_by_cwe = {}
|
|
603
|
+
for finding in group:
|
|
604
|
+
members_by_cwe.setdefault(finding["cwe"], []).append(finding)
|
|
605
|
+
priority = FAMILY_PRIORITY[family]
|
|
606
|
+
primary = next(cwe for cwe in priority if cwe in members_by_cwe)
|
|
607
|
+
representative = members_by_cwe[primary][0]
|
|
608
|
+
consolidated = []
|
|
609
|
+
for cwe in priority:
|
|
610
|
+
consolidated.extend(member for member in members_by_cwe.get(cwe, []) if member is not representative)
|
|
611
|
+
return {
|
|
612
|
+
**representative,
|
|
613
|
+
"cwe": primary,
|
|
614
|
+
"line": min(member["line"] for member in group),
|
|
615
|
+
"severity": max(
|
|
616
|
+
(member["severity"] for member in group),
|
|
617
|
+
key=lambda value: SEVERITY_RANK.get(value, 0),
|
|
618
|
+
),
|
|
619
|
+
"confidence": max(
|
|
620
|
+
(member.get("confidence", "MEDIUM") for member in group),
|
|
621
|
+
key=lambda value: CONFIDENCE_ORDER.get(value, 0),
|
|
622
|
+
),
|
|
623
|
+
"message": _family_message(family, primary, members_by_cwe),
|
|
624
|
+
"consolidated_from": [f"{member['cwe']}@{member['line']}" for member in consolidated],
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def consolidate_findings(findings):
|
|
629
|
+
"""Collapse same-site CWE families into one finding, preserving scan order."""
|
|
630
|
+
buckets = {}
|
|
631
|
+
for finding in findings:
|
|
632
|
+
family = FAMILY_OF_CWE.get(finding["cwe"])
|
|
633
|
+
if family:
|
|
634
|
+
buckets.setdefault((family, finding["file"], finding["line"]), []).append(finding)
|
|
635
|
+
|
|
636
|
+
emitted = set()
|
|
637
|
+
consolidated = []
|
|
638
|
+
for finding in findings:
|
|
639
|
+
family = FAMILY_OF_CWE.get(finding["cwe"])
|
|
640
|
+
if not family:
|
|
641
|
+
consolidated.append(finding)
|
|
642
|
+
continue
|
|
643
|
+
key = (family, finding["file"], finding["line"])
|
|
644
|
+
if key in emitted:
|
|
645
|
+
continue
|
|
646
|
+
emitted.add(key)
|
|
647
|
+
consolidated.append(_consolidate_family(family, buckets[key]))
|
|
648
|
+
return sorted(consolidated, key=lambda item: (item["file"], item["line"], item["cwe"]))
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def _findings_for(tracker, edges):
|
|
652
|
+
sinks = {sink.id: sink for sink in tracker.sinks}
|
|
653
|
+
by_identity = {}
|
|
654
|
+
for edge in edges:
|
|
655
|
+
sink = sinks.get(edge.target_id)
|
|
656
|
+
if sink is None:
|
|
657
|
+
continue
|
|
658
|
+
cwe = sink.metadata.get("cwe")
|
|
659
|
+
if not cwe and edge.proof_graph is not None:
|
|
660
|
+
cwe = edge.proof_graph.cwe
|
|
661
|
+
cwe = cwe or "UNKNOWN_CWE"
|
|
662
|
+
rule = get_rule(cwe)
|
|
663
|
+
confidence_label = "CONFIRMED" if edge.kind == "CONFIRMED_DATA_FLOW" else "POTENTIAL"
|
|
664
|
+
severity = rule.get_severity(confidence_label).upper() if rule else "HIGH"
|
|
665
|
+
# HIGH follows the engine's own CONFIRMED label. Phase collectors record a sink with a
|
|
666
|
+
# synthetic source id (`OS_COMMAND_EXECUTION`, `WEAK_HASH_ALGORITHM`) and still say
|
|
667
|
+
# CONFIRMED, so this over-claims rather than under-claims: a `--min-confidence high`
|
|
668
|
+
# run may show a pattern row, but it can never hide a traced flow behind a filter.
|
|
669
|
+
sink_type = sink.metadata.get("sink_type") or sink.metadata.get("category")
|
|
670
|
+
if sink_type in LOW_CONFIDENCE_SINK_TYPES:
|
|
671
|
+
confidence = "LOW"
|
|
672
|
+
elif edge.kind == "CONFIRMED_DATA_FLOW":
|
|
673
|
+
confidence = "HIGH"
|
|
674
|
+
else:
|
|
675
|
+
confidence = "MEDIUM"
|
|
676
|
+
location = sink.location
|
|
677
|
+
file_path = location.file.replace("\\", "/")
|
|
678
|
+
finding = {
|
|
679
|
+
"file": file_path,
|
|
680
|
+
"line": location.line_start,
|
|
681
|
+
"cwe": cwe,
|
|
682
|
+
"severity": severity,
|
|
683
|
+
"confidence": confidence,
|
|
684
|
+
"category": sink.metadata.get("category") or (rule.category if rule else "Security"),
|
|
685
|
+
"message": sink.metadata.get("message") or \
|
|
686
|
+
f"{cwe}: {sink.metadata.get('operation') or sanitize_symbol_display(sink.symbol)}",
|
|
687
|
+
}
|
|
688
|
+
# Triage-only shape label (CWE-611 today). It rides beside the message, never inside it,
|
|
689
|
+
# so the canonical keys the baselines compare cannot move because of it.
|
|
690
|
+
xml_input = sink.metadata.get("xml_input")
|
|
691
|
+
if xml_input:
|
|
692
|
+
finding["xml_input"] = xml_input
|
|
693
|
+
identity = (finding["file"], finding["line"], finding["cwe"])
|
|
694
|
+
existing = by_identity.get(identity)
|
|
695
|
+
if existing is None:
|
|
696
|
+
by_identity[identity] = finding
|
|
697
|
+
elif CONFIDENCE_ORDER[confidence] > CONFIDENCE_ORDER[existing["confidence"]]:
|
|
698
|
+
# One site can be reached by several edges; the loudest label wins so a
|
|
699
|
+
# confidence filter can never hide a confirmed flow behind a weaker edge.
|
|
700
|
+
existing["confidence"] = confidence
|
|
701
|
+
return sorted(by_identity.values(),
|
|
702
|
+
key=lambda item: (item["file"], item["line"], item["cwe"]))
|
|
703
|
+
|
|
704
|
+
|
|
705
|
+
def _parse_line_ranges(spec):
|
|
706
|
+
"""Parse a --lines value like '4-5' or '4-5,10' into inclusive (start, end) ranges.
|
|
707
|
+
|
|
708
|
+
Returns None when the spec is empty/absent, meaning a full-file scan.
|
|
709
|
+
Raises ValueError on malformed input so the CLI can report it cleanly.
|
|
710
|
+
"""
|
|
711
|
+
if not spec:
|
|
712
|
+
return None
|
|
713
|
+
ranges = []
|
|
714
|
+
for part in str(spec).split(","):
|
|
715
|
+
part = part.strip()
|
|
716
|
+
if not part:
|
|
717
|
+
continue
|
|
718
|
+
bounds = part.split("-", 1)
|
|
719
|
+
try:
|
|
720
|
+
start = int(bounds[0].strip())
|
|
721
|
+
end = int(bounds[1].strip()) if len(bounds) > 1 else start
|
|
722
|
+
except ValueError as exc:
|
|
723
|
+
raise ValueError(f"Invalid --lines range {part!r}; expected START[-END]") from exc
|
|
724
|
+
if start < 1 or end < start:
|
|
725
|
+
raise ValueError(f"Invalid --lines range {part!r}; lines must be positive and ordered")
|
|
726
|
+
ranges.append((start, end))
|
|
727
|
+
return ranges or None
|
|
728
|
+
|
|
729
|
+
|
|
730
|
+
def _in_line_ranges(line, ranges):
|
|
731
|
+
return ranges is None or any(start <= line <= end for start, end in ranges)
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
def _scope_findings_to_lines(findings, ranges):
|
|
735
|
+
"""Keep findings whose sink line intersects the requested ranges.
|
|
736
|
+
|
|
737
|
+
Findings without a usable line number are kept: dropping them would turn an
|
|
738
|
+
incremental scan into a false-negative source.
|
|
739
|
+
"""
|
|
740
|
+
if ranges is None:
|
|
741
|
+
return findings
|
|
742
|
+
scoped = []
|
|
743
|
+
for finding in findings:
|
|
744
|
+
line = finding.get("line")
|
|
745
|
+
if not isinstance(line, int):
|
|
746
|
+
scoped.append(finding)
|
|
747
|
+
continue
|
|
748
|
+
if _in_line_ranges(line, ranges):
|
|
749
|
+
scoped.append(finding)
|
|
750
|
+
return scoped
|
|
751
|
+
|
|
752
|
+
|
|
753
|
+
def _apply_inline_suppressions(findings, sources):
|
|
754
|
+
"""Drop findings named by a `# tcs: ignore[CWE-XXX]` comment on the finding's own line.
|
|
755
|
+
|
|
756
|
+
`sources` maps every scanned path to its text (Python, templates and IaC alike).
|
|
757
|
+
A finding whose file is absent from that map is kept: an unreadable source can
|
|
758
|
+
never suppress anything.
|
|
759
|
+
"""
|
|
760
|
+
if not findings or not sources:
|
|
761
|
+
return findings
|
|
762
|
+
resolve_line_suppressions(findings, sources)
|
|
763
|
+
return [finding for finding in findings if not finding.get("suppressed")]
|
|
764
|
+
|
|
765
|
+
|
|
766
|
+
def _scope_findings_to_diff(findings, intervals, repo_root):
|
|
767
|
+
"""Keep findings reported on added/modified lines of the working tree versus a ref.
|
|
768
|
+
|
|
769
|
+
`intervals` is keyed by repo-relative paths; findings carry CWD-relative keys, so both
|
|
770
|
+
sides are mapped through `_file_key`. A finding without a usable line number in a
|
|
771
|
+
changed file is kept, mirroring `_scope_findings_to_lines`.
|
|
772
|
+
"""
|
|
773
|
+
cwd = Path.cwd().resolve()
|
|
774
|
+
changed = {}
|
|
775
|
+
for rel_path, spans in intervals.items():
|
|
776
|
+
try:
|
|
777
|
+
changed[_file_key((repo_root / rel_path).resolve(), cwd)] = spans
|
|
778
|
+
except OSError:
|
|
779
|
+
continue
|
|
780
|
+
kept = []
|
|
781
|
+
for finding in findings:
|
|
782
|
+
spans = changed.get(finding.get("file"))
|
|
783
|
+
if spans is None:
|
|
784
|
+
continue
|
|
785
|
+
line = finding.get("line")
|
|
786
|
+
if not isinstance(line, int) or any(start <= line <= end for start, end in spans):
|
|
787
|
+
kept.append(finding)
|
|
788
|
+
return kept
|
|
789
|
+
|
|
790
|
+
|
|
791
|
+
def _filter_by_confidence(findings, minimum):
|
|
792
|
+
"""Hide findings below `minimum` from the report; detection itself is untouched."""
|
|
793
|
+
if not minimum:
|
|
794
|
+
return findings
|
|
795
|
+
floor = CONFIDENCE_ORDER[minimum.upper()]
|
|
796
|
+
return [
|
|
797
|
+
finding for finding in findings
|
|
798
|
+
if CONFIDENCE_ORDER.get(str(finding.get("confidence") or "MEDIUM").upper(), 2) >= floor
|
|
799
|
+
]
|
|
800
|
+
|
|
801
|
+
|
|
802
|
+
# The floor a plain `cli.py scan` table starts at when nobody asked for one. It is a reading order for
|
|
803
|
+
# a person, not a detection decision, so it is applied to the table only and always announced.
|
|
804
|
+
TABLE_DEFAULT_CONFIDENCE = "high"
|
|
805
|
+
|
|
806
|
+
|
|
807
|
+
def _split_by_table_default(findings):
|
|
808
|
+
"""Return (rows a default table shows, rows it hides) at `TABLE_DEFAULT_CONFIDENCE`."""
|
|
809
|
+
floor = CONFIDENCE_ORDER[TABLE_DEFAULT_CONFIDENCE.upper()]
|
|
810
|
+
visible, hidden = [], []
|
|
811
|
+
for finding in findings:
|
|
812
|
+
label = str(finding.get("confidence") or "MEDIUM").upper()
|
|
813
|
+
(visible if CONFIDENCE_ORDER.get(label, 2) >= floor else hidden).append(finding)
|
|
814
|
+
return visible, hidden
|
|
815
|
+
|
|
816
|
+
|
|
817
|
+
def non_production_scope_note() -> str:
|
|
818
|
+
"""Name the scope a directory walk narrowed by default.
|
|
819
|
+
|
|
820
|
+
Held-back files are invisible to the report, which is the one way a scope choice can be mistaken
|
|
821
|
+
for a clean tree -- and a hardcoded secret in `tests/fixture.py` or `examples/app.py` is exactly
|
|
822
|
+
the finding that lives there. So the count is always stated when the walk pruned anything, and the
|
|
823
|
+
way back is named. A path given on the command line is always walked in full, which is why this
|
|
824
|
+
note never appears for `scan tests/`.
|
|
825
|
+
"""
|
|
826
|
+
dirs = _SCOPE_STATS["skipped_dirs"]
|
|
827
|
+
if not dirs:
|
|
828
|
+
return ""
|
|
829
|
+
files = _SCOPE_STATS["skipped_files"]
|
|
830
|
+
return (f"Note: Held back {files} python file(s) in {dirs} non-production "
|
|
831
|
+
f"director{'y' if dirs == 1 else 'ies'} (tests/, testutils/, fixtures/, "
|
|
832
|
+
f"examples/, samples/, debug/); pass --include-tests to scan them.")
|
|
833
|
+
|
|
834
|
+
|
|
835
|
+
def table_default_note(hidden) -> str:
|
|
836
|
+
"""One line naming what the default table floor hid, so a short table is never read as a clean scan.
|
|
837
|
+
|
|
838
|
+
A CRITICAL-severity row can sit under the floor when its flow was matched structurally rather than
|
|
839
|
+
traced, so the count of hidden criticals is called out separately: that is the case where a reader
|
|
840
|
+
would otherwise conclude the scan had nothing dangerous in it.
|
|
841
|
+
"""
|
|
842
|
+
if not hidden:
|
|
843
|
+
return ""
|
|
844
|
+
counts = {}
|
|
845
|
+
for finding in hidden:
|
|
846
|
+
label = str(finding.get("confidence") or "MEDIUM").upper()
|
|
847
|
+
counts[label] = counts.get(label, 0) + 1
|
|
848
|
+
parts = [f"{counts[label]} {label}" for label in ("HIGH", "MEDIUM", "LOW") if counts.get(label)]
|
|
849
|
+
critical = sum(1 for finding in hidden if finding.get("severity") == "CRITICAL")
|
|
850
|
+
tail = ", all CRITICAL severity" if critical == len(hidden) else ""
|
|
851
|
+
if 0 < critical < len(hidden):
|
|
852
|
+
tail = f", {critical} of them CRITICAL severity"
|
|
853
|
+
noun = "finding" if len(hidden) == 1 else "findings"
|
|
854
|
+
return (f"Note: {' and '.join(parts)} {noun} hidden by the default table filter{tail} "
|
|
855
|
+
"(pass --min-confidence all to view all findings).")
|
|
856
|
+
|
|
857
|
+
|
|
858
|
+
def _apply_config_filters(findings, config, path_excludes, scan_root=None, aux_scopes=None):
|
|
859
|
+
"""Apply root `.tcs.yaml` to findings: path exclusions and per-rule path overrides.
|
|
860
|
+
|
|
861
|
+
`path_excludes` is False when the user named a single file on the command line --
|
|
862
|
+
then only the rule-level overrides apply, because an explicitly named file is
|
|
863
|
+
always analysed (same contract as `--exclude`).
|
|
864
|
+
|
|
865
|
+
`scan_root` is the path the user named, as seen from the working directory: collected
|
|
866
|
+
keys carry it as a prefix, and `TcsConfig` needs it to honour an anchored pattern such
|
|
867
|
+
as `docs/**` when the scan is aimed at a subdirectory instead of the repository root.
|
|
868
|
+
|
|
869
|
+
`aux_scopes` is the container/CI-IaC discovery map. Per-rule overrides are a *Python* noise
|
|
870
|
+
policy -- the shipped CWE-78 globs name CI scripts and example trees, where shell-by-design
|
|
871
|
+
is not injection -- and `**/.github/**` over a workflow YAML would delete the GHA injection
|
|
872
|
+
rule at its only source. Those documents answer to their own structural falsifiers instead.
|
|
873
|
+
A global `exclude_paths` entry still never-reads them, and templates are unaffected.
|
|
874
|
+
"""
|
|
875
|
+
if config is None:
|
|
876
|
+
return findings
|
|
877
|
+
security_docs = aux_scopes or {}
|
|
878
|
+
kept = []
|
|
879
|
+
for finding in findings:
|
|
880
|
+
path_key = str(finding.get("file", ""))
|
|
881
|
+
if path_excludes and config.excludes_path(path_key, scan_root):
|
|
882
|
+
continue
|
|
883
|
+
if path_key not in security_docs and config.rule_excluded(
|
|
884
|
+
str(finding.get("cwe", "")), path_key, scan_root):
|
|
885
|
+
continue
|
|
886
|
+
kept.append(finding)
|
|
887
|
+
return kept
|
|
888
|
+
|
|
889
|
+
|
|
890
|
+
def _statement_span(source_code, line_number):
|
|
891
|
+
"""Return the (start, end) line span of the smallest statement covering line_number.
|
|
892
|
+
|
|
893
|
+
Falls back to (line_number, line_number) when the source is unparseable or no
|
|
894
|
+
statement covers the requested line.
|
|
895
|
+
"""
|
|
896
|
+
try:
|
|
897
|
+
tree = ast.parse(source_code)
|
|
898
|
+
except (SyntaxError, ValueError, RecursionError):
|
|
899
|
+
return line_number, line_number
|
|
900
|
+
|
|
901
|
+
candidates = [
|
|
902
|
+
node
|
|
903
|
+
for node in ast.walk(tree)
|
|
904
|
+
if isinstance(node, ast.stmt)
|
|
905
|
+
and getattr(node, "lineno", 0)
|
|
906
|
+
and node.lineno <= line_number <= (node.end_lineno or node.lineno)
|
|
907
|
+
]
|
|
908
|
+
if not candidates:
|
|
909
|
+
return line_number, line_number
|
|
910
|
+
|
|
911
|
+
stmt = min(candidates, key=lambda n: (n.end_lineno or n.lineno) - n.lineno)
|
|
912
|
+
return stmt.lineno, (stmt.end_lineno or stmt.lineno)
|
|
913
|
+
|
|
914
|
+
|
|
915
|
+
def _generate_autofix(finding, source_code, file_path, engine=None):
|
|
916
|
+
"""
|
|
917
|
+
Generate an autofix object for a finding by running the RemediationEngine.
|
|
918
|
+
|
|
919
|
+
Returns a dict with autofix schema or None if no deterministic fix is available.
|
|
920
|
+
|
|
921
|
+
Autofix schema:
|
|
922
|
+
{
|
|
923
|
+
"type": "ast_patch",
|
|
924
|
+
"description": str,
|
|
925
|
+
"replacement_text": str,
|
|
926
|
+
"range": {
|
|
927
|
+
"start_line": int,
|
|
928
|
+
"start_col": int,
|
|
929
|
+
"end_line": int,
|
|
930
|
+
"end_col": int
|
|
931
|
+
},
|
|
932
|
+
"diff": str (optional),
|
|
933
|
+
"verification_passed": bool
|
|
934
|
+
}
|
|
935
|
+
"""
|
|
936
|
+
if engine is None:
|
|
937
|
+
engine = RemediationEngine()
|
|
938
|
+
|
|
939
|
+
# Map CLI finding to remediation engine format
|
|
940
|
+
remediation_finding = {
|
|
941
|
+
"id": f"TimeCodeSecurity-{finding['cwe'].replace('-', '')}-{finding['line']}",
|
|
942
|
+
"cwe": finding["cwe"],
|
|
943
|
+
"line_number": finding["line"],
|
|
944
|
+
"file": finding["file"],
|
|
945
|
+
"code_snippet": "", # Will be extracted from source
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
try:
|
|
949
|
+
record = engine.remediate(remediation_finding, source_code, file_path)
|
|
950
|
+
except Exception:
|
|
951
|
+
# If remediation fails entirely, return None
|
|
952
|
+
return None
|
|
953
|
+
|
|
954
|
+
# Only attach autofix for successful patches
|
|
955
|
+
if record.patch_status.value != "SUCCESS":
|
|
956
|
+
return None
|
|
957
|
+
|
|
958
|
+
# The autofix replaces the whole original statement, so derive the range from
|
|
959
|
+
# the statement span in the UNPATCHED source. The snippet's own line count is
|
|
960
|
+
# not used: a multi-line replacement still overwrites a single-line statement.
|
|
961
|
+
stmt_start, stmt_end = _statement_span(source_code, record.line_number)
|
|
962
|
+
start_line = stmt_start
|
|
963
|
+
end_line = stmt_end
|
|
964
|
+
|
|
965
|
+
source_lines = source_code.split("\n")
|
|
966
|
+
if 0 < end_line <= len(source_lines):
|
|
967
|
+
end_col = len(source_lines[end_line - 1].rstrip("\r"))
|
|
968
|
+
else:
|
|
969
|
+
end_col = 0
|
|
970
|
+
|
|
971
|
+
autofix = {
|
|
972
|
+
"type": "ast_patch",
|
|
973
|
+
"description": f"Apply {record.remediation_rule.value} transformation for {record.cwe}",
|
|
974
|
+
"replacement_text": record.patched_code_snippet,
|
|
975
|
+
"range": {
|
|
976
|
+
"start_line": start_line,
|
|
977
|
+
"start_col": 0,
|
|
978
|
+
"end_line": end_line,
|
|
979
|
+
"end_col": end_col
|
|
980
|
+
},
|
|
981
|
+
"diff": record.unified_diff,
|
|
982
|
+
"verification_passed": record.verification_passed
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
return autofix
|
|
986
|
+
|
|
987
|
+
|
|
988
|
+
def _attach_autofixes(findings, source_cache, autofix_engine=None):
|
|
989
|
+
"""
|
|
990
|
+
Attach autofix objects to findings where deterministic fixes are available.
|
|
991
|
+
|
|
992
|
+
Args:
|
|
993
|
+
findings: List of finding dicts
|
|
994
|
+
source_cache: Dict mapping file paths to source code strings
|
|
995
|
+
autofix_engine: Optional RemediationEngine instance
|
|
996
|
+
|
|
997
|
+
Returns:
|
|
998
|
+
List of findings with autofix objects attached where applicable
|
|
999
|
+
"""
|
|
1000
|
+
if autofix_engine is None:
|
|
1001
|
+
autofix_engine = RemediationEngine()
|
|
1002
|
+
|
|
1003
|
+
enhanced_findings = []
|
|
1004
|
+
for finding in findings:
|
|
1005
|
+
file_path = finding["file"]
|
|
1006
|
+
source_code = source_cache.get(file_path)
|
|
1007
|
+
|
|
1008
|
+
if source_code:
|
|
1009
|
+
autofix = _generate_autofix(finding, source_code, file_path, autofix_engine)
|
|
1010
|
+
if autofix:
|
|
1011
|
+
finding_copy = finding.copy()
|
|
1012
|
+
finding_copy["autofix"] = autofix
|
|
1013
|
+
enhanced_findings.append(finding_copy)
|
|
1014
|
+
else:
|
|
1015
|
+
enhanced_findings.append(finding)
|
|
1016
|
+
else:
|
|
1017
|
+
enhanced_findings.append(finding)
|
|
1018
|
+
|
|
1019
|
+
return enhanced_findings
|
|
1020
|
+
|
|
1021
|
+
|
|
1022
|
+
def _ascii_table(findings):
|
|
1023
|
+
headers = ("#", "File", "Line", "CWE", "Severity", "Category")
|
|
1024
|
+
max_widths = (3, 15, 4, 8, 8, 22)
|
|
1025
|
+
rows = [
|
|
1026
|
+
(str(index), item["file"], str(item["line"]), item["cwe"], item["severity"], item["category"].replace("_", " "))
|
|
1027
|
+
for index, item in enumerate(findings, 1)
|
|
1028
|
+
]
|
|
1029
|
+
widths = [
|
|
1030
|
+
min(max_widths[index], max([len(headers[index])] + [len(row[index]) for row in rows]))
|
|
1031
|
+
for index in range(len(headers))
|
|
1032
|
+
]
|
|
1033
|
+
|
|
1034
|
+
def fit(value, width, keep_end=False):
|
|
1035
|
+
if len(value) <= width:
|
|
1036
|
+
return value.ljust(width)
|
|
1037
|
+
if keep_end:
|
|
1038
|
+
return "..." + value[-(width - 3):]
|
|
1039
|
+
return value[:width - 3] + "..."
|
|
1040
|
+
|
|
1041
|
+
border = "+" + "+".join("-" * (width + 2) for width in widths) + "+"
|
|
1042
|
+
lines = [border, "| " + " | ".join(headers[i].ljust(widths[i]) for i in range(len(headers))) + " |", border]
|
|
1043
|
+
for row in rows:
|
|
1044
|
+
cells = [fit(row[index], widths[index], keep_end=index == 1) for index in range(len(headers))]
|
|
1045
|
+
lines.append("| " + " | ".join(cells) + " |")
|
|
1046
|
+
lines.append(border)
|
|
1047
|
+
return "\n".join(lines)
|
|
1048
|
+
|
|
1049
|
+
|
|
1050
|
+
def _sarif_tags(cwe, related):
|
|
1051
|
+
"""Tag list for one rule/result: its own CWE plus any secondary CWEs it stands for.
|
|
1052
|
+
|
|
1053
|
+
A deduplicated, order-preserving list, so the same set of secondary ids always renders the
|
|
1054
|
+
same bytes and a CWE that already appears as the primary id is never repeated.
|
|
1055
|
+
"""
|
|
1056
|
+
tags = ["security", str(cwe).lower()]
|
|
1057
|
+
for related_cwe in related or ():
|
|
1058
|
+
tag = str(related_cwe).lower()
|
|
1059
|
+
if tag not in tags:
|
|
1060
|
+
tags.append(tag)
|
|
1061
|
+
return tags
|
|
1062
|
+
|
|
1063
|
+
|
|
1064
|
+
def _sarif_rule(cwe, category, severity, related=()):
|
|
1065
|
+
"""Build one driver rule carrying the same required members the SARIF adapter emits."""
|
|
1066
|
+
digits = re.search(r"\d+", str(cwe))
|
|
1067
|
+
rule_meta = get_rule(cwe)
|
|
1068
|
+
short = str(category).replace("_", " ")
|
|
1069
|
+
return {
|
|
1070
|
+
"id": cwe,
|
|
1071
|
+
"name": str(cwe).replace("-", "_"),
|
|
1072
|
+
"shortDescription": {"text": short},
|
|
1073
|
+
"fullDescription": {"text": rule_meta.name if rule_meta else short},
|
|
1074
|
+
"helpUri": (
|
|
1075
|
+
f"https://cwe.mitre.org/data/definitions/{digits.group()}.html"
|
|
1076
|
+
if digits else "https://cwe.mitre.org/data/definitions/index.html"
|
|
1077
|
+
),
|
|
1078
|
+
"defaultConfiguration": {
|
|
1079
|
+
"level": "error" if severity in ("CRITICAL", "HIGH") else "warning"
|
|
1080
|
+
},
|
|
1081
|
+
"properties": {"tags": _sarif_tags(cwe, related)},
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
|
|
1085
|
+
def _sarif_uri(path, cwd):
|
|
1086
|
+
"""Normalize a filesystem path into a valid SARIF artifactLocation URI."""
|
|
1087
|
+
resolved = Path(str(path).replace("\\", "/")).resolve()
|
|
1088
|
+
try:
|
|
1089
|
+
return resolved.relative_to(cwd).as_posix()
|
|
1090
|
+
except ValueError:
|
|
1091
|
+
return resolved.as_uri()
|
|
1092
|
+
|
|
1093
|
+
|
|
1094
|
+
def _sarif_code_flows(flow_trace, cwd):
|
|
1095
|
+
"""Wrap a cross-file trace in SARIF codeFlows/threadFlows for GHAS and VS Code viewers."""
|
|
1096
|
+
locations = []
|
|
1097
|
+
for step, hop in enumerate(flow_trace, 1):
|
|
1098
|
+
locations.append({
|
|
1099
|
+
"location": {
|
|
1100
|
+
"message": {"text": f"{hop['role']} : {hop['label']}"},
|
|
1101
|
+
"physicalLocation": {
|
|
1102
|
+
"artifactLocation": {"uri": _sarif_uri(hop["file"], cwd)},
|
|
1103
|
+
"region": {"startLine": max(1, hop["line"]), "startColumn": 1},
|
|
1104
|
+
},
|
|
1105
|
+
},
|
|
1106
|
+
"executionOrder": step,
|
|
1107
|
+
"importance": "essential",
|
|
1108
|
+
})
|
|
1109
|
+
return [{
|
|
1110
|
+
"message": {"text": "Cross-file taint execution path"},
|
|
1111
|
+
"threadFlows": [{"locations": locations}],
|
|
1112
|
+
}]
|
|
1113
|
+
|
|
1114
|
+
|
|
1115
|
+
def _sarif_document(findings, cwd, execution_successful=True):
|
|
1116
|
+
"""Build the SARIF 2.1.0 run for `findings`.
|
|
1117
|
+
|
|
1118
|
+
`execution_successful` mirrors reality: the crash and timeout-flush callers pass False, because
|
|
1119
|
+
their JSON siblings already report `status: incomplete` / `status: partial`, and both already
|
|
1120
|
+
exit non-zero -- claiming a successful run in the artefact would contradict them.
|
|
1121
|
+
"""
|
|
1122
|
+
rules = []
|
|
1123
|
+
rule_indexes = {}
|
|
1124
|
+
# Rules are keyed by CWE alone, so the secondary ids of *every* finding carrying that CWE
|
|
1125
|
+
# are unioned up front. Reading only the first-seen finding would drop a later finding's
|
|
1126
|
+
# related CWEs and leave the rule under-indexed.
|
|
1127
|
+
related_by_cwe = {}
|
|
1128
|
+
for finding in findings:
|
|
1129
|
+
for related_cwe in finding.get("related_cwes") or ():
|
|
1130
|
+
collected = related_by_cwe.setdefault(finding["cwe"], [])
|
|
1131
|
+
if related_cwe not in collected:
|
|
1132
|
+
collected.append(related_cwe)
|
|
1133
|
+
for finding in findings:
|
|
1134
|
+
cwe = finding["cwe"]
|
|
1135
|
+
if cwe in rule_indexes:
|
|
1136
|
+
continue
|
|
1137
|
+
rule_indexes[cwe] = len(rules)
|
|
1138
|
+
rules.append(_sarif_rule(cwe, finding["category"], finding["severity"],
|
|
1139
|
+
related_by_cwe.get(cwe)))
|
|
1140
|
+
|
|
1141
|
+
level_by_severity = {"CRITICAL": "error", "HIGH": "error", "MEDIUM": "warning", "LOW": "note", "UNKNOWN": "note"}
|
|
1142
|
+
results = []
|
|
1143
|
+
for finding in findings:
|
|
1144
|
+
result = {
|
|
1145
|
+
"ruleId": finding["cwe"],
|
|
1146
|
+
"ruleIndex": rule_indexes[finding["cwe"]],
|
|
1147
|
+
"level": level_by_severity.get(finding["severity"], "warning"),
|
|
1148
|
+
"message": {"text": finding["message"]},
|
|
1149
|
+
"locations": [{
|
|
1150
|
+
"physicalLocation": {
|
|
1151
|
+
"artifactLocation": {"uri": _sarif_uri(finding["file"], cwd)},
|
|
1152
|
+
"region": {"startLine": max(1, finding["line"]), "startColumn": 1},
|
|
1153
|
+
}
|
|
1154
|
+
}],
|
|
1155
|
+
}
|
|
1156
|
+
# Mirrors the rule tag list so a reader that only walks results still sees the
|
|
1157
|
+
# secondary CWE. Emitted only when a finding actually carries one, which keeps the
|
|
1158
|
+
# bytes of every existing finding identical.
|
|
1159
|
+
properties = {}
|
|
1160
|
+
related = finding.get("related_cwes")
|
|
1161
|
+
if related:
|
|
1162
|
+
properties["tags"] = _sarif_tags(finding["cwe"], related)
|
|
1163
|
+
# CWE-611 triage hint: which syntactic shape the parser argument had. It exists only
|
|
1164
|
+
# here, never in the message, so SARIF text consumers keep reading the same sentence.
|
|
1165
|
+
xml_input = finding.get("xml_input")
|
|
1166
|
+
if xml_input:
|
|
1167
|
+
properties["xml_input"] = xml_input
|
|
1168
|
+
if properties:
|
|
1169
|
+
result["properties"] = properties
|
|
1170
|
+
flow_trace = finding.get("flow_trace")
|
|
1171
|
+
if flow_trace:
|
|
1172
|
+
result["codeFlows"] = _sarif_code_flows(flow_trace, cwd)
|
|
1173
|
+
results.append(result)
|
|
1174
|
+
|
|
1175
|
+
return {
|
|
1176
|
+
"$schema": SARIF_SCHEMA,
|
|
1177
|
+
"version": "2.1.0",
|
|
1178
|
+
"runs": [{
|
|
1179
|
+
"tool": {"driver": {"name": "TimeCodeSecurity", "rules": rules}},
|
|
1180
|
+
"results": results,
|
|
1181
|
+
"invocations": [{"executionSuccessful": bool(execution_successful)}],
|
|
1182
|
+
}],
|
|
1183
|
+
}
|
|
1184
|
+
|
|
1185
|
+
|
|
1186
|
+
def _scope_breakdown(findings, aux_scopes=None):
|
|
1187
|
+
counts = {scope: 0 for scope in SCOPE_ORDER}
|
|
1188
|
+
for finding in findings:
|
|
1189
|
+
counts[_scope_of(finding["file"], aux_scopes)] += 1
|
|
1190
|
+
return counts
|
|
1191
|
+
|
|
1192
|
+
|
|
1193
|
+
def _print_summary(file_count, duration_ms, findings, file_counts=None,
|
|
1194
|
+
discovered_files=None, fast_path_skipped_files=0,
|
|
1195
|
+
unparseable_files=0, parse_governor=None, aux_scopes=None):
|
|
1196
|
+
counts = {severity: 0 for severity in SEVERITIES}
|
|
1197
|
+
scope_counts = _scope_breakdown(findings, aux_scopes)
|
|
1198
|
+
for finding in findings:
|
|
1199
|
+
severity = finding["severity"]
|
|
1200
|
+
counts[severity if severity in counts else "UNKNOWN"] += 1
|
|
1201
|
+
if discovered_files is not None:
|
|
1202
|
+
print(f"\nTotal discovered files: {discovered_files}")
|
|
1203
|
+
print(f"Total scanned files: {file_count}")
|
|
1204
|
+
if file_counts:
|
|
1205
|
+
print("Files scanned by scope: " + ", ".join(
|
|
1206
|
+
f"{SCOPE_LABELS[scope]}: {file_counts[scope]}" for scope in SCOPE_ORDER
|
|
1207
|
+
))
|
|
1208
|
+
skipped = discovered_files - file_count if discovered_files is not None else 0
|
|
1209
|
+
if skipped:
|
|
1210
|
+
print(f"Files not scanned: {skipped} "
|
|
1211
|
+
f"({fast_path_skipped_files} fast-path skipped, "
|
|
1212
|
+
f"{unparseable_files} unparseable)")
|
|
1213
|
+
if parse_governor:
|
|
1214
|
+
workers = parse_governor.get("workers_final", parse_governor.get("workers_used"))
|
|
1215
|
+
print(f"Parser: {parse_governor.get('parse_mode', 'unknown')} mode, "
|
|
1216
|
+
f"{workers} worker(s), "
|
|
1217
|
+
f"{parse_governor.get('governor_events', 0)} governor reduction(s)")
|
|
1218
|
+
if parse_governor.get("execution_mode") == "serial_fallback":
|
|
1219
|
+
# A degraded run must announce itself on stdout: the reason, and the fact that
|
|
1220
|
+
# coverage did not shrink. Silent serial re-parsing would look like a normal scan.
|
|
1221
|
+
print(f"[!] EXECUTION MODE: serial_fallback - the parallel parse pool failed "
|
|
1222
|
+
f"({parse_governor.get('parallel_fallback_reason')}). All "
|
|
1223
|
+
f"{file_count} scanned file(s) were re-parsed serially; coverage unchanged, "
|
|
1224
|
+
f"scan slower.")
|
|
1225
|
+
print(f"Scan duration: {duration_ms:.2f} ms")
|
|
1226
|
+
print("Findings by severity: " + ", ".join(f"{severity.title()}: {counts[severity]}" for severity in SEVERITIES))
|
|
1227
|
+
print("Findings by scope: " + ", ".join(
|
|
1228
|
+
f"{SCOPE_LABELS[scope]}: {scope_counts[scope]}" for scope in SCOPE_ORDER
|
|
1229
|
+
))
|
|
1230
|
+
|
|
1231
|
+
|
|
1232
|
+
def _cross_trace_report(findings, cwd):
|
|
1233
|
+
"""Render the multi-hop attack path for every cross-file finding, or '' if none."""
|
|
1234
|
+
traced = [
|
|
1235
|
+
(index, finding)
|
|
1236
|
+
for index, finding in enumerate(findings, 1)
|
|
1237
|
+
if finding.get("category") == "cross_file" and finding.get("flow_trace")
|
|
1238
|
+
]
|
|
1239
|
+
if not traced:
|
|
1240
|
+
return ""
|
|
1241
|
+
|
|
1242
|
+
rule = "-" * 80
|
|
1243
|
+
lines = ["", rule, " TIMECODESECURITY - CROSS-FILE EXPLOIT CHAINS", rule]
|
|
1244
|
+
for index, finding in traced:
|
|
1245
|
+
rule_meta = get_rule(finding["cwe"])
|
|
1246
|
+
label = (rule_meta.category if rule_meta else finding.get("category", "")).replace("_", " ").upper()
|
|
1247
|
+
lines.append(f"[TRACE] Finding #{index}: {finding['cwe']} ({label})")
|
|
1248
|
+
for step, hop in enumerate(finding["flow_trace"], 1):
|
|
1249
|
+
location = _file_key(Path(hop["file"]), cwd)
|
|
1250
|
+
lines.append(f" {step}. {hop['role'].ljust(7)}: {location}:{hop['line']} ({hop['label']})")
|
|
1251
|
+
lines.append(rule)
|
|
1252
|
+
return "\n".join(lines)
|
|
1253
|
+
|
|
1254
|
+
|
|
1255
|
+
|
|
1256
|
+
class CrossFinding(dict):
|
|
1257
|
+
"""Hybrid finding: acts as dict and object simultaneously."""
|
|
1258
|
+
def __init__(self, cwe, file_path, lineno, message, severity="HIGH", flow_trace=None,
|
|
1259
|
+
confidence="HIGH"):
|
|
1260
|
+
super().__init__(
|
|
1261
|
+
cwe=cwe,
|
|
1262
|
+
rule_id=cwe,
|
|
1263
|
+
severity="HIGH",
|
|
1264
|
+
confidence=confidence,
|
|
1265
|
+
file_path=file_path,
|
|
1266
|
+
file=file_path,
|
|
1267
|
+
path=file_path,
|
|
1268
|
+
lineno=lineno,
|
|
1269
|
+
line=lineno,
|
|
1270
|
+
message=message,
|
|
1271
|
+
desc=message,
|
|
1272
|
+
description=message,
|
|
1273
|
+
category="cross_file",
|
|
1274
|
+
flow_trace=flow_trace or [],
|
|
1275
|
+
)
|
|
1276
|
+
self.category = "cross_file"
|
|
1277
|
+
self.confidence = confidence
|
|
1278
|
+
self.flow_trace = flow_trace or []
|
|
1279
|
+
self.cwe = cwe
|
|
1280
|
+
self.rule_id = cwe
|
|
1281
|
+
self.severity = "HIGH"
|
|
1282
|
+
self.file_path = file_path
|
|
1283
|
+
self.file = file_path
|
|
1284
|
+
self.path = file_path
|
|
1285
|
+
self.lineno = lineno
|
|
1286
|
+
self.line = lineno
|
|
1287
|
+
self.message = message
|
|
1288
|
+
self.desc = message
|
|
1289
|
+
self.description = message
|
|
1290
|
+
|
|
1291
|
+
def _cross_scan_reuse_trees(tracker):
|
|
1292
|
+
"""The scanner's already-parsed ASTs, offered to the cross-file indexer for a second use.
|
|
1293
|
+
|
|
1294
|
+
Reuse preconditions, each one checked before this dict is handed over:
|
|
1295
|
+
- parser mode is not `treesitter`: the engine reparses with the built-in `ast` module and
|
|
1296
|
+
a treesitter-produced tree is not what `GlobalSymbolIndex._index_file` walks;
|
|
1297
|
+
- files the scanner skipped (unparseable), recovered (block-pass substituted) or rescued
|
|
1298
|
+
are excluded - the engine's own disk parse would build a *different* tree for them,
|
|
1299
|
+
and the baseline behaviour for those files is the fresh parse, not the scanner's one;
|
|
1300
|
+
- the indexer still re-reads every path and reuses its tree only when the on-disk source
|
|
1301
|
+
equals the scanner's snapshot byte-for-byte, so a file changed mid-scan falls back.
|
|
1302
|
+
|
|
1303
|
+
Returns None when nothing qualifies; the engine then behaves exactly as before.
|
|
1304
|
+
"""
|
|
1305
|
+
if tracker is None or not getattr(tracker, "modules", None):
|
|
1306
|
+
return None
|
|
1307
|
+
if getattr(tracker, "execution_mode", None) == "treesitter":
|
|
1308
|
+
return None
|
|
1309
|
+
cwd = Path.cwd()
|
|
1310
|
+
excluded = (set(tracker.skipped_files) | set(tracker.recovered_files)
|
|
1311
|
+
| set(tracker.rescued_files))
|
|
1312
|
+
reuse = {}
|
|
1313
|
+
for mod, tree in tracker.modules.items():
|
|
1314
|
+
key = tracker.file_paths.get(mod)
|
|
1315
|
+
if key is None or key in excluded or tree is None:
|
|
1316
|
+
continue
|
|
1317
|
+
try:
|
|
1318
|
+
source = tracker.files[key]
|
|
1319
|
+
reuse[str((cwd / key).resolve())] = (source, tree)
|
|
1320
|
+
except (KeyError, OSError):
|
|
1321
|
+
continue
|
|
1322
|
+
return reuse or None
|
|
1323
|
+
|
|
1324
|
+
|
|
1325
|
+
def _run_cross_scan(target_path, reuse_trees=None):
|
|
1326
|
+
"""Executes CrossFileTaintEngine. Returns (findings, engine).
|
|
1327
|
+
|
|
1328
|
+
An engine failure is fatal rather than empty: swallowing the exception made a broken
|
|
1329
|
+
cross-file analysis indistinguishable from a clean scan, silently dropping both the
|
|
1330
|
+
cross-file findings and the sanitizer suppression that depends on the engine.
|
|
1331
|
+
"""
|
|
1332
|
+
try:
|
|
1333
|
+
engine = CrossFileTaintEngine(target_path, reuse_trees=reuse_trees)
|
|
1334
|
+
raw_findings = engine.run()
|
|
1335
|
+
except Exception as exc:
|
|
1336
|
+
print(f"[FATAL] CrossFileTaintEngine crashed: {exc}", file=sys.stderr)
|
|
1337
|
+
traceback.print_exc()
|
|
1338
|
+
raise SystemExit(1)
|
|
1339
|
+
|
|
1340
|
+
results = []
|
|
1341
|
+
for cf in raw_findings:
|
|
1342
|
+
msg = f"Cross-file leak: {cf.caller_func} flows into {cf.callee_func} ({cf.callee_sink_name}) at {cf.callee_file}:{cf.callee_sink_lineno}"
|
|
1343
|
+
results.append(CrossFinding(
|
|
1344
|
+
cwe=cf.cwe,
|
|
1345
|
+
file_path=cf.caller_file,
|
|
1346
|
+
lineno=cf.caller_lineno,
|
|
1347
|
+
message=msg,
|
|
1348
|
+
severity="HIGH",
|
|
1349
|
+
flow_trace=[
|
|
1350
|
+
{"role": hop.role, "file": hop.file_path, "line": hop.lineno, "label": hop.label}
|
|
1351
|
+
for hop in getattr(cf, "flow_trace", None) or []
|
|
1352
|
+
],
|
|
1353
|
+
))
|
|
1354
|
+
return results, engine
|
|
1355
|
+
|
|
1356
|
+
|
|
1357
|
+
def _suppress_cross_file_sanitized(findings, engine):
|
|
1358
|
+
"""Drop single-file findings whose tainted arguments a cross-file sanitizer neutralized.
|
|
1359
|
+
|
|
1360
|
+
The AST scanner has no visibility into callee contracts, so `cmd = format_command(x)`
|
|
1361
|
+
followed by `os.system(cmd)` is reported as injectable even when the helper escapes its
|
|
1362
|
+
argument. Cross-file findings are left untouched: that engine already honours contracts.
|
|
1363
|
+
|
|
1364
|
+
A failure here is fatal, not ignored: returning the findings unchanged would silently
|
|
1365
|
+
re-introduce every sanitizer false positive and still report a successful scan.
|
|
1366
|
+
"""
|
|
1367
|
+
try:
|
|
1368
|
+
sanitized_sites = {
|
|
1369
|
+
(_normalize_compare_path(file_path), lineno)
|
|
1370
|
+
for file_path, lineno in engine.sanitized_sink_locations()
|
|
1371
|
+
}
|
|
1372
|
+
except Exception as exc:
|
|
1373
|
+
print(f"[FATAL] CrossFileTaintEngine sanitizer analysis crashed: {exc}", file=sys.stderr)
|
|
1374
|
+
traceback.print_exc()
|
|
1375
|
+
raise SystemExit(1)
|
|
1376
|
+
if not sanitized_sites:
|
|
1377
|
+
return findings
|
|
1378
|
+
|
|
1379
|
+
kept = []
|
|
1380
|
+
for finding in findings:
|
|
1381
|
+
if finding.get("category") == "cross_file":
|
|
1382
|
+
kept.append(finding)
|
|
1383
|
+
continue
|
|
1384
|
+
identity = (_normalize_compare_path(finding["file"]), finding["line"])
|
|
1385
|
+
if identity in sanitized_sites:
|
|
1386
|
+
continue
|
|
1387
|
+
kept.append(finding)
|
|
1388
|
+
return kept
|
|
1389
|
+
|
|
1390
|
+
def _js_ts_findings(path):
|
|
1391
|
+
"""Map the isolated tree-sitter JS/TS findings onto this module's finding shape."""
|
|
1392
|
+
mapped = []
|
|
1393
|
+
for finding in run_js_ts_scan_isolated(path, base_dir=Path.cwd().resolve()):
|
|
1394
|
+
mapped.append({
|
|
1395
|
+
"file": str(finding.get("file", "")).replace("\\", "/"),
|
|
1396
|
+
"line": finding.get("line_number") or 1,
|
|
1397
|
+
"cwe": finding.get("cwe") or "UNKNOWN_CWE",
|
|
1398
|
+
"severity": str(finding.get("severity") or "HIGH").upper(),
|
|
1399
|
+
"confidence": "MEDIUM",
|
|
1400
|
+
"category": finding.get("category") or "Security",
|
|
1401
|
+
"message": finding.get("message") or "",
|
|
1402
|
+
})
|
|
1403
|
+
return sorted(mapped, key=lambda item: (item["file"], item["line"], item["cwe"]))
|
|
1404
|
+
|
|
1405
|
+
|
|
1406
|
+
def _scan_failure(args, exc):
|
|
1407
|
+
"""A crashed scan still owes CI parsable artefacts, then reports exit 2.
|
|
1408
|
+
|
|
1409
|
+
Without this the process dies on a traceback: stdout JSON stays empty (0-byte file
|
|
1410
|
+
after redirection) and the SARIF target never exists, which GitHub's uploader
|
|
1411
|
+
reports as an unrelated failure.
|
|
1412
|
+
"""
|
|
1413
|
+
message = f"{type(exc).__name__}: {exc}"
|
|
1414
|
+
print(f"[FATAL] TimeCodeSecurity scan aborted: {message}", file=sys.stderr)
|
|
1415
|
+
if getattr(args, "format", "") == "json":
|
|
1416
|
+
print(json.dumps({
|
|
1417
|
+
"scope": getattr(args, "scope", "all"),
|
|
1418
|
+
"status": "incomplete",
|
|
1419
|
+
"error": message,
|
|
1420
|
+
"total_discovered_files": 0,
|
|
1421
|
+
"scanned_files": 0,
|
|
1422
|
+
"fast_path_skipped_files": 0,
|
|
1423
|
+
"unparseable_files": 0,
|
|
1424
|
+
"findings": [],
|
|
1425
|
+
}, indent=2))
|
|
1426
|
+
if getattr(args, "sarif", None):
|
|
1427
|
+
try:
|
|
1428
|
+
document = bound_sarif_document(
|
|
1429
|
+
_sarif_document([], Path.cwd().resolve(), execution_successful=False))
|
|
1430
|
+
atomic_write_text(Path(args.sarif), json.dumps(document, indent=2) + "\n")
|
|
1431
|
+
except OSError as write_exc:
|
|
1432
|
+
print(f"Error writing fallback SARIF file: {write_exc}", file=sys.stderr)
|
|
1433
|
+
return 2
|
|
1434
|
+
|
|
1435
|
+
|
|
1436
|
+
# A runner that kills the process at a step timeout leaves no SARIF at all, and the referee can only
|
|
1437
|
+
# report that as "0 findings" - indistinguishable from a clean scan. The handlers below turn that
|
|
1438
|
+
# truncation into a real (partial) report: whatever the scan already holds gets written to the
|
|
1439
|
+
# requested --sarif / --format json targets. Two limits are stated plainly: findings the engine has
|
|
1440
|
+
# not produced yet cannot be flushed, and GitHub's own step timeout does not reliably signal child
|
|
1441
|
+
# processes, so a caller must bound the run with `timeout --signal=TERM` to actually reach this path.
|
|
1442
|
+
_PARTIAL = {
|
|
1443
|
+
"args": None,
|
|
1444
|
+
"findings": [],
|
|
1445
|
+
"tracker": None,
|
|
1446
|
+
"files": None,
|
|
1447
|
+
"written": False,
|
|
1448
|
+
"current_file": None,
|
|
1449
|
+
"t0": None,
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1452
|
+
|
|
1453
|
+
def _partial_findings():
|
|
1454
|
+
"""Findings as far as the scan has got: stage list first, live tracker index second."""
|
|
1455
|
+
findings = list(_PARTIAL["findings"])
|
|
1456
|
+
if findings:
|
|
1457
|
+
return findings
|
|
1458
|
+
tracker = _PARTIAL["tracker"]
|
|
1459
|
+
if tracker is None:
|
|
1460
|
+
return []
|
|
1461
|
+
try:
|
|
1462
|
+
return _findings_for(tracker, list(tracker.edges))
|
|
1463
|
+
except Exception as exc:
|
|
1464
|
+
# A half-built index must not hide the artefact: report an empty partial run instead.
|
|
1465
|
+
print(f"[warning] Partial index unreadable: {type(exc).__name__}: {exc}",
|
|
1466
|
+
file=sys.stderr, flush=True)
|
|
1467
|
+
return []
|
|
1468
|
+
|
|
1469
|
+
|
|
1470
|
+
def _print_termination_telemetry():
|
|
1471
|
+
"""Say where the scan stopped, on stderr, before the partial payload is emitted.
|
|
1472
|
+
|
|
1473
|
+
A partial SARIF/JSON file only carries the findings that existed at the moment of the
|
|
1474
|
+
kill; it cannot say how far the engine had got. Without this line a timeout at file 3 of
|
|
1475
|
+
9,000 and a timeout inside the final cascade look identical to whoever reads the artefact.
|
|
1476
|
+
Everything here is observational: it writes to stderr and never touches the JSON shape or
|
|
1477
|
+
the SARIF document.
|
|
1478
|
+
"""
|
|
1479
|
+
tracker = _PARTIAL["tracker"]
|
|
1480
|
+
point = None
|
|
1481
|
+
if tracker is not None:
|
|
1482
|
+
try:
|
|
1483
|
+
point = tracker.termination_checkpoint()
|
|
1484
|
+
except Exception as exc:
|
|
1485
|
+
print(f"[telemetry] Checkpoint unavailable: {type(exc).__name__}: {exc}",
|
|
1486
|
+
file=sys.stderr, flush=True)
|
|
1487
|
+
if point is None:
|
|
1488
|
+
print("[telemetry] Active phase: pre-analysis (discovery/parse), analyze() not started",
|
|
1489
|
+
file=sys.stderr, flush=True)
|
|
1490
|
+
else:
|
|
1491
|
+
print(f"[telemetry] Active phase: {point['phase']}", file=sys.stderr, flush=True)
|
|
1492
|
+
profile = point.get("collector_profile_s") or {}
|
|
1493
|
+
if profile:
|
|
1494
|
+
ordered = sorted(profile.items(), key=lambda kv: (-kv[1], kv[0]))
|
|
1495
|
+
print("[telemetry] Collector timings so far (s): "
|
|
1496
|
+
+ " ".join(f"{name}={secs:.2f}" for name, secs in ordered),
|
|
1497
|
+
file=sys.stderr, flush=True)
|
|
1498
|
+
t0 = _PARTIAL.get("t0")
|
|
1499
|
+
if t0 is not None:
|
|
1500
|
+
line = f"[telemetry] Elapsed since scan start: {time.perf_counter() - t0:.2f}s"
|
|
1501
|
+
if point is not None and point.get("analyze_elapsed_s") is not None:
|
|
1502
|
+
line += f" (analyze(): {point['analyze_elapsed_s']:.2f}s)"
|
|
1503
|
+
print(line, file=sys.stderr, flush=True)
|
|
1504
|
+
|
|
1505
|
+
|
|
1506
|
+
def _flush_partial(reason):
|
|
1507
|
+
"""Serialize the current scan state into the requested artefacts, once."""
|
|
1508
|
+
args = _PARTIAL["args"]
|
|
1509
|
+
if args is None or _PARTIAL["written"]:
|
|
1510
|
+
return []
|
|
1511
|
+
_PARTIAL["written"] = True
|
|
1512
|
+
# Telemetry first: stdout carries the JSON payload, so the checkpoint has to be on stderr
|
|
1513
|
+
# and flushed before it, or a consumer piping stdout never sees it in order.
|
|
1514
|
+
_print_termination_telemetry()
|
|
1515
|
+
findings = _partial_findings()
|
|
1516
|
+
files = _PARTIAL["files"] or {}
|
|
1517
|
+
if getattr(args, "format", "") == "json":
|
|
1518
|
+
print(json.dumps({
|
|
1519
|
+
"scope": getattr(args, "scope", "all"),
|
|
1520
|
+
"status": "partial",
|
|
1521
|
+
"terminated_by": reason,
|
|
1522
|
+
"total_discovered_files": len(files),
|
|
1523
|
+
"scanned_files": len(files),
|
|
1524
|
+
"findings": findings,
|
|
1525
|
+
}, indent=2), flush=True)
|
|
1526
|
+
if getattr(args, "sarif", None):
|
|
1527
|
+
try:
|
|
1528
|
+
document = bound_sarif_document(
|
|
1529
|
+
_sarif_document(findings, Path.cwd().resolve(), execution_successful=False))
|
|
1530
|
+
atomic_write_text(Path(args.sarif), json.dumps(document, indent=2) + "\n")
|
|
1531
|
+
except OSError as exc:
|
|
1532
|
+
print(f"[warning] Partial SARIF write failed: {exc}", file=sys.stderr, flush=True)
|
|
1533
|
+
return findings
|
|
1534
|
+
print("[warning] Caught termination signal; flushed partial findings to SARIF",
|
|
1535
|
+
file=sys.stderr, flush=True)
|
|
1536
|
+
return findings
|
|
1537
|
+
|
|
1538
|
+
|
|
1539
|
+
def _install_partial_flush(args):
|
|
1540
|
+
"""Arm SIGTERM/SIGINT for this scan run, then let the caller work normally."""
|
|
1541
|
+
_PARTIAL.update({"args": args, "findings": [], "tracker": None, "files": None,
|
|
1542
|
+
"written": False, "current_file": None, "t0": time.perf_counter()})
|
|
1543
|
+
|
|
1544
|
+
def _handle(signum, _frame):
|
|
1545
|
+
reason = signal.Signals(signum).name
|
|
1546
|
+
# Progress lines are throttled to one per PROGRESS_FILE_EVERY files, so the file the scan
|
|
1547
|
+
# is actually inside comes from _PARTIAL["current_file"] rather than the last log line.
|
|
1548
|
+
current = _PARTIAL["current_file"]
|
|
1549
|
+
where = f" while reading {current}" if current else " mid-scan"
|
|
1550
|
+
print(f"[warning] Caught {reason}{where}; flushing what is known so far",
|
|
1551
|
+
file=sys.stderr, flush=True)
|
|
1552
|
+
_flush_partial(reason)
|
|
1553
|
+
# 128 + signum is the shell convention for "died of that signal": 143 SIGTERM, 130 SIGINT.
|
|
1554
|
+
raise SystemExit(128 + signum)
|
|
1555
|
+
|
|
1556
|
+
for sig in (signal.SIGTERM, signal.SIGINT):
|
|
1557
|
+
try:
|
|
1558
|
+
signal.signal(sig, _handle)
|
|
1559
|
+
except (AttributeError, OSError, ValueError):
|
|
1560
|
+
# No such signal on this platform, or not the main thread.
|
|
1561
|
+
pass
|
|
1562
|
+
|
|
1563
|
+
|
|
1564
|
+
def _parse_parser_arg(parser_str: Optional[str]) -> str:
|
|
1565
|
+
"""Normalise --parser into one of PARSER_MODES. Anything unexpected is the default, never a crash.
|
|
1566
|
+
|
|
1567
|
+
argparse restricts the choices for a real command line; this exists for callers that build an args
|
|
1568
|
+
namespace themselves, where `getattr(args, "parser", None)` comes back None.
|
|
1569
|
+
"""
|
|
1570
|
+
if isinstance(parser_str, str) and parser_str.strip().lower() in PARSER_MODES:
|
|
1571
|
+
return parser_str.strip().lower()
|
|
1572
|
+
return DEFAULT_PARSER_MODE
|
|
1573
|
+
|
|
1574
|
+
|
|
1575
|
+
def _parse_workers_arg(workers_str: Optional[str]) -> int:
|
|
1576
|
+
"""Parse --workers argument into integer worker count.
|
|
1577
|
+
|
|
1578
|
+
Args:
|
|
1579
|
+
workers_str: String value from CLI ('1', '2', '4', '8', 'auto', or None)
|
|
1580
|
+
|
|
1581
|
+
Returns:
|
|
1582
|
+
Integer worker count (defaults to os.cpu_count() for full multi-core utilization)
|
|
1583
|
+
"""
|
|
1584
|
+
if workers_str is None:
|
|
1585
|
+
workers_str = "auto"
|
|
1586
|
+
|
|
1587
|
+
workers_str = workers_str.strip().lower()
|
|
1588
|
+
|
|
1589
|
+
if workers_str == "auto":
|
|
1590
|
+
try:
|
|
1591
|
+
import os
|
|
1592
|
+
cpu_count = os.cpu_count() or 4
|
|
1593
|
+
# Use all available CPU cores (dynamic multi-core concurrency)
|
|
1594
|
+
return max(1, cpu_count)
|
|
1595
|
+
except Exception:
|
|
1596
|
+
return 4
|
|
1597
|
+
|
|
1598
|
+
try:
|
|
1599
|
+
n = int(workers_str)
|
|
1600
|
+
if n < 1:
|
|
1601
|
+
print("Warning: --workers must be >= 1, using 1", file=sys.stderr)
|
|
1602
|
+
return 1
|
|
1603
|
+
if n > 64:
|
|
1604
|
+
print(f"Warning: --workers={n} is excessive, capping at 32", file=sys.stderr)
|
|
1605
|
+
return 32
|
|
1606
|
+
return n
|
|
1607
|
+
except ValueError:
|
|
1608
|
+
print(f"Warning: invalid --workers value '{workers_str}', using auto-detect", file=sys.stderr)
|
|
1609
|
+
try:
|
|
1610
|
+
import os
|
|
1611
|
+
return os.cpu_count() or 4
|
|
1612
|
+
except Exception:
|
|
1613
|
+
return 4
|
|
1614
|
+
|
|
1615
|
+
|
|
1616
|
+
def _maybe_warn_js_ts(args):
|
|
1617
|
+
"""Warn about a missing JS/TS grammar only when JS/TS was actually part of this run.
|
|
1618
|
+
|
|
1619
|
+
Pure-Python scans used to inherit an import-time warning from js_scanner that the user could
|
|
1620
|
+
do nothing with. The grammar is already imported fine in most runs, so that case costs one
|
|
1621
|
+
boolean; only a broken install pays the (early-exit) tree walk that decides whether the
|
|
1622
|
+
target holds any JS/TS file.
|
|
1623
|
+
"""
|
|
1624
|
+
if _JS_TS_SCANNER_AVAILABLE and js_ts_available():
|
|
1625
|
+
return
|
|
1626
|
+
if not getattr(args, "include_js", False) and not has_js_ts_files(args.path):
|
|
1627
|
+
return
|
|
1628
|
+
warn_js_ts_unavailable()
|
|
1629
|
+
|
|
1630
|
+
|
|
1631
|
+
def _scan(args):
|
|
1632
|
+
scope = args.scope
|
|
1633
|
+
# Phase-1 outer clock: anchored at the earliest scan entry point, i.e. BEFORE
|
|
1634
|
+
# `_maybe_warn_js_ts`, auxiliary collection and `_collect_files`. The pre-existing
|
|
1635
|
+
# `started`/`duration_ms` anchor (after file collection) is left untouched so the
|
|
1636
|
+
# JSON schema keeps its documented meaning; Outer(...) TOTAL is the wider boundary.
|
|
1637
|
+
_clock = _StageClock("scan")
|
|
1638
|
+
_install_partial_flush(args)
|
|
1639
|
+
_maybe_warn_js_ts(args)
|
|
1640
|
+
|
|
1641
|
+
# Parse workers argument and pass to TaintTracker
|
|
1642
|
+
workers = _parse_workers_arg(getattr(args, "workers", None))
|
|
1643
|
+
ux = _ux_enabled(args)
|
|
1644
|
+
if ux:
|
|
1645
|
+
banner.print_banner(args.path, scope=scope, workers=workers, stream=sys.stderr)
|
|
1646
|
+
|
|
1647
|
+
try:
|
|
1648
|
+
line_ranges = _parse_line_ranges(getattr(args, "lines", None))
|
|
1649
|
+
exclude_patterns = _parse_exclude_patterns(getattr(args, "exclude", None))
|
|
1650
|
+
# Root configuration is optional and read from the working directory only, so a
|
|
1651
|
+
# checkout without `.tcs.yaml` keeps the byte-identical default scan.
|
|
1652
|
+
config = load_config()
|
|
1653
|
+
# Collected keys are spelled relative to the working directory, so the root is
|
|
1654
|
+
# resolved once here and reused by every `.tcs.yaml` match below.
|
|
1655
|
+
_config_root = _file_key(args.path, Path.cwd().resolve())
|
|
1656
|
+
config_excludes = (
|
|
1657
|
+
(lambda key: config.excludes_path(key, _config_root))
|
|
1658
|
+
if config is not None else None
|
|
1659
|
+
)
|
|
1660
|
+
tree_scan = args.path.is_dir()
|
|
1661
|
+
# Pruning non-production trees is the default for a directory walk; `--include-tests` opts
|
|
1662
|
+
# back in, and `--skip-non-production` is accepted but no longer changes anything. Only a
|
|
1663
|
+
# walk prunes: a file or directory named on the command line is analysed in full.
|
|
1664
|
+
skip_non_production = tree_scan and not bool(getattr(args, "include_tests", False))
|
|
1665
|
+
_SCOPE_STATS["skipped_dirs"] = 0
|
|
1666
|
+
_SCOPE_STATS["skipped_files"] = 0
|
|
1667
|
+
templates, iac, manifests, secrets, aux_scopes = _collect_auxiliary_files(
|
|
1668
|
+
args.path, exclude_patterns, config_excludes, skip_non_production)
|
|
1669
|
+
if scope not in ("all", "html"):
|
|
1670
|
+
templates = {}
|
|
1671
|
+
if scope not in ("all", "sca"):
|
|
1672
|
+
manifests = {}
|
|
1673
|
+
if scope not in ("all", "secrets"):
|
|
1674
|
+
secrets = {}
|
|
1675
|
+
if scope in ("docker", "iac"):
|
|
1676
|
+
iac = {key: source for key, source in iac.items() if aux_scopes.get(key) == scope}
|
|
1677
|
+
elif scope != "all":
|
|
1678
|
+
iac = {}
|
|
1679
|
+
aux_scopes = {key: value for key, value in aux_scopes.items()
|
|
1680
|
+
if scope == "all" or value == scope}
|
|
1681
|
+
_clock.mark("AuxCollect")
|
|
1682
|
+
tracker = None
|
|
1683
|
+
files = {}
|
|
1684
|
+
if scope in ("all", "python"):
|
|
1685
|
+
try:
|
|
1686
|
+
files = _collect_files(args.path, exclude_patterns=exclude_patterns,
|
|
1687
|
+
config_excludes=config_excludes,
|
|
1688
|
+
skip_non_production=skip_non_production)
|
|
1689
|
+
except ValueError:
|
|
1690
|
+
if not templates and not iac and not manifests and not secrets:
|
|
1691
|
+
raise
|
|
1692
|
+
_clock.mark("CollectPy")
|
|
1693
|
+
if ux:
|
|
1694
|
+
auxiliary = len(templates) + len(iac) + len(manifests) + len(secrets)
|
|
1695
|
+
banner.print_phase(1, 4, "discovery", stream=sys.stderr)
|
|
1696
|
+
banner.print_step(1, "files discovered",
|
|
1697
|
+
f"{len(files)} python, {auxiliary} auxiliary", stream=sys.stderr)
|
|
1698
|
+
|
|
1699
|
+
# TimeCodeSecurity: Capture pre-scan memory stats
|
|
1700
|
+
try:
|
|
1701
|
+
import psutil
|
|
1702
|
+
mem_before = psutil.virtual_memory()
|
|
1703
|
+
available_ram_before_gb = mem_before.available / (1024 ** 3)
|
|
1704
|
+
rss_before_mb = psutil.Process().memory_info().rss / (1024 ** 2)
|
|
1705
|
+
except ImportError:
|
|
1706
|
+
available_ram_before_gb = 0
|
|
1707
|
+
rss_before_mb = 0
|
|
1708
|
+
|
|
1709
|
+
if ux:
|
|
1710
|
+
banner.print_phase(2, 4, "analysis", stream=sys.stderr)
|
|
1711
|
+
for worker_id in range(workers if files else 0):
|
|
1712
|
+
banner.print_worker(worker_id, "ready", stream=sys.stderr)
|
|
1713
|
+
started = time.perf_counter()
|
|
1714
|
+
if files:
|
|
1715
|
+
_PARTIAL["files"] = files
|
|
1716
|
+
_parser_mode = _parse_parser_arg(getattr(args, "parser", None))
|
|
1717
|
+
# Default mode keeps the progress line byte-for-byte as it was; a chosen mode names itself,
|
|
1718
|
+
# because "with 8 worker(s)" is not what `treesitter` is doing (it runs single-process).
|
|
1719
|
+
_progress(f"parsing {len(files)} python file(s) with {workers} worker(s)"
|
|
1720
|
+
+ ("" if _parser_mode == DEFAULT_PARSER_MODE else f", parser={_parser_mode}"))
|
|
1721
|
+
tracker = TaintTracker(
|
|
1722
|
+
files=files,
|
|
1723
|
+
max_workers=workers,
|
|
1724
|
+
parser_mode=_parser_mode,
|
|
1725
|
+
audit_all=getattr(args, "audit_all", False),
|
|
1726
|
+
)
|
|
1727
|
+
_PARTIAL["tracker"] = tracker
|
|
1728
|
+
_clock.mark("Parse+Init")
|
|
1729
|
+
for fpath, reason in sorted(tracker.skipped_files.items()):
|
|
1730
|
+
print(f"Warning: skipping unparseable file {fpath}: {reason}", file=sys.stderr)
|
|
1731
|
+
_, _, edges = tracker.analyze()
|
|
1732
|
+
_clock.mark("Analyze")
|
|
1733
|
+
ast_findings = _findings_for(tracker, edges)
|
|
1734
|
+
_clock.mark("ASTFindings")
|
|
1735
|
+
_progress(f"parsed + traced, {len(ast_findings)} AST finding(s) so far")
|
|
1736
|
+
else:
|
|
1737
|
+
ast_findings = []
|
|
1738
|
+
_clock.mark("Parse+Init")
|
|
1739
|
+
_clock.mark("Analyze")
|
|
1740
|
+
_clock.mark("ASTFindings")
|
|
1741
|
+
if ux:
|
|
1742
|
+
banner.print_step(2, "ast findings", f"{len(ast_findings)} row(s)", stream=sys.stderr)
|
|
1743
|
+
aux_file_counts = {"docker": 0, "iac": 0, "sca": 0, "secrets": 0}
|
|
1744
|
+
for aux_scope in aux_scopes.values():
|
|
1745
|
+
aux_file_counts[aux_scope] += 1
|
|
1746
|
+
file_counts = {
|
|
1747
|
+
"python": len(tracker.modules) if tracker else 0,
|
|
1748
|
+
"docker": aux_file_counts["docker"],
|
|
1749
|
+
"iac": aux_file_counts["iac"],
|
|
1750
|
+
"sca": aux_file_counts["sca"],
|
|
1751
|
+
"secrets": aux_file_counts["secrets"],
|
|
1752
|
+
"html": len(templates),
|
|
1753
|
+
}
|
|
1754
|
+
scanned_files = sum(file_counts.values())
|
|
1755
|
+
parse_stats = getattr(tracker, "parse_stats", None) or {}
|
|
1756
|
+
python_discovered = parse_stats.get("discovered", len(files))
|
|
1757
|
+
fast_path_skipped_files = parse_stats.get("fast_path_skipped", 0)
|
|
1758
|
+
unparseable_files = parse_stats.get(
|
|
1759
|
+
"unparseable", len(tracker.skipped_files) if tracker else 0)
|
|
1760
|
+
scope_breakdown = {
|
|
1761
|
+
"python": {
|
|
1762
|
+
"discovered": python_discovered,
|
|
1763
|
+
"scanned": file_counts["python"],
|
|
1764
|
+
"fast_path_skipped": fast_path_skipped_files,
|
|
1765
|
+
"unparseable": unparseable_files,
|
|
1766
|
+
},
|
|
1767
|
+
"docker": {
|
|
1768
|
+
"discovered": aux_file_counts["docker"],
|
|
1769
|
+
"scanned": file_counts["docker"],
|
|
1770
|
+
},
|
|
1771
|
+
"iac": {
|
|
1772
|
+
"discovered": aux_file_counts["iac"],
|
|
1773
|
+
"scanned": file_counts["iac"],
|
|
1774
|
+
},
|
|
1775
|
+
"sca": {
|
|
1776
|
+
"discovered": aux_file_counts["sca"],
|
|
1777
|
+
"scanned": file_counts["sca"],
|
|
1778
|
+
},
|
|
1779
|
+
"secrets": {
|
|
1780
|
+
"discovered": aux_file_counts["secrets"],
|
|
1781
|
+
"scanned": file_counts["secrets"],
|
|
1782
|
+
},
|
|
1783
|
+
"html": {
|
|
1784
|
+
"discovered": len(templates),
|
|
1785
|
+
"scanned": file_counts["html"],
|
|
1786
|
+
},
|
|
1787
|
+
}
|
|
1788
|
+
total_discovered_files = sum(bucket["discovered"] for bucket in scope_breakdown.values())
|
|
1789
|
+
parse_governor = {
|
|
1790
|
+
key: parse_stats[key]
|
|
1791
|
+
for key in ("parse_mode", "workers_used", "workers_initial",
|
|
1792
|
+
"workers_final", "governor_events",
|
|
1793
|
+
"execution_mode", "parallel_fallback_reason",
|
|
1794
|
+
# Phase 3: which reader was allowed, and how many modules it actually produced
|
|
1795
|
+
# (a shadow run can be checked against those two numbers instead of a log).
|
|
1796
|
+
"parser_mode", "parsed_by_packer")
|
|
1797
|
+
if key in parse_stats
|
|
1798
|
+
}
|
|
1799
|
+
# The engine attribute is the authority; parse_stats carries it only when a tracker ran.
|
|
1800
|
+
parse_governor.setdefault("execution_mode", getattr(tracker, "execution_mode", "not_started"))
|
|
1801
|
+
parse_governor.setdefault("parallel_fallback_reason",
|
|
1802
|
+
getattr(tracker, "parallel_fallback_reason", None))
|
|
1803
|
+
# The legacy auditor keeps owning every CWE-1357 row and the Dockerfile CWE-250/CWE-269
|
|
1804
|
+
# pair, byte-identical. The modular scanners add the container and CI/IaC rule families on
|
|
1805
|
+
# top and are handed the legacy rows so one root-user defect is never booked twice.
|
|
1806
|
+
legacy_iac = audit_iac_files(iac, audit_all=getattr(args, "audit_all", False))
|
|
1807
|
+
sca_findings = audit_sca_files(manifests)
|
|
1808
|
+
if sca_findings:
|
|
1809
|
+
# The import walk is paid for only when a vulnerable pin exists, so a Python-only tree
|
|
1810
|
+
# -- every latency gate's workload -- does no extra work at all.
|
|
1811
|
+
tag_reachability(sca_findings, _imported_roots(tracker))
|
|
1812
|
+
findings = _merge_findings(
|
|
1813
|
+
ast_findings,
|
|
1814
|
+
audit_templates(templates),
|
|
1815
|
+
legacy_iac,
|
|
1816
|
+
audit_security_files(iac, legacy_findings=legacy_iac),
|
|
1817
|
+
sca_findings,
|
|
1818
|
+
audit_secret_files(secrets),
|
|
1819
|
+
)
|
|
1820
|
+
findings = consolidate_findings(findings)
|
|
1821
|
+
# Template/IaC auditors report a structural match and no data-flow provenance, which
|
|
1822
|
+
# is exactly MEDIUM; AST findings arrive with their own label.
|
|
1823
|
+
for finding in findings:
|
|
1824
|
+
finding.setdefault("confidence", "MEDIUM")
|
|
1825
|
+
_clock.mark("Merge+Consolidate")
|
|
1826
|
+
_PARTIAL["findings"] = findings
|
|
1827
|
+
if scope in ("all", "python"):
|
|
1828
|
+
cross_findings, cross_engine = _run_cross_scan(
|
|
1829
|
+
args.path, reuse_trees=_cross_scan_reuse_trees(tracker))
|
|
1830
|
+
_clock.mark("CrossScan")
|
|
1831
|
+
findings = _suppress_cross_file_sanitized(findings, cross_engine)
|
|
1832
|
+
_clock.mark("Suppression")
|
|
1833
|
+
findings.extend(cross_findings)
|
|
1834
|
+
_PARTIAL["findings"] = findings
|
|
1835
|
+
else:
|
|
1836
|
+
_clock.mark("CrossScan")
|
|
1837
|
+
_clock.mark("Suppression")
|
|
1838
|
+
if ux:
|
|
1839
|
+
banner.print_phase(3, 4, "correlation", stream=sys.stderr)
|
|
1840
|
+
banner.print_step(3, "cross-file + suppression", f"{len(findings)} row(s)",
|
|
1841
|
+
stream=sys.stderr)
|
|
1842
|
+
|
|
1843
|
+
# Incremental scan: keep only diagnostics intersecting the modified lines.
|
|
1844
|
+
# Applied before autofix generation so patching work is scoped too.
|
|
1845
|
+
findings = _scope_findings_to_lines(findings, line_ranges)
|
|
1846
|
+
|
|
1847
|
+
# Opt-in tree-sitter JS/TS pass. Off by default: the grammar is a native
|
|
1848
|
+
# extension whose access violation takes the whole process down.
|
|
1849
|
+
if getattr(args, "include_js", False) and _JS_TS_SCANNER_AVAILABLE and js_ts_available():
|
|
1850
|
+
findings = _merge_findings(findings, _js_ts_findings(args.path))
|
|
1851
|
+
|
|
1852
|
+
# Root `.tcs.yaml`: path exclusions and per-rule path overrides from the working
|
|
1853
|
+
# directory. Applied after every producer so cross-file and JS rows obey it too.
|
|
1854
|
+
findings = _apply_config_filters(findings, config, path_excludes=tree_scan,
|
|
1855
|
+
scan_root=_config_root, aux_scopes=aux_scopes)
|
|
1856
|
+
findings = _apply_inline_suppressions(
|
|
1857
|
+
findings, {**files, **templates, **iac, **manifests, **secrets})
|
|
1858
|
+
|
|
1859
|
+
# --diff narrows *reporting* to added/modified lines. The analysis above already
|
|
1860
|
+
# saw the whole tree, so cross-file context is never truncated.
|
|
1861
|
+
diff_ref = getattr(args, "diff", None)
|
|
1862
|
+
if diff_ref:
|
|
1863
|
+
_diff_repo_root = resolve_repo_root()
|
|
1864
|
+
findings = _scope_findings_to_diff(
|
|
1865
|
+
findings, changed_line_intervals(_diff_repo_root, diff_ref), _diff_repo_root)
|
|
1866
|
+
|
|
1867
|
+
# Display-only confidence filter; the engine's emission set is already final.
|
|
1868
|
+
requested_minimum = getattr(args, "min_confidence", None)
|
|
1869
|
+
confidence_minimum = None if requested_minimum in (None, "all") else requested_minimum
|
|
1870
|
+
findings = _filter_by_confidence(findings, confidence_minimum)
|
|
1871
|
+
# The default high floor belongs to the table alone, and only when nobody named a level: an
|
|
1872
|
+
# explicit `all` is a request to see everything, in the table too. `findings` stays the
|
|
1873
|
+
# complete emission set either way, so JSON, SARIF, the autofix pass and the exit code never
|
|
1874
|
+
# see a floor nobody asked for, and `confidence_filter` keeps reporting null for a machine run.
|
|
1875
|
+
hidden_rows = []
|
|
1876
|
+
display_rows = findings
|
|
1877
|
+
if requested_minimum is None and getattr(args, "format", "table") == "table":
|
|
1878
|
+
display_rows, hidden_rows = _split_by_table_default(findings)
|
|
1879
|
+
_PARTIAL["findings"] = findings
|
|
1880
|
+
|
|
1881
|
+
# Generate autofix suggestions if requested (JSON only)
|
|
1882
|
+
if args.with_autofix and args.format == "json" and files:
|
|
1883
|
+
# Build source cache from scanned files
|
|
1884
|
+
source_cache = {}
|
|
1885
|
+
for file_path in files:
|
|
1886
|
+
try:
|
|
1887
|
+
with open(file_path, 'r', encoding='utf-8') as f:
|
|
1888
|
+
source_cache[file_path.replace("\\", "/")] = f.read()
|
|
1889
|
+
except (OSError, UnicodeDecodeError):
|
|
1890
|
+
pass
|
|
1891
|
+
|
|
1892
|
+
# Attach autofixes where deterministic fixes are available
|
|
1893
|
+
autofix_engine = RemediationEngine()
|
|
1894
|
+
findings = _attach_autofixes(findings, source_cache, autofix_engine)
|
|
1895
|
+
_clock.mark("LineScope+JS+Autofix")
|
|
1896
|
+
|
|
1897
|
+
duration_ms = (time.perf_counter() - started) * 1000
|
|
1898
|
+
|
|
1899
|
+
# TimeCodeSecurity: Capture post-scan memory stats
|
|
1900
|
+
try:
|
|
1901
|
+
import psutil
|
|
1902
|
+
mem_after = psutil.virtual_memory()
|
|
1903
|
+
available_ram_after_gb = mem_after.available / (1024 ** 3)
|
|
1904
|
+
rss_after_mb = psutil.Process().memory_info().rss / (1024 ** 2)
|
|
1905
|
+
peak_rss_mb = rss_after_mb
|
|
1906
|
+
available_ram_gb = min(available_ram_before_gb, available_ram_after_gb)
|
|
1907
|
+
except ImportError:
|
|
1908
|
+
peak_rss_mb = 0
|
|
1909
|
+
available_ram_gb = 0
|
|
1910
|
+
except (OSError, ValueError) as exc:
|
|
1911
|
+
return _scan_failure(args, exc)
|
|
1912
|
+
|
|
1913
|
+
if ux:
|
|
1914
|
+
banner.print_phase(4, 4, "reporting", stream=sys.stderr)
|
|
1915
|
+
if args.format == "json":
|
|
1916
|
+
print(json.dumps({
|
|
1917
|
+
"scope": scope,
|
|
1918
|
+
"line_filter": [list(r) for r in line_ranges] if line_ranges else None,
|
|
1919
|
+
"diff_ref": diff_ref,
|
|
1920
|
+
"confidence_filter": confidence_minimum,
|
|
1921
|
+
"total_discovered_files": total_discovered_files,
|
|
1922
|
+
"scanned_files": scanned_files,
|
|
1923
|
+
"scanned_files_by_scope": file_counts,
|
|
1924
|
+
"fast_path_skipped_files": fast_path_skipped_files,
|
|
1925
|
+
"unparseable_files": unparseable_files,
|
|
1926
|
+
"scope_breakdown": scope_breakdown,
|
|
1927
|
+
"parse_governor": parse_governor,
|
|
1928
|
+
"execution_mode": parse_governor.get("execution_mode", "not_started"),
|
|
1929
|
+
"parallel_fallback_reason": parse_governor.get("parallel_fallback_reason"),
|
|
1930
|
+
"duration_ms": round(duration_ms, 2),
|
|
1931
|
+
"findings_by_scope": _scope_breakdown(findings, aux_scopes),
|
|
1932
|
+
"findings": findings,
|
|
1933
|
+
}, indent=2))
|
|
1934
|
+
# A scope note belongs on stderr for a machine run: the payload and its keys stay exactly
|
|
1935
|
+
# as they are, so nothing that parses this output changes behaviour.
|
|
1936
|
+
scope_note = non_production_scope_note()
|
|
1937
|
+
if scope_note:
|
|
1938
|
+
print(scope_note, file=sys.stderr)
|
|
1939
|
+
else:
|
|
1940
|
+
if display_rows:
|
|
1941
|
+
print(_ascii_table(display_rows))
|
|
1942
|
+
elif hidden_rows:
|
|
1943
|
+
print(f"[+] No findings at the default {TABLE_DEFAULT_CONFIDENCE.upper()} confidence "
|
|
1944
|
+
f"floor; {len(hidden_rows)} row(s) are held back, listed in the note below.")
|
|
1945
|
+
else:
|
|
1946
|
+
if _SCOPE_STATS["skipped_files"]:
|
|
1947
|
+
# Nothing was reported, but the walk did not read everything: "Clean scan!" over a
|
|
1948
|
+
# tree whose tests/ and examples/ were pruned would be the most misleading line
|
|
1949
|
+
# this tool can print.
|
|
1950
|
+
print(f"[+] No findings in the {scanned_files} file(s) that were read; "
|
|
1951
|
+
f"{_SCOPE_STATS['skipped_files']} non-production file(s) were held back "
|
|
1952
|
+
f"-- see the note below.")
|
|
1953
|
+
else:
|
|
1954
|
+
print("[+] No vulnerabilities found. Clean scan!")
|
|
1955
|
+
note = table_default_note(hidden_rows)
|
|
1956
|
+
if note:
|
|
1957
|
+
print(note)
|
|
1958
|
+
scope_note = non_production_scope_note()
|
|
1959
|
+
if scope_note:
|
|
1960
|
+
print(scope_note)
|
|
1961
|
+
_print_summary(scanned_files, duration_ms, display_rows, file_counts,
|
|
1962
|
+
discovered_files=total_discovered_files,
|
|
1963
|
+
fast_path_skipped_files=fast_path_skipped_files,
|
|
1964
|
+
unparseable_files=unparseable_files,
|
|
1965
|
+
parse_governor=parse_governor, aux_scopes=aux_scopes)
|
|
1966
|
+
traces = _cross_trace_report(findings, Path.cwd().resolve())
|
|
1967
|
+
if traces:
|
|
1968
|
+
print(traces)
|
|
1969
|
+
_clock.mark("StdoutOutput")
|
|
1970
|
+
|
|
1971
|
+
if args.sarif:
|
|
1972
|
+
try:
|
|
1973
|
+
document = bound_sarif_document(_sarif_document(findings, Path.cwd().resolve()))
|
|
1974
|
+
sarif_path = Path(args.sarif)
|
|
1975
|
+
atomic_write_text(sarif_path, json.dumps(document, indent=2) + "\n")
|
|
1976
|
+
sarif_size_mb = sarif_path.stat().st_size / (1024 ** 2) if sarif_path.exists() else 0
|
|
1977
|
+
except OSError as exc:
|
|
1978
|
+
print(f"Error writing SARIF file: {exc}", file=sys.stderr)
|
|
1979
|
+
return 2
|
|
1980
|
+
else:
|
|
1981
|
+
sarif_size_mb = 0
|
|
1982
|
+
_clock.mark("SARIFDump")
|
|
1983
|
+
|
|
1984
|
+
sbom_path_text = getattr(args, "sbom", None)
|
|
1985
|
+
if sbom_path_text:
|
|
1986
|
+
try:
|
|
1987
|
+
document = build_sbom(collect_components(manifests), findings,
|
|
1988
|
+
root_name=Path(args.path).resolve().name or "target")
|
|
1989
|
+
sbom_path = Path(sbom_path_text)
|
|
1990
|
+
atomic_write_text(sbom_path, json.dumps(document, indent=2) + "\n")
|
|
1991
|
+
except OSError as exc:
|
|
1992
|
+
print(f"Error writing SBOM file: {exc}", file=sys.stderr)
|
|
1993
|
+
return 2
|
|
1994
|
+
_clock.mark("SBOMDump")
|
|
1995
|
+
|
|
1996
|
+
# TimeCodeSecurity: Print performance metrics to stderr
|
|
1997
|
+
if files and tracker:
|
|
1998
|
+
try:
|
|
1999
|
+
from tcs.parallel_scanner import HARD_SAFETY_THRESHOLD_MB
|
|
2000
|
+
|
|
2001
|
+
print("\n" + "="*70, file=sys.stderr)
|
|
2002
|
+
print("TimeCodeSecurity - Performance Metrics", file=sys.stderr)
|
|
2003
|
+
print("="*70, file=sys.stderr)
|
|
2004
|
+
print(f"Wall-Clock Scan Time: {duration_ms/1000:.2f}s ({duration_ms:.0f}ms)", file=sys.stderr)
|
|
2005
|
+
print(f"Parser Mode: {parse_governor.get('parse_mode', 'unknown')}"
|
|
2006
|
+
f" (execution: {parse_governor.get('execution_mode', 'unknown')})", file=sys.stderr)
|
|
2007
|
+
print(f"Worker Count Utilized: {parse_governor.get('workers_final', parse_governor.get('workers_used', 'n/a'))}"
|
|
2008
|
+
f" (governor start: {parse_governor.get('workers_initial', parse_governor.get('workers_used', 'n/a'))}, "
|
|
2009
|
+
f"reductions: {parse_governor.get('governor_events', 0)})", file=sys.stderr)
|
|
2010
|
+
print(f"Governor RAM Floor: {HARD_SAFETY_THRESHOLD_MB} MB", file=sys.stderr)
|
|
2011
|
+
print(f"Peak Memory RSS: {peak_rss_mb:.1f} MB", file=sys.stderr)
|
|
2012
|
+
print(f"Available RAM (min): {available_ram_gb:.2f} GB", file=sys.stderr)
|
|
2013
|
+
print(f"Total Findings: {len(findings)}", file=sys.stderr)
|
|
2014
|
+
if args.sarif:
|
|
2015
|
+
print(f"SARIF File Size: {sarif_size_mb:.2f} MB {'✓' if sarif_size_mb < 10 else '⚠ EXCEEDS 10 MB'}", file=sys.stderr)
|
|
2016
|
+
print(f"Files Discovered: {total_discovered_files} "
|
|
2017
|
+
f"(python: {python_discovered}, docker: {aux_file_counts['docker']}, "
|
|
2018
|
+
f"iac: {aux_file_counts['iac']}, sca: {aux_file_counts['sca']}, "
|
|
2019
|
+
f"secrets: {aux_file_counts['secrets']}, "
|
|
2020
|
+
f"html: {len(templates)})", file=sys.stderr)
|
|
2021
|
+
print(f"Files Scanned: {scanned_files} "
|
|
2022
|
+
f"(python: {file_counts['python']}, docker: {file_counts['docker']}, "
|
|
2023
|
+
f"iac: {file_counts['iac']}, sca: {file_counts['sca']}, "
|
|
2024
|
+
f"secrets: {file_counts['secrets']}, "
|
|
2025
|
+
f"html: {file_counts['html']})", file=sys.stderr)
|
|
2026
|
+
print(f"Zero-Risk Fast-Path Skip: {fast_path_skipped_files} skipped", file=sys.stderr)
|
|
2027
|
+
print(f"Unparseable Files: {unparseable_files} skipped", file=sys.stderr)
|
|
2028
|
+
print("="*70, file=sys.stderr)
|
|
2029
|
+
except ImportError:
|
|
2030
|
+
pass # Parallel scanner not available, skip metrics
|
|
2031
|
+
|
|
2032
|
+
_clock.mark("MetricsBlock")
|
|
2033
|
+
# Outer TOTAL spans the whole scan command: entry (before auxiliary/python collection)
|
|
2034
|
+
# to just before exit-code computation. Stages before `started` (AuxCollect, CollectPy)
|
|
2035
|
+
# are inside TOTAL but NOT inside duration_ms - that is the CLI-vs-UI boundary gap.
|
|
2036
|
+
_clock.report()
|
|
2037
|
+
if ux:
|
|
2038
|
+
banner.print_completion(duration_ms / 1000, len(findings),
|
|
2039
|
+
sarif_path=str(args.sarif) if args.sarif else None,
|
|
2040
|
+
sbom_path=sbom_path_text or None, stream=sys.stderr)
|
|
2041
|
+
|
|
2042
|
+
# Exit-code parity with `tcs_cli.py`: a scan that finds something exits 1, so CI can gate
|
|
2043
|
+
# on either entry point. 2 stays reserved for an internal error (see the crash handler).
|
|
2044
|
+
# `--fail-on-critical` narrows what blocks: only Critical/High findings count.
|
|
2045
|
+
if args.fail_on_critical:
|
|
2046
|
+
blocking = [item for item in findings if item["severity"] in ("CRITICAL", "HIGH")]
|
|
2047
|
+
else:
|
|
2048
|
+
blocking = findings
|
|
2049
|
+
return 1 if blocking else 0
|
|
2050
|
+
|
|
2051
|
+
|
|
2052
|
+
def _normalize_compare_path(value):
|
|
2053
|
+
return os.path.normcase(str(Path(value).resolve()))
|
|
2054
|
+
|
|
2055
|
+
|
|
2056
|
+
def _normalize_cwes(value):
|
|
2057
|
+
if isinstance(value, dict):
|
|
2058
|
+
value = value.get("id", "")
|
|
2059
|
+
values = value if isinstance(value, list) else [value]
|
|
2060
|
+
cwes = set()
|
|
2061
|
+
for item in values:
|
|
2062
|
+
if isinstance(item, dict):
|
|
2063
|
+
item = item.get("id", "")
|
|
2064
|
+
text = str(item or "").upper()
|
|
2065
|
+
cwes.update(f"CWE-{number}" for number in re.findall(r"\bCWE[-_ ]?(\d+)\b", text))
|
|
2066
|
+
if text.isdigit():
|
|
2067
|
+
cwes.add(f"CWE-{text}")
|
|
2068
|
+
return sorted(cwes) if cwes else ["UNKNOWN_CWE"]
|
|
2069
|
+
|
|
2070
|
+
|
|
2071
|
+
def _competitor_findings(tool, data):
|
|
2072
|
+
findings = []
|
|
2073
|
+
if tool == "semgrep":
|
|
2074
|
+
for item in data.get("results", []):
|
|
2075
|
+
metadata = item.get("extra", {}).get("metadata", {})
|
|
2076
|
+
cwes = _normalize_cwes(metadata.get("cwe"))
|
|
2077
|
+
if cwes == ["UNKNOWN_CWE"]:
|
|
2078
|
+
cwes = _normalize_cwes(item.get("check_id", ""))
|
|
2079
|
+
path = item.get("path", "")
|
|
2080
|
+
line = item.get("start", {}).get("line", 0)
|
|
2081
|
+
rule = item.get("check_id", "")
|
|
2082
|
+
message = item.get("extra", {}).get("message", "")
|
|
2083
|
+
for cwe in cwes:
|
|
2084
|
+
findings.append({"file": _normalize_compare_path(path), "line": line, "cwe": cwe, "rule": rule, "message": message})
|
|
2085
|
+
else:
|
|
2086
|
+
for item in data.get("results", []):
|
|
2087
|
+
cwes = _normalize_cwes(item.get("issue_cwe"))
|
|
2088
|
+
path = item.get("filename", "")
|
|
2089
|
+
line = item.get("line_number", 0)
|
|
2090
|
+
rule = item.get("test_id", "")
|
|
2091
|
+
message = item.get("issue_text", "")
|
|
2092
|
+
for cwe in cwes:
|
|
2093
|
+
findings.append({"file": _normalize_compare_path(path), "line": line, "cwe": cwe, "rule": rule, "message": message})
|
|
2094
|
+
return findings
|
|
2095
|
+
|
|
2096
|
+
|
|
2097
|
+
def _run_competitor(tool, scan_path):
|
|
2098
|
+
executable = shutil.which(tool)
|
|
2099
|
+
if executable is None:
|
|
2100
|
+
print(f"Error: {tool} is not installed or not available on PATH.", file=sys.stderr)
|
|
2101
|
+
return None, None, 127
|
|
2102
|
+
if tool == "semgrep":
|
|
2103
|
+
command = [executable, "scan", "--json", "--config", "auto", str(scan_path)]
|
|
2104
|
+
else:
|
|
2105
|
+
command = [executable, "-r", "-f", "json", str(scan_path)]
|
|
2106
|
+
|
|
2107
|
+
started = time.perf_counter()
|
|
2108
|
+
try:
|
|
2109
|
+
completed = subprocess.run(command, capture_output=True, text=True, encoding="utf-8", errors="replace", check=False)
|
|
2110
|
+
except OSError as exc:
|
|
2111
|
+
print(f"Error: unable to run {tool}: {exc}", file=sys.stderr)
|
|
2112
|
+
return None, None, 2
|
|
2113
|
+
duration_ms = (time.perf_counter() - started) * 1000
|
|
2114
|
+
try:
|
|
2115
|
+
data = json.loads(completed.stdout)
|
|
2116
|
+
except (json.JSONDecodeError, TypeError):
|
|
2117
|
+
detail = completed.stderr.strip() or completed.stdout.strip() or f"process exited with status {completed.returncode}"
|
|
2118
|
+
print(f"Error: {tool} did not return valid JSON: {detail}", file=sys.stderr)
|
|
2119
|
+
return None, None, 2
|
|
2120
|
+
return _competitor_findings(tool, data), duration_ms, 0
|
|
2121
|
+
|
|
2122
|
+
|
|
2123
|
+
def _compare_findings(tcs_findings, competitor_findings, tolerance):
|
|
2124
|
+
matched = []
|
|
2125
|
+
matched_tcs = set()
|
|
2126
|
+
competitor_only = []
|
|
2127
|
+
for competitor in competitor_findings:
|
|
2128
|
+
candidates = [
|
|
2129
|
+
(index, finding) for index, finding in enumerate(tcs_findings)
|
|
2130
|
+
if finding["_normalized_file"] == competitor["file"]
|
|
2131
|
+
and finding["cwe"] == competitor["cwe"]
|
|
2132
|
+
and abs(finding["line"] - competitor["line"]) <= tolerance
|
|
2133
|
+
]
|
|
2134
|
+
if candidates:
|
|
2135
|
+
tcs_index, finding = min(candidates, key=lambda pair: abs(pair[1]["line"] - competitor["line"]))
|
|
2136
|
+
matched_tcs.add(tcs_index)
|
|
2137
|
+
matched.append({"cwe": competitor["cwe"], "file": finding["file"], "tcs_line": finding["line"], "competitor_line": competitor["line"], "rule": competitor["rule"]})
|
|
2138
|
+
else:
|
|
2139
|
+
competitor_only.append(competitor)
|
|
2140
|
+
|
|
2141
|
+
tcs_only = [finding for index, finding in enumerate(tcs_findings) if index not in matched_tcs]
|
|
2142
|
+
return matched, tcs_only, competitor_only
|
|
2143
|
+
|
|
2144
|
+
|
|
2145
|
+
def _cwe_breakdown(matched, tcs_only, competitor_only):
|
|
2146
|
+
cwes = sorted({item["cwe"] for item in matched + tcs_only + competitor_only})
|
|
2147
|
+
return [
|
|
2148
|
+
{
|
|
2149
|
+
"cwe": cwe,
|
|
2150
|
+
"both": sum(item["cwe"] == cwe for item in matched),
|
|
2151
|
+
"tcs_only": sum(item["cwe"] == cwe for item in tcs_only),
|
|
2152
|
+
"competitor_only": sum(item["cwe"] == cwe for item in competitor_only),
|
|
2153
|
+
}
|
|
2154
|
+
for cwe in cwes
|
|
2155
|
+
]
|
|
2156
|
+
|
|
2157
|
+
|
|
2158
|
+
def _print_compare_scoreboard(tool, tcs_ms, competitor_ms, matched, tcs_only, competitor_only, breakdown):
|
|
2159
|
+
summary_rows = [
|
|
2160
|
+
("Execution speed", f"TimeCodeSecurity {tcs_ms:.2f} ms vs {tool.title()} {competitor_ms:.2f} ms"),
|
|
2161
|
+
("Matched Findings (both)", str(len(matched))),
|
|
2162
|
+
("TimeCodeSecurity Exclusive Findings", str(len(tcs_only))),
|
|
2163
|
+
(f"{tool.title()} Only Findings", str(len(competitor_only))),
|
|
2164
|
+
]
|
|
2165
|
+
summary_headers = ("Metric", "Result")
|
|
2166
|
+
summary_widths = [max(len(summary_headers[index]), *(len(row[index]) for row in summary_rows)) for index in range(len(summary_headers))]
|
|
2167
|
+
summary_border = "+" + "+".join("-" * (width + 2) for width in summary_widths) + "+"
|
|
2168
|
+
print("Comparison Scoreboard")
|
|
2169
|
+
print(summary_border)
|
|
2170
|
+
print("| " + " | ".join(summary_headers[index].ljust(summary_widths[index]) for index in range(len(summary_headers))) + " |")
|
|
2171
|
+
print(summary_border)
|
|
2172
|
+
for row in summary_rows:
|
|
2173
|
+
print("| " + " | ".join(row[index].ljust(summary_widths[index]) for index in range(len(summary_headers))) + " |")
|
|
2174
|
+
print(summary_border)
|
|
2175
|
+
rows = [(item["cwe"], str(item["both"]), str(item["tcs_only"]), str(item["competitor_only"])) for item in breakdown]
|
|
2176
|
+
headers = ("CWE", "Both", "TimeCodeSecurity only", f"{tool.title()} only")
|
|
2177
|
+
widths = [max(len(headers[index]), *(len(row[index]) for row in rows)) for index in range(len(headers))]
|
|
2178
|
+
border = "+" + "+".join("-" * (width + 2) for width in widths) + "+"
|
|
2179
|
+
print("\nCWE Breakdown")
|
|
2180
|
+
print(border)
|
|
2181
|
+
print("| " + " | ".join(headers[index].ljust(widths[index]) for index in range(len(headers))) + " |")
|
|
2182
|
+
print(border)
|
|
2183
|
+
for row in rows:
|
|
2184
|
+
print("| " + " | ".join(row[index].ljust(widths[index]) for index in range(len(headers))) + " |")
|
|
2185
|
+
print(border)
|
|
2186
|
+
|
|
2187
|
+
|
|
2188
|
+
def _compare(args):
|
|
2189
|
+
try:
|
|
2190
|
+
exclude_patterns = _parse_exclude_patterns(getattr(args, "exclude", None))
|
|
2191
|
+
templates, iac, manifests, secrets, _aux_scopes = _collect_auxiliary_files(
|
|
2192
|
+
args.path, exclude_patterns)
|
|
2193
|
+
|
|
2194
|
+
try:
|
|
2195
|
+
files = _collect_files(args.path, exclude_patterns=exclude_patterns)
|
|
2196
|
+
except ValueError:
|
|
2197
|
+
if not templates and not iac and not manifests and not secrets:
|
|
2198
|
+
raise
|
|
2199
|
+
files = {}
|
|
2200
|
+
started = time.perf_counter()
|
|
2201
|
+
if files:
|
|
2202
|
+
tracker = TaintTracker(files=files)
|
|
2203
|
+
_, _, edges = tracker.analyze()
|
|
2204
|
+
ast_findings = _findings_for(tracker, edges)
|
|
2205
|
+
else:
|
|
2206
|
+
ast_findings = []
|
|
2207
|
+
tracker = None
|
|
2208
|
+
legacy_iac = audit_iac_files(iac)
|
|
2209
|
+
sca_findings = audit_sca_files(manifests)
|
|
2210
|
+
if sca_findings:
|
|
2211
|
+
tag_reachability(sca_findings, _imported_roots(tracker))
|
|
2212
|
+
tcs_findings = _merge_findings(ast_findings, audit_templates(templates), legacy_iac,
|
|
2213
|
+
audit_security_files(iac, legacy_findings=legacy_iac),
|
|
2214
|
+
sca_findings, audit_secret_files(secrets))
|
|
2215
|
+
tcs_ms = (time.perf_counter() - started) * 1000
|
|
2216
|
+
except (OSError, ValueError) as exc:
|
|
2217
|
+
print(f"Error: {exc}", file=sys.stderr)
|
|
2218
|
+
return 2
|
|
2219
|
+
|
|
2220
|
+
for finding in tcs_findings:
|
|
2221
|
+
finding["_normalized_file"] = _normalize_compare_path(finding["file"])
|
|
2222
|
+
competitor_findings, competitor_ms, status = _run_competitor(args.vs, args.path)
|
|
2223
|
+
if status:
|
|
2224
|
+
return status
|
|
2225
|
+
|
|
2226
|
+
matched, tcs_only, competitor_only = _compare_findings(tcs_findings, competitor_findings, args.tolerance)
|
|
2227
|
+
breakdown = _cwe_breakdown(matched, tcs_only, competitor_only)
|
|
2228
|
+
_print_compare_scoreboard(args.vs, tcs_ms, competitor_ms, matched, tcs_only, competitor_only, breakdown)
|
|
2229
|
+
|
|
2230
|
+
report = {
|
|
2231
|
+
"competitor": args.vs,
|
|
2232
|
+
"tolerance": args.tolerance,
|
|
2233
|
+
"execution_ms": {"tcs": round(tcs_ms, 2), "competitor": round(competitor_ms, 2)},
|
|
2234
|
+
"counts": {"both": len(matched), "tcs_only": len(tcs_only), "competitor_only": len(competitor_only)},
|
|
2235
|
+
"cwe_breakdown": breakdown,
|
|
2236
|
+
"matched": matched,
|
|
2237
|
+
"tcs_exclusive": [{key: value for key, value in item.items() if not key.startswith("_")} for item in tcs_only],
|
|
2238
|
+
"competitor_only_findings": competitor_only,
|
|
2239
|
+
}
|
|
2240
|
+
if args.output_json:
|
|
2241
|
+
try:
|
|
2242
|
+
atomic_write_text(Path(args.output_json), json.dumps(report, indent=2) + "\n")
|
|
2243
|
+
except OSError as exc:
|
|
2244
|
+
print(f"Error writing comparison report: {exc}", file=sys.stderr)
|
|
2245
|
+
return 2
|
|
2246
|
+
return 0
|
|
2247
|
+
|
|
2248
|
+
|
|
2249
|
+
def main(argv=None):
|
|
2250
|
+
parser = argparse.ArgumentParser(prog="cli.py", description="Standalone TimeCodeSecurity scanner")
|
|
2251
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
2252
|
+
scan_parser = commands.add_parser("scan", help="Scan a Python file or directory")
|
|
2253
|
+
scan_parser.add_argument("path", type=Path, help="Python file or directory to scan")
|
|
2254
|
+
scan_parser.add_argument("--sarif", metavar="OUTPUT_PATH", help="Write findings as SARIF 2.1.0 JSON")
|
|
2255
|
+
scan_parser.add_argument("--sbom", metavar="OUTPUT_PATH",
|
|
2256
|
+
help="Write the dependency inventory and its advisories as a "
|
|
2257
|
+
"CycloneDX 1.5 JSON SBOM (stdout output is unchanged)")
|
|
2258
|
+
scan_parser.add_argument("--fail-on-critical", action="store_true", help="Only exit 1 on a Critical or High finding (default: any finding exits 1, matching tcs_cli.py)")
|
|
2259
|
+
scan_parser.add_argument("--format", choices=("table", "json"), default="table", help="Output format (default: table)")
|
|
2260
|
+
scan_parser.add_argument(
|
|
2261
|
+
"--scope", choices=("all", "python", "docker", "iac", "sca", "secrets", "html"),
|
|
2262
|
+
default="all",
|
|
2263
|
+
help="Limit analysis to Python code, container images/compose, CI-CD & IaC "
|
|
2264
|
+
"(workflows, Kubernetes, Terraform), dependency manifests/lockfiles (SCA), "
|
|
2265
|
+
"config/env documents carrying credentials, "
|
|
2266
|
+
"or HTML templates (default: all)",
|
|
2267
|
+
)
|
|
2268
|
+
scan_parser.add_argument(
|
|
2269
|
+
"--with-autofix", action="store_true", default=False,
|
|
2270
|
+
help="Include deterministic autofix suggestions in JSON output (only for supported CWEs)",
|
|
2271
|
+
)
|
|
2272
|
+
scan_parser.add_argument(
|
|
2273
|
+
"--lines", metavar="START[-END][,START[-END]...]", default=None,
|
|
2274
|
+
help="Incremental scan: restrict findings to the given 1-indexed line ranges, e.g. --lines 4-5",
|
|
2275
|
+
)
|
|
2276
|
+
scan_parser.add_argument(
|
|
2277
|
+
"--diff", nargs="?", const="HEAD", default=None, metavar="REF",
|
|
2278
|
+
help="Report only findings on lines added or modified in the working tree versus REF "
|
|
2279
|
+
"(default REF: HEAD). The whole tree is still analysed, so cross-file context is "
|
|
2280
|
+
"never truncated. Requires a git repository",
|
|
2281
|
+
)
|
|
2282
|
+
scan_parser.add_argument(
|
|
2283
|
+
"--min-confidence", choices=("all", "low", "medium", "high"), default=None,
|
|
2284
|
+
help="Only show findings at or above this confidence level: high = an explicit "
|
|
2285
|
+
"source->sink flow, medium = pattern/structural match, low = heuristic comparison. "
|
|
2286
|
+
"`all` shows every finding. Default: the human-readable table starts at high, every "
|
|
2287
|
+
"machine format starts at all. Detection is unchanged",
|
|
2288
|
+
)
|
|
2289
|
+
scan_parser.add_argument(
|
|
2290
|
+
"--include-js", dest="include_js", action="store_true", default=False,
|
|
2291
|
+
help="Also run the tree-sitter JS/TS scanner (CWE-79/CWE-95 in .js/.jsx/.ts/.tsx). "
|
|
2292
|
+
"Off by default so the analysis stays scoped to Python",
|
|
2293
|
+
)
|
|
2294
|
+
scan_parser.add_argument(
|
|
2295
|
+
"--workers", type=str, default="auto",
|
|
2296
|
+
help="Number of parallel worker processes for scanning. Use integer (1-64) or 'auto' (default: auto = all available CPU cores via os.cpu_count())",
|
|
2297
|
+
)
|
|
2298
|
+
scan_parser.add_argument(
|
|
2299
|
+
"--exclude", type=str, default=None,
|
|
2300
|
+
help="Comma-separated glob patterns to exclude from scanning (e.g., tests,test_*,*_test.py,migrations)",
|
|
2301
|
+
)
|
|
2302
|
+
scan_parser.add_argument(
|
|
2303
|
+
"--include-tests", action="store_true",
|
|
2304
|
+
help="Scan tests/, test/, __tests__/, examples/, example/ and samples/ too. Directory walks "
|
|
2305
|
+
"prune those trees before descending by default, which is where fixtures and samples "
|
|
2306
|
+
"live -- and also where a hardcoded secret is classically committed, so use this flag "
|
|
2307
|
+
"for a full-coverage sweep. A file or directory named on the command line is always "
|
|
2308
|
+
"scanned in full, with or without it.",
|
|
2309
|
+
)
|
|
2310
|
+
scan_parser.add_argument(
|
|
2311
|
+
"--skip-non-production", action="store_true",
|
|
2312
|
+
help="Accepted for compatibility with scripts written before skipping became the default; "
|
|
2313
|
+
"it now asks for what already happens and changes nothing.",
|
|
2314
|
+
)
|
|
2315
|
+
scan_parser.add_argument(
|
|
2316
|
+
"--parser", choices=PARSER_MODES, default=DEFAULT_PARSER_MODE,
|
|
2317
|
+
help="Which reader builds the ASTs. 'hybrid' (default) = native ast.parse, with the "
|
|
2318
|
+
"tree-sitter rescue shield only on a file that raises SyntaxError; 'treesitter' = the "
|
|
2319
|
+
"tolerant tree-sitter CST reader for every file, single-process, CPython still catches "
|
|
2320
|
+
"what it declines; 'ast' = native ast.parse and the block-level sweep, tree-sitter "
|
|
2321
|
+
"never invoked",
|
|
2322
|
+
)
|
|
2323
|
+
scan_parser.add_argument(
|
|
2324
|
+
"--audit-all", action="store_true", default=False,
|
|
2325
|
+
help="Emit speculative POTENTIAL findings and non-actionable IaC notes",
|
|
2326
|
+
)
|
|
2327
|
+
compare_parser = commands.add_parser("compare", help="Compare TimeCodeSecurity findings with Semgrep or Bandit")
|
|
2328
|
+
compare_parser.add_argument("path", type=Path, help="Python file or directory to compare")
|
|
2329
|
+
compare_parser.add_argument("--vs", choices=("semgrep", "bandit"), default="semgrep", help="Competitor scanner (default: semgrep)")
|
|
2330
|
+
compare_parser.add_argument("--tolerance", type=int, default=5, help="Maximum line distance for a match (default: 5)")
|
|
2331
|
+
compare_parser.add_argument("--output-json", metavar="REPORT_PATH", help="Write detailed comparison report as JSON")
|
|
2332
|
+
args = parser.parse_args(argv)
|
|
2333
|
+
if args.command == "scan":
|
|
2334
|
+
try:
|
|
2335
|
+
return _scan(args)
|
|
2336
|
+
except Exception as exc:
|
|
2337
|
+
# Anything unhandled is an internal error: emit the fallback artefacts and
|
|
2338
|
+
# exit 2 instead of dying on a traceback with an empty report.
|
|
2339
|
+
traceback.print_exc()
|
|
2340
|
+
return _scan_failure(args, exc)
|
|
2341
|
+
if args.tolerance < 0:
|
|
2342
|
+
parser.error("--tolerance must be zero or greater")
|
|
2343
|
+
return _compare(args)
|
|
2344
|
+
|
|
2345
|
+
|
|
2346
|
+
if __name__ == "__main__":
|
|
2347
|
+
sys.exit(main())
|