timecs 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app.py +3014 -0
- ast_scanner.py +36187 -0
- banner.py +146 -0
- ci_reporter.py +570 -0
- cli.py +2347 -0
- config_loader.py +307 -0
- cross_file_engine.py +1781 -0
- delivery_models.py +145 -0
- delivery_repository.py +78 -0
- diff_scan.py +127 -0
- durable_delivery_sink.py +103 -0
- event_bus.py +155 -0
- event_publisher.py +345 -0
- event_taxonomy.py +313 -0
- function_summarizer.py +354 -0
- html_auditor.py +274 -0
- iac_auditor.py +400 -0
- in_app_adapter.py +115 -0
- js_scanner.py +895 -0
- manifest_parser.py +960 -0
- master_rules_bank.py +389 -0
- notification_models.py +346 -0
- notification_policy.py +309 -0
- notification_repository.py +118 -0
- notification_routes.py +394 -0
- notification_service.py +199 -0
- osv_client.py +437 -0
- remediation/__init__.py +39 -0
- remediation/base.py +102 -0
- remediation/contracts.py +196 -0
- remediation/cwe1188_binding.py +78 -0
- remediation/cwe1336_jinja.py +91 -0
- remediation/cwe22_path_traversal.py +348 -0
- remediation/cwe295_ssl.py +87 -0
- remediation/cwe327_weak_hash.py +78 -0
- remediation/cwe377_tempfile.py +77 -0
- remediation/cwe489_debug.py +80 -0
- remediation/cwe502_yaml.py +89 -0
- remediation/cwe614_cookie.py +91 -0
- remediation/cwe78_cmdi.py +415 -0
- remediation/cwe798_credentials.py +153 -0
- remediation/cwe89_sqli.py +425 -0
- remediation/cwe916_password_hash.py +131 -0
- remediation/cwe95_eval.py +77 -0
- remediation/dispatcher.py +114 -0
- remediation/orchestrator.py +541 -0
- remediation/patch_engine.py +213 -0
- remediation/verification.py +253 -0
- rule_engine.py +1535 -0
- sarif_adapter.py +719 -0
- sarif_exporter.py +88 -0
- sca_reachability/__init__.py +28 -0
- sca_reachability/ast_analyzer.py +366 -0
- sca_reachability/call_graph.py +203 -0
- sca_reachability/contracts.py +365 -0
- sca_reachability/dist_import_map.py +46 -0
- sca_reachability/engine.py +603 -0
- sca_reachability/resolver.py +169 -0
- scanners/__init__.py +70 -0
- scanners/container_scanner.py +323 -0
- scanners/cve_index.py +521 -0
- scanners/iac_scanner.py +328 -0
- scanners/sbom.py +183 -0
- scanners/sca_scanner.py +667 -0
- scanners/secret_scanner.py +345 -0
- scanners/yaml_lite.py +217 -0
- secret_filters.py +435 -0
- secret_scanner.py +943 -0
- sqlite_delivery_repository.py +277 -0
- sqlite_notification_repository.py +441 -0
- ssrf_guard.py +276 -0
- staged_scanner.py +492 -0
- suppression_resolver.py +282 -0
- symbol_indexer.py +424 -0
- tcs/__init__.py +1 -0
- tcs/analysis/__init__.py +1 -0
- tcs/analysis/def_use.py +408 -0
- tcs/analysis/function_contracts.py +290 -0
- tcs/parallel_analyzer.py +276 -0
- tcs/parallel_scanner.py +630 -0
- tcs/treesitter_packer.py +1446 -0
- tcs_cli.py +2086 -0
- tcs_config.py +218 -0
- tcs_gui.py +2123 -0
- timecs-2.0.0.dist-info/METADATA +142 -0
- timecs-2.0.0.dist-info/RECORD +94 -0
- timecs-2.0.0.dist-info/WHEEL +5 -0
- timecs-2.0.0.dist-info/entry_points.txt +4 -0
- timecs-2.0.0.dist-info/licenses/LICENSE +201 -0
- timecs-2.0.0.dist-info/top_level.txt +49 -0
- version_matcher.py +934 -0
- webhook_adapter.py +495 -0
- webhook_client.py +483 -0
- webhook_formatters.py +245 -0
tcs/parallel_analyzer.py
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
"""TimeCodeSecurity — File-level parallel scanner with deterministic merge.
|
|
2
|
+
|
|
3
|
+
This module parallelizes scanning across multiple files while ensuring:
|
|
4
|
+
- All 14 structural inspectors run on every file
|
|
5
|
+
- Full taint tracking engine executes per-worker
|
|
6
|
+
- Findings are deterministically merged and deduplicated
|
|
7
|
+
- Zero accuracy regression between --workers 1 and --workers N
|
|
8
|
+
|
|
9
|
+
Architecture:
|
|
10
|
+
- Each worker gets a subset of files
|
|
11
|
+
- Worker creates independent TaintTracker instance
|
|
12
|
+
- Worker runs full analyze() pipeline (all phases)
|
|
13
|
+
- Main process merges findings with deterministic sort + dedup
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import ast
|
|
18
|
+
import json
|
|
19
|
+
import os
|
|
20
|
+
import sys
|
|
21
|
+
import time
|
|
22
|
+
from concurrent.futures import ProcessPoolExecutor, as_completed
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from typing import Any, Dict, List, Optional, Set, Tuple
|
|
25
|
+
|
|
26
|
+
try:
|
|
27
|
+
import psutil
|
|
28
|
+
except ImportError:
|
|
29
|
+
# Declared optional on purpose. A hard module-level `import psutil` made this whole module
|
|
30
|
+
# unimportable wherever psutil was absent, and `ast_scanner._phase2_worker_count` then fell
|
|
31
|
+
# back to 1 worker silently -- that is how a 2-vCPU / 7 GB runner spent 283 s collecting
|
|
32
|
+
# statements serially. `available_ram_mb()` covers the same decision without it.
|
|
33
|
+
psutil = None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# ─── Worker RAM Budget for Analysis ──────────────────────────────────────────
|
|
37
|
+
|
|
38
|
+
ANALYSIS_WORKER_RAM_BUDGET_MB = 600 # Budget 600 MB per analysis worker
|
|
39
|
+
HARD_SAFETY_THRESHOLD_MB = 900 # Drop to 1 worker if RAM < 900 MB
|
|
40
|
+
CI_TWO_CORE_RAM_MB = 1600 # Minimum free RAM before any second worker is opened
|
|
41
|
+
MAX_PHASE2_WORKERS = 4 # Phase 2 caps here; the parent holds every parsed tree
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def available_ram_mb():
|
|
45
|
+
"""Free RAM in MB, or None when it cannot be measured.
|
|
46
|
+
|
|
47
|
+
psutil is the accurate source. Without it, Linux reports the same number in
|
|
48
|
+
`/proc/meminfo` as MemAvailable. Other platforms without psutil return None, and every caller
|
|
49
|
+
treats "unknown" as "no evidence of headroom" and stays on one worker.
|
|
50
|
+
"""
|
|
51
|
+
if psutil is not None:
|
|
52
|
+
return psutil.virtual_memory().available / (1024 * 1024)
|
|
53
|
+
try:
|
|
54
|
+
with open("/proc/meminfo", encoding="ascii") as handle:
|
|
55
|
+
for line in handle:
|
|
56
|
+
if line.startswith("MemAvailable:"):
|
|
57
|
+
return float(line.split()[1]) / 1024.0
|
|
58
|
+
except (OSError, ValueError, IndexError):
|
|
59
|
+
return None
|
|
60
|
+
return None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def compute_analysis_workers() -> int:
|
|
64
|
+
"""Compute safe worker count for analysis phase (heavier than parsing)."""
|
|
65
|
+
free_ram_mb = available_ram_mb()
|
|
66
|
+
if free_ram_mb is None:
|
|
67
|
+
return 1
|
|
68
|
+
# Hard safety threshold
|
|
69
|
+
if free_ram_mb < HARD_SAFETY_THRESHOLD_MB:
|
|
70
|
+
return 1
|
|
71
|
+
|
|
72
|
+
cpu_count = os.cpu_count() or 4
|
|
73
|
+
# The old rule kept 2 cores free (`cpu_count - 2`), which collapses a 2-vCPU GitHub runner to
|
|
74
|
+
# zero and then to one worker, so the runner never used the machine it was given. `cpu_count`
|
|
75
|
+
# capped at 4 keeps a core for the parent, and the RAM terms below still bound the pool:
|
|
76
|
+
# every worker needs ANALYSIS_WORKER_RAM_BUDGET_MB of its own, and multi-worker only starts
|
|
77
|
+
# once free RAM clears CI_TWO_CORE_RAM_MB.
|
|
78
|
+
cpu_capacity = min(cpu_count, MAX_PHASE2_WORKERS) if free_ram_mb >= CI_TWO_CORE_RAM_MB else 1
|
|
79
|
+
ram_capacity = max(1, int(free_ram_mb // ANALYSIS_WORKER_RAM_BUDGET_MB))
|
|
80
|
+
return max(1, min(cpu_capacity, ram_capacity))
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _scan_file_batch(args: Tuple[List[str], str, Dict[str, Any]]) -> Dict[str, Any]:
|
|
84
|
+
"""Scan a batch of files in a worker process.
|
|
85
|
+
|
|
86
|
+
This runs the FULL TaintTracker pipeline on assigned files.
|
|
87
|
+
|
|
88
|
+
Args:
|
|
89
|
+
args: (file_paths_list, scope, options_dict) tuple
|
|
90
|
+
|
|
91
|
+
Returns:
|
|
92
|
+
Dict with keys: findings (list of finding dicts), error (str or None)
|
|
93
|
+
"""
|
|
94
|
+
file_paths, scope, options = args
|
|
95
|
+
|
|
96
|
+
try:
|
|
97
|
+
# Import here to ensure clean worker state
|
|
98
|
+
from ast_scanner import TaintTracker
|
|
99
|
+
|
|
100
|
+
# Collect source code for all files
|
|
101
|
+
files = {}
|
|
102
|
+
for fpath in file_paths:
|
|
103
|
+
try:
|
|
104
|
+
with open(fpath, 'r', encoding='utf-8') as f:
|
|
105
|
+
files[fpath] = f.read()
|
|
106
|
+
except (OSError, UnicodeDecodeError):
|
|
107
|
+
continue
|
|
108
|
+
|
|
109
|
+
if not files:
|
|
110
|
+
return {
|
|
111
|
+
"findings": [],
|
|
112
|
+
"error": None,
|
|
113
|
+
"file_count": len(file_paths),
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
# Create TaintTracker and run full analysis
|
|
117
|
+
tracker = TaintTracker(files=files)
|
|
118
|
+
_, _, edges = tracker.analyze()
|
|
119
|
+
|
|
120
|
+
# Extract findings using the same logic as cli.py
|
|
121
|
+
findings = _extract_findings_from_tracker(tracker, edges)
|
|
122
|
+
|
|
123
|
+
return {
|
|
124
|
+
"findings": findings,
|
|
125
|
+
"error": None,
|
|
126
|
+
"file_count": len(file_paths),
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
except Exception as exc:
|
|
130
|
+
return {
|
|
131
|
+
"findings": [],
|
|
132
|
+
"error": f"{type(exc).__name__}: {exc}"[:200],
|
|
133
|
+
"file_count": len(file_paths),
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _extract_findings_from_tracker(tracker, edges) -> List[Dict[str, Any]]:
|
|
138
|
+
"""Extract findings from TaintTracker matching cli.py logic.
|
|
139
|
+
|
|
140
|
+
This mirrors the _findings_for() function in cli.py to ensure parity.
|
|
141
|
+
"""
|
|
142
|
+
findings = []
|
|
143
|
+
|
|
144
|
+
# Iterate through all sinks and generate findings
|
|
145
|
+
for sink in tracker.sinks:
|
|
146
|
+
finding = {
|
|
147
|
+
"file": getattr(sink, 'location', {}).get('file', ''),
|
|
148
|
+
"line": getattr(sink, 'location', {}).get('line', 0),
|
|
149
|
+
"column": getattr(sink, 'location', {}).get('column', 0),
|
|
150
|
+
"cwe": sink.metadata.get('cwe', '') if hasattr(sink, 'metadata') else '',
|
|
151
|
+
"severity": sink.metadata.get('severity', 'MEDIUM') if hasattr(sink, 'metadata') else 'MEDIUM',
|
|
152
|
+
"rule_id": sink.metadata.get('operation', '') if hasattr(sink, 'metadata') else '',
|
|
153
|
+
"message": f"{sink.metadata.get('operation', 'Vulnerability')} detected",
|
|
154
|
+
}
|
|
155
|
+
findings.append(finding)
|
|
156
|
+
|
|
157
|
+
return findings
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def parallel_scan_files(
|
|
161
|
+
file_paths: List[str],
|
|
162
|
+
scope: str = "python",
|
|
163
|
+
max_workers: Optional[int] = None,
|
|
164
|
+
) -> Tuple[List[Dict[str, Any]], int]:
|
|
165
|
+
"""Scan files in parallel with deterministic merge.
|
|
166
|
+
|
|
167
|
+
Args:
|
|
168
|
+
file_paths: List of absolute file paths to scan
|
|
169
|
+
scope: Scan scope ("python", "all", etc.)
|
|
170
|
+
max_workers: Override worker count (None = auto-compute)
|
|
171
|
+
|
|
172
|
+
Returns:
|
|
173
|
+
(merged_findings, total_files_scanned) where findings are sorted
|
|
174
|
+
deterministically by (file, line, cwe, rule_id)
|
|
175
|
+
"""
|
|
176
|
+
if max_workers is None:
|
|
177
|
+
max_workers = compute_analysis_workers()
|
|
178
|
+
|
|
179
|
+
total_files = len(file_paths)
|
|
180
|
+
|
|
181
|
+
print(
|
|
182
|
+
f"TimeCodeSecurity [PARALLEL] Scanning {total_files} files with {max_workers} workers",
|
|
183
|
+
file=sys.stderr,
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
# For small workloads, skip parallelism
|
|
187
|
+
if total_files <= 10 or max_workers == 1:
|
|
188
|
+
result = _scan_file_batch((file_paths, scope, {}))
|
|
189
|
+
if result["error"]:
|
|
190
|
+
print(
|
|
191
|
+
f"TimeCodeSecurity [WARN] Batch scan failed: {result['error']}",
|
|
192
|
+
file=sys.stderr,
|
|
193
|
+
)
|
|
194
|
+
return result.get("findings", []), result.get("file_count", 0)
|
|
195
|
+
|
|
196
|
+
# Partition files into batches (one per worker ideally)
|
|
197
|
+
batch_size = max(1, total_files // max_workers)
|
|
198
|
+
batches = []
|
|
199
|
+
for i in range(0, total_files, batch_size):
|
|
200
|
+
batch = file_paths[i:i + batch_size]
|
|
201
|
+
batches.append(batch)
|
|
202
|
+
|
|
203
|
+
all_findings = []
|
|
204
|
+
total_scanned = 0
|
|
205
|
+
start_time = time.time()
|
|
206
|
+
|
|
207
|
+
# Process batches in parallel
|
|
208
|
+
with ProcessPoolExecutor(max_workers=max_workers) as executor:
|
|
209
|
+
future_to_batch = {
|
|
210
|
+
executor.submit(_scan_file_batch, (batch, scope, {})): idx
|
|
211
|
+
for idx, batch in enumerate(batches)
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
for future in as_completed(future_to_batch):
|
|
215
|
+
batch_idx = future_to_batch[future]
|
|
216
|
+
try:
|
|
217
|
+
result = future.result(timeout=300) # 5-minute timeout per batch
|
|
218
|
+
if result["error"]:
|
|
219
|
+
print(
|
|
220
|
+
f"TimeCodeSecurity [WARN] Batch {batch_idx} failed: {result['error']}",
|
|
221
|
+
file=sys.stderr,
|
|
222
|
+
)
|
|
223
|
+
else:
|
|
224
|
+
all_findings.extend(result["findings"])
|
|
225
|
+
total_scanned += result.get("file_count", 0)
|
|
226
|
+
except Exception as exc:
|
|
227
|
+
print(
|
|
228
|
+
f"TimeCodeSecurity [WARN] Batch {batch_idx} exception: {exc}",
|
|
229
|
+
file=sys.stderr,
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
elapsed = time.time() - start_time
|
|
233
|
+
print(
|
|
234
|
+
f"TimeCodeSecurity [PARALLEL] Completed in {elapsed:.2f}s "
|
|
235
|
+
f"({len(all_findings)} findings from {total_scanned} files)",
|
|
236
|
+
file=sys.stderr,
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
# Deterministic sort and deduplication
|
|
240
|
+
merged_findings = _merge_and_deduplicate_findings(all_findings)
|
|
241
|
+
|
|
242
|
+
return merged_findings, total_scanned
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _merge_and_deduplicate_findings(findings: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
246
|
+
"""Merge findings from multiple workers with deterministic sort and dedup.
|
|
247
|
+
|
|
248
|
+
Deduplicates by (file, line, cwe, rule_id) tuple.
|
|
249
|
+
Sorts by (file, line, cwe, rule_id) for reproducibility.
|
|
250
|
+
"""
|
|
251
|
+
# Deduplicate
|
|
252
|
+
seen = set()
|
|
253
|
+
unique_findings = []
|
|
254
|
+
|
|
255
|
+
for finding in findings:
|
|
256
|
+
key = (
|
|
257
|
+
finding.get("file", ""),
|
|
258
|
+
finding.get("line", 0),
|
|
259
|
+
finding.get("cwe", ""),
|
|
260
|
+
finding.get("rule_id", ""),
|
|
261
|
+
)
|
|
262
|
+
if key not in seen:
|
|
263
|
+
seen.add(key)
|
|
264
|
+
unique_findings.append(finding)
|
|
265
|
+
|
|
266
|
+
# Deterministic sort
|
|
267
|
+
unique_findings.sort(
|
|
268
|
+
key=lambda x: (x.get("file", ""), x.get("line", 0), x.get("cwe", ""), x.get("rule_id", ""))
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
return unique_findings
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
if __name__ == "__main__":
|
|
275
|
+
# Safety guard for Windows multiprocessing
|
|
276
|
+
print("TimeCodeSecurity parallel scanner module loaded successfully.")
|