timecs 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app.py +3014 -0
- ast_scanner.py +36187 -0
- banner.py +146 -0
- ci_reporter.py +570 -0
- cli.py +2347 -0
- config_loader.py +307 -0
- cross_file_engine.py +1781 -0
- delivery_models.py +145 -0
- delivery_repository.py +78 -0
- diff_scan.py +127 -0
- durable_delivery_sink.py +103 -0
- event_bus.py +155 -0
- event_publisher.py +345 -0
- event_taxonomy.py +313 -0
- function_summarizer.py +354 -0
- html_auditor.py +274 -0
- iac_auditor.py +400 -0
- in_app_adapter.py +115 -0
- js_scanner.py +895 -0
- manifest_parser.py +960 -0
- master_rules_bank.py +389 -0
- notification_models.py +346 -0
- notification_policy.py +309 -0
- notification_repository.py +118 -0
- notification_routes.py +394 -0
- notification_service.py +199 -0
- osv_client.py +437 -0
- remediation/__init__.py +39 -0
- remediation/base.py +102 -0
- remediation/contracts.py +196 -0
- remediation/cwe1188_binding.py +78 -0
- remediation/cwe1336_jinja.py +91 -0
- remediation/cwe22_path_traversal.py +348 -0
- remediation/cwe295_ssl.py +87 -0
- remediation/cwe327_weak_hash.py +78 -0
- remediation/cwe377_tempfile.py +77 -0
- remediation/cwe489_debug.py +80 -0
- remediation/cwe502_yaml.py +89 -0
- remediation/cwe614_cookie.py +91 -0
- remediation/cwe78_cmdi.py +415 -0
- remediation/cwe798_credentials.py +153 -0
- remediation/cwe89_sqli.py +425 -0
- remediation/cwe916_password_hash.py +131 -0
- remediation/cwe95_eval.py +77 -0
- remediation/dispatcher.py +114 -0
- remediation/orchestrator.py +541 -0
- remediation/patch_engine.py +213 -0
- remediation/verification.py +253 -0
- rule_engine.py +1535 -0
- sarif_adapter.py +719 -0
- sarif_exporter.py +88 -0
- sca_reachability/__init__.py +28 -0
- sca_reachability/ast_analyzer.py +366 -0
- sca_reachability/call_graph.py +203 -0
- sca_reachability/contracts.py +365 -0
- sca_reachability/dist_import_map.py +46 -0
- sca_reachability/engine.py +603 -0
- sca_reachability/resolver.py +169 -0
- scanners/__init__.py +70 -0
- scanners/container_scanner.py +323 -0
- scanners/cve_index.py +521 -0
- scanners/iac_scanner.py +328 -0
- scanners/sbom.py +183 -0
- scanners/sca_scanner.py +667 -0
- scanners/secret_scanner.py +345 -0
- scanners/yaml_lite.py +217 -0
- secret_filters.py +435 -0
- secret_scanner.py +943 -0
- sqlite_delivery_repository.py +277 -0
- sqlite_notification_repository.py +441 -0
- ssrf_guard.py +276 -0
- staged_scanner.py +492 -0
- suppression_resolver.py +282 -0
- symbol_indexer.py +424 -0
- tcs/__init__.py +1 -0
- tcs/analysis/__init__.py +1 -0
- tcs/analysis/def_use.py +408 -0
- tcs/analysis/function_contracts.py +290 -0
- tcs/parallel_analyzer.py +276 -0
- tcs/parallel_scanner.py +630 -0
- tcs/treesitter_packer.py +1446 -0
- tcs_cli.py +2086 -0
- tcs_config.py +218 -0
- tcs_gui.py +2123 -0
- timecs-2.0.0.dist-info/METADATA +142 -0
- timecs-2.0.0.dist-info/RECORD +94 -0
- timecs-2.0.0.dist-info/WHEEL +5 -0
- timecs-2.0.0.dist-info/entry_points.txt +4 -0
- timecs-2.0.0.dist-info/licenses/LICENSE +201 -0
- timecs-2.0.0.dist-info/top_level.txt +49 -0
- version_matcher.py +934 -0
- webhook_adapter.py +495 -0
- webhook_client.py +483 -0
- webhook_formatters.py +245 -0
tcs/parallel_scanner.py
ADDED
|
@@ -0,0 +1,630 @@
|
|
|
1
|
+
"""TimeCodeSecurity — Safe parallel scanner with hard resource governance.
|
|
2
|
+
|
|
3
|
+
This module provides CPU-aware, memory-governed parallel scanning that:
|
|
4
|
+
- Never freezes Windows by keeping ≥2 cores free for OS/DWM
|
|
5
|
+
- Throttles workers when RAM drops below 1.2 GB
|
|
6
|
+
- Enforces per-file timeouts (5 s) to prevent hangs
|
|
7
|
+
- Returns only lightweight primitives (no AST serialization)
|
|
8
|
+
- Uses fast-path pre-filtering for zero-risk files
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import ast
|
|
13
|
+
import os
|
|
14
|
+
import pathlib
|
|
15
|
+
import sys
|
|
16
|
+
import time
|
|
17
|
+
from concurrent.futures import ProcessPoolExecutor
|
|
18
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
19
|
+
|
|
20
|
+
try:
|
|
21
|
+
import psutil
|
|
22
|
+
except ImportError:
|
|
23
|
+
# `ast_scanner` imports this module inside `try/except ImportError` to set
|
|
24
|
+
# PARALLEL_PARSING_AVAILABLE, so one undeclared optional dependency here silently disabled
|
|
25
|
+
# parallel parsing *and* the Phase 2 fan-out that rides on it. RAM is read through
|
|
26
|
+
# `tcs.parallel_analyzer.available_ram_mb()`, which works with or without psutil.
|
|
27
|
+
psutil = None
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
# ─── Hardware Governance Constants ───────────────────────────────────────────
|
|
31
|
+
|
|
32
|
+
MIN_FREE_CORES = 2 # Keep at least 2 cores free for Windows OS + DWM
|
|
33
|
+
WORKER_RAM_BUDGET_MB = 550 # Budget 550 MB RAM per active worker process
|
|
34
|
+
HARD_SAFETY_THRESHOLD_MB = 900 # If available RAM < 900 MB, drop to 1 worker immediately
|
|
35
|
+
PER_FILE_TIMEOUT_S = 4.0 # Strict per-file watchdog timeout (seconds)
|
|
36
|
+
# Extra head-room on top of the queue-aware deadline below. The parent itself does work between
|
|
37
|
+
# collections (merging results, reading files), so a deadline measured exactly to the millisecond
|
|
38
|
+
# would blame that bookkeeping on the worker. 1.0s on a 4.0s parse budget, nothing more.
|
|
39
|
+
QUEUE_BUDGET_SLACK_S = 1.0
|
|
40
|
+
BATCH_SIZE = 40 # Files per batch (balance IPC vs overhead)
|
|
41
|
+
MAX_WORKERS_CEILING = 12 # Allow up to 12 workers on cloud runners with ample RAM
|
|
42
|
+
CI_TWO_CORE_RAM_MB = 1600 # 2-core CI runner gets its second worker above this much RAM
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def compute_safe_workers() -> int:
|
|
46
|
+
"""Compute safe worker count using dynamic RAM-aware hardware governor.
|
|
47
|
+
|
|
48
|
+
This replaces the static min(4, ...) hardcode with adaptive scaling:
|
|
49
|
+
- On low-RAM laptops (~1-2 GB free): settles on 1-2 workers safely
|
|
50
|
+
- On cloud runners (8-32 GB RAM): scales up to 6-12 workers for max throughput
|
|
51
|
+
|
|
52
|
+
Safety guarantees:
|
|
53
|
+
- Never freezes OS by keeping ≥2 cores free
|
|
54
|
+
- Hard safety threshold at 900 MB available RAM
|
|
55
|
+
- Budgets 550 MB per worker to prevent OOM
|
|
56
|
+
"""
|
|
57
|
+
try:
|
|
58
|
+
from tcs.parallel_analyzer import available_ram_mb
|
|
59
|
+
free_ram_mb = available_ram_mb()
|
|
60
|
+
if free_ram_mb is None:
|
|
61
|
+
# No measurable free RAM means no evidence the pool fits, so the 900 MB floor is
|
|
62
|
+
# honoured by default rather than assumed.
|
|
63
|
+
print("TimeCodeSecurity [GOVERNOR] free RAM unmeasurable: staying on 1 worker.",
|
|
64
|
+
file=sys.stderr)
|
|
65
|
+
return 1
|
|
66
|
+
|
|
67
|
+
# Calculate capacity based on both CPU and RAM
|
|
68
|
+
cpu_count = os.cpu_count() or 4
|
|
69
|
+
cpu_capacity = max(1, cpu_count - MIN_FREE_CORES)
|
|
70
|
+
ram_capacity = max(1, int(free_ram_mb // WORKER_RAM_BUDGET_MB))
|
|
71
|
+
|
|
72
|
+
# Dynamic worker calculation with hard ceiling
|
|
73
|
+
max_workers = max(1, min(cpu_capacity, ram_capacity, MAX_WORKERS_CEILING))
|
|
74
|
+
|
|
75
|
+
# Hard safety threshold: if RAM critically low, force single-worker mode
|
|
76
|
+
if free_ram_mb < HARD_SAFETY_THRESHOLD_MB:
|
|
77
|
+
print(
|
|
78
|
+
f"TimeCodeSecurity [GOVERNOR] CRITICAL: Available RAM {free_ram_mb:.0f} MB "
|
|
79
|
+
f"< {HARD_SAFETY_THRESHOLD_MB} MB threshold. Forcing single-worker mode.",
|
|
80
|
+
file=sys.stderr,
|
|
81
|
+
)
|
|
82
|
+
return 1
|
|
83
|
+
|
|
84
|
+
# Same carve-out as `tcs/parallel_analyzer.compute_analysis_workers`: a 2-core runner would
|
|
85
|
+
# otherwise collapse to 1 worker through `cpu_count - MIN_FREE_CORES` and never use the
|
|
86
|
+
# machine CI gave it. The 900 MB floor above still wins whenever free RAM drops.
|
|
87
|
+
if cpu_count == 2 and max_workers == 1 and free_ram_mb >= CI_TWO_CORE_RAM_MB:
|
|
88
|
+
return 2
|
|
89
|
+
|
|
90
|
+
return max_workers
|
|
91
|
+
|
|
92
|
+
except Exception as exc:
|
|
93
|
+
# Fallback to conservative defaults if psutil fails
|
|
94
|
+
print(f"Warning: Governor error ({exc}), falling back to 2 workers", file=sys.stderr)
|
|
95
|
+
return 2
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def compute_adaptive_timeout(available_ram_mb: float) -> float:
|
|
99
|
+
"""Compute per-file timeout based on available memory.
|
|
100
|
+
|
|
101
|
+
Under low memory, processes swap and take longer. Increase timeout proportionally.
|
|
102
|
+
Base timeout is 4.0s (PER_FILE_TIMEOUT_S).
|
|
103
|
+
"""
|
|
104
|
+
if available_ram_mb < HARD_SAFETY_THRESHOLD_MB:
|
|
105
|
+
# Critical pressure: allow up to 12s per file
|
|
106
|
+
return PER_FILE_TIMEOUT_S * 3.0
|
|
107
|
+
elif available_ram_mb < 1500:
|
|
108
|
+
# Moderate pressure: allow up to 8s per file
|
|
109
|
+
return PER_FILE_TIMEOUT_S * 2.0
|
|
110
|
+
else:
|
|
111
|
+
# Normal conditions: base timeout
|
|
112
|
+
return PER_FILE_TIMEOUT_S
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def get_memory_stats() -> Dict[str, float]:
|
|
116
|
+
"""Get current memory statistics in MB/GB."""
|
|
117
|
+
try:
|
|
118
|
+
mem = psutil.virtual_memory()
|
|
119
|
+
return {
|
|
120
|
+
"total_gb": mem.total / (1024 ** 3),
|
|
121
|
+
"available_gb": mem.available / (1024 ** 3),
|
|
122
|
+
"used_gb": mem.used / (1024 ** 3),
|
|
123
|
+
"percent_used": mem.percent,
|
|
124
|
+
}
|
|
125
|
+
except Exception:
|
|
126
|
+
return {"total_gb": 0, "available_gb": 0, "used_gb": 0, "percent_used": 0}
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _is_zero_risk_file(filepath: str, source: str) -> Tuple[bool, Optional[ast.Module]]:
|
|
130
|
+
"""Fast-path pre-filter: skip full AST analysis for inert files.
|
|
131
|
+
|
|
132
|
+
Zero-risk files include:
|
|
133
|
+
- Empty __init__.py files
|
|
134
|
+
- Files containing only static docstrings/comments
|
|
135
|
+
- Pure localization string files
|
|
136
|
+
|
|
137
|
+
Returns `(is_zero_risk, tree)`. The tree is the module the decision was taken on, handed back
|
|
138
|
+
so the caller never parses the same source a second time. It is `None` when there is no tree to
|
|
139
|
+
hand on: an inert file (the tree is released with the decision) and a source whose parse raised,
|
|
140
|
+
which the caller reports exactly as before.
|
|
141
|
+
"""
|
|
142
|
+
stripped = source.strip()
|
|
143
|
+
|
|
144
|
+
# Empty file
|
|
145
|
+
if not stripped:
|
|
146
|
+
return True, None
|
|
147
|
+
|
|
148
|
+
# Check if it's an __init__.py
|
|
149
|
+
basename = os.path.basename(filepath)
|
|
150
|
+
if basename == "__init__.py" and len(stripped) < 100:
|
|
151
|
+
# Very small __init__.py — likely just imports or empty
|
|
152
|
+
lines = [l.strip() for l in stripped.splitlines() if l.strip()]
|
|
153
|
+
non_empty = [l for l in lines if not l.startswith('#') and not l.startswith('"""') and not l.startswith("'''")]
|
|
154
|
+
if not non_empty:
|
|
155
|
+
return True, None
|
|
156
|
+
|
|
157
|
+
# Only comments/docstrings?
|
|
158
|
+
try:
|
|
159
|
+
tree = ast.parse(source, filename=filepath)
|
|
160
|
+
except (SyntaxError, ValueError):
|
|
161
|
+
# Parse errors handled elsewhere
|
|
162
|
+
return False, None
|
|
163
|
+
# If nothing substantive remains, it's low-risk
|
|
164
|
+
if isinstance(tree, ast.Module):
|
|
165
|
+
body_items = []
|
|
166
|
+
for node in tree.body:
|
|
167
|
+
if isinstance(node, ast.Expr) and isinstance(node.value, (ast.Constant, ast.Str)):
|
|
168
|
+
continue # Docstring
|
|
169
|
+
elif isinstance(node, (ast.Import, ast.ImportFrom)):
|
|
170
|
+
continue # Pure imports
|
|
171
|
+
else:
|
|
172
|
+
body_items.append(node)
|
|
173
|
+
|
|
174
|
+
if not body_items:
|
|
175
|
+
return True, None
|
|
176
|
+
|
|
177
|
+
return False, tree
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _parse_single_file(args: Tuple[str, Optional[str]]) -> Dict[str, Any]:
|
|
181
|
+
"""Parse a single Python file and return lightweight metadata.
|
|
182
|
+
|
|
183
|
+
This runs in a worker process. Returns ONLY primitive types.
|
|
184
|
+
|
|
185
|
+
Args:
|
|
186
|
+
args: (filepath, source_code) tuple. A None source means "stream it from disk here":
|
|
187
|
+
the parent never buffers the repository for a fan-out it does not need.
|
|
188
|
+
|
|
189
|
+
Returns:
|
|
190
|
+
Dict with keys: filepath, mod_name, tree_serialized, skipped, error
|
|
191
|
+
"""
|
|
192
|
+
filepath, source = args
|
|
193
|
+
if source is None:
|
|
194
|
+
try:
|
|
195
|
+
source = pathlib.Path(filepath).read_text(encoding="utf-8", errors="replace")
|
|
196
|
+
except (OSError, ValueError) as exc:
|
|
197
|
+
return {
|
|
198
|
+
"filepath": filepath,
|
|
199
|
+
"mod_name": None,
|
|
200
|
+
"tree": None,
|
|
201
|
+
"skipped_fast_path": False,
|
|
202
|
+
"error": f"{type(exc).__name__}: {exc}"[:200],
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
# Fast-path: skip zero-risk files early. The tree the pre-filter parsed to make that decision
|
|
206
|
+
# is carried through here, so a file costs one `ast.parse` instead of two.
|
|
207
|
+
zero_risk, tree = _is_zero_risk_file(filepath, source)
|
|
208
|
+
if zero_risk:
|
|
209
|
+
return {
|
|
210
|
+
"filepath": filepath,
|
|
211
|
+
"mod_name": None,
|
|
212
|
+
"tree": None,
|
|
213
|
+
"skipped_fast_path": True,
|
|
214
|
+
"error": None,
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
# Compute module name
|
|
218
|
+
mod_name = filepath.replace("\\\\", "/").replace(".py", "").replace("/", ".")
|
|
219
|
+
if mod_name.endswith(".__init__"):
|
|
220
|
+
mod_name = mod_name[:-9]
|
|
221
|
+
|
|
222
|
+
# Parse AST
|
|
223
|
+
try:
|
|
224
|
+
# `tree` is None only when the pre-filter's own parse raised; re-parsing here re-raises the
|
|
225
|
+
# identical exception, so an unparseable file is reported exactly as it was before.
|
|
226
|
+
if tree is None:
|
|
227
|
+
tree = ast.parse(source, filename=filepath)
|
|
228
|
+
|
|
229
|
+
# Link parent pointers (required for TaintTracker)
|
|
230
|
+
for parent in ast.walk(tree):
|
|
231
|
+
for child in ast.iter_child_nodes(parent):
|
|
232
|
+
child.parent = parent
|
|
233
|
+
|
|
234
|
+
return {
|
|
235
|
+
"filepath": filepath,
|
|
236
|
+
"mod_name": mod_name,
|
|
237
|
+
"tree": tree, # Will be pickled back to main process
|
|
238
|
+
"skipped_fast_path": False,
|
|
239
|
+
"error": None,
|
|
240
|
+
}
|
|
241
|
+
except (SyntaxError, ValueError, UnicodeDecodeError) as exc:
|
|
242
|
+
return {
|
|
243
|
+
"filepath": filepath,
|
|
244
|
+
"mod_name": mod_name,
|
|
245
|
+
"tree": None,
|
|
246
|
+
"skipped_fast_path": False,
|
|
247
|
+
"error": f"{type(exc).__name__}: {exc}"[:200],
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def parse_files_parallel(
|
|
252
|
+
files: Dict[str, str],
|
|
253
|
+
max_workers: Optional[int] = None,
|
|
254
|
+
) -> Tuple[Dict[str, ast.AST], Dict[str, str], Dict[str, str], Dict[str, Any]]:
|
|
255
|
+
"""Parse files in parallel with dynamic hardware governance.
|
|
256
|
+
|
|
257
|
+
Args:
|
|
258
|
+
files: Dict mapping filepath → source code
|
|
259
|
+
max_workers: Override safe worker count (None = auto-compute via governor)
|
|
260
|
+
|
|
261
|
+
Returns:
|
|
262
|
+
(modules, file_paths, skipped_files, parse_stats) matching TaintTracker init.
|
|
263
|
+
parse_stats reports every file the discovery walk handed over and where it went,
|
|
264
|
+
so "scanned_files" is never mistaken for "files considered".
|
|
265
|
+
"""
|
|
266
|
+
if max_workers is None:
|
|
267
|
+
max_workers = compute_safe_workers()
|
|
268
|
+
|
|
269
|
+
workers_initial = max_workers
|
|
270
|
+
governor_events = 0
|
|
271
|
+
|
|
272
|
+
# Get current memory stats for logging
|
|
273
|
+
mem_stats = get_memory_stats()
|
|
274
|
+
free_ram_mb = mem_stats["available_gb"] * 1024
|
|
275
|
+
|
|
276
|
+
print(
|
|
277
|
+
f"TimeCodeSecurity [GOVERNOR] Workers: {max_workers} | "
|
|
278
|
+
f"Free RAM: {free_ram_mb:.0f} MB | "
|
|
279
|
+
f"Cores: {os.cpu_count() or 'unknown'}",
|
|
280
|
+
file=sys.stderr,
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
# Prepare file list
|
|
284
|
+
file_items = list(files.items())
|
|
285
|
+
total_files = len(file_items)
|
|
286
|
+
|
|
287
|
+
# For small workloads, skip parallelism overhead
|
|
288
|
+
if total_files <= BATCH_SIZE:
|
|
289
|
+
return _parse_files_sequential(files)
|
|
290
|
+
|
|
291
|
+
modules: Dict[str, ast.AST] = {}
|
|
292
|
+
file_paths: Dict[str, str] = {}
|
|
293
|
+
skipped_files: Dict[str, str] = {}
|
|
294
|
+
fast_path_skipped = 0
|
|
295
|
+
start_time = time.time()
|
|
296
|
+
|
|
297
|
+
def _merge(result: Dict[str, Any]) -> None:
|
|
298
|
+
# The same three-way branch the batch loop used to run, just fed in file order.
|
|
299
|
+
nonlocal fast_path_skipped
|
|
300
|
+
if result["error"]:
|
|
301
|
+
skipped_files[result["filepath"]] = result["error"]
|
|
302
|
+
elif result["skipped_fast_path"]:
|
|
303
|
+
fast_path_skipped += 1
|
|
304
|
+
else:
|
|
305
|
+
modules[result["mod_name"]] = result["tree"]
|
|
306
|
+
file_paths[result["mod_name"]] = result["filepath"]
|
|
307
|
+
|
|
308
|
+
# A governor that lands on one worker used to keep the fan-out anyway: a brand new
|
|
309
|
+
# ProcessPoolExecutor for every BATCH_SIZE slice, each file pickled into the child and its AST
|
|
310
|
+
# pickled back. Measured on the 609-file corpus: 4.91 s that way against 0.15 s in process, for
|
|
311
|
+
# identical trees - and the per-slice spawn multiplies by 267 slices on a 10k-file repository.
|
|
312
|
+
# So one worker parses in the parent, and several workers share ONE pool for the whole run
|
|
313
|
+
# instead of paying a fresh interpreter per slice.
|
|
314
|
+
governor_events = 0
|
|
315
|
+
in_process = max_workers <= 1
|
|
316
|
+
slices = [file_items[i:i + BATCH_SIZE] for i in range(0, total_files, BATCH_SIZE)]
|
|
317
|
+
|
|
318
|
+
if in_process:
|
|
319
|
+
for filepath, source in file_items:
|
|
320
|
+
_merge(_parse_here(filepath, source))
|
|
321
|
+
max_workers = 1
|
|
322
|
+
else:
|
|
323
|
+
executor: Optional[ProcessPoolExecutor] = None
|
|
324
|
+
try:
|
|
325
|
+
for slice_idx, batch in enumerate(slices):
|
|
326
|
+
# Re-check memory between slices and adjust the pool dynamically. Once the run has
|
|
327
|
+
# fallen back to the parent there is no pool left to protect, so the check stops.
|
|
328
|
+
if executor is not None and slice_idx > 0 and slice_idx % 2 == 0:
|
|
329
|
+
current_free_mb = get_memory_stats()["available_gb"] * 1024
|
|
330
|
+
if current_free_mb < HARD_SAFETY_THRESHOLD_MB:
|
|
331
|
+
print(
|
|
332
|
+
f"TimeCodeSecurity [GOVERNOR] CRITICAL: RAM dropped to "
|
|
333
|
+
f"{current_free_mb:.0f} MB. Finishing this run in the parent.",
|
|
334
|
+
file=sys.stderr,
|
|
335
|
+
)
|
|
336
|
+
# A ProcessPoolExecutor cannot shrink, and "1 worker" through a pool is the
|
|
337
|
+
# pickling round-trip this change removes. The remaining slices run here:
|
|
338
|
+
# one process instead of N is the strict RAM win the governor is for.
|
|
339
|
+
# Nothing is in flight at this point (`_parse_slice` collected every
|
|
340
|
+
# future), so waiting costs nothing and leaves no orphan worker behind.
|
|
341
|
+
executor.shutdown(wait=True)
|
|
342
|
+
executor = None
|
|
343
|
+
in_process = True
|
|
344
|
+
max_workers = 1
|
|
345
|
+
governor_events += 1
|
|
346
|
+
elif current_free_mb < 1500 and max_workers > 2:
|
|
347
|
+
print(
|
|
348
|
+
f"TimeCodeSecurity [GOVERNOR] Memory pressure "
|
|
349
|
+
f"({current_free_mb:.0f} MB). Reducing workers from "
|
|
350
|
+
f"{max_workers} to {max_workers - 1}.",
|
|
351
|
+
file=sys.stderr,
|
|
352
|
+
)
|
|
353
|
+
max_workers = max_workers - 1
|
|
354
|
+
governor_events += 1
|
|
355
|
+
# Tear the pool down so the next slice rebuilds it at the smaller size: a
|
|
356
|
+
# live ProcessPoolExecutor cannot give workers back, and a log line claiming
|
|
357
|
+
# a reduction that did not happen would be the governor lying about RAM.
|
|
358
|
+
executor.shutdown(wait=True)
|
|
359
|
+
executor = None
|
|
360
|
+
if in_process:
|
|
361
|
+
for filepath, source in batch:
|
|
362
|
+
_merge(_parse_here(filepath, source))
|
|
363
|
+
continue
|
|
364
|
+
if executor is None:
|
|
365
|
+
executor = ProcessPoolExecutor(max_workers=max_workers)
|
|
366
|
+
free_mb = get_memory_stats()["available_gb"] * 1024
|
|
367
|
+
for result in _parse_slice(executor, batch, free_mb, max_workers):
|
|
368
|
+
_merge(result)
|
|
369
|
+
finally:
|
|
370
|
+
if executor is not None:
|
|
371
|
+
executor.shutdown(wait=True)
|
|
372
|
+
|
|
373
|
+
elapsed = time.time() - start_time
|
|
374
|
+
|
|
375
|
+
print(
|
|
376
|
+
f"TimeCodeSecurity: Parsed {len(modules)} files in {elapsed:.2f}s "
|
|
377
|
+
f"({fast_path_skipped} fast-path skipped, {len(skipped_files)} errors)",
|
|
378
|
+
file=sys.stderr,
|
|
379
|
+
)
|
|
380
|
+
|
|
381
|
+
parse_stats = {
|
|
382
|
+
"discovered": total_files,
|
|
383
|
+
"parsed": len(modules),
|
|
384
|
+
"fast_path_skipped": fast_path_skipped,
|
|
385
|
+
"unparseable": len(skipped_files),
|
|
386
|
+
"parse_mode": "inline" if in_process else "parallel",
|
|
387
|
+
"workers_initial": workers_initial,
|
|
388
|
+
"workers_final": max_workers,
|
|
389
|
+
"governor_events": governor_events,
|
|
390
|
+
}
|
|
391
|
+
return modules, file_paths, skipped_files, parse_stats
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _parse_here(filepath: str, source: Optional[str]) -> Dict[str, Any]:
|
|
397
|
+
"""Run the worker body in the parent. It cannot raise: it is where a lost worker lands."""
|
|
398
|
+
try:
|
|
399
|
+
return _parse_single_file((filepath, source))
|
|
400
|
+
except Exception as exc:
|
|
401
|
+
return {
|
|
402
|
+
"filepath": filepath,
|
|
403
|
+
"mod_name": None,
|
|
404
|
+
"tree": None,
|
|
405
|
+
"skipped_fast_path": False,
|
|
406
|
+
"error": f"{type(exc).__name__}: {exc}"[:200],
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def _parse_slice(
|
|
411
|
+
executor: ProcessPoolExecutor,
|
|
412
|
+
batch: List[Tuple[str, Optional[str]]],
|
|
413
|
+
free_ram_mb: float,
|
|
414
|
+
workers: int = 1,
|
|
415
|
+
) -> List[Dict[str, Any]]:
|
|
416
|
+
"""Parse one slice on the shared pool and hand the results back in file order.
|
|
417
|
+
|
|
418
|
+
Two properties the per-slice pool did not have:
|
|
419
|
+
- the executor is created once for the whole run, so a 10k-file repository pays one spawn
|
|
420
|
+
instead of one per 40 files;
|
|
421
|
+
- results are collected by index, not by `as_completed`. Completion order made `modules`
|
|
422
|
+
insertion order - and therefore the sink ids every later phase hands out from it - depend on
|
|
423
|
+
which worker happened to finish first. Files now always land in discovery order.
|
|
424
|
+
|
|
425
|
+
Only one slice is ever in flight, so the returned ASTs are bounded by BATCH_SIZE per parent
|
|
426
|
+
instead of the whole tree sitting in the result queue.
|
|
427
|
+
|
|
428
|
+
Queue wait is NOT parse time. `future.result(timeout=…)` was previously given the bare
|
|
429
|
+
per-file parse budget, so a file that simply sat behind `idx // workers` predecessors in the
|
|
430
|
+
FIFO queue burned its own watchdog while doing nothing: Home Assistant's
|
|
431
|
+
`tests/components/mqtt/test_init.py` (2,797 bytes, 0.8 ms to parse) was dropped that way at
|
|
432
|
+
14,893 MB free RAM, next to a "(low memory)" message that had never looked at RAM. Each file
|
|
433
|
+
now gets the same parse budget, extended only by the queue depth it must wait behind - a
|
|
434
|
+
genuinely stuck worker still hits a bounded deadline, a merely queued file never does.
|
|
435
|
+
"""
|
|
436
|
+
per_file_timeout = compute_adaptive_timeout(free_ram_mb)
|
|
437
|
+
workers = max(1, int(workers))
|
|
438
|
+
_slice_t0 = time.perf_counter()
|
|
439
|
+
futures = [executor.submit(_parse_single_file, (filepath, source))
|
|
440
|
+
for filepath, source in batch]
|
|
441
|
+
results: List[Dict[str, Any]] = [None] * len(batch)
|
|
442
|
+
timed_out: List[Tuple[int, Any, str, Optional[str]]] = []
|
|
443
|
+
for idx, ((filepath, source), future) in enumerate(zip(batch, futures)):
|
|
444
|
+
# Files are collected in submission order, and the pool is FIFO, so the task at `idx`
|
|
445
|
+
# cannot even START before `idx // workers` predecessors have finished. Its deadline is
|
|
446
|
+
# therefore the parse budget multiplied by that queue depth.
|
|
447
|
+
budget = per_file_timeout * (1 + idx // workers) + QUEUE_BUDGET_SLACK_S
|
|
448
|
+
try:
|
|
449
|
+
results[idx] = future.result(
|
|
450
|
+
timeout=max(0.0, budget - (time.perf_counter() - _slice_t0)))
|
|
451
|
+
except TimeoutError:
|
|
452
|
+
timed_out.append((idx, future, filepath, source, budget))
|
|
453
|
+
except Exception as exc:
|
|
454
|
+
# A dead worker is a local fault, not a reason to lose a file: re-parse it here.
|
|
455
|
+
print(
|
|
456
|
+
f"TimeCodeSecurity [GOVERNOR] worker lost for {filepath} "
|
|
457
|
+
f"({type(exc).__name__}); re-parsing in parent",
|
|
458
|
+
file=sys.stderr,
|
|
459
|
+
)
|
|
460
|
+
results[idx] = _parse_here(filepath, source)
|
|
461
|
+
# A second look at the timed-out files, costing zero wall time: later files in the slice
|
|
462
|
+
# were collected while earlier workers kept running, so a future that first reported a
|
|
463
|
+
# timeout may be done by now. Take those results; the parse completed anyway, and dropping
|
|
464
|
+
# it lost a security-relevant file (Home Assistant's websocket_api/test_init.py) for no time
|
|
465
|
+
# saved. Anything still running keeps the original skip policy, unchanged.
|
|
466
|
+
for idx, future, filepath, source, budget in timed_out:
|
|
467
|
+
if future.done():
|
|
468
|
+
try:
|
|
469
|
+
results[idx] = future.result()
|
|
470
|
+
print(
|
|
471
|
+
f"TimeCodeSecurity [INFO] Recovered timed-out parse: {filepath}",
|
|
472
|
+
file=sys.stderr,
|
|
473
|
+
)
|
|
474
|
+
continue
|
|
475
|
+
except Exception as exc:
|
|
476
|
+
# Completed-with-error is the same local fault the first pass already treats as a
|
|
477
|
+
# lost worker (`_parse_single_file` itself never raises - an exception here means
|
|
478
|
+
# the pool or the result transport failed). Route it to the identical, pre-existing
|
|
479
|
+
# fallback: `_parse_here` cannot raise and fail-closes, so a genuine parser failure
|
|
480
|
+
# still lands as an error entry, never as a silently successful parse.
|
|
481
|
+
print(
|
|
482
|
+
f"TimeCodeSecurity [GOVERNOR] worker lost for {filepath} "
|
|
483
|
+
f"({type(exc).__name__}); re-parsing in parent",
|
|
484
|
+
file=sys.stderr,
|
|
485
|
+
)
|
|
486
|
+
results[idx] = _parse_here(filepath, source)
|
|
487
|
+
continue
|
|
488
|
+
print(
|
|
489
|
+
f"TimeCodeSecurity [WARN] File analysis timeout skipped: {filepath}",
|
|
490
|
+
file=sys.stderr,
|
|
491
|
+
)
|
|
492
|
+
results[idx] = {
|
|
493
|
+
"filepath": filepath,
|
|
494
|
+
"mod_name": None,
|
|
495
|
+
"tree": None,
|
|
496
|
+
"skipped_fast_path": False,
|
|
497
|
+
# Truthful accounting: the budget, why it is that size, and the RAM the governor saw.
|
|
498
|
+
# The old string blamed "(low memory)" unconditionally, which is what made a 14.9 GB
|
|
499
|
+
# runner log two files as low-memory skips.
|
|
500
|
+
"error": (f"Timeout after {budget:.1f}s deadline "
|
|
501
|
+
f"(parse limit {per_file_timeout:.1f}s x queue depth "
|
|
502
|
+
f"{1 + idx // workers}, {workers} worker(s), "
|
|
503
|
+
f"{free_ram_mb:.0f} MB free RAM)"),
|
|
504
|
+
}
|
|
505
|
+
return results
|
|
506
|
+
|
|
507
|
+
def _parse_files_sequential(
|
|
508
|
+
files: Dict[str, str],
|
|
509
|
+
) -> Tuple[Dict[str, ast.AST], Dict[str, str], Dict[str, str], Dict[str, Any]]:
|
|
510
|
+
"""Fallback sequential parsing for small workloads."""
|
|
511
|
+
modules: Dict[str, ast.AST] = {}
|
|
512
|
+
file_paths: Dict[str, str] = {}
|
|
513
|
+
skipped_files: Dict[str, str] = {}
|
|
514
|
+
fast_path_skipped = 0
|
|
515
|
+
|
|
516
|
+
for filepath, source in files.items():
|
|
517
|
+
zero_risk, tree = _is_zero_risk_file(filepath, source)
|
|
518
|
+
if zero_risk:
|
|
519
|
+
fast_path_skipped += 1
|
|
520
|
+
continue
|
|
521
|
+
|
|
522
|
+
mod_name = filepath.replace("\\\\", "/").replace(".py", "").replace("/", ".")
|
|
523
|
+
if mod_name.endswith(".__init__"):
|
|
524
|
+
mod_name = mod_name[:-9]
|
|
525
|
+
|
|
526
|
+
try:
|
|
527
|
+
# Same single-parse rule as the fan-out body: this path must hand back trees identical
|
|
528
|
+
# to it, and a second `ast.parse` would only differ in object identity.
|
|
529
|
+
if tree is None:
|
|
530
|
+
tree = ast.parse(source, filename=filepath)
|
|
531
|
+
for parent in ast.walk(tree):
|
|
532
|
+
for child in ast.iter_child_nodes(parent):
|
|
533
|
+
child.parent = parent
|
|
534
|
+
modules[mod_name] = tree
|
|
535
|
+
file_paths[mod_name] = filepath
|
|
536
|
+
except (SyntaxError, ValueError, UnicodeDecodeError) as exc:
|
|
537
|
+
skipped_files[filepath] = f"{type(exc).__name__}: {exc}"[:200]
|
|
538
|
+
|
|
539
|
+
parse_stats = {
|
|
540
|
+
"discovered": len(files),
|
|
541
|
+
"parsed": len(modules),
|
|
542
|
+
"fast_path_skipped": fast_path_skipped,
|
|
543
|
+
"unparseable": len(skipped_files),
|
|
544
|
+
"workers_used": 1,
|
|
545
|
+
"governor_events": 0,
|
|
546
|
+
"parse_mode": "sequential",
|
|
547
|
+
}
|
|
548
|
+
return modules, file_paths, skipped_files, parse_stats
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def _read_and_parse_path(path: str) -> Dict[str, Any]:
|
|
552
|
+
"""Worker entry: read one file from disk and parse it, so the parent never does either.
|
|
553
|
+
|
|
554
|
+
The path, not the source text, is what crosses the process boundary. A second serial
|
|
555
|
+
read + `ast.parse` pass over the whole tree (the cross-file symbol indexer's) used to run in
|
|
556
|
+
the parent while every other core sat idle; passing paths is what lets that pass fan out
|
|
557
|
+
without pickling the repository's source through the queue first.
|
|
558
|
+
"""
|
|
559
|
+
entry = {"path": path, "source": None, "tree": None, "error": None}
|
|
560
|
+
try:
|
|
561
|
+
source = pathlib.Path(path).read_text(encoding="utf-8", errors="replace")
|
|
562
|
+
except (OSError, ValueError) as exc:
|
|
563
|
+
entry["error"] = f"read {type(exc).__name__}: {exc}"[:200]
|
|
564
|
+
return entry
|
|
565
|
+
entry["source"] = source
|
|
566
|
+
try:
|
|
567
|
+
entry["tree"] = ast.parse(source, filename=path)
|
|
568
|
+
except (SyntaxError, ValueError, UnicodeDecodeError) as exc:
|
|
569
|
+
entry["error"] = f"parse {type(exc).__name__}: {exc}"[:200]
|
|
570
|
+
return entry
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
def _read_and_parse_here(path: str) -> Dict[str, Any]:
|
|
574
|
+
"""Same contract in-process, and it cannot raise: this is the path a worker failure lands on.
|
|
575
|
+
|
|
576
|
+
A lost worker must never mean a lost file. The serial re-do here is what keeps the parallel
|
|
577
|
+
fan-out findings-identical to the loop it replaced.
|
|
578
|
+
"""
|
|
579
|
+
try:
|
|
580
|
+
return _read_and_parse_path(path)
|
|
581
|
+
except Exception as exc:
|
|
582
|
+
return {"path": path, "source": None, "tree": None,
|
|
583
|
+
"error": f"{type(exc).__name__}: {exc}"[:200]}
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
def read_and_parse_paths(paths: List[str],
|
|
587
|
+
max_workers: Optional[int] = None) -> List[Dict[str, Any]]:
|
|
588
|
+
"""Read and parse *paths* in parallel, returning one entry per path **in the given order**.
|
|
589
|
+
|
|
590
|
+
Ordering is the whole contract: callers build dicts from these results and later phases walk
|
|
591
|
+
dict insertion order, so a completion-order fan-out would reshuffle findings. Entries come
|
|
592
|
+
back by index, never by whoever finished first.
|
|
593
|
+
|
|
594
|
+
Each entry is `{"path", "source", "tree", "error"}`; `tree` is None exactly when the serial
|
|
595
|
+
loop would have `continue`d, so the caller's decisions are unchanged.
|
|
596
|
+
"""
|
|
597
|
+
if not paths:
|
|
598
|
+
return []
|
|
599
|
+
workers = compute_safe_workers() if max_workers is None else max(1, int(max_workers))
|
|
600
|
+
if workers <= 1:
|
|
601
|
+
return [_read_and_parse_here(p) for p in paths]
|
|
602
|
+
available_mb = get_memory_stats()["available_gb"] * 1024
|
|
603
|
+
# Generous, because a timeout here is not a skip: the file is re-read and re-parsed in the
|
|
604
|
+
# parent below. The watchdog exists to stop a wedged worker, not to drop work.
|
|
605
|
+
file_timeout = max(PER_FILE_TIMEOUT_S * 4, compute_adaptive_timeout(available_mb) * 4)
|
|
606
|
+
try:
|
|
607
|
+
with ProcessPoolExecutor(max_workers=workers) as executor:
|
|
608
|
+
futures = [executor.submit(_read_and_parse_path, p) for p in paths]
|
|
609
|
+
entries: List[Dict[str, Any]] = []
|
|
610
|
+
for path, future in zip(paths, futures):
|
|
611
|
+
try:
|
|
612
|
+
entries.append(future.result(timeout=file_timeout))
|
|
613
|
+
except Exception as exc:
|
|
614
|
+
print(f"TimeCodeSecurity [GOVERNOR] worker lost for {path} "
|
|
615
|
+
f"({type(exc).__name__}: {exc}); re-parsing in parent",
|
|
616
|
+
file=sys.stderr)
|
|
617
|
+
entries.append(_read_and_parse_here(path))
|
|
618
|
+
return entries
|
|
619
|
+
except Exception as exc:
|
|
620
|
+
print(f"TimeCodeSecurity [GOVERNOR] path fan-out unavailable "
|
|
621
|
+
f"({type(exc).__name__}: {exc}); serial read+parse of {len(paths)} file(s)",
|
|
622
|
+
file=sys.stderr)
|
|
623
|
+
return [_read_and_parse_here(p) for p in paths]
|
|
624
|
+
|
|
625
|
+
|
|
626
|
+
if __name__ == "__main__":
|
|
627
|
+
# Safety guard: prevent accidental execution outside intended context
|
|
628
|
+
print("TimeCodeSecurity parallel scanner module loaded successfully.")
|
|
629
|
+
print(f"Safe worker count: {compute_safe_workers()}")
|
|
630
|
+
print(f"Memory stats: {get_memory_stats()}")
|