timecs 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. app.py +3014 -0
  2. ast_scanner.py +36187 -0
  3. banner.py +146 -0
  4. ci_reporter.py +570 -0
  5. cli.py +2347 -0
  6. config_loader.py +307 -0
  7. cross_file_engine.py +1781 -0
  8. delivery_models.py +145 -0
  9. delivery_repository.py +78 -0
  10. diff_scan.py +127 -0
  11. durable_delivery_sink.py +103 -0
  12. event_bus.py +155 -0
  13. event_publisher.py +345 -0
  14. event_taxonomy.py +313 -0
  15. function_summarizer.py +354 -0
  16. html_auditor.py +274 -0
  17. iac_auditor.py +400 -0
  18. in_app_adapter.py +115 -0
  19. js_scanner.py +895 -0
  20. manifest_parser.py +960 -0
  21. master_rules_bank.py +389 -0
  22. notification_models.py +346 -0
  23. notification_policy.py +309 -0
  24. notification_repository.py +118 -0
  25. notification_routes.py +394 -0
  26. notification_service.py +199 -0
  27. osv_client.py +437 -0
  28. remediation/__init__.py +39 -0
  29. remediation/base.py +102 -0
  30. remediation/contracts.py +196 -0
  31. remediation/cwe1188_binding.py +78 -0
  32. remediation/cwe1336_jinja.py +91 -0
  33. remediation/cwe22_path_traversal.py +348 -0
  34. remediation/cwe295_ssl.py +87 -0
  35. remediation/cwe327_weak_hash.py +78 -0
  36. remediation/cwe377_tempfile.py +77 -0
  37. remediation/cwe489_debug.py +80 -0
  38. remediation/cwe502_yaml.py +89 -0
  39. remediation/cwe614_cookie.py +91 -0
  40. remediation/cwe78_cmdi.py +415 -0
  41. remediation/cwe798_credentials.py +153 -0
  42. remediation/cwe89_sqli.py +425 -0
  43. remediation/cwe916_password_hash.py +131 -0
  44. remediation/cwe95_eval.py +77 -0
  45. remediation/dispatcher.py +114 -0
  46. remediation/orchestrator.py +541 -0
  47. remediation/patch_engine.py +213 -0
  48. remediation/verification.py +253 -0
  49. rule_engine.py +1535 -0
  50. sarif_adapter.py +719 -0
  51. sarif_exporter.py +88 -0
  52. sca_reachability/__init__.py +28 -0
  53. sca_reachability/ast_analyzer.py +366 -0
  54. sca_reachability/call_graph.py +203 -0
  55. sca_reachability/contracts.py +365 -0
  56. sca_reachability/dist_import_map.py +46 -0
  57. sca_reachability/engine.py +603 -0
  58. sca_reachability/resolver.py +169 -0
  59. scanners/__init__.py +70 -0
  60. scanners/container_scanner.py +323 -0
  61. scanners/cve_index.py +521 -0
  62. scanners/iac_scanner.py +328 -0
  63. scanners/sbom.py +183 -0
  64. scanners/sca_scanner.py +667 -0
  65. scanners/secret_scanner.py +345 -0
  66. scanners/yaml_lite.py +217 -0
  67. secret_filters.py +435 -0
  68. secret_scanner.py +943 -0
  69. sqlite_delivery_repository.py +277 -0
  70. sqlite_notification_repository.py +441 -0
  71. ssrf_guard.py +276 -0
  72. staged_scanner.py +492 -0
  73. suppression_resolver.py +282 -0
  74. symbol_indexer.py +424 -0
  75. tcs/__init__.py +1 -0
  76. tcs/analysis/__init__.py +1 -0
  77. tcs/analysis/def_use.py +408 -0
  78. tcs/analysis/function_contracts.py +290 -0
  79. tcs/parallel_analyzer.py +276 -0
  80. tcs/parallel_scanner.py +630 -0
  81. tcs/treesitter_packer.py +1446 -0
  82. tcs_cli.py +2086 -0
  83. tcs_config.py +218 -0
  84. tcs_gui.py +2123 -0
  85. timecs-2.0.0.dist-info/METADATA +142 -0
  86. timecs-2.0.0.dist-info/RECORD +94 -0
  87. timecs-2.0.0.dist-info/WHEEL +5 -0
  88. timecs-2.0.0.dist-info/entry_points.txt +4 -0
  89. timecs-2.0.0.dist-info/licenses/LICENSE +201 -0
  90. timecs-2.0.0.dist-info/top_level.txt +49 -0
  91. version_matcher.py +934 -0
  92. webhook_adapter.py +495 -0
  93. webhook_client.py +483 -0
  94. webhook_formatters.py +245 -0
@@ -0,0 +1,630 @@
1
+ """TimeCodeSecurity — Safe parallel scanner with hard resource governance.
2
+
3
+ This module provides CPU-aware, memory-governed parallel scanning that:
4
+ - Never freezes Windows by keeping ≥2 cores free for OS/DWM
5
+ - Throttles workers when RAM drops below 1.2 GB
6
+ - Enforces per-file timeouts (5 s) to prevent hangs
7
+ - Returns only lightweight primitives (no AST serialization)
8
+ - Uses fast-path pre-filtering for zero-risk files
9
+ """
10
+ from __future__ import annotations
11
+
12
+ import ast
13
+ import os
14
+ import pathlib
15
+ import sys
16
+ import time
17
+ from concurrent.futures import ProcessPoolExecutor
18
+ from typing import Any, Dict, List, Optional, Tuple
19
+
20
+ try:
21
+ import psutil
22
+ except ImportError:
23
+ # `ast_scanner` imports this module inside `try/except ImportError` to set
24
+ # PARALLEL_PARSING_AVAILABLE, so one undeclared optional dependency here silently disabled
25
+ # parallel parsing *and* the Phase 2 fan-out that rides on it. RAM is read through
26
+ # `tcs.parallel_analyzer.available_ram_mb()`, which works with or without psutil.
27
+ psutil = None
28
+
29
+
30
+ # ─── Hardware Governance Constants ───────────────────────────────────────────
31
+
32
+ MIN_FREE_CORES = 2 # Keep at least 2 cores free for Windows OS + DWM
33
+ WORKER_RAM_BUDGET_MB = 550 # Budget 550 MB RAM per active worker process
34
+ HARD_SAFETY_THRESHOLD_MB = 900 # If available RAM < 900 MB, drop to 1 worker immediately
35
+ PER_FILE_TIMEOUT_S = 4.0 # Strict per-file watchdog timeout (seconds)
36
+ # Extra head-room on top of the queue-aware deadline below. The parent itself does work between
37
+ # collections (merging results, reading files), so a deadline measured exactly to the millisecond
38
+ # would blame that bookkeeping on the worker. 1.0s on a 4.0s parse budget, nothing more.
39
+ QUEUE_BUDGET_SLACK_S = 1.0
40
+ BATCH_SIZE = 40 # Files per batch (balance IPC vs overhead)
41
+ MAX_WORKERS_CEILING = 12 # Allow up to 12 workers on cloud runners with ample RAM
42
+ CI_TWO_CORE_RAM_MB = 1600 # 2-core CI runner gets its second worker above this much RAM
43
+
44
+
45
+ def compute_safe_workers() -> int:
46
+ """Compute safe worker count using dynamic RAM-aware hardware governor.
47
+
48
+ This replaces the static min(4, ...) hardcode with adaptive scaling:
49
+ - On low-RAM laptops (~1-2 GB free): settles on 1-2 workers safely
50
+ - On cloud runners (8-32 GB RAM): scales up to 6-12 workers for max throughput
51
+
52
+ Safety guarantees:
53
+ - Never freezes OS by keeping ≥2 cores free
54
+ - Hard safety threshold at 900 MB available RAM
55
+ - Budgets 550 MB per worker to prevent OOM
56
+ """
57
+ try:
58
+ from tcs.parallel_analyzer import available_ram_mb
59
+ free_ram_mb = available_ram_mb()
60
+ if free_ram_mb is None:
61
+ # No measurable free RAM means no evidence the pool fits, so the 900 MB floor is
62
+ # honoured by default rather than assumed.
63
+ print("TimeCodeSecurity [GOVERNOR] free RAM unmeasurable: staying on 1 worker.",
64
+ file=sys.stderr)
65
+ return 1
66
+
67
+ # Calculate capacity based on both CPU and RAM
68
+ cpu_count = os.cpu_count() or 4
69
+ cpu_capacity = max(1, cpu_count - MIN_FREE_CORES)
70
+ ram_capacity = max(1, int(free_ram_mb // WORKER_RAM_BUDGET_MB))
71
+
72
+ # Dynamic worker calculation with hard ceiling
73
+ max_workers = max(1, min(cpu_capacity, ram_capacity, MAX_WORKERS_CEILING))
74
+
75
+ # Hard safety threshold: if RAM critically low, force single-worker mode
76
+ if free_ram_mb < HARD_SAFETY_THRESHOLD_MB:
77
+ print(
78
+ f"TimeCodeSecurity [GOVERNOR] CRITICAL: Available RAM {free_ram_mb:.0f} MB "
79
+ f"< {HARD_SAFETY_THRESHOLD_MB} MB threshold. Forcing single-worker mode.",
80
+ file=sys.stderr,
81
+ )
82
+ return 1
83
+
84
+ # Same carve-out as `tcs/parallel_analyzer.compute_analysis_workers`: a 2-core runner would
85
+ # otherwise collapse to 1 worker through `cpu_count - MIN_FREE_CORES` and never use the
86
+ # machine CI gave it. The 900 MB floor above still wins whenever free RAM drops.
87
+ if cpu_count == 2 and max_workers == 1 and free_ram_mb >= CI_TWO_CORE_RAM_MB:
88
+ return 2
89
+
90
+ return max_workers
91
+
92
+ except Exception as exc:
93
+ # Fallback to conservative defaults if psutil fails
94
+ print(f"Warning: Governor error ({exc}), falling back to 2 workers", file=sys.stderr)
95
+ return 2
96
+
97
+
98
+ def compute_adaptive_timeout(available_ram_mb: float) -> float:
99
+ """Compute per-file timeout based on available memory.
100
+
101
+ Under low memory, processes swap and take longer. Increase timeout proportionally.
102
+ Base timeout is 4.0s (PER_FILE_TIMEOUT_S).
103
+ """
104
+ if available_ram_mb < HARD_SAFETY_THRESHOLD_MB:
105
+ # Critical pressure: allow up to 12s per file
106
+ return PER_FILE_TIMEOUT_S * 3.0
107
+ elif available_ram_mb < 1500:
108
+ # Moderate pressure: allow up to 8s per file
109
+ return PER_FILE_TIMEOUT_S * 2.0
110
+ else:
111
+ # Normal conditions: base timeout
112
+ return PER_FILE_TIMEOUT_S
113
+
114
+
115
+ def get_memory_stats() -> Dict[str, float]:
116
+ """Get current memory statistics in MB/GB."""
117
+ try:
118
+ mem = psutil.virtual_memory()
119
+ return {
120
+ "total_gb": mem.total / (1024 ** 3),
121
+ "available_gb": mem.available / (1024 ** 3),
122
+ "used_gb": mem.used / (1024 ** 3),
123
+ "percent_used": mem.percent,
124
+ }
125
+ except Exception:
126
+ return {"total_gb": 0, "available_gb": 0, "used_gb": 0, "percent_used": 0}
127
+
128
+
129
+ def _is_zero_risk_file(filepath: str, source: str) -> Tuple[bool, Optional[ast.Module]]:
130
+ """Fast-path pre-filter: skip full AST analysis for inert files.
131
+
132
+ Zero-risk files include:
133
+ - Empty __init__.py files
134
+ - Files containing only static docstrings/comments
135
+ - Pure localization string files
136
+
137
+ Returns `(is_zero_risk, tree)`. The tree is the module the decision was taken on, handed back
138
+ so the caller never parses the same source a second time. It is `None` when there is no tree to
139
+ hand on: an inert file (the tree is released with the decision) and a source whose parse raised,
140
+ which the caller reports exactly as before.
141
+ """
142
+ stripped = source.strip()
143
+
144
+ # Empty file
145
+ if not stripped:
146
+ return True, None
147
+
148
+ # Check if it's an __init__.py
149
+ basename = os.path.basename(filepath)
150
+ if basename == "__init__.py" and len(stripped) < 100:
151
+ # Very small __init__.py — likely just imports or empty
152
+ lines = [l.strip() for l in stripped.splitlines() if l.strip()]
153
+ non_empty = [l for l in lines if not l.startswith('#') and not l.startswith('"""') and not l.startswith("'''")]
154
+ if not non_empty:
155
+ return True, None
156
+
157
+ # Only comments/docstrings?
158
+ try:
159
+ tree = ast.parse(source, filename=filepath)
160
+ except (SyntaxError, ValueError):
161
+ # Parse errors handled elsewhere
162
+ return False, None
163
+ # If nothing substantive remains, it's low-risk
164
+ if isinstance(tree, ast.Module):
165
+ body_items = []
166
+ for node in tree.body:
167
+ if isinstance(node, ast.Expr) and isinstance(node.value, (ast.Constant, ast.Str)):
168
+ continue # Docstring
169
+ elif isinstance(node, (ast.Import, ast.ImportFrom)):
170
+ continue # Pure imports
171
+ else:
172
+ body_items.append(node)
173
+
174
+ if not body_items:
175
+ return True, None
176
+
177
+ return False, tree
178
+
179
+
180
+ def _parse_single_file(args: Tuple[str, Optional[str]]) -> Dict[str, Any]:
181
+ """Parse a single Python file and return lightweight metadata.
182
+
183
+ This runs in a worker process. Returns ONLY primitive types.
184
+
185
+ Args:
186
+ args: (filepath, source_code) tuple. A None source means "stream it from disk here":
187
+ the parent never buffers the repository for a fan-out it does not need.
188
+
189
+ Returns:
190
+ Dict with keys: filepath, mod_name, tree_serialized, skipped, error
191
+ """
192
+ filepath, source = args
193
+ if source is None:
194
+ try:
195
+ source = pathlib.Path(filepath).read_text(encoding="utf-8", errors="replace")
196
+ except (OSError, ValueError) as exc:
197
+ return {
198
+ "filepath": filepath,
199
+ "mod_name": None,
200
+ "tree": None,
201
+ "skipped_fast_path": False,
202
+ "error": f"{type(exc).__name__}: {exc}"[:200],
203
+ }
204
+
205
+ # Fast-path: skip zero-risk files early. The tree the pre-filter parsed to make that decision
206
+ # is carried through here, so a file costs one `ast.parse` instead of two.
207
+ zero_risk, tree = _is_zero_risk_file(filepath, source)
208
+ if zero_risk:
209
+ return {
210
+ "filepath": filepath,
211
+ "mod_name": None,
212
+ "tree": None,
213
+ "skipped_fast_path": True,
214
+ "error": None,
215
+ }
216
+
217
+ # Compute module name
218
+ mod_name = filepath.replace("\\\\", "/").replace(".py", "").replace("/", ".")
219
+ if mod_name.endswith(".__init__"):
220
+ mod_name = mod_name[:-9]
221
+
222
+ # Parse AST
223
+ try:
224
+ # `tree` is None only when the pre-filter's own parse raised; re-parsing here re-raises the
225
+ # identical exception, so an unparseable file is reported exactly as it was before.
226
+ if tree is None:
227
+ tree = ast.parse(source, filename=filepath)
228
+
229
+ # Link parent pointers (required for TaintTracker)
230
+ for parent in ast.walk(tree):
231
+ for child in ast.iter_child_nodes(parent):
232
+ child.parent = parent
233
+
234
+ return {
235
+ "filepath": filepath,
236
+ "mod_name": mod_name,
237
+ "tree": tree, # Will be pickled back to main process
238
+ "skipped_fast_path": False,
239
+ "error": None,
240
+ }
241
+ except (SyntaxError, ValueError, UnicodeDecodeError) as exc:
242
+ return {
243
+ "filepath": filepath,
244
+ "mod_name": mod_name,
245
+ "tree": None,
246
+ "skipped_fast_path": False,
247
+ "error": f"{type(exc).__name__}: {exc}"[:200],
248
+ }
249
+
250
+
251
+ def parse_files_parallel(
252
+ files: Dict[str, str],
253
+ max_workers: Optional[int] = None,
254
+ ) -> Tuple[Dict[str, ast.AST], Dict[str, str], Dict[str, str], Dict[str, Any]]:
255
+ """Parse files in parallel with dynamic hardware governance.
256
+
257
+ Args:
258
+ files: Dict mapping filepath → source code
259
+ max_workers: Override safe worker count (None = auto-compute via governor)
260
+
261
+ Returns:
262
+ (modules, file_paths, skipped_files, parse_stats) matching TaintTracker init.
263
+ parse_stats reports every file the discovery walk handed over and where it went,
264
+ so "scanned_files" is never mistaken for "files considered".
265
+ """
266
+ if max_workers is None:
267
+ max_workers = compute_safe_workers()
268
+
269
+ workers_initial = max_workers
270
+ governor_events = 0
271
+
272
+ # Get current memory stats for logging
273
+ mem_stats = get_memory_stats()
274
+ free_ram_mb = mem_stats["available_gb"] * 1024
275
+
276
+ print(
277
+ f"TimeCodeSecurity [GOVERNOR] Workers: {max_workers} | "
278
+ f"Free RAM: {free_ram_mb:.0f} MB | "
279
+ f"Cores: {os.cpu_count() or 'unknown'}",
280
+ file=sys.stderr,
281
+ )
282
+
283
+ # Prepare file list
284
+ file_items = list(files.items())
285
+ total_files = len(file_items)
286
+
287
+ # For small workloads, skip parallelism overhead
288
+ if total_files <= BATCH_SIZE:
289
+ return _parse_files_sequential(files)
290
+
291
+ modules: Dict[str, ast.AST] = {}
292
+ file_paths: Dict[str, str] = {}
293
+ skipped_files: Dict[str, str] = {}
294
+ fast_path_skipped = 0
295
+ start_time = time.time()
296
+
297
+ def _merge(result: Dict[str, Any]) -> None:
298
+ # The same three-way branch the batch loop used to run, just fed in file order.
299
+ nonlocal fast_path_skipped
300
+ if result["error"]:
301
+ skipped_files[result["filepath"]] = result["error"]
302
+ elif result["skipped_fast_path"]:
303
+ fast_path_skipped += 1
304
+ else:
305
+ modules[result["mod_name"]] = result["tree"]
306
+ file_paths[result["mod_name"]] = result["filepath"]
307
+
308
+ # A governor that lands on one worker used to keep the fan-out anyway: a brand new
309
+ # ProcessPoolExecutor for every BATCH_SIZE slice, each file pickled into the child and its AST
310
+ # pickled back. Measured on the 609-file corpus: 4.91 s that way against 0.15 s in process, for
311
+ # identical trees - and the per-slice spawn multiplies by 267 slices on a 10k-file repository.
312
+ # So one worker parses in the parent, and several workers share ONE pool for the whole run
313
+ # instead of paying a fresh interpreter per slice.
314
+ governor_events = 0
315
+ in_process = max_workers <= 1
316
+ slices = [file_items[i:i + BATCH_SIZE] for i in range(0, total_files, BATCH_SIZE)]
317
+
318
+ if in_process:
319
+ for filepath, source in file_items:
320
+ _merge(_parse_here(filepath, source))
321
+ max_workers = 1
322
+ else:
323
+ executor: Optional[ProcessPoolExecutor] = None
324
+ try:
325
+ for slice_idx, batch in enumerate(slices):
326
+ # Re-check memory between slices and adjust the pool dynamically. Once the run has
327
+ # fallen back to the parent there is no pool left to protect, so the check stops.
328
+ if executor is not None and slice_idx > 0 and slice_idx % 2 == 0:
329
+ current_free_mb = get_memory_stats()["available_gb"] * 1024
330
+ if current_free_mb < HARD_SAFETY_THRESHOLD_MB:
331
+ print(
332
+ f"TimeCodeSecurity [GOVERNOR] CRITICAL: RAM dropped to "
333
+ f"{current_free_mb:.0f} MB. Finishing this run in the parent.",
334
+ file=sys.stderr,
335
+ )
336
+ # A ProcessPoolExecutor cannot shrink, and "1 worker" through a pool is the
337
+ # pickling round-trip this change removes. The remaining slices run here:
338
+ # one process instead of N is the strict RAM win the governor is for.
339
+ # Nothing is in flight at this point (`_parse_slice` collected every
340
+ # future), so waiting costs nothing and leaves no orphan worker behind.
341
+ executor.shutdown(wait=True)
342
+ executor = None
343
+ in_process = True
344
+ max_workers = 1
345
+ governor_events += 1
346
+ elif current_free_mb < 1500 and max_workers > 2:
347
+ print(
348
+ f"TimeCodeSecurity [GOVERNOR] Memory pressure "
349
+ f"({current_free_mb:.0f} MB). Reducing workers from "
350
+ f"{max_workers} to {max_workers - 1}.",
351
+ file=sys.stderr,
352
+ )
353
+ max_workers = max_workers - 1
354
+ governor_events += 1
355
+ # Tear the pool down so the next slice rebuilds it at the smaller size: a
356
+ # live ProcessPoolExecutor cannot give workers back, and a log line claiming
357
+ # a reduction that did not happen would be the governor lying about RAM.
358
+ executor.shutdown(wait=True)
359
+ executor = None
360
+ if in_process:
361
+ for filepath, source in batch:
362
+ _merge(_parse_here(filepath, source))
363
+ continue
364
+ if executor is None:
365
+ executor = ProcessPoolExecutor(max_workers=max_workers)
366
+ free_mb = get_memory_stats()["available_gb"] * 1024
367
+ for result in _parse_slice(executor, batch, free_mb, max_workers):
368
+ _merge(result)
369
+ finally:
370
+ if executor is not None:
371
+ executor.shutdown(wait=True)
372
+
373
+ elapsed = time.time() - start_time
374
+
375
+ print(
376
+ f"TimeCodeSecurity: Parsed {len(modules)} files in {elapsed:.2f}s "
377
+ f"({fast_path_skipped} fast-path skipped, {len(skipped_files)} errors)",
378
+ file=sys.stderr,
379
+ )
380
+
381
+ parse_stats = {
382
+ "discovered": total_files,
383
+ "parsed": len(modules),
384
+ "fast_path_skipped": fast_path_skipped,
385
+ "unparseable": len(skipped_files),
386
+ "parse_mode": "inline" if in_process else "parallel",
387
+ "workers_initial": workers_initial,
388
+ "workers_final": max_workers,
389
+ "governor_events": governor_events,
390
+ }
391
+ return modules, file_paths, skipped_files, parse_stats
392
+
393
+
394
+
395
+
396
+ def _parse_here(filepath: str, source: Optional[str]) -> Dict[str, Any]:
397
+ """Run the worker body in the parent. It cannot raise: it is where a lost worker lands."""
398
+ try:
399
+ return _parse_single_file((filepath, source))
400
+ except Exception as exc:
401
+ return {
402
+ "filepath": filepath,
403
+ "mod_name": None,
404
+ "tree": None,
405
+ "skipped_fast_path": False,
406
+ "error": f"{type(exc).__name__}: {exc}"[:200],
407
+ }
408
+
409
+
410
+ def _parse_slice(
411
+ executor: ProcessPoolExecutor,
412
+ batch: List[Tuple[str, Optional[str]]],
413
+ free_ram_mb: float,
414
+ workers: int = 1,
415
+ ) -> List[Dict[str, Any]]:
416
+ """Parse one slice on the shared pool and hand the results back in file order.
417
+
418
+ Two properties the per-slice pool did not have:
419
+ - the executor is created once for the whole run, so a 10k-file repository pays one spawn
420
+ instead of one per 40 files;
421
+ - results are collected by index, not by `as_completed`. Completion order made `modules`
422
+ insertion order - and therefore the sink ids every later phase hands out from it - depend on
423
+ which worker happened to finish first. Files now always land in discovery order.
424
+
425
+ Only one slice is ever in flight, so the returned ASTs are bounded by BATCH_SIZE per parent
426
+ instead of the whole tree sitting in the result queue.
427
+
428
+ Queue wait is NOT parse time. `future.result(timeout=…)` was previously given the bare
429
+ per-file parse budget, so a file that simply sat behind `idx // workers` predecessors in the
430
+ FIFO queue burned its own watchdog while doing nothing: Home Assistant's
431
+ `tests/components/mqtt/test_init.py` (2,797 bytes, 0.8 ms to parse) was dropped that way at
432
+ 14,893 MB free RAM, next to a "(low memory)" message that had never looked at RAM. Each file
433
+ now gets the same parse budget, extended only by the queue depth it must wait behind - a
434
+ genuinely stuck worker still hits a bounded deadline, a merely queued file never does.
435
+ """
436
+ per_file_timeout = compute_adaptive_timeout(free_ram_mb)
437
+ workers = max(1, int(workers))
438
+ _slice_t0 = time.perf_counter()
439
+ futures = [executor.submit(_parse_single_file, (filepath, source))
440
+ for filepath, source in batch]
441
+ results: List[Dict[str, Any]] = [None] * len(batch)
442
+ timed_out: List[Tuple[int, Any, str, Optional[str]]] = []
443
+ for idx, ((filepath, source), future) in enumerate(zip(batch, futures)):
444
+ # Files are collected in submission order, and the pool is FIFO, so the task at `idx`
445
+ # cannot even START before `idx // workers` predecessors have finished. Its deadline is
446
+ # therefore the parse budget multiplied by that queue depth.
447
+ budget = per_file_timeout * (1 + idx // workers) + QUEUE_BUDGET_SLACK_S
448
+ try:
449
+ results[idx] = future.result(
450
+ timeout=max(0.0, budget - (time.perf_counter() - _slice_t0)))
451
+ except TimeoutError:
452
+ timed_out.append((idx, future, filepath, source, budget))
453
+ except Exception as exc:
454
+ # A dead worker is a local fault, not a reason to lose a file: re-parse it here.
455
+ print(
456
+ f"TimeCodeSecurity [GOVERNOR] worker lost for {filepath} "
457
+ f"({type(exc).__name__}); re-parsing in parent",
458
+ file=sys.stderr,
459
+ )
460
+ results[idx] = _parse_here(filepath, source)
461
+ # A second look at the timed-out files, costing zero wall time: later files in the slice
462
+ # were collected while earlier workers kept running, so a future that first reported a
463
+ # timeout may be done by now. Take those results; the parse completed anyway, and dropping
464
+ # it lost a security-relevant file (Home Assistant's websocket_api/test_init.py) for no time
465
+ # saved. Anything still running keeps the original skip policy, unchanged.
466
+ for idx, future, filepath, source, budget in timed_out:
467
+ if future.done():
468
+ try:
469
+ results[idx] = future.result()
470
+ print(
471
+ f"TimeCodeSecurity [INFO] Recovered timed-out parse: {filepath}",
472
+ file=sys.stderr,
473
+ )
474
+ continue
475
+ except Exception as exc:
476
+ # Completed-with-error is the same local fault the first pass already treats as a
477
+ # lost worker (`_parse_single_file` itself never raises - an exception here means
478
+ # the pool or the result transport failed). Route it to the identical, pre-existing
479
+ # fallback: `_parse_here` cannot raise and fail-closes, so a genuine parser failure
480
+ # still lands as an error entry, never as a silently successful parse.
481
+ print(
482
+ f"TimeCodeSecurity [GOVERNOR] worker lost for {filepath} "
483
+ f"({type(exc).__name__}); re-parsing in parent",
484
+ file=sys.stderr,
485
+ )
486
+ results[idx] = _parse_here(filepath, source)
487
+ continue
488
+ print(
489
+ f"TimeCodeSecurity [WARN] File analysis timeout skipped: {filepath}",
490
+ file=sys.stderr,
491
+ )
492
+ results[idx] = {
493
+ "filepath": filepath,
494
+ "mod_name": None,
495
+ "tree": None,
496
+ "skipped_fast_path": False,
497
+ # Truthful accounting: the budget, why it is that size, and the RAM the governor saw.
498
+ # The old string blamed "(low memory)" unconditionally, which is what made a 14.9 GB
499
+ # runner log two files as low-memory skips.
500
+ "error": (f"Timeout after {budget:.1f}s deadline "
501
+ f"(parse limit {per_file_timeout:.1f}s x queue depth "
502
+ f"{1 + idx // workers}, {workers} worker(s), "
503
+ f"{free_ram_mb:.0f} MB free RAM)"),
504
+ }
505
+ return results
506
+
507
+ def _parse_files_sequential(
508
+ files: Dict[str, str],
509
+ ) -> Tuple[Dict[str, ast.AST], Dict[str, str], Dict[str, str], Dict[str, Any]]:
510
+ """Fallback sequential parsing for small workloads."""
511
+ modules: Dict[str, ast.AST] = {}
512
+ file_paths: Dict[str, str] = {}
513
+ skipped_files: Dict[str, str] = {}
514
+ fast_path_skipped = 0
515
+
516
+ for filepath, source in files.items():
517
+ zero_risk, tree = _is_zero_risk_file(filepath, source)
518
+ if zero_risk:
519
+ fast_path_skipped += 1
520
+ continue
521
+
522
+ mod_name = filepath.replace("\\\\", "/").replace(".py", "").replace("/", ".")
523
+ if mod_name.endswith(".__init__"):
524
+ mod_name = mod_name[:-9]
525
+
526
+ try:
527
+ # Same single-parse rule as the fan-out body: this path must hand back trees identical
528
+ # to it, and a second `ast.parse` would only differ in object identity.
529
+ if tree is None:
530
+ tree = ast.parse(source, filename=filepath)
531
+ for parent in ast.walk(tree):
532
+ for child in ast.iter_child_nodes(parent):
533
+ child.parent = parent
534
+ modules[mod_name] = tree
535
+ file_paths[mod_name] = filepath
536
+ except (SyntaxError, ValueError, UnicodeDecodeError) as exc:
537
+ skipped_files[filepath] = f"{type(exc).__name__}: {exc}"[:200]
538
+
539
+ parse_stats = {
540
+ "discovered": len(files),
541
+ "parsed": len(modules),
542
+ "fast_path_skipped": fast_path_skipped,
543
+ "unparseable": len(skipped_files),
544
+ "workers_used": 1,
545
+ "governor_events": 0,
546
+ "parse_mode": "sequential",
547
+ }
548
+ return modules, file_paths, skipped_files, parse_stats
549
+
550
+
551
+ def _read_and_parse_path(path: str) -> Dict[str, Any]:
552
+ """Worker entry: read one file from disk and parse it, so the parent never does either.
553
+
554
+ The path, not the source text, is what crosses the process boundary. A second serial
555
+ read + `ast.parse` pass over the whole tree (the cross-file symbol indexer's) used to run in
556
+ the parent while every other core sat idle; passing paths is what lets that pass fan out
557
+ without pickling the repository's source through the queue first.
558
+ """
559
+ entry = {"path": path, "source": None, "tree": None, "error": None}
560
+ try:
561
+ source = pathlib.Path(path).read_text(encoding="utf-8", errors="replace")
562
+ except (OSError, ValueError) as exc:
563
+ entry["error"] = f"read {type(exc).__name__}: {exc}"[:200]
564
+ return entry
565
+ entry["source"] = source
566
+ try:
567
+ entry["tree"] = ast.parse(source, filename=path)
568
+ except (SyntaxError, ValueError, UnicodeDecodeError) as exc:
569
+ entry["error"] = f"parse {type(exc).__name__}: {exc}"[:200]
570
+ return entry
571
+
572
+
573
+ def _read_and_parse_here(path: str) -> Dict[str, Any]:
574
+ """Same contract in-process, and it cannot raise: this is the path a worker failure lands on.
575
+
576
+ A lost worker must never mean a lost file. The serial re-do here is what keeps the parallel
577
+ fan-out findings-identical to the loop it replaced.
578
+ """
579
+ try:
580
+ return _read_and_parse_path(path)
581
+ except Exception as exc:
582
+ return {"path": path, "source": None, "tree": None,
583
+ "error": f"{type(exc).__name__}: {exc}"[:200]}
584
+
585
+
586
+ def read_and_parse_paths(paths: List[str],
587
+ max_workers: Optional[int] = None) -> List[Dict[str, Any]]:
588
+ """Read and parse *paths* in parallel, returning one entry per path **in the given order**.
589
+
590
+ Ordering is the whole contract: callers build dicts from these results and later phases walk
591
+ dict insertion order, so a completion-order fan-out would reshuffle findings. Entries come
592
+ back by index, never by whoever finished first.
593
+
594
+ Each entry is `{"path", "source", "tree", "error"}`; `tree` is None exactly when the serial
595
+ loop would have `continue`d, so the caller's decisions are unchanged.
596
+ """
597
+ if not paths:
598
+ return []
599
+ workers = compute_safe_workers() if max_workers is None else max(1, int(max_workers))
600
+ if workers <= 1:
601
+ return [_read_and_parse_here(p) for p in paths]
602
+ available_mb = get_memory_stats()["available_gb"] * 1024
603
+ # Generous, because a timeout here is not a skip: the file is re-read and re-parsed in the
604
+ # parent below. The watchdog exists to stop a wedged worker, not to drop work.
605
+ file_timeout = max(PER_FILE_TIMEOUT_S * 4, compute_adaptive_timeout(available_mb) * 4)
606
+ try:
607
+ with ProcessPoolExecutor(max_workers=workers) as executor:
608
+ futures = [executor.submit(_read_and_parse_path, p) for p in paths]
609
+ entries: List[Dict[str, Any]] = []
610
+ for path, future in zip(paths, futures):
611
+ try:
612
+ entries.append(future.result(timeout=file_timeout))
613
+ except Exception as exc:
614
+ print(f"TimeCodeSecurity [GOVERNOR] worker lost for {path} "
615
+ f"({type(exc).__name__}: {exc}); re-parsing in parent",
616
+ file=sys.stderr)
617
+ entries.append(_read_and_parse_here(path))
618
+ return entries
619
+ except Exception as exc:
620
+ print(f"TimeCodeSecurity [GOVERNOR] path fan-out unavailable "
621
+ f"({type(exc).__name__}: {exc}); serial read+parse of {len(paths)} file(s)",
622
+ file=sys.stderr)
623
+ return [_read_and_parse_here(p) for p in paths]
624
+
625
+
626
+ if __name__ == "__main__":
627
+ # Safety guard: prevent accidental execution outside intended context
628
+ print("TimeCodeSecurity parallel scanner module loaded successfully.")
629
+ print(f"Safe worker count: {compute_safe_workers()}")
630
+ print(f"Memory stats: {get_memory_stats()}")