timecs 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. app.py +3014 -0
  2. ast_scanner.py +36187 -0
  3. banner.py +146 -0
  4. ci_reporter.py +570 -0
  5. cli.py +2347 -0
  6. config_loader.py +307 -0
  7. cross_file_engine.py +1781 -0
  8. delivery_models.py +145 -0
  9. delivery_repository.py +78 -0
  10. diff_scan.py +127 -0
  11. durable_delivery_sink.py +103 -0
  12. event_bus.py +155 -0
  13. event_publisher.py +345 -0
  14. event_taxonomy.py +313 -0
  15. function_summarizer.py +354 -0
  16. html_auditor.py +274 -0
  17. iac_auditor.py +400 -0
  18. in_app_adapter.py +115 -0
  19. js_scanner.py +895 -0
  20. manifest_parser.py +960 -0
  21. master_rules_bank.py +389 -0
  22. notification_models.py +346 -0
  23. notification_policy.py +309 -0
  24. notification_repository.py +118 -0
  25. notification_routes.py +394 -0
  26. notification_service.py +199 -0
  27. osv_client.py +437 -0
  28. remediation/__init__.py +39 -0
  29. remediation/base.py +102 -0
  30. remediation/contracts.py +196 -0
  31. remediation/cwe1188_binding.py +78 -0
  32. remediation/cwe1336_jinja.py +91 -0
  33. remediation/cwe22_path_traversal.py +348 -0
  34. remediation/cwe295_ssl.py +87 -0
  35. remediation/cwe327_weak_hash.py +78 -0
  36. remediation/cwe377_tempfile.py +77 -0
  37. remediation/cwe489_debug.py +80 -0
  38. remediation/cwe502_yaml.py +89 -0
  39. remediation/cwe614_cookie.py +91 -0
  40. remediation/cwe78_cmdi.py +415 -0
  41. remediation/cwe798_credentials.py +153 -0
  42. remediation/cwe89_sqli.py +425 -0
  43. remediation/cwe916_password_hash.py +131 -0
  44. remediation/cwe95_eval.py +77 -0
  45. remediation/dispatcher.py +114 -0
  46. remediation/orchestrator.py +541 -0
  47. remediation/patch_engine.py +213 -0
  48. remediation/verification.py +253 -0
  49. rule_engine.py +1535 -0
  50. sarif_adapter.py +719 -0
  51. sarif_exporter.py +88 -0
  52. sca_reachability/__init__.py +28 -0
  53. sca_reachability/ast_analyzer.py +366 -0
  54. sca_reachability/call_graph.py +203 -0
  55. sca_reachability/contracts.py +365 -0
  56. sca_reachability/dist_import_map.py +46 -0
  57. sca_reachability/engine.py +603 -0
  58. sca_reachability/resolver.py +169 -0
  59. scanners/__init__.py +70 -0
  60. scanners/container_scanner.py +323 -0
  61. scanners/cve_index.py +521 -0
  62. scanners/iac_scanner.py +328 -0
  63. scanners/sbom.py +183 -0
  64. scanners/sca_scanner.py +667 -0
  65. scanners/secret_scanner.py +345 -0
  66. scanners/yaml_lite.py +217 -0
  67. secret_filters.py +435 -0
  68. secret_scanner.py +943 -0
  69. sqlite_delivery_repository.py +277 -0
  70. sqlite_notification_repository.py +441 -0
  71. ssrf_guard.py +276 -0
  72. staged_scanner.py +492 -0
  73. suppression_resolver.py +282 -0
  74. symbol_indexer.py +424 -0
  75. tcs/__init__.py +1 -0
  76. tcs/analysis/__init__.py +1 -0
  77. tcs/analysis/def_use.py +408 -0
  78. tcs/analysis/function_contracts.py +290 -0
  79. tcs/parallel_analyzer.py +276 -0
  80. tcs/parallel_scanner.py +630 -0
  81. tcs/treesitter_packer.py +1446 -0
  82. tcs_cli.py +2086 -0
  83. tcs_config.py +218 -0
  84. tcs_gui.py +2123 -0
  85. timecs-2.0.0.dist-info/METADATA +142 -0
  86. timecs-2.0.0.dist-info/RECORD +94 -0
  87. timecs-2.0.0.dist-info/WHEEL +5 -0
  88. timecs-2.0.0.dist-info/entry_points.txt +4 -0
  89. timecs-2.0.0.dist-info/licenses/LICENSE +201 -0
  90. timecs-2.0.0.dist-info/top_level.txt +49 -0
  91. version_matcher.py +934 -0
  92. webhook_adapter.py +495 -0
  93. webhook_client.py +483 -0
  94. webhook_formatters.py +245 -0
@@ -0,0 +1,276 @@
1
+ """TimeCodeSecurity — File-level parallel scanner with deterministic merge.
2
+
3
+ This module parallelizes scanning across multiple files while ensuring:
4
+ - All 14 structural inspectors run on every file
5
+ - Full taint tracking engine executes per-worker
6
+ - Findings are deterministically merged and deduplicated
7
+ - Zero accuracy regression between --workers 1 and --workers N
8
+
9
+ Architecture:
10
+ - Each worker gets a subset of files
11
+ - Worker creates independent TaintTracker instance
12
+ - Worker runs full analyze() pipeline (all phases)
13
+ - Main process merges findings with deterministic sort + dedup
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import ast
18
+ import json
19
+ import os
20
+ import sys
21
+ import time
22
+ from concurrent.futures import ProcessPoolExecutor, as_completed
23
+ from pathlib import Path
24
+ from typing import Any, Dict, List, Optional, Set, Tuple
25
+
26
+ try:
27
+ import psutil
28
+ except ImportError:
29
+ # Declared optional on purpose. A hard module-level `import psutil` made this whole module
30
+ # unimportable wherever psutil was absent, and `ast_scanner._phase2_worker_count` then fell
31
+ # back to 1 worker silently -- that is how a 2-vCPU / 7 GB runner spent 283 s collecting
32
+ # statements serially. `available_ram_mb()` covers the same decision without it.
33
+ psutil = None
34
+
35
+
36
+ # ─── Worker RAM Budget for Analysis ──────────────────────────────────────────
37
+
38
+ ANALYSIS_WORKER_RAM_BUDGET_MB = 600 # Budget 600 MB per analysis worker
39
+ HARD_SAFETY_THRESHOLD_MB = 900 # Drop to 1 worker if RAM < 900 MB
40
+ CI_TWO_CORE_RAM_MB = 1600 # Minimum free RAM before any second worker is opened
41
+ MAX_PHASE2_WORKERS = 4 # Phase 2 caps here; the parent holds every parsed tree
42
+
43
+
44
+ def available_ram_mb():
45
+ """Free RAM in MB, or None when it cannot be measured.
46
+
47
+ psutil is the accurate source. Without it, Linux reports the same number in
48
+ `/proc/meminfo` as MemAvailable. Other platforms without psutil return None, and every caller
49
+ treats "unknown" as "no evidence of headroom" and stays on one worker.
50
+ """
51
+ if psutil is not None:
52
+ return psutil.virtual_memory().available / (1024 * 1024)
53
+ try:
54
+ with open("/proc/meminfo", encoding="ascii") as handle:
55
+ for line in handle:
56
+ if line.startswith("MemAvailable:"):
57
+ return float(line.split()[1]) / 1024.0
58
+ except (OSError, ValueError, IndexError):
59
+ return None
60
+ return None
61
+
62
+
63
+ def compute_analysis_workers() -> int:
64
+ """Compute safe worker count for analysis phase (heavier than parsing)."""
65
+ free_ram_mb = available_ram_mb()
66
+ if free_ram_mb is None:
67
+ return 1
68
+ # Hard safety threshold
69
+ if free_ram_mb < HARD_SAFETY_THRESHOLD_MB:
70
+ return 1
71
+
72
+ cpu_count = os.cpu_count() or 4
73
+ # The old rule kept 2 cores free (`cpu_count - 2`), which collapses a 2-vCPU GitHub runner to
74
+ # zero and then to one worker, so the runner never used the machine it was given. `cpu_count`
75
+ # capped at 4 keeps a core for the parent, and the RAM terms below still bound the pool:
76
+ # every worker needs ANALYSIS_WORKER_RAM_BUDGET_MB of its own, and multi-worker only starts
77
+ # once free RAM clears CI_TWO_CORE_RAM_MB.
78
+ cpu_capacity = min(cpu_count, MAX_PHASE2_WORKERS) if free_ram_mb >= CI_TWO_CORE_RAM_MB else 1
79
+ ram_capacity = max(1, int(free_ram_mb // ANALYSIS_WORKER_RAM_BUDGET_MB))
80
+ return max(1, min(cpu_capacity, ram_capacity))
81
+
82
+
83
+ def _scan_file_batch(args: Tuple[List[str], str, Dict[str, Any]]) -> Dict[str, Any]:
84
+ """Scan a batch of files in a worker process.
85
+
86
+ This runs the FULL TaintTracker pipeline on assigned files.
87
+
88
+ Args:
89
+ args: (file_paths_list, scope, options_dict) tuple
90
+
91
+ Returns:
92
+ Dict with keys: findings (list of finding dicts), error (str or None)
93
+ """
94
+ file_paths, scope, options = args
95
+
96
+ try:
97
+ # Import here to ensure clean worker state
98
+ from ast_scanner import TaintTracker
99
+
100
+ # Collect source code for all files
101
+ files = {}
102
+ for fpath in file_paths:
103
+ try:
104
+ with open(fpath, 'r', encoding='utf-8') as f:
105
+ files[fpath] = f.read()
106
+ except (OSError, UnicodeDecodeError):
107
+ continue
108
+
109
+ if not files:
110
+ return {
111
+ "findings": [],
112
+ "error": None,
113
+ "file_count": len(file_paths),
114
+ }
115
+
116
+ # Create TaintTracker and run full analysis
117
+ tracker = TaintTracker(files=files)
118
+ _, _, edges = tracker.analyze()
119
+
120
+ # Extract findings using the same logic as cli.py
121
+ findings = _extract_findings_from_tracker(tracker, edges)
122
+
123
+ return {
124
+ "findings": findings,
125
+ "error": None,
126
+ "file_count": len(file_paths),
127
+ }
128
+
129
+ except Exception as exc:
130
+ return {
131
+ "findings": [],
132
+ "error": f"{type(exc).__name__}: {exc}"[:200],
133
+ "file_count": len(file_paths),
134
+ }
135
+
136
+
137
+ def _extract_findings_from_tracker(tracker, edges) -> List[Dict[str, Any]]:
138
+ """Extract findings from TaintTracker matching cli.py logic.
139
+
140
+ This mirrors the _findings_for() function in cli.py to ensure parity.
141
+ """
142
+ findings = []
143
+
144
+ # Iterate through all sinks and generate findings
145
+ for sink in tracker.sinks:
146
+ finding = {
147
+ "file": getattr(sink, 'location', {}).get('file', ''),
148
+ "line": getattr(sink, 'location', {}).get('line', 0),
149
+ "column": getattr(sink, 'location', {}).get('column', 0),
150
+ "cwe": sink.metadata.get('cwe', '') if hasattr(sink, 'metadata') else '',
151
+ "severity": sink.metadata.get('severity', 'MEDIUM') if hasattr(sink, 'metadata') else 'MEDIUM',
152
+ "rule_id": sink.metadata.get('operation', '') if hasattr(sink, 'metadata') else '',
153
+ "message": f"{sink.metadata.get('operation', 'Vulnerability')} detected",
154
+ }
155
+ findings.append(finding)
156
+
157
+ return findings
158
+
159
+
160
+ def parallel_scan_files(
161
+ file_paths: List[str],
162
+ scope: str = "python",
163
+ max_workers: Optional[int] = None,
164
+ ) -> Tuple[List[Dict[str, Any]], int]:
165
+ """Scan files in parallel with deterministic merge.
166
+
167
+ Args:
168
+ file_paths: List of absolute file paths to scan
169
+ scope: Scan scope ("python", "all", etc.)
170
+ max_workers: Override worker count (None = auto-compute)
171
+
172
+ Returns:
173
+ (merged_findings, total_files_scanned) where findings are sorted
174
+ deterministically by (file, line, cwe, rule_id)
175
+ """
176
+ if max_workers is None:
177
+ max_workers = compute_analysis_workers()
178
+
179
+ total_files = len(file_paths)
180
+
181
+ print(
182
+ f"TimeCodeSecurity [PARALLEL] Scanning {total_files} files with {max_workers} workers",
183
+ file=sys.stderr,
184
+ )
185
+
186
+ # For small workloads, skip parallelism
187
+ if total_files <= 10 or max_workers == 1:
188
+ result = _scan_file_batch((file_paths, scope, {}))
189
+ if result["error"]:
190
+ print(
191
+ f"TimeCodeSecurity [WARN] Batch scan failed: {result['error']}",
192
+ file=sys.stderr,
193
+ )
194
+ return result.get("findings", []), result.get("file_count", 0)
195
+
196
+ # Partition files into batches (one per worker ideally)
197
+ batch_size = max(1, total_files // max_workers)
198
+ batches = []
199
+ for i in range(0, total_files, batch_size):
200
+ batch = file_paths[i:i + batch_size]
201
+ batches.append(batch)
202
+
203
+ all_findings = []
204
+ total_scanned = 0
205
+ start_time = time.time()
206
+
207
+ # Process batches in parallel
208
+ with ProcessPoolExecutor(max_workers=max_workers) as executor:
209
+ future_to_batch = {
210
+ executor.submit(_scan_file_batch, (batch, scope, {})): idx
211
+ for idx, batch in enumerate(batches)
212
+ }
213
+
214
+ for future in as_completed(future_to_batch):
215
+ batch_idx = future_to_batch[future]
216
+ try:
217
+ result = future.result(timeout=300) # 5-minute timeout per batch
218
+ if result["error"]:
219
+ print(
220
+ f"TimeCodeSecurity [WARN] Batch {batch_idx} failed: {result['error']}",
221
+ file=sys.stderr,
222
+ )
223
+ else:
224
+ all_findings.extend(result["findings"])
225
+ total_scanned += result.get("file_count", 0)
226
+ except Exception as exc:
227
+ print(
228
+ f"TimeCodeSecurity [WARN] Batch {batch_idx} exception: {exc}",
229
+ file=sys.stderr,
230
+ )
231
+
232
+ elapsed = time.time() - start_time
233
+ print(
234
+ f"TimeCodeSecurity [PARALLEL] Completed in {elapsed:.2f}s "
235
+ f"({len(all_findings)} findings from {total_scanned} files)",
236
+ file=sys.stderr,
237
+ )
238
+
239
+ # Deterministic sort and deduplication
240
+ merged_findings = _merge_and_deduplicate_findings(all_findings)
241
+
242
+ return merged_findings, total_scanned
243
+
244
+
245
+ def _merge_and_deduplicate_findings(findings: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
246
+ """Merge findings from multiple workers with deterministic sort and dedup.
247
+
248
+ Deduplicates by (file, line, cwe, rule_id) tuple.
249
+ Sorts by (file, line, cwe, rule_id) for reproducibility.
250
+ """
251
+ # Deduplicate
252
+ seen = set()
253
+ unique_findings = []
254
+
255
+ for finding in findings:
256
+ key = (
257
+ finding.get("file", ""),
258
+ finding.get("line", 0),
259
+ finding.get("cwe", ""),
260
+ finding.get("rule_id", ""),
261
+ )
262
+ if key not in seen:
263
+ seen.add(key)
264
+ unique_findings.append(finding)
265
+
266
+ # Deterministic sort
267
+ unique_findings.sort(
268
+ key=lambda x: (x.get("file", ""), x.get("line", 0), x.get("cwe", ""), x.get("rule_id", ""))
269
+ )
270
+
271
+ return unique_findings
272
+
273
+
274
+ if __name__ == "__main__":
275
+ # Safety guard for Windows multiprocessing
276
+ print("TimeCodeSecurity parallel scanner module loaded successfully.")