timecs 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. app.py +3014 -0
  2. ast_scanner.py +36187 -0
  3. banner.py +146 -0
  4. ci_reporter.py +570 -0
  5. cli.py +2347 -0
  6. config_loader.py +307 -0
  7. cross_file_engine.py +1781 -0
  8. delivery_models.py +145 -0
  9. delivery_repository.py +78 -0
  10. diff_scan.py +127 -0
  11. durable_delivery_sink.py +103 -0
  12. event_bus.py +155 -0
  13. event_publisher.py +345 -0
  14. event_taxonomy.py +313 -0
  15. function_summarizer.py +354 -0
  16. html_auditor.py +274 -0
  17. iac_auditor.py +400 -0
  18. in_app_adapter.py +115 -0
  19. js_scanner.py +895 -0
  20. manifest_parser.py +960 -0
  21. master_rules_bank.py +389 -0
  22. notification_models.py +346 -0
  23. notification_policy.py +309 -0
  24. notification_repository.py +118 -0
  25. notification_routes.py +394 -0
  26. notification_service.py +199 -0
  27. osv_client.py +437 -0
  28. remediation/__init__.py +39 -0
  29. remediation/base.py +102 -0
  30. remediation/contracts.py +196 -0
  31. remediation/cwe1188_binding.py +78 -0
  32. remediation/cwe1336_jinja.py +91 -0
  33. remediation/cwe22_path_traversal.py +348 -0
  34. remediation/cwe295_ssl.py +87 -0
  35. remediation/cwe327_weak_hash.py +78 -0
  36. remediation/cwe377_tempfile.py +77 -0
  37. remediation/cwe489_debug.py +80 -0
  38. remediation/cwe502_yaml.py +89 -0
  39. remediation/cwe614_cookie.py +91 -0
  40. remediation/cwe78_cmdi.py +415 -0
  41. remediation/cwe798_credentials.py +153 -0
  42. remediation/cwe89_sqli.py +425 -0
  43. remediation/cwe916_password_hash.py +131 -0
  44. remediation/cwe95_eval.py +77 -0
  45. remediation/dispatcher.py +114 -0
  46. remediation/orchestrator.py +541 -0
  47. remediation/patch_engine.py +213 -0
  48. remediation/verification.py +253 -0
  49. rule_engine.py +1535 -0
  50. sarif_adapter.py +719 -0
  51. sarif_exporter.py +88 -0
  52. sca_reachability/__init__.py +28 -0
  53. sca_reachability/ast_analyzer.py +366 -0
  54. sca_reachability/call_graph.py +203 -0
  55. sca_reachability/contracts.py +365 -0
  56. sca_reachability/dist_import_map.py +46 -0
  57. sca_reachability/engine.py +603 -0
  58. sca_reachability/resolver.py +169 -0
  59. scanners/__init__.py +70 -0
  60. scanners/container_scanner.py +323 -0
  61. scanners/cve_index.py +521 -0
  62. scanners/iac_scanner.py +328 -0
  63. scanners/sbom.py +183 -0
  64. scanners/sca_scanner.py +667 -0
  65. scanners/secret_scanner.py +345 -0
  66. scanners/yaml_lite.py +217 -0
  67. secret_filters.py +435 -0
  68. secret_scanner.py +943 -0
  69. sqlite_delivery_repository.py +277 -0
  70. sqlite_notification_repository.py +441 -0
  71. ssrf_guard.py +276 -0
  72. staged_scanner.py +492 -0
  73. suppression_resolver.py +282 -0
  74. symbol_indexer.py +424 -0
  75. tcs/__init__.py +1 -0
  76. tcs/analysis/__init__.py +1 -0
  77. tcs/analysis/def_use.py +408 -0
  78. tcs/analysis/function_contracts.py +290 -0
  79. tcs/parallel_analyzer.py +276 -0
  80. tcs/parallel_scanner.py +630 -0
  81. tcs/treesitter_packer.py +1446 -0
  82. tcs_cli.py +2086 -0
  83. tcs_config.py +218 -0
  84. tcs_gui.py +2123 -0
  85. timecs-2.0.0.dist-info/METADATA +142 -0
  86. timecs-2.0.0.dist-info/RECORD +94 -0
  87. timecs-2.0.0.dist-info/WHEEL +5 -0
  88. timecs-2.0.0.dist-info/entry_points.txt +4 -0
  89. timecs-2.0.0.dist-info/licenses/LICENSE +201 -0
  90. timecs-2.0.0.dist-info/top_level.txt +49 -0
  91. version_matcher.py +934 -0
  92. webhook_adapter.py +495 -0
  93. webhook_client.py +483 -0
  94. webhook_formatters.py +245 -0
symbol_indexer.py ADDED
@@ -0,0 +1,424 @@
1
+ """
2
+ TimeCodeSecurity - Phase 4: Global Symbol & Export Indexer
3
+ Deterministic AST-based multi-file symbol and import resolver.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import ast
9
+ import os
10
+ import sys
11
+ import time
12
+ from dataclasses import dataclass, field
13
+ from pathlib import Path
14
+ from typing import Dict, List, Optional, Set, Tuple
15
+
16
+ # Phase-1 instrumentation only (never read by a decision): the fallback resolver below is a
17
+ # linear scan over every registered module name, so on a 10k-module repository each miss costs
18
+ # 10k `endswith` tests. These counters say how often that happens and how long it costs.
19
+ # `build()` resets them, so the numbers belong to one indexing run.
20
+ _MATCH_STATS = {"calls": 0, "exact": 0, "suffix": 0, "miss": 0, "index_size": 0}
21
+
22
+
23
+ def resolver_stats_line() -> str:
24
+ s = _MATCH_STATS
25
+ return (f"resolver: calls={s['calls']} exact={s['exact']} suffix={s['suffix']} "
26
+ f"miss={s['miss']} suffix_index_size={s['index_size']}")
27
+
28
+ # The cross-file index re-reads and re-parses the tree the scanner already parsed. On a big repository
29
+ # that duplicated pass is worth handing to the same RAM-governed worker pool the scanner uses, but the
30
+ # import is optional: `symbol_indexer` is imported by standalone tooling and tests where the worker
31
+ # package is not on the path, and those callers must keep getting the serial behaviour, not an ImportError.
32
+ try:
33
+ from tcs.parallel_scanner import read_and_parse_paths
34
+
35
+ _FANOUT_AVAILABLE = True
36
+ except Exception: # pragma: no cover - environment dependent
37
+ _FANOUT_AVAILABLE = False
38
+
39
+ # Spawning a worker costs ~1.4 s, and 600 files parse serially in ~1 s, so a small tree must never pay
40
+ # for a pool. Only fan out once the duplicated pass is long enough to be worth the spawn.
41
+ PARALLEL_READ_MIN_FILES = 2000
42
+
43
+
44
+ def _read_and_parse_one(path: str) -> dict:
45
+ """Read + parse one file into the worker entry shape; never raises."""
46
+ try:
47
+ source = Path(path).read_text(encoding="utf-8", errors="replace")
48
+ return {"path": path, "source": source,
49
+ "tree": ast.parse(source, filename=path), "error": None}
50
+ except Exception as exc:
51
+ return {"path": path, "source": None, "tree": None,
52
+ "error": f"{type(exc).__name__}: {exc}"[:200]}
53
+
54
+
55
+ @dataclass(slots=True)
56
+ class SymbolDefinition:
57
+ name: str
58
+ kind: str # "function", "async_function", "class", "variable"
59
+ file_path: str
60
+ lineno: int
61
+ params: list[str] = field(default_factory=list)
62
+
63
+
64
+ @dataclass(slots=True)
65
+ class ImportBinding:
66
+ alias: str # Local identifier used in caller file
67
+ source_module: str # Canonical module path (e.g., 'introduction.apis')
68
+ imported_name: str # Original symbol name in target module
69
+ lineno: int = 0
70
+ is_star: bool = False
71
+
72
+
73
+ @dataclass
74
+ class ModuleIndex:
75
+ module_name: str
76
+ file_path: Path
77
+ definitions: dict[str, SymbolDefinition] = field(default_factory=dict)
78
+ imports: dict[str, ImportBinding] = field(default_factory=dict)
79
+ star_imports: list[str] = field(default_factory=list)
80
+
81
+
82
+ class GlobalSymbolIndex:
83
+ """Project-wide AST symbol table and import graph resolver."""
84
+
85
+ def __init__(self, root_dir: str | Path):
86
+ self.root_dir = Path(root_dir).resolve()
87
+ self.modules: dict[str, ModuleIndex] = {}
88
+ self.file_to_module: dict[Path, str] = {}
89
+ self.module_to_file: dict[str, Path] = {}
90
+ self._ast_cache: dict[str, ast.AST] = {}
91
+ # suffix -> first-registered module whose dotted name ends with that suffix.
92
+ # Rebuilt once per build(); replaces the per-call linear scan over `modules`.
93
+ self._suffix_index: dict[str, str] = {}
94
+
95
+ def _file_to_modname(self, file_path: Path) -> str:
96
+ """Derive standard dotted module name from relative project path."""
97
+ rel = file_path.resolve().relative_to(self.root_dir)
98
+ parts = list(rel.with_suffix("").parts)
99
+ if parts and parts[-1] == "__init__":
100
+ parts.pop()
101
+ # Interned because this name becomes the key of `modules`, `module_to_file` and every
102
+ # later import-resolution lookup. One canonical object per name means the interpreter
103
+ # hashes the string once instead of once per dict lookup, and equal names compare by
104
+ # pointer. The returned value is still the same dotted string.
105
+ return sys.intern(".".join(parts))
106
+
107
+ def build(self, max_workers: Optional[int] = None,
108
+ reuse_trees: Optional[dict[str, tuple[str, ast.AST]]] = None,
109
+ on_indexed=None) -> GlobalSymbolIndex:
110
+ """Pass 1: Discover and index all Python definitions and imports.
111
+
112
+ `reuse_trees` optionally maps a resolved absolute path string to the scanner's own
113
+ `(source, tree)` pair. A tree is reused only when re-reading that path from disk
114
+ yields a source byte-for-byte equal to the cached snapshot - so a file the scanner
115
+ parsed is exactly the file this index would have parsed. Any miss, read error or
116
+ content mismatch falls back to the original read+parse path, unchanged.
117
+
118
+ `on_indexed(mod_name, file_path, tree)` turns this into a streaming pass: every
119
+ module name is still registered first, but files are parsed one at a time, handed to
120
+ the callback, and released. The index then stores no ASTs at all (`_ast_cache` stays
121
+ empty), which is what lets the cross-file engine run on a compact fact table. The
122
+ callback runs outside the `_index_file` try, because a consumer failure would silently
123
+ drop facts and must be loud instead.
124
+ """
125
+ _t_walk0 = time.perf_counter()
126
+ # Observational reset so resolver counters belong to this engine run.
127
+ for _k in _MATCH_STATS:
128
+ _MATCH_STATS[_k] = 0
129
+ self._suffix_index = {}
130
+ py_files: list[Path] = []
131
+ for root, dirs, files in os.walk(self.root_dir, followlinks=False):
132
+ # Symlinked dirs are pruned before descending: a link into an ancestor
133
+ # would otherwise make this crawl loop and never reach the analysis.
134
+ dirs[:] = [d for d in dirs if not os.path.islink(os.path.join(root, d))]
135
+ for f in files:
136
+ if f.endswith(".py"):
137
+ full_p = Path(root) / f
138
+ # Skip common virtual environments and cache directories
139
+ parts = full_p.parts
140
+ if any(x in parts for x in (".venv", "venv", "env", "__pycache__", ".git")):
141
+ continue
142
+ py_files.append(full_p)
143
+
144
+ # 1. Register module namespace
145
+ for p in py_files:
146
+ mod_name = self._file_to_modname(p)
147
+ self.file_to_module[p] = mod_name
148
+ self.module_to_file[mod_name] = p
149
+ self.modules[mod_name] = ModuleIndex(module_name=mod_name, file_path=p)
150
+ self._build_suffix_index()
151
+
152
+ # Streaming branch: registration above is already complete, so every symbol reference
153
+ # resolves; from here one tree is alive at a time and none is retained.
154
+ if on_indexed is not None:
155
+ _t_stream0 = time.perf_counter()
156
+ reused = 0
157
+ parent_chars = 0
158
+ trees = 0
159
+ for p in py_files:
160
+ mod_name = self.file_to_module[p]
161
+ entry = None
162
+ if reuse_trees:
163
+ cached = reuse_trees.get(str(p))
164
+ if cached is not None and cached[1] is not None:
165
+ try:
166
+ if Path(str(p)).read_text(
167
+ encoding="utf-8", errors="replace") == cached[0]:
168
+ entry = {"tree": cached[1], "source": cached[0]}
169
+ reused += 1
170
+ except Exception:
171
+ entry = None
172
+ if entry is None:
173
+ entry = _read_and_parse_one(str(p))
174
+ parent_chars += len(entry["source"] or "")
175
+ tree = entry["tree"]
176
+ if tree is None:
177
+ continue
178
+ trees += 1
179
+ try:
180
+ self._index_file(mod_name, p, tree)
181
+ except Exception:
182
+ # Same contract as the batched path: a file with an odd top-level
183
+ # statement is still summarised and harvested from its tree.
184
+ pass
185
+ on_indexed(mod_name, p, tree)
186
+ del tree, entry
187
+ _t_stream1 = time.perf_counter()
188
+ print(f"TimeCodeSecurity [PROFILE] SymbolIndex: walk={_t_stream0 - _t_walk0:.2f}s "
189
+ f"py_files={len(py_files)} stream={_t_stream1 - _t_stream0:.2f}s "
190
+ f"trees={trees} reused={reused} parent_chars={parent_chars} "
191
+ f"modules={len(self.modules)} "
192
+ f"TOTAL={_t_stream1 - _t_walk0:.2f}s", file=sys.stderr)
193
+ return self
194
+
195
+ # 2. Parse and index symbols.
196
+ # This pass reads and parses the whole tree a *second* time - the scanner already parsed it -
197
+ # and it did so in the parent process, serially. Measured on the 609-file benchmark corpus
198
+ # that is 0.97 s, so it is not the reason a Home Assistant scan is slow; it is nevertheless
199
+ # pure duplicated work that scales linearly with the tree (~17 s at 10,648 files), so the
200
+ # read+parse goes through the same RAM-governed fan-out the scanner uses. `py_files` order is
201
+ # preserved because `modules` and `_ast_cache` are built by inserting these results and the
202
+ # cross-file engine walks that insertion order.
203
+ _t_walk1 = time.perf_counter()
204
+ paths = [str(p) for p in py_files]
205
+ reusable: dict[int, ast.AST] = {}
206
+ if reuse_trees:
207
+ for i, path_str in enumerate(paths):
208
+ cached = reuse_trees.get(path_str)
209
+ if cached is None:
210
+ continue
211
+ source, tree = cached
212
+ if tree is None:
213
+ continue
214
+ try:
215
+ if Path(path_str).read_text(encoding="utf-8", errors="replace") != source:
216
+ continue
217
+ except Exception:
218
+ continue
219
+ reusable[i] = tree
220
+ parsed = self._read_and_parse([p for i, p in enumerate(paths)
221
+ if i not in reusable], max_workers)
222
+ _it = iter(parsed)
223
+ entries = [({"tree": reusable[i], "source": ""} if i in reusable else next(_it))
224
+ for i in range(len(paths))]
225
+ _t_read1 = time.perf_counter()
226
+ # Phase-1 counters: this is the duplicated read+parse pass over the whole tree
227
+ # (`source` only exists in-process on the serial path; the fan-out returns trees, so
228
+ # char totals cover what the parent itself decoded).
229
+ _reread_chars = sum(len(e["source"]) for e in entries if e.get("source"))
230
+ _trees_built = sum(1 for e in entries if e["tree"] is not None)
231
+ for p, entry in zip(py_files, entries):
232
+ mod_name = self.file_to_module[p]
233
+ if entry["tree"] is None:
234
+ continue
235
+ try:
236
+ self._ast_cache[mod_name] = entry["tree"]
237
+ self._index_file(mod_name, p, entry["tree"])
238
+ except Exception:
239
+ # Unchanged from the serial loop: a tree cached before `_index_file` raised stays
240
+ # cached, so a file with an odd top-level statement is still analysable downstream.
241
+ continue
242
+
243
+ _t_idx1 = time.perf_counter()
244
+ print(f"TimeCodeSecurity [PROFILE] SymbolIndex: walk={_t_walk1 - _t_walk0:.2f}s "
245
+ f"py_files={len(py_files)} read_parse={_t_read1 - _t_walk1:.2f}s "
246
+ f"trees={_trees_built} reused={len(reusable)} parent_chars={_reread_chars} "
247
+ f"index={_t_idx1 - _t_read1:.2f}s modules={len(self.modules)} "
248
+ f"TOTAL={_t_idx1 - _t_walk0:.2f}s", file=sys.stderr)
249
+ return self
250
+
251
+ @staticmethod
252
+ def _read_and_parse(paths: List[str], max_workers: Optional[int]) -> List[dict]:
253
+ """Read + parse every path in path order: through the governor when it is importable."""
254
+ if _FANOUT_AVAILABLE and len(paths) >= PARALLEL_READ_MIN_FILES:
255
+ return read_and_parse_paths(paths, max_workers=max_workers)
256
+ return [_read_and_parse_one(p) for p in paths]
257
+
258
+ def _index_file(self, mod_name: str, file_path: Path, tree: ast.AST) -> None:
259
+ idx = self.modules[mod_name]
260
+
261
+ for stmt in tree.body:
262
+ # Functions
263
+ if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef)):
264
+ params = [a.arg for a in stmt.args.args]
265
+ kind = "async_function" if isinstance(stmt, ast.AsyncFunctionDef) else "function"
266
+ idx.definitions[stmt.name] = SymbolDefinition(
267
+ name=stmt.name,
268
+ kind=kind,
269
+ file_path=str(file_path),
270
+ lineno=stmt.lineno,
271
+ params=params,
272
+ )
273
+
274
+ # Classes
275
+ elif isinstance(stmt, ast.ClassDef):
276
+ idx.definitions[stmt.name] = SymbolDefinition(
277
+ name=stmt.name,
278
+ kind="class",
279
+ file_path=str(file_path),
280
+ lineno=stmt.lineno,
281
+ )
282
+ for item in stmt.body:
283
+ if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)):
284
+ method_params = [a.arg for a in item.args.args]
285
+ qualname = f"{stmt.name}.{item.name}"
286
+ idx.definitions[qualname] = SymbolDefinition(
287
+ name=qualname,
288
+ kind="method",
289
+ file_path=str(file_path),
290
+ lineno=item.lineno,
291
+ params=method_params,
292
+ )
293
+
294
+ # Imports: import a.b as c
295
+ elif isinstance(stmt, ast.Import):
296
+ for alias in stmt.names:
297
+ local_name = alias.asname or alias.name
298
+ idx.imports[local_name] = ImportBinding(
299
+ alias=local_name,
300
+ source_module=alias.name,
301
+ imported_name="",
302
+ lineno=stmt.lineno,
303
+ )
304
+
305
+ # From Imports: from a.b import c as d
306
+ elif isinstance(stmt, ast.ImportFrom):
307
+ target_mod = self._resolve_from_module(mod_name, stmt.module, stmt.level)
308
+ for alias in stmt.names:
309
+ if alias.name == "*":
310
+ idx.star_imports.append(target_mod)
311
+ else:
312
+ local_name = alias.asname or alias.name
313
+ idx.imports[local_name] = ImportBinding(
314
+ alias=local_name,
315
+ source_module=target_mod,
316
+ imported_name=alias.name,
317
+ lineno=stmt.lineno,
318
+ )
319
+
320
+ def _resolve_from_module(self, current_mod: str, rel_module: Optional[str], level: int) -> str:
321
+ """Resolve Python relative import dots (level) to canonical module name."""
322
+ if level == 0:
323
+ return rel_module or ""
324
+
325
+ parts = current_mod.split(".")
326
+ # Relative hop
327
+ if level <= len(parts):
328
+ base = parts[:-level]
329
+ else:
330
+ base = []
331
+
332
+ if rel_module:
333
+ base.append(rel_module)
334
+ return ".".join(base)
335
+
336
+ def resolve_symbol(self, caller_mod: str, symbol_name: str) -> Optional[SymbolDefinition]:
337
+ """
338
+ Cross-file resolver:
339
+ Given a caller module and a referenced symbol (e.g., 'helper' or 'api.run'),
340
+ returns the exact definition in the origin module.
341
+ """
342
+ mod_idx = self.modules.get(caller_mod)
343
+ if not mod_idx:
344
+ return None
345
+
346
+ # 1. Check local definitions first
347
+ if symbol_name in mod_idx.definitions:
348
+ return mod_idx.definitions[symbol_name]
349
+
350
+ # 2. Check explicit import bindings
351
+ if symbol_name in mod_idx.imports:
352
+ binding = mod_idx.imports[symbol_name]
353
+ target_mod = self._match_registered_module(binding.source_module)
354
+ if target_mod and target_mod in self.modules:
355
+ target_idx = self.modules[target_mod]
356
+ lookup_name = binding.imported_name or symbol_name
357
+ if lookup_name in target_idx.definitions:
358
+ return target_idx.definitions[lookup_name]
359
+
360
+ # 3. Check star imports
361
+ for star_mod in mod_idx.star_imports:
362
+ target_mod = self._match_registered_module(star_mod)
363
+ if target_mod and target_mod in self.modules:
364
+ if symbol_name in self.modules[target_mod].definitions:
365
+ return self.modules[target_mod].definitions[symbol_name]
366
+
367
+ return None
368
+
369
+ def _match_registered_module(self, module_candidate: str) -> Optional[str]:
370
+ """Fuzzy match module name against root-prefixed or stripped module paths.
371
+
372
+ The suffix index reproduces the old linear scan exactly: `self.modules` keeps
373
+ insertion order, `_build_suffix_index` inserts each dotted suffix only once (first
374
+ module wins), and the indexed key is the same `mod.endswith(f".{candidate}")` test
375
+ the loop applied - i.e. `candidate` must match a proper dotted suffix, never a bare
376
+ trailing segment. Lookup is O(len(parts)) instead of O(N modules) per call.
377
+ """
378
+ st = _MATCH_STATS
379
+ st["calls"] += 1
380
+ if module_candidate in self.modules:
381
+ st["exact"] += 1
382
+ return module_candidate
383
+ match = self._suffix_index.get(module_candidate)
384
+ if match is not None:
385
+ st["suffix"] += 1
386
+ else:
387
+ st["miss"] += 1
388
+ return match
389
+
390
+ def _build_suffix_index(self) -> None:
391
+ """Index every proper dotted suffix of each registered module name.
392
+
393
+ `mod == candidate` is not included here - exact names hit `self.modules` directly -
394
+ matching the old loop where the equality branch preceded the suffix match.
395
+ """
396
+ idx: dict[str, str] = {}
397
+ for mod in self.modules:
398
+ parts = mod.split(".")
399
+ for i in range(1, len(parts)):
400
+ suffix = ".".join(parts[i:])
401
+ if suffix not in idx:
402
+ idx[suffix] = mod
403
+ self._suffix_index = idx
404
+ _MATCH_STATS["index_size"] = len(idx)
405
+
406
+
407
+ if __name__ == "__main__":
408
+ import time
409
+
410
+ target_dir = sys.argv[1] if len(sys.argv) > 1 else "."
411
+ print(f"[*] Indexing project symbols in: {target_dir}")
412
+ t0 = time.perf_counter()
413
+ indexer = GlobalSymbolIndex(target_dir).build()
414
+ elapsed = (time.perf_counter() - t0) * 1000
415
+
416
+ total_files = len(indexer.modules)
417
+ total_defs = sum(len(m.definitions) for m in indexer.modules.values())
418
+ total_imports = sum(len(m.imports) for m in indexer.modules.values())
419
+
420
+ print("\n--- GLOBAL SYMBOL INDEX SUMMARY ---")
421
+ print(f"Total Python Modules Indexed: {total_files}")
422
+ print(f"Total Definitions (Func/Class): {total_defs}")
423
+ print(f"Total Cross-Module Imports: {total_imports}")
424
+ print(f"Indexing Time: {elapsed:.2f} ms")
tcs/__init__.py ADDED
@@ -0,0 +1 @@
1
+ # TCS Analysis Framework
@@ -0,0 +1 @@
1
+ # TCS Analysis Modules