codegraphcontext-rust 0.5.5__cp311-cp311-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codegraphcontext/__init__.py +2 -0
- codegraphcontext/__main__.py +5 -0
- codegraphcontext/_cgc_rust.cp311-win_amd64.pyd +0 -0
- codegraphcontext/cli/__init__.py +1 -0
- codegraphcontext/cli/cli_helpers.py +774 -0
- codegraphcontext/cli/config_manager.py +1051 -0
- codegraphcontext/cli/main.py +2386 -0
- codegraphcontext/cli/registry_commands.py +404 -0
- codegraphcontext/cli/setup_macos.py +93 -0
- codegraphcontext/cli/setup_wizard.py +991 -0
- codegraphcontext/cli/visualizer.py +50 -0
- codegraphcontext/core/__init__.py +165 -0
- codegraphcontext/core/bundle_registry.py +181 -0
- codegraphcontext/core/cgc_bundle.py +858 -0
- codegraphcontext/core/cgcignore.py +118 -0
- codegraphcontext/core/database.py +273 -0
- codegraphcontext/core/database_falkordb.py +480 -0
- codegraphcontext/core/database_falkordb_remote.py +199 -0
- codegraphcontext/core/database_kuzu.py +636 -0
- codegraphcontext/core/falkor_worker.py +132 -0
- codegraphcontext/core/jobs.py +132 -0
- codegraphcontext/core/watcher.py +260 -0
- codegraphcontext/prompts.py +125 -0
- codegraphcontext/server.py +471 -0
- codegraphcontext/tool_definitions.py +220 -0
- codegraphcontext/tools/__init__.py +0 -0
- codegraphcontext/tools/advanced_language_query_tool.py +103 -0
- codegraphcontext/tools/code_finder.py +1160 -0
- codegraphcontext/tools/graph_builder.py +337 -0
- codegraphcontext/tools/handlers/analysis_handlers.py +114 -0
- codegraphcontext/tools/handlers/indexing_handlers.py +116 -0
- codegraphcontext/tools/handlers/management_handlers.py +339 -0
- codegraphcontext/tools/handlers/query_handlers.py +83 -0
- codegraphcontext/tools/handlers/watcher_handlers.py +83 -0
- codegraphcontext/tools/indexing/__init__.py +1 -0
- codegraphcontext/tools/indexing/constants.py +25 -0
- codegraphcontext/tools/indexing/discovery.py +64 -0
- codegraphcontext/tools/indexing/engine.py +109 -0
- codegraphcontext/tools/indexing/persistence/__init__.py +3 -0
- codegraphcontext/tools/indexing/persistence/parquet_loader.py +643 -0
- codegraphcontext/tools/indexing/persistence/writer.py +1114 -0
- codegraphcontext/tools/indexing/pipeline.py +171 -0
- codegraphcontext/tools/indexing/pre_scan.py +105 -0
- codegraphcontext/tools/indexing/resolution/__init__.py +8 -0
- codegraphcontext/tools/indexing/resolution/calls.py +253 -0
- codegraphcontext/tools/indexing/resolution/inheritance.py +91 -0
- codegraphcontext/tools/indexing/sanitize.py +41 -0
- codegraphcontext/tools/indexing/schema.py +79 -0
- codegraphcontext/tools/indexing/schema_contract.py +44 -0
- codegraphcontext/tools/indexing/scip_pipeline.py +140 -0
- codegraphcontext/tools/languages/c.py +563 -0
- codegraphcontext/tools/languages/cpp.py +616 -0
- codegraphcontext/tools/languages/csharp.py +551 -0
- codegraphcontext/tools/languages/dart.py +378 -0
- codegraphcontext/tools/languages/elixir.py +460 -0
- codegraphcontext/tools/languages/go.py +499 -0
- codegraphcontext/tools/languages/haskell.py +426 -0
- codegraphcontext/tools/languages/java.py +500 -0
- codegraphcontext/tools/languages/javascript.py +589 -0
- codegraphcontext/tools/languages/kotlin.py +640 -0
- codegraphcontext/tools/languages/perl.py +260 -0
- codegraphcontext/tools/languages/php.py +519 -0
- codegraphcontext/tools/languages/python.py +576 -0
- codegraphcontext/tools/languages/ruby.py +537 -0
- codegraphcontext/tools/languages/rust.py +295 -0
- codegraphcontext/tools/languages/scala.py +519 -0
- codegraphcontext/tools/languages/swift.py +491 -0
- codegraphcontext/tools/languages/typescript.py +575 -0
- codegraphcontext/tools/languages/typescriptjsx.py +151 -0
- codegraphcontext/tools/package_resolver.py +472 -0
- codegraphcontext/tools/query_tool_languages/c_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/cpp_toolkit.py +87 -0
- codegraphcontext/tools/query_tool_languages/csharp_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/dart_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/go_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/haskell_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/java_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/javascript_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/perl_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/python_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/ruby_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/rust_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/scala_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/swift_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/typescript_toolkit.py +5 -0
- codegraphcontext/tools/scip_indexer.py +467 -0
- codegraphcontext/tools/scip_pb2.py +2455 -0
- codegraphcontext/tools/system.py +133 -0
- codegraphcontext/tools/tree_sitter_parser.py +103 -0
- codegraphcontext/utils/debug_log.py +90 -0
- codegraphcontext/utils/path_ignore.py +55 -0
- codegraphcontext/utils/repo_path.py +27 -0
- codegraphcontext/utils/tree_sitter_manager.py +285 -0
- codegraphcontext/utils/visualize_graph.py +0 -0
- codegraphcontext/viz/server.py +282 -0
- codegraphcontext_rust-0.5.5.dist-info/METADATA +537 -0
- codegraphcontext_rust-0.5.5.dist-info/RECORD +101 -0
- codegraphcontext_rust-0.5.5.dist-info/WHEEL +4 -0
- codegraphcontext_rust-0.5.5.dist-info/entry_points.txt +3 -0
- codegraphcontext_rust-0.5.5.dist-info/licenses/LICENSE +21 -0
- codegraphcontext_rust-0.5.5.dist-info/sboms/cgc-python.cyclonedx.json +2333 -0
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""Orchestrates full-repo indexing (Tree-sitter path)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import time
|
|
7
|
+
from datetime import datetime
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any, Callable, Dict, List, Optional
|
|
10
|
+
|
|
11
|
+
from ...core.jobs import JobManager, JobStatus
|
|
12
|
+
from ...utils.debug_log import debug_log, error_logger, info_logger
|
|
13
|
+
from .discovery import discover_files_to_index
|
|
14
|
+
from .persistence.writer import GraphWriter
|
|
15
|
+
from .pre_scan import pre_scan_for_imports
|
|
16
|
+
from .resolution.calls import build_function_call_groups
|
|
17
|
+
from .resolution.inheritance import build_inheritance_and_csharp_files
|
|
18
|
+
|
|
19
|
+
# Try to load Rust engine for accelerated parsing
|
|
20
|
+
try:
|
|
21
|
+
from .engine import (
|
|
22
|
+
RUST_AVAILABLE,
|
|
23
|
+
_RUST_SUPPORTED_LANGS,
|
|
24
|
+
_rust_parse_files_parallel,
|
|
25
|
+
_rust_pre_scan,
|
|
26
|
+
_rust_parse_and_prescan,
|
|
27
|
+
_rust_resolve_calls,
|
|
28
|
+
_rust_resolve_inheritance,
|
|
29
|
+
)
|
|
30
|
+
except ImportError:
|
|
31
|
+
RUST_AVAILABLE = False
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
async def run_tree_sitter_index_async(
|
|
35
|
+
path: Path,
|
|
36
|
+
is_dependency: bool,
|
|
37
|
+
job_id: Optional[str],
|
|
38
|
+
cgcignore_path: Optional[str],
|
|
39
|
+
writer: GraphWriter,
|
|
40
|
+
job_manager: JobManager,
|
|
41
|
+
parsers: Dict[str, str],
|
|
42
|
+
get_parser: Callable[[str], Any],
|
|
43
|
+
parse_file: Callable[[Path, Path, bool], Dict[str, Any]],
|
|
44
|
+
add_minimal_file_node: Callable[[Path, Path, bool], None],
|
|
45
|
+
) -> None:
|
|
46
|
+
"""Parse all discovered files, write symbols, then inheritance + CALLS.
|
|
47
|
+
|
|
48
|
+
When the Rust engine is available, files with supported languages are
|
|
49
|
+
parsed in parallel via Rust for significant speedup. Unsupported
|
|
50
|
+
languages (e.g. Kotlin, Perl) fall back to the Python parser.
|
|
51
|
+
"""
|
|
52
|
+
if job_id:
|
|
53
|
+
job_manager.update_job(job_id, status=JobStatus.RUNNING)
|
|
54
|
+
|
|
55
|
+
writer.add_repository_to_graph(path, is_dependency)
|
|
56
|
+
repo_name = path.name
|
|
57
|
+
|
|
58
|
+
files, _ignore_root = discover_files_to_index(path, cgcignore_path)
|
|
59
|
+
|
|
60
|
+
if job_id:
|
|
61
|
+
job_manager.update_job(job_id, total_files=len(files))
|
|
62
|
+
|
|
63
|
+
all_file_data: List[Dict[str, Any]] = []
|
|
64
|
+
resolved_repo_path_str = str(path.resolve()) if path.is_dir() else str(path.parent.resolve())
|
|
65
|
+
repo_path_resolved = path.resolve() if path.is_dir() else path.parent.resolve()
|
|
66
|
+
|
|
67
|
+
# --- Combined pre-scan + parsing phase ---
|
|
68
|
+
if RUST_AVAILABLE:
|
|
69
|
+
# Split files into Rust-supported and fallback
|
|
70
|
+
rust_files = []
|
|
71
|
+
fallback_files = []
|
|
72
|
+
for file in files:
|
|
73
|
+
if not file.is_file():
|
|
74
|
+
continue
|
|
75
|
+
lang = parsers.get(file.suffix)
|
|
76
|
+
if lang and lang in _RUST_SUPPORTED_LANGS and file.suffix != ".ipynb":
|
|
77
|
+
rust_files.append((file, lang))
|
|
78
|
+
else:
|
|
79
|
+
fallback_files.append(file)
|
|
80
|
+
|
|
81
|
+
# Combined parse + pre-scan in one parallel pass (saves ~12% time)
|
|
82
|
+
imports_map = {}
|
|
83
|
+
if rust_files:
|
|
84
|
+
info_logger(f"Rust combined parse+prescan for {len(rust_files)} files...")
|
|
85
|
+
t_parse = time.time()
|
|
86
|
+
specs = [(str(f), lang, is_dependency) for f, lang in rust_files]
|
|
87
|
+
rust_results, imports_map = _rust_parse_and_prescan(specs)
|
|
88
|
+
info_logger(f"Rust parse+prescan done in {time.time() - t_parse:.1f}s "
|
|
89
|
+
f"({len(imports_map)} symbols)")
|
|
90
|
+
|
|
91
|
+
for file_data in rust_results:
|
|
92
|
+
if "error" not in file_data:
|
|
93
|
+
file_data["repo_path"] = str(repo_path_resolved)
|
|
94
|
+
writer.add_file_to_graph(file_data, repo_name, imports_map, repo_path_str=resolved_repo_path_str)
|
|
95
|
+
all_file_data.append(file_data)
|
|
96
|
+
|
|
97
|
+
if job_id:
|
|
98
|
+
job_manager.update_job(job_id, processed_files=len(rust_files))
|
|
99
|
+
|
|
100
|
+
# Fallback for unsupported files (.ipynb, Perl, etc.)
|
|
101
|
+
processed_count = len(rust_files)
|
|
102
|
+
for file in fallback_files:
|
|
103
|
+
if job_id:
|
|
104
|
+
job_manager.update_job(job_id, current_file=str(file))
|
|
105
|
+
file_data = parse_file(repo_path_resolved, file, is_dependency)
|
|
106
|
+
if "error" not in file_data:
|
|
107
|
+
writer.add_file_to_graph(file_data, repo_name, imports_map, repo_path_str=resolved_repo_path_str)
|
|
108
|
+
all_file_data.append(file_data)
|
|
109
|
+
elif not file_data.get("unsupported"):
|
|
110
|
+
add_minimal_file_node(file, repo_path_resolved, is_dependency)
|
|
111
|
+
processed_count += 1
|
|
112
|
+
|
|
113
|
+
if job_id:
|
|
114
|
+
job_manager.update_job(job_id, processed_files=processed_count)
|
|
115
|
+
if processed_count % 50 == 0:
|
|
116
|
+
await asyncio.sleep(0)
|
|
117
|
+
else:
|
|
118
|
+
# Original Python-only path: separate pre-scan
|
|
119
|
+
debug_log("Starting pre-scan to build imports map...")
|
|
120
|
+
imports_map = pre_scan_for_imports(files, parsers.keys(), get_parser)
|
|
121
|
+
debug_log(f"Pre-scan complete. Found {len(imports_map)} definitions.")
|
|
122
|
+
processed_count = 0
|
|
123
|
+
for file in files:
|
|
124
|
+
if not file.is_file():
|
|
125
|
+
continue
|
|
126
|
+
if job_id:
|
|
127
|
+
job_manager.update_job(job_id, current_file=str(file))
|
|
128
|
+
repo_path = path.resolve() if path.is_dir() else file.parent.resolve()
|
|
129
|
+
file_data = parse_file(repo_path, file, is_dependency)
|
|
130
|
+
if "error" not in file_data:
|
|
131
|
+
writer.add_file_to_graph(file_data, repo_name, imports_map, repo_path_str=resolved_repo_path_str)
|
|
132
|
+
all_file_data.append(file_data)
|
|
133
|
+
elif not file_data.get("unsupported"):
|
|
134
|
+
add_minimal_file_node(file, repo_path, is_dependency)
|
|
135
|
+
processed_count += 1
|
|
136
|
+
|
|
137
|
+
if job_id:
|
|
138
|
+
job_manager.update_job(job_id, processed_files=processed_count)
|
|
139
|
+
if processed_count % 50 == 0:
|
|
140
|
+
await asyncio.sleep(0)
|
|
141
|
+
|
|
142
|
+
info_logger(
|
|
143
|
+
f"File processing complete. {len(all_file_data)} files parsed. "
|
|
144
|
+
f"Starting post-processing phase (inheritance + function calls)..."
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
# --- Post-processing phase ---
|
|
148
|
+
t0 = time.time()
|
|
149
|
+
|
|
150
|
+
if RUST_AVAILABLE:
|
|
151
|
+
info_logger(f"[INHERITS] Rust-accelerated inheritance resolution ({len(all_file_data)} files)...")
|
|
152
|
+
inheritance_batch, csharp_files = _rust_resolve_inheritance(all_file_data, imports_map)
|
|
153
|
+
else:
|
|
154
|
+
info_logger(f"[INHERITS] Resolving inheritance links across {len(all_file_data)} files...")
|
|
155
|
+
inheritance_batch, csharp_files = build_inheritance_and_csharp_files(all_file_data, imports_map)
|
|
156
|
+
writer.write_inheritance_links(inheritance_batch, csharp_files, imports_map)
|
|
157
|
+
t1 = time.time()
|
|
158
|
+
info_logger(f"Inheritance links created in {t1 - t0:.1f}s. Starting function calls...")
|
|
159
|
+
|
|
160
|
+
if RUST_AVAILABLE:
|
|
161
|
+
from ...cli.config_manager import get_config_value
|
|
162
|
+
skip_external = (get_config_value("SKIP_EXTERNAL_RESOLUTION") or "false").lower() == "true"
|
|
163
|
+
groups = _rust_resolve_calls(all_file_data, imports_map, skip_external)
|
|
164
|
+
else:
|
|
165
|
+
groups = build_function_call_groups(all_file_data, imports_map, None)
|
|
166
|
+
writer.write_function_call_groups(*groups)
|
|
167
|
+
t2 = time.time()
|
|
168
|
+
info_logger(f"Function calls created in {t2 - t1:.1f}s. Total post-processing: {t2 - t0:.1f}s")
|
|
169
|
+
|
|
170
|
+
if job_id:
|
|
171
|
+
job_manager.update_job(job_id, status=JobStatus.COMPLETED, end_time=datetime.now())
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""Build global imports_map via language-specific pre-scan (registry dispatch)."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any, Callable, Dict, List, Optional
|
|
5
|
+
|
|
6
|
+
# (extension -> callable(files, get_parser_for_ext) -> dict updates)
|
|
7
|
+
_PreScanFn = Callable[[List[Path], Callable[[str], Any]], dict]
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _register_prescans() -> Dict[str, _PreScanFn]:
|
|
11
|
+
from ..languages import python as python_lang_module
|
|
12
|
+
from ..languages import javascript as js_lang_module
|
|
13
|
+
from ..languages import go as go_lang_module
|
|
14
|
+
from ..languages import typescript as ts_lang_module
|
|
15
|
+
from ..languages import typescriptjsx as tsx_lang_module
|
|
16
|
+
from ..languages import cpp as cpp_lang_module
|
|
17
|
+
from ..languages import rust as rust_lang_module
|
|
18
|
+
from ..languages import c as c_lang_module
|
|
19
|
+
from ..languages import java as java_lang_module
|
|
20
|
+
from ..languages import ruby as ruby_lang_module
|
|
21
|
+
from ..languages import csharp as csharp_lang_module
|
|
22
|
+
from ..languages import kotlin as kotlin_lang_module
|
|
23
|
+
from ..languages import scala as scala_lang_module
|
|
24
|
+
from ..languages import swift as swift_lang_module
|
|
25
|
+
from ..languages import dart as dart_lang_module
|
|
26
|
+
from ..languages import perl as perl_lang_module
|
|
27
|
+
from ..languages import php as php_lang_module
|
|
28
|
+
from ..languages import haskell as haskell_lang_module
|
|
29
|
+
from ..languages import elixir as elixir_lang_module
|
|
30
|
+
|
|
31
|
+
def make_py(ext: str) -> _PreScanFn:
|
|
32
|
+
def scan(files: List[Path], gp: Callable[[str], Any]) -> dict:
|
|
33
|
+
return python_lang_module.pre_scan_python(files, gp(ext))
|
|
34
|
+
|
|
35
|
+
return scan
|
|
36
|
+
|
|
37
|
+
def make_js(ext: str) -> _PreScanFn:
|
|
38
|
+
def scan(files: List[Path], gp: Callable[[str], Any]) -> dict:
|
|
39
|
+
return js_lang_module.pre_scan_javascript(files, gp(ext))
|
|
40
|
+
|
|
41
|
+
return scan
|
|
42
|
+
|
|
43
|
+
return {
|
|
44
|
+
".py": make_py(".py"),
|
|
45
|
+
".ipynb": make_py(".ipynb"),
|
|
46
|
+
".js": make_js(".js"),
|
|
47
|
+
".jsx": make_js(".jsx"),
|
|
48
|
+
".mjs": make_js(".mjs"),
|
|
49
|
+
".cjs": make_js(".cjs"),
|
|
50
|
+
".go": lambda files, gp: go_lang_module.pre_scan_go(files, gp(".go")),
|
|
51
|
+
".ts": lambda files, gp: ts_lang_module.pre_scan_typescript(files, gp(".ts")),
|
|
52
|
+
".tsx": lambda files, gp: tsx_lang_module.pre_scan_typescript(files, gp(".tsx")),
|
|
53
|
+
".cpp": lambda files, gp: cpp_lang_module.pre_scan_cpp(files, gp(".cpp")),
|
|
54
|
+
".h": lambda files, gp: cpp_lang_module.pre_scan_cpp(files, gp(".h")),
|
|
55
|
+
".hpp": lambda files, gp: cpp_lang_module.pre_scan_cpp(files, gp(".hpp")),
|
|
56
|
+
".hh": lambda files, gp: cpp_lang_module.pre_scan_cpp(files, gp(".hh")),
|
|
57
|
+
".rs": lambda files, gp: rust_lang_module.pre_scan_rust(files, gp(".rs")),
|
|
58
|
+
".c": lambda files, gp: c_lang_module.pre_scan_c(files, gp(".c")),
|
|
59
|
+
".java": lambda files, gp: java_lang_module.pre_scan_java(files, gp(".java")),
|
|
60
|
+
".rb": lambda files, gp: ruby_lang_module.pre_scan_ruby(files, gp(".rb")),
|
|
61
|
+
".cs": lambda files, gp: csharp_lang_module.pre_scan_csharp(files, gp(".cs")),
|
|
62
|
+
".kt": lambda files, gp: kotlin_lang_module.pre_scan_kotlin(files, gp(".kt")),
|
|
63
|
+
".scala": lambda files, gp: scala_lang_module.pre_scan_scala(files, gp(".scala")),
|
|
64
|
+
".sc": lambda files, gp: scala_lang_module.pre_scan_scala(files, gp(".sc")),
|
|
65
|
+
".swift": lambda files, gp: swift_lang_module.pre_scan_swift(files, gp(".swift")),
|
|
66
|
+
".dart": lambda files, gp: dart_lang_module.pre_scan_dart(files, gp(".dart")),
|
|
67
|
+
".pl": lambda files, gp: perl_lang_module.pre_scan_perl(files, gp(".pl")),
|
|
68
|
+
".pm": lambda files, gp: perl_lang_module.pre_scan_perl(files, gp(".pm")),
|
|
69
|
+
".php": lambda files, gp: php_lang_module.pre_scan_php(files, gp(".php")),
|
|
70
|
+
".hs": lambda files, gp: haskell_lang_module.pre_scan_haskell(files, gp(".hs")),
|
|
71
|
+
".ex": lambda files, gp: elixir_lang_module.pre_scan_elixir(files, gp(".ex")),
|
|
72
|
+
".exs": lambda files, gp: elixir_lang_module.pre_scan_elixir(files, gp(".exs")),
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
_PRESCAN_REGISTRY: Optional[Dict[str, _PreScanFn]] = None
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _get_registry() -> Dict[str, _PreScanFn]:
|
|
80
|
+
global _PRESCAN_REGISTRY
|
|
81
|
+
if _PRESCAN_REGISTRY is None:
|
|
82
|
+
_PRESCAN_REGISTRY = _register_prescans()
|
|
83
|
+
return _PRESCAN_REGISTRY
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def pre_scan_for_imports(
|
|
87
|
+
files: List[Path],
|
|
88
|
+
parsers_keys: Any,
|
|
89
|
+
get_parser: Callable[[str], Any],
|
|
90
|
+
) -> dict:
|
|
91
|
+
"""Dispatch pre-scan by file extension; *parsers_keys* is the set of supported extensions (e.g. graph_builder.parsers.keys())."""
|
|
92
|
+
imports_map: dict = {}
|
|
93
|
+
files_by_ext: Dict[str, List[Path]] = {}
|
|
94
|
+
for file in files:
|
|
95
|
+
if file.suffix in parsers_keys:
|
|
96
|
+
ext = file.suffix
|
|
97
|
+
files_by_ext.setdefault(ext, []).append(file)
|
|
98
|
+
|
|
99
|
+
registry = _get_registry()
|
|
100
|
+
for ext, file_list in files_by_ext.items():
|
|
101
|
+
scanner = registry.get(ext)
|
|
102
|
+
if scanner:
|
|
103
|
+
imports_map.update(scanner(file_list, get_parser))
|
|
104
|
+
|
|
105
|
+
return imports_map
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
"""Heuristic resolution of function calls into CALLS edge payloads (no DB I/O)."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
5
|
+
|
|
6
|
+
from ....cli.config_manager import get_config_value
|
|
7
|
+
from ....utils.debug_log import info_logger
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def resolve_function_call(
|
|
11
|
+
call: Dict[str, Any],
|
|
12
|
+
caller_file_path: str,
|
|
13
|
+
local_names: set,
|
|
14
|
+
local_imports: dict,
|
|
15
|
+
imports_map: dict,
|
|
16
|
+
skip_external: bool,
|
|
17
|
+
) -> Optional[Dict[str, Any]]:
|
|
18
|
+
"""Resolve a single function call to its target. Returns call params dict or None if skipped."""
|
|
19
|
+
called_name = call["name"]
|
|
20
|
+
if called_name in __builtins__:
|
|
21
|
+
return None
|
|
22
|
+
|
|
23
|
+
resolved_path = None
|
|
24
|
+
full_call = call.get("full_name", called_name)
|
|
25
|
+
base_obj = full_call.split(".")[0] if "." in full_call else None
|
|
26
|
+
|
|
27
|
+
is_chained_call = full_call.count(".") > 1 if "." in full_call else False
|
|
28
|
+
|
|
29
|
+
if is_chained_call and base_obj in ("self", "this", "super", "super()", "cls", "@"):
|
|
30
|
+
lookup_name = called_name
|
|
31
|
+
else:
|
|
32
|
+
lookup_name = base_obj if base_obj else called_name
|
|
33
|
+
|
|
34
|
+
if base_obj in ("self", "this", "super", "super()", "cls", "@") and not is_chained_call:
|
|
35
|
+
resolved_path = caller_file_path
|
|
36
|
+
elif lookup_name in local_names:
|
|
37
|
+
resolved_path = caller_file_path
|
|
38
|
+
elif call.get("inferred_obj_type"):
|
|
39
|
+
obj_type = call["inferred_obj_type"]
|
|
40
|
+
possible_paths = imports_map.get(obj_type, [])
|
|
41
|
+
if len(possible_paths) > 0:
|
|
42
|
+
resolved_path = possible_paths[0]
|
|
43
|
+
|
|
44
|
+
if not resolved_path:
|
|
45
|
+
possible_paths = imports_map.get(lookup_name, [])
|
|
46
|
+
if len(possible_paths) == 1:
|
|
47
|
+
resolved_path = possible_paths[0]
|
|
48
|
+
elif len(possible_paths) > 1:
|
|
49
|
+
if lookup_name in local_imports:
|
|
50
|
+
full_import_name = local_imports[lookup_name]
|
|
51
|
+
if full_import_name in imports_map:
|
|
52
|
+
direct_paths = imports_map[full_import_name]
|
|
53
|
+
if direct_paths and len(direct_paths) == 1:
|
|
54
|
+
resolved_path = direct_paths[0]
|
|
55
|
+
if not resolved_path:
|
|
56
|
+
for path in possible_paths:
|
|
57
|
+
if full_import_name.replace(".", "/") in path:
|
|
58
|
+
resolved_path = path
|
|
59
|
+
break
|
|
60
|
+
|
|
61
|
+
if not resolved_path:
|
|
62
|
+
is_unresolved_external = True
|
|
63
|
+
else:
|
|
64
|
+
is_unresolved_external = False
|
|
65
|
+
|
|
66
|
+
if not resolved_path:
|
|
67
|
+
possible_paths = imports_map.get(lookup_name, [])
|
|
68
|
+
if len(possible_paths) > 0:
|
|
69
|
+
if lookup_name in local_imports:
|
|
70
|
+
pass
|
|
71
|
+
else:
|
|
72
|
+
pass
|
|
73
|
+
if not resolved_path:
|
|
74
|
+
if called_name in local_names:
|
|
75
|
+
resolved_path = caller_file_path
|
|
76
|
+
is_unresolved_external = False
|
|
77
|
+
elif called_name in imports_map and imports_map[called_name]:
|
|
78
|
+
candidates = imports_map[called_name]
|
|
79
|
+
for path in candidates:
|
|
80
|
+
for imp_name in local_imports.values():
|
|
81
|
+
if imp_name.replace(".", "/") in path:
|
|
82
|
+
resolved_path = path
|
|
83
|
+
is_unresolved_external = False
|
|
84
|
+
break
|
|
85
|
+
if resolved_path:
|
|
86
|
+
break
|
|
87
|
+
if not resolved_path:
|
|
88
|
+
resolved_path = candidates[0]
|
|
89
|
+
else:
|
|
90
|
+
resolved_path = caller_file_path
|
|
91
|
+
|
|
92
|
+
if skip_external and is_unresolved_external:
|
|
93
|
+
return None
|
|
94
|
+
|
|
95
|
+
caller_context = call.get("context")
|
|
96
|
+
if caller_context and len(caller_context) == 3 and caller_context[0] is not None:
|
|
97
|
+
caller_name, _, caller_line_number = caller_context
|
|
98
|
+
return {
|
|
99
|
+
"type": "function",
|
|
100
|
+
"caller_name": caller_name,
|
|
101
|
+
"caller_file_path": caller_file_path,
|
|
102
|
+
"caller_line_number": caller_line_number,
|
|
103
|
+
"called_name": called_name,
|
|
104
|
+
"called_file_path": resolved_path,
|
|
105
|
+
"line_number": call["line_number"],
|
|
106
|
+
"args": call.get("args", []),
|
|
107
|
+
"full_call_name": call.get("full_name", called_name),
|
|
108
|
+
}
|
|
109
|
+
return {
|
|
110
|
+
"type": "file",
|
|
111
|
+
"caller_file_path": caller_file_path,
|
|
112
|
+
"called_name": called_name,
|
|
113
|
+
"called_file_path": resolved_path,
|
|
114
|
+
"line_number": call["line_number"],
|
|
115
|
+
"args": call.get("args", []),
|
|
116
|
+
"full_call_name": call.get("full_name", called_name),
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def build_function_call_groups(
|
|
121
|
+
all_file_data: List[Dict[str, Any]],
|
|
122
|
+
imports_map: dict,
|
|
123
|
+
file_class_lookup: Optional[Dict[str, set]] = None,
|
|
124
|
+
) -> Tuple[
|
|
125
|
+
List[Dict],
|
|
126
|
+
List[Dict],
|
|
127
|
+
List[Dict],
|
|
128
|
+
List[Dict],
|
|
129
|
+
List[Dict],
|
|
130
|
+
List[Dict],
|
|
131
|
+
]:
|
|
132
|
+
"""Phase 1 of CALLS linking: resolve and bucket by (caller_label, called_label) pair."""
|
|
133
|
+
skip_external = (get_config_value("SKIP_EXTERNAL_RESOLUTION") or "false").lower() == "true"
|
|
134
|
+
|
|
135
|
+
if file_class_lookup is None:
|
|
136
|
+
file_class_lookup = {}
|
|
137
|
+
for fd in all_file_data:
|
|
138
|
+
fp = str(Path(fd["path"]).resolve())
|
|
139
|
+
file_class_lookup[fp] = {c["name"] for c in fd.get("classes", [])}
|
|
140
|
+
|
|
141
|
+
info_logger(f"[CALLS] Resolving function calls across {len(all_file_data)} files...")
|
|
142
|
+
fn_to_fn: List[Dict] = []
|
|
143
|
+
fn_to_cls: List[Dict] = []
|
|
144
|
+
cls_to_fn: List[Dict] = []
|
|
145
|
+
cls_to_cls: List[Dict] = []
|
|
146
|
+
file_to_fn: List[Dict] = []
|
|
147
|
+
file_to_cls: List[Dict] = []
|
|
148
|
+
|
|
149
|
+
# Pre-build per-language-extension filtered imports_map views.
|
|
150
|
+
# When a caller is Java, we only want to resolve against Java files —
|
|
151
|
+
# this prevents false-positive CALLS edges from names that coincidentally
|
|
152
|
+
# exist in another language's file (e.g. Java `add()` -> JS `add`).
|
|
153
|
+
_lang_imports_cache: Dict[str, dict] = {}
|
|
154
|
+
|
|
155
|
+
def _get_lang_imports(caller_lang: str) -> dict:
|
|
156
|
+
if caller_lang not in _lang_imports_cache:
|
|
157
|
+
# Map language name to typical file extensions
|
|
158
|
+
_LANG_EXTS: Dict[str, set] = {
|
|
159
|
+
"java": {".java"},
|
|
160
|
+
"python": {".py", ".ipynb"},
|
|
161
|
+
"javascript": {".js", ".jsx", ".mjs", ".cjs"},
|
|
162
|
+
"typescript": {".ts", ".tsx"},
|
|
163
|
+
"go": {".go"},
|
|
164
|
+
"rust": {".rs"},
|
|
165
|
+
"cpp": {".cpp", ".h", ".hpp", ".hh"},
|
|
166
|
+
"c": {".c"},
|
|
167
|
+
"c_sharp": {".cs"},
|
|
168
|
+
"kotlin": {".kt"},
|
|
169
|
+
"scala": {".scala", ".sc"},
|
|
170
|
+
"ruby": {".rb"},
|
|
171
|
+
"swift": {".swift"},
|
|
172
|
+
"php": {".php"},
|
|
173
|
+
"dart": {".dart"},
|
|
174
|
+
"perl": {".pl", ".pm"},
|
|
175
|
+
"haskell": {".hs"},
|
|
176
|
+
"elixir": {".ex", ".exs"},
|
|
177
|
+
}
|
|
178
|
+
exts = _LANG_EXTS.get(caller_lang)
|
|
179
|
+
if not exts:
|
|
180
|
+
# Unknown language — use full imports_map unchanged
|
|
181
|
+
_lang_imports_cache[caller_lang] = imports_map
|
|
182
|
+
else:
|
|
183
|
+
filtered: dict = {}
|
|
184
|
+
for name, paths in imports_map.items():
|
|
185
|
+
same_lang = [p for p in paths if Path(p).suffix in exts]
|
|
186
|
+
if same_lang:
|
|
187
|
+
filtered[name] = same_lang
|
|
188
|
+
elif paths:
|
|
189
|
+
# Keep non-file entries (e.g. package names with no extension)
|
|
190
|
+
if not any(Path(p).suffix for p in paths):
|
|
191
|
+
filtered[name] = paths
|
|
192
|
+
_lang_imports_cache[caller_lang] = filtered
|
|
193
|
+
return _lang_imports_cache[caller_lang]
|
|
194
|
+
|
|
195
|
+
for idx, file_data in enumerate(all_file_data):
|
|
196
|
+
caller_file_path = str(Path(file_data["path"]).resolve())
|
|
197
|
+
func_names = {f["name"] for f in file_data.get("functions", [])}
|
|
198
|
+
class_names = {c["name"] for c in file_data.get("classes", [])}
|
|
199
|
+
local_names = func_names | class_names
|
|
200
|
+
local_imports = {
|
|
201
|
+
imp.get("alias") or imp["name"].split(".")[-1]: imp["name"]
|
|
202
|
+
for imp in file_data.get("imports", [])
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
caller_lang = file_data.get("lang", "")
|
|
206
|
+
effective_imports_map = _get_lang_imports(caller_lang) if caller_lang else imports_map
|
|
207
|
+
|
|
208
|
+
for call in file_data.get("function_calls", []):
|
|
209
|
+
resolved = resolve_function_call(
|
|
210
|
+
call, caller_file_path, local_names, local_imports, effective_imports_map, skip_external
|
|
211
|
+
)
|
|
212
|
+
if not resolved:
|
|
213
|
+
continue
|
|
214
|
+
|
|
215
|
+
called_path = resolved.get("called_file_path", "")
|
|
216
|
+
called_name = resolved["called_name"]
|
|
217
|
+
called_is_class = called_name in file_class_lookup.get(called_path, set())
|
|
218
|
+
|
|
219
|
+
if resolved["type"] == "file":
|
|
220
|
+
if called_is_class:
|
|
221
|
+
file_to_cls.append(resolved)
|
|
222
|
+
else:
|
|
223
|
+
file_to_fn.append(resolved)
|
|
224
|
+
else:
|
|
225
|
+
caller_name = resolved["caller_name"]
|
|
226
|
+
caller_is_class = caller_name in class_names
|
|
227
|
+
if caller_is_class:
|
|
228
|
+
(cls_to_cls if called_is_class else cls_to_fn).append(resolved)
|
|
229
|
+
else:
|
|
230
|
+
(fn_to_cls if called_is_class else fn_to_fn).append(resolved)
|
|
231
|
+
|
|
232
|
+
if (idx + 1) % 1000 == 0:
|
|
233
|
+
total = len(fn_to_fn) + len(fn_to_cls) + len(cls_to_fn) + len(cls_to_cls)
|
|
234
|
+
file_total = len(file_to_fn) + len(file_to_cls)
|
|
235
|
+
info_logger(
|
|
236
|
+
f"[CALLS] Resolved {idx + 1}/{len(all_file_data)} files... "
|
|
237
|
+
f"({total} fn/cls calls, {file_total} file calls)"
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
total_all = (
|
|
241
|
+
len(fn_to_fn)
|
|
242
|
+
+ len(fn_to_cls)
|
|
243
|
+
+ len(cls_to_fn)
|
|
244
|
+
+ len(cls_to_cls)
|
|
245
|
+
+ len(file_to_fn)
|
|
246
|
+
+ len(file_to_cls)
|
|
247
|
+
)
|
|
248
|
+
info_logger(
|
|
249
|
+
f"[CALLS] Resolution complete: fn→fn={len(fn_to_fn)}, fn→cls={len(fn_to_cls)}, "
|
|
250
|
+
f"cls→fn={len(cls_to_fn)}, cls→cls={len(cls_to_cls)}, "
|
|
251
|
+
f"file→fn={len(file_to_fn)}, file→cls={len(file_to_cls)}. Total={total_all}"
|
|
252
|
+
)
|
|
253
|
+
return fn_to_fn, fn_to_cls, cls_to_fn, cls_to_cls, file_to_fn, file_to_cls
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""Resolve class inheritance into INHERITS row payloads (no DB I/O for non-C# batch)."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def resolve_inheritance_link(
|
|
8
|
+
class_item: Dict[str, Any],
|
|
9
|
+
base_class_str: str,
|
|
10
|
+
caller_file_path: str,
|
|
11
|
+
local_class_names: set,
|
|
12
|
+
local_imports: dict,
|
|
13
|
+
imports_map: dict,
|
|
14
|
+
) -> Optional[Dict[str, Any]]:
|
|
15
|
+
"""Resolve a single inheritance link. Returns row dict or None."""
|
|
16
|
+
if base_class_str == "object":
|
|
17
|
+
return None
|
|
18
|
+
|
|
19
|
+
resolved_path = None
|
|
20
|
+
target_class_name = base_class_str.split(".")[-1]
|
|
21
|
+
|
|
22
|
+
if "." in base_class_str:
|
|
23
|
+
lookup_name = base_class_str.split(".")[0]
|
|
24
|
+
if lookup_name in local_imports:
|
|
25
|
+
full_import_name = local_imports[lookup_name]
|
|
26
|
+
possible_paths = imports_map.get(target_class_name, [])
|
|
27
|
+
for path in possible_paths:
|
|
28
|
+
if full_import_name.replace(".", "/") in path:
|
|
29
|
+
resolved_path = path
|
|
30
|
+
break
|
|
31
|
+
else:
|
|
32
|
+
lookup_name = base_class_str
|
|
33
|
+
if lookup_name in local_class_names:
|
|
34
|
+
resolved_path = caller_file_path
|
|
35
|
+
elif lookup_name in local_imports:
|
|
36
|
+
full_import_name = local_imports[lookup_name]
|
|
37
|
+
possible_paths = imports_map.get(target_class_name, [])
|
|
38
|
+
for path in possible_paths:
|
|
39
|
+
if full_import_name.replace(".", "/") in path:
|
|
40
|
+
resolved_path = path
|
|
41
|
+
break
|
|
42
|
+
elif lookup_name in imports_map:
|
|
43
|
+
possible_paths = imports_map[lookup_name]
|
|
44
|
+
if len(possible_paths) == 1:
|
|
45
|
+
resolved_path = possible_paths[0]
|
|
46
|
+
|
|
47
|
+
if resolved_path:
|
|
48
|
+
return {
|
|
49
|
+
"child_name": class_item["name"],
|
|
50
|
+
"path": caller_file_path,
|
|
51
|
+
"parent_name": target_class_name,
|
|
52
|
+
"resolved_parent_file_path": resolved_path,
|
|
53
|
+
}
|
|
54
|
+
return None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def build_inheritance_and_csharp_files(
|
|
58
|
+
all_file_data: List[Dict[str, Any]], imports_map: dict
|
|
59
|
+
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
|
|
60
|
+
"""Returns (inheritance_batch_rows, csharp_file_data_list)."""
|
|
61
|
+
inheritance_batch: List[Dict[str, Any]] = []
|
|
62
|
+
csharp_files: List[Dict[str, Any]] = []
|
|
63
|
+
|
|
64
|
+
for file_data in all_file_data:
|
|
65
|
+
if file_data.get("lang") == "c_sharp":
|
|
66
|
+
csharp_files.append(file_data)
|
|
67
|
+
continue
|
|
68
|
+
|
|
69
|
+
caller_file_path = str(Path(file_data["path"]).resolve())
|
|
70
|
+
local_class_names = {c["name"] for c in file_data.get("classes", [])}
|
|
71
|
+
local_imports = {
|
|
72
|
+
imp.get("alias") or imp["name"].split(".")[-1]: imp["name"]
|
|
73
|
+
for imp in file_data.get("imports", [])
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
for class_item in file_data.get("classes", []):
|
|
77
|
+
if not class_item.get("bases"):
|
|
78
|
+
continue
|
|
79
|
+
for base_class_str in class_item["bases"]:
|
|
80
|
+
resolved = resolve_inheritance_link(
|
|
81
|
+
class_item,
|
|
82
|
+
base_class_str,
|
|
83
|
+
caller_file_path,
|
|
84
|
+
local_class_names,
|
|
85
|
+
local_imports,
|
|
86
|
+
imports_map,
|
|
87
|
+
)
|
|
88
|
+
if resolved:
|
|
89
|
+
inheritance_batch.append(resolved)
|
|
90
|
+
|
|
91
|
+
return inheritance_batch, csharp_files
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Property coercion for graph backends (shared helpers for dialects / writers)."""
|
|
2
|
+
|
|
3
|
+
from typing import Any, Dict
|
|
4
|
+
|
|
5
|
+
# Neo4j RANGE indexes have an ~8 kB key-size limit. Long C++ template function names
|
|
6
|
+
# can exceed this. Cap string properties at 4096 chars.
|
|
7
|
+
MAX_STR_LEN = 4096
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def sanitize_props(props: Dict[str, Any]) -> Dict[str, Any]:
|
|
11
|
+
"""Return a copy of *props* with values coerced to database-safe types.
|
|
12
|
+
|
|
13
|
+
FalkorDB and KùzuDB only accept node properties that are primitives
|
|
14
|
+
(str, int, float, bool, None) or flat lists of primitives. Complex
|
|
15
|
+
values are serialized to JSON. Strings are truncated to MAX_STR_LEN.
|
|
16
|
+
"""
|
|
17
|
+
import json
|
|
18
|
+
|
|
19
|
+
MAX = MAX_STR_LEN
|
|
20
|
+
|
|
21
|
+
def _is_primitive(v):
|
|
22
|
+
return isinstance(v, (str, int, float, bool)) or v is None
|
|
23
|
+
|
|
24
|
+
def _is_flat_list(v):
|
|
25
|
+
return isinstance(v, list) and all(_is_primitive(item) for item in v)
|
|
26
|
+
|
|
27
|
+
def _coerce(v):
|
|
28
|
+
if isinstance(v, str):
|
|
29
|
+
return v[:MAX] if len(v) > MAX else v
|
|
30
|
+
if _is_primitive(v):
|
|
31
|
+
return v
|
|
32
|
+
if _is_flat_list(v):
|
|
33
|
+
return [s[:MAX] if isinstance(s, str) and len(s) > MAX else s for s in v]
|
|
34
|
+
try:
|
|
35
|
+
serialized = json.dumps(v, default=str)
|
|
36
|
+
return serialized[:MAX] if len(serialized) > MAX else serialized
|
|
37
|
+
except Exception:
|
|
38
|
+
s = str(v)
|
|
39
|
+
return s[:MAX] if len(s) > MAX else s
|
|
40
|
+
|
|
41
|
+
return {k: _coerce(v) for k, v in props.items()}
|