codegraphcontext-rust 0.5.5__cp311-cp311-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codegraphcontext/__init__.py +2 -0
- codegraphcontext/__main__.py +5 -0
- codegraphcontext/_cgc_rust.cp311-win_amd64.pyd +0 -0
- codegraphcontext/cli/__init__.py +1 -0
- codegraphcontext/cli/cli_helpers.py +774 -0
- codegraphcontext/cli/config_manager.py +1051 -0
- codegraphcontext/cli/main.py +2386 -0
- codegraphcontext/cli/registry_commands.py +404 -0
- codegraphcontext/cli/setup_macos.py +93 -0
- codegraphcontext/cli/setup_wizard.py +991 -0
- codegraphcontext/cli/visualizer.py +50 -0
- codegraphcontext/core/__init__.py +165 -0
- codegraphcontext/core/bundle_registry.py +181 -0
- codegraphcontext/core/cgc_bundle.py +858 -0
- codegraphcontext/core/cgcignore.py +118 -0
- codegraphcontext/core/database.py +273 -0
- codegraphcontext/core/database_falkordb.py +480 -0
- codegraphcontext/core/database_falkordb_remote.py +199 -0
- codegraphcontext/core/database_kuzu.py +636 -0
- codegraphcontext/core/falkor_worker.py +132 -0
- codegraphcontext/core/jobs.py +132 -0
- codegraphcontext/core/watcher.py +260 -0
- codegraphcontext/prompts.py +125 -0
- codegraphcontext/server.py +471 -0
- codegraphcontext/tool_definitions.py +220 -0
- codegraphcontext/tools/__init__.py +0 -0
- codegraphcontext/tools/advanced_language_query_tool.py +103 -0
- codegraphcontext/tools/code_finder.py +1160 -0
- codegraphcontext/tools/graph_builder.py +337 -0
- codegraphcontext/tools/handlers/analysis_handlers.py +114 -0
- codegraphcontext/tools/handlers/indexing_handlers.py +116 -0
- codegraphcontext/tools/handlers/management_handlers.py +339 -0
- codegraphcontext/tools/handlers/query_handlers.py +83 -0
- codegraphcontext/tools/handlers/watcher_handlers.py +83 -0
- codegraphcontext/tools/indexing/__init__.py +1 -0
- codegraphcontext/tools/indexing/constants.py +25 -0
- codegraphcontext/tools/indexing/discovery.py +64 -0
- codegraphcontext/tools/indexing/engine.py +109 -0
- codegraphcontext/tools/indexing/persistence/__init__.py +3 -0
- codegraphcontext/tools/indexing/persistence/parquet_loader.py +643 -0
- codegraphcontext/tools/indexing/persistence/writer.py +1114 -0
- codegraphcontext/tools/indexing/pipeline.py +171 -0
- codegraphcontext/tools/indexing/pre_scan.py +105 -0
- codegraphcontext/tools/indexing/resolution/__init__.py +8 -0
- codegraphcontext/tools/indexing/resolution/calls.py +253 -0
- codegraphcontext/tools/indexing/resolution/inheritance.py +91 -0
- codegraphcontext/tools/indexing/sanitize.py +41 -0
- codegraphcontext/tools/indexing/schema.py +79 -0
- codegraphcontext/tools/indexing/schema_contract.py +44 -0
- codegraphcontext/tools/indexing/scip_pipeline.py +140 -0
- codegraphcontext/tools/languages/c.py +563 -0
- codegraphcontext/tools/languages/cpp.py +616 -0
- codegraphcontext/tools/languages/csharp.py +551 -0
- codegraphcontext/tools/languages/dart.py +378 -0
- codegraphcontext/tools/languages/elixir.py +460 -0
- codegraphcontext/tools/languages/go.py +499 -0
- codegraphcontext/tools/languages/haskell.py +426 -0
- codegraphcontext/tools/languages/java.py +500 -0
- codegraphcontext/tools/languages/javascript.py +589 -0
- codegraphcontext/tools/languages/kotlin.py +640 -0
- codegraphcontext/tools/languages/perl.py +260 -0
- codegraphcontext/tools/languages/php.py +519 -0
- codegraphcontext/tools/languages/python.py +576 -0
- codegraphcontext/tools/languages/ruby.py +537 -0
- codegraphcontext/tools/languages/rust.py +295 -0
- codegraphcontext/tools/languages/scala.py +519 -0
- codegraphcontext/tools/languages/swift.py +491 -0
- codegraphcontext/tools/languages/typescript.py +575 -0
- codegraphcontext/tools/languages/typescriptjsx.py +151 -0
- codegraphcontext/tools/package_resolver.py +472 -0
- codegraphcontext/tools/query_tool_languages/c_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/cpp_toolkit.py +87 -0
- codegraphcontext/tools/query_tool_languages/csharp_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/dart_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/go_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/haskell_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/java_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/javascript_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/perl_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/python_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/ruby_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/rust_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/scala_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/swift_toolkit.py +5 -0
- codegraphcontext/tools/query_tool_languages/typescript_toolkit.py +5 -0
- codegraphcontext/tools/scip_indexer.py +467 -0
- codegraphcontext/tools/scip_pb2.py +2455 -0
- codegraphcontext/tools/system.py +133 -0
- codegraphcontext/tools/tree_sitter_parser.py +103 -0
- codegraphcontext/utils/debug_log.py +90 -0
- codegraphcontext/utils/path_ignore.py +55 -0
- codegraphcontext/utils/repo_path.py +27 -0
- codegraphcontext/utils/tree_sitter_manager.py +285 -0
- codegraphcontext/utils/visualize_graph.py +0 -0
- codegraphcontext/viz/server.py +282 -0
- codegraphcontext_rust-0.5.5.dist-info/METADATA +537 -0
- codegraphcontext_rust-0.5.5.dist-info/RECORD +101 -0
- codegraphcontext_rust-0.5.5.dist-info/WHEEL +4 -0
- codegraphcontext_rust-0.5.5.dist-info/entry_points.txt +3 -0
- codegraphcontext_rust-0.5.5.dist-info/licenses/LICENSE +21 -0
- codegraphcontext_rust-0.5.5.dist-info/sboms/cgc-python.cyclonedx.json +2333 -0
|
@@ -0,0 +1,1114 @@
|
|
|
1
|
+
"""All graph DB writes for indexing (single persistence entry point)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, Callable, Dict, List, Optional, Tuple
|
|
8
|
+
|
|
9
|
+
from ....utils.debug_log import info_logger, warning_logger
|
|
10
|
+
from ..sanitize import sanitize_props
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class GraphWriter:
|
|
14
|
+
"""Persists repository/file/symbol nodes and relationships via the Neo4j-like driver API."""
|
|
15
|
+
|
|
16
|
+
def __init__(self, driver: Any):
|
|
17
|
+
self.driver = driver
|
|
18
|
+
|
|
19
|
+
def add_repository_to_graph(self, repo_path: Path, is_dependency: bool = False) -> None:
|
|
20
|
+
repo_name = repo_path.name
|
|
21
|
+
repo_path_str = str(repo_path.resolve())
|
|
22
|
+
with self.driver.session() as session:
|
|
23
|
+
session.run(
|
|
24
|
+
"""
|
|
25
|
+
MERGE (r:Repository {path: $path})
|
|
26
|
+
SET r.name = $name, r.is_dependency = $is_dependency
|
|
27
|
+
""",
|
|
28
|
+
path=repo_path_str,
|
|
29
|
+
name=repo_name,
|
|
30
|
+
is_dependency=is_dependency,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
def add_file_to_graph(
|
|
34
|
+
self,
|
|
35
|
+
file_data: Dict[str, Any],
|
|
36
|
+
repo_name: str,
|
|
37
|
+
imports_map: dict,
|
|
38
|
+
repo_path_str: Optional[str] = None,
|
|
39
|
+
) -> None:
|
|
40
|
+
file_path_str = str(Path(file_data["path"]).resolve())
|
|
41
|
+
file_name = Path(file_path_str).name
|
|
42
|
+
is_dependency = file_data.get("is_dependency", False)
|
|
43
|
+
lang = file_data.get("lang")
|
|
44
|
+
|
|
45
|
+
with self.driver.session() as session:
|
|
46
|
+
if repo_path_str:
|
|
47
|
+
resolved_repo_str = repo_path_str
|
|
48
|
+
else:
|
|
49
|
+
repo_result = session.run(
|
|
50
|
+
"MATCH (r:Repository {path: $repo_path}) RETURN r.path as path",
|
|
51
|
+
repo_path=str(Path(file_data["repo_path"]).resolve()),
|
|
52
|
+
).single()
|
|
53
|
+
resolved_repo_str = (
|
|
54
|
+
repo_result["path"] if repo_result else str(Path(file_data["repo_path"]).resolve())
|
|
55
|
+
)
|
|
56
|
+
if not repo_result:
|
|
57
|
+
warning_logger(
|
|
58
|
+
f"Repository node not found for {file_data['repo_path']} during indexing of {file_name}."
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
try:
|
|
62
|
+
relative_path = str(Path(file_path_str).relative_to(Path(resolved_repo_str)))
|
|
63
|
+
except ValueError:
|
|
64
|
+
relative_path = file_name
|
|
65
|
+
|
|
66
|
+
session.run(
|
|
67
|
+
"""
|
|
68
|
+
MERGE (f:File {path: $path})
|
|
69
|
+
SET f.name = $name, f.relative_path = $relative_path, f.is_dependency = $is_dependency
|
|
70
|
+
""",
|
|
71
|
+
path=file_path_str,
|
|
72
|
+
name=file_name,
|
|
73
|
+
relative_path=relative_path,
|
|
74
|
+
is_dependency=is_dependency,
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
file_path_obj = Path(file_path_str)
|
|
78
|
+
repo_path_obj = Path(resolved_repo_str)
|
|
79
|
+
relative_path_to_file = file_path_obj.relative_to(repo_path_obj)
|
|
80
|
+
parent_path = resolved_repo_str
|
|
81
|
+
parent_label = "Repository"
|
|
82
|
+
for part in relative_path_to_file.parts[:-1]:
|
|
83
|
+
current_path_str = str(Path(parent_path) / part)
|
|
84
|
+
session.run(
|
|
85
|
+
f"""
|
|
86
|
+
MATCH (p:{parent_label} {{path: $parent_path}})
|
|
87
|
+
MERGE (d:Directory {{path: $current_path}})
|
|
88
|
+
SET d.name = $part
|
|
89
|
+
MERGE (p)-[:CONTAINS]->(d)
|
|
90
|
+
""",
|
|
91
|
+
parent_path=parent_path,
|
|
92
|
+
current_path=current_path_str,
|
|
93
|
+
part=part,
|
|
94
|
+
)
|
|
95
|
+
parent_path = current_path_str
|
|
96
|
+
parent_label = "Directory"
|
|
97
|
+
session.run(
|
|
98
|
+
f"""
|
|
99
|
+
MATCH (p:{parent_label} {{path: $parent_path}})
|
|
100
|
+
MATCH (f:File {{path: $path}})
|
|
101
|
+
MERGE (p)-[:CONTAINS]->(f)
|
|
102
|
+
""",
|
|
103
|
+
parent_path=parent_path,
|
|
104
|
+
path=file_path_str,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
item_mappings = [
|
|
108
|
+
(file_data.get("functions", []), "Function"),
|
|
109
|
+
(file_data.get("classes", []), "Class"),
|
|
110
|
+
(file_data.get("traits", []), "Trait"),
|
|
111
|
+
(file_data.get("variables", []), "Variable"),
|
|
112
|
+
(file_data.get("interfaces", []), "Interface"),
|
|
113
|
+
(file_data.get("macros", []), "Macro"),
|
|
114
|
+
(file_data.get("structs", []), "Struct"),
|
|
115
|
+
(file_data.get("enums", []), "Enum"),
|
|
116
|
+
(file_data.get("unions", []), "Union"),
|
|
117
|
+
(file_data.get("records", []), "Record"),
|
|
118
|
+
(file_data.get("properties", []), "Property"),
|
|
119
|
+
]
|
|
120
|
+
params_batch: List[Dict[str, Any]] = []
|
|
121
|
+
class_fn_batch: List[Dict[str, Any]] = []
|
|
122
|
+
nested_fn_batch: List[Dict[str, Any]] = []
|
|
123
|
+
|
|
124
|
+
for item_list, label in item_mappings:
|
|
125
|
+
if not item_list:
|
|
126
|
+
continue
|
|
127
|
+
batch: List[Dict[str, Any]] = []
|
|
128
|
+
for item in item_list:
|
|
129
|
+
row = dict(item)
|
|
130
|
+
if label == "Function" and "cyclomatic_complexity" not in row:
|
|
131
|
+
row["cyclomatic_complexity"] = 1
|
|
132
|
+
batch.append(sanitize_props(row))
|
|
133
|
+
if label == "Function":
|
|
134
|
+
for arg_name in item.get("args", []):
|
|
135
|
+
params_batch.append(
|
|
136
|
+
{
|
|
137
|
+
"func_name": item["name"],
|
|
138
|
+
"line_number": item["line_number"],
|
|
139
|
+
"arg_name": arg_name,
|
|
140
|
+
}
|
|
141
|
+
)
|
|
142
|
+
if item.get("class_context"):
|
|
143
|
+
class_fn_batch.append(
|
|
144
|
+
{
|
|
145
|
+
"class_name": item["class_context"],
|
|
146
|
+
"func_name": item["name"],
|
|
147
|
+
"func_line": item["line_number"],
|
|
148
|
+
}
|
|
149
|
+
)
|
|
150
|
+
if item.get("context_type") == "function_definition":
|
|
151
|
+
nested_fn_batch.append(
|
|
152
|
+
{
|
|
153
|
+
"outer": item["context"],
|
|
154
|
+
"inner_name": item["name"],
|
|
155
|
+
"inner_line": item["line_number"],
|
|
156
|
+
}
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
if batch:
|
|
160
|
+
import json as _json
|
|
161
|
+
|
|
162
|
+
all_keys = set()
|
|
163
|
+
for b in batch:
|
|
164
|
+
all_keys.update(b.keys())
|
|
165
|
+
|
|
166
|
+
for k in all_keys:
|
|
167
|
+
counts: Dict[str, int] = {}
|
|
168
|
+
for b in batch:
|
|
169
|
+
v = b.get(k)
|
|
170
|
+
if v is not None:
|
|
171
|
+
tname = type(v).__name__
|
|
172
|
+
counts[tname] = counts.get(tname, 0) + 1
|
|
173
|
+
|
|
174
|
+
dominant = max(counts, key=counts.get) if counts else "str"
|
|
175
|
+
|
|
176
|
+
for b in batch:
|
|
177
|
+
v = b.get(k)
|
|
178
|
+
if dominant == "list":
|
|
179
|
+
if isinstance(v, list):
|
|
180
|
+
b[k] = [str(x) for x in v] if v else [""]
|
|
181
|
+
elif isinstance(v, str) and v:
|
|
182
|
+
try:
|
|
183
|
+
p = _json.loads(v)
|
|
184
|
+
b[k] = [str(x) for x in p] if isinstance(p, list) and p else [""]
|
|
185
|
+
except Exception:
|
|
186
|
+
b[k] = [v]
|
|
187
|
+
else:
|
|
188
|
+
b[k] = [""]
|
|
189
|
+
elif dominant == "int":
|
|
190
|
+
if v is None or v == "":
|
|
191
|
+
b[k] = 0
|
|
192
|
+
elif not isinstance(v, int):
|
|
193
|
+
try:
|
|
194
|
+
b[k] = int(v)
|
|
195
|
+
except Exception:
|
|
196
|
+
b[k] = 0
|
|
197
|
+
elif dominant == "bool":
|
|
198
|
+
b[k] = bool(v) if v is not None else False
|
|
199
|
+
else:
|
|
200
|
+
if v is None:
|
|
201
|
+
b[k] = ""
|
|
202
|
+
elif isinstance(v, list):
|
|
203
|
+
b[k] = _json.dumps(v)
|
|
204
|
+
elif not isinstance(v, str):
|
|
205
|
+
b[k] = str(v)
|
|
206
|
+
|
|
207
|
+
key_order = sorted(all_keys)
|
|
208
|
+
batch[:] = [{k: b[k] for k in key_order} for b in batch]
|
|
209
|
+
|
|
210
|
+
session.run(
|
|
211
|
+
f"""
|
|
212
|
+
UNWIND $batch AS row
|
|
213
|
+
MERGE (n:{label} {{name: row.name, path: $file_path, line_number: row.line_number}})
|
|
214
|
+
SET n += row
|
|
215
|
+
""",
|
|
216
|
+
batch=batch,
|
|
217
|
+
file_path=file_path_str,
|
|
218
|
+
)
|
|
219
|
+
session.run(
|
|
220
|
+
f"""
|
|
221
|
+
UNWIND $batch AS row
|
|
222
|
+
MATCH (f:File {{path: $file_path}})
|
|
223
|
+
MATCH (n:{label} {{name: row.name, path: $file_path, line_number: row.line_number}})
|
|
224
|
+
MERGE (f)-[:CONTAINS]->(n)
|
|
225
|
+
""",
|
|
226
|
+
batch=batch,
|
|
227
|
+
file_path=file_path_str,
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
if params_batch:
|
|
231
|
+
session.run(
|
|
232
|
+
"""
|
|
233
|
+
UNWIND $batch AS row
|
|
234
|
+
MATCH (fn:Function {name: row.func_name, path: $file_path, line_number: row.line_number})
|
|
235
|
+
MERGE (p:Parameter {name: row.arg_name, path: $file_path, function_line_number: row.line_number})
|
|
236
|
+
MERGE (fn)-[:HAS_PARAMETER]->(p)
|
|
237
|
+
""",
|
|
238
|
+
batch=params_batch,
|
|
239
|
+
file_path=file_path_str,
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
if class_fn_batch:
|
|
243
|
+
session.run(
|
|
244
|
+
"""
|
|
245
|
+
UNWIND $batch AS row
|
|
246
|
+
MATCH (c:Class {name: row.class_name, path: $file_path})
|
|
247
|
+
MATCH (fn:Function {name: row.func_name, path: $file_path, line_number: row.func_line})
|
|
248
|
+
MERGE (c)-[:CONTAINS]->(fn)
|
|
249
|
+
""",
|
|
250
|
+
batch=class_fn_batch,
|
|
251
|
+
file_path=file_path_str,
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
if nested_fn_batch:
|
|
255
|
+
session.run(
|
|
256
|
+
"""
|
|
257
|
+
UNWIND $batch AS row
|
|
258
|
+
MATCH (outer:Function {name: row.outer, path: $file_path})
|
|
259
|
+
MATCH (inner:Function {name: row.inner_name, path: $file_path, line_number: row.inner_line})
|
|
260
|
+
MERGE (outer)-[:CONTAINS]->(inner)
|
|
261
|
+
""",
|
|
262
|
+
batch=nested_fn_batch,
|
|
263
|
+
file_path=file_path_str,
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
ruby_modules = file_data.get("modules", [])
|
|
267
|
+
if ruby_modules:
|
|
268
|
+
session.run(
|
|
269
|
+
"""
|
|
270
|
+
UNWIND $batch AS row
|
|
271
|
+
MERGE (mod:Module {name: row.name})
|
|
272
|
+
ON CREATE SET mod.lang = row.lang
|
|
273
|
+
ON MATCH SET mod.lang = coalesce(mod.lang, row.lang)
|
|
274
|
+
""",
|
|
275
|
+
batch=[{"name": m["name"], "lang": lang} for m in ruby_modules],
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
js_imports = []
|
|
279
|
+
other_imports = []
|
|
280
|
+
for imp in file_data.get("imports", []):
|
|
281
|
+
if lang == "javascript":
|
|
282
|
+
module_name = imp.get("source")
|
|
283
|
+
if module_name:
|
|
284
|
+
js_imports.append(
|
|
285
|
+
{
|
|
286
|
+
"module_name": module_name,
|
|
287
|
+
"imported_name": imp.get("name", "*"),
|
|
288
|
+
"alias": imp.get("alias"),
|
|
289
|
+
"line_number": imp.get("line_number"),
|
|
290
|
+
}
|
|
291
|
+
)
|
|
292
|
+
else:
|
|
293
|
+
other_imports.append(imp)
|
|
294
|
+
|
|
295
|
+
if js_imports:
|
|
296
|
+
session.run(
|
|
297
|
+
"""
|
|
298
|
+
UNWIND $batch AS row
|
|
299
|
+
MATCH (f:File {path: $file_path})
|
|
300
|
+
MERGE (m:Module {name: row.module_name})
|
|
301
|
+
MERGE (f)-[r:IMPORTS]->(m)
|
|
302
|
+
SET r.imported_name = row.imported_name,
|
|
303
|
+
r.alias = row.alias,
|
|
304
|
+
r.line_number = row.line_number
|
|
305
|
+
""",
|
|
306
|
+
batch=js_imports,
|
|
307
|
+
file_path=file_path_str,
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
if other_imports:
|
|
311
|
+
session.run(
|
|
312
|
+
"""
|
|
313
|
+
UNWIND $batch AS row
|
|
314
|
+
MATCH (f:File {path: $file_path})
|
|
315
|
+
MERGE (m:Module {name: row.name})
|
|
316
|
+
SET m.alias = row.alias,
|
|
317
|
+
m.full_import_name = coalesce(row.full_import_name, m.full_import_name)
|
|
318
|
+
MERGE (f)-[r:IMPORTS]->(m)
|
|
319
|
+
SET r.line_number = row.line_number,
|
|
320
|
+
r.alias = row.alias
|
|
321
|
+
""",
|
|
322
|
+
batch=other_imports,
|
|
323
|
+
file_path=file_path_str,
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
module_inclusions = file_data.get("module_inclusions", [])
|
|
327
|
+
if module_inclusions:
|
|
328
|
+
session.run(
|
|
329
|
+
"""
|
|
330
|
+
UNWIND $batch AS row
|
|
331
|
+
MATCH (c:Class {name: row.class_name, path: $file_path})
|
|
332
|
+
MERGE (m:Module {name: row.module_name})
|
|
333
|
+
MERGE (c)-[:INCLUDES]->(m)
|
|
334
|
+
""",
|
|
335
|
+
batch=[
|
|
336
|
+
{"class_name": i["class"], "module_name": i["module"]} for i in module_inclusions
|
|
337
|
+
],
|
|
338
|
+
file_path=file_path_str,
|
|
339
|
+
)
|
|
340
|
+
|
|
341
|
+
# ── Batch write (10-20x faster than per-file add_file_to_graph) ──────────
|
|
342
|
+
|
|
343
|
+
def add_files_batch_to_graph(
|
|
344
|
+
self,
|
|
345
|
+
all_file_data: List[Dict[str, Any]],
|
|
346
|
+
repo_name: str,
|
|
347
|
+
imports_map: dict,
|
|
348
|
+
repo_path_str: str,
|
|
349
|
+
on_progress: Optional[Callable] = None,
|
|
350
|
+
) -> int:
|
|
351
|
+
"""Write all parsed files to graph in batched queries.
|
|
352
|
+
|
|
353
|
+
Instead of N×12 individual queries, this runs ~12 batched UNWIND
|
|
354
|
+
queries total, reducing KuzuDB transaction overhead dramatically.
|
|
355
|
+
|
|
356
|
+
Returns the number of files successfully written.
|
|
357
|
+
"""
|
|
358
|
+
import json as _json
|
|
359
|
+
|
|
360
|
+
repo_path_obj = Path(repo_path_str)
|
|
361
|
+
total = len(all_file_data)
|
|
362
|
+
|
|
363
|
+
# ── Collect all data into flat batches ──────────────────────────
|
|
364
|
+
file_batch = []
|
|
365
|
+
dir_batch = [] # (parent_path, parent_label, dir_path, dir_name)
|
|
366
|
+
file_parent_batch = [] # (parent_path, parent_label, file_path)
|
|
367
|
+
symbol_batches: Dict[str, List[Dict]] = {} # label → [rows]
|
|
368
|
+
symbol_file_batches: Dict[str, List[Dict]] = {} # label → [(file_path, name, line)]
|
|
369
|
+
params_batch = []
|
|
370
|
+
class_fn_batch = []
|
|
371
|
+
nested_fn_batch = []
|
|
372
|
+
js_imports_batch = []
|
|
373
|
+
other_imports_batch = []
|
|
374
|
+
|
|
375
|
+
for idx, file_data in enumerate(all_file_data):
|
|
376
|
+
file_path_str = str(Path(file_data["path"]).resolve())
|
|
377
|
+
file_name = Path(file_path_str).name
|
|
378
|
+
is_dep = file_data.get("is_dependency", False)
|
|
379
|
+
lang = file_data.get("lang")
|
|
380
|
+
|
|
381
|
+
try:
|
|
382
|
+
rel_path = str(Path(file_path_str).relative_to(repo_path_obj))
|
|
383
|
+
except ValueError:
|
|
384
|
+
rel_path = file_name
|
|
385
|
+
|
|
386
|
+
file_batch.append({
|
|
387
|
+
"path": file_path_str, "name": file_name,
|
|
388
|
+
"relative_path": rel_path, "is_dependency": is_dep,
|
|
389
|
+
})
|
|
390
|
+
|
|
391
|
+
# Directory chain
|
|
392
|
+
try:
|
|
393
|
+
rel_to_file = Path(file_path_str).relative_to(repo_path_obj)
|
|
394
|
+
except ValueError:
|
|
395
|
+
rel_to_file = Path(file_name)
|
|
396
|
+
|
|
397
|
+
parent_path = repo_path_str
|
|
398
|
+
parent_label = "Repository"
|
|
399
|
+
for part in rel_to_file.parts[:-1]:
|
|
400
|
+
cur = str(Path(parent_path) / part)
|
|
401
|
+
dir_batch.append({
|
|
402
|
+
"parent_path": parent_path, "parent_label": parent_label,
|
|
403
|
+
"dir_path": cur, "dir_name": part,
|
|
404
|
+
})
|
|
405
|
+
parent_path = cur
|
|
406
|
+
parent_label = "Directory"
|
|
407
|
+
file_parent_batch.append({
|
|
408
|
+
"parent_path": parent_path, "parent_label": parent_label,
|
|
409
|
+
"file_path": file_path_str,
|
|
410
|
+
})
|
|
411
|
+
|
|
412
|
+
# Symbols
|
|
413
|
+
item_mappings = [
|
|
414
|
+
(file_data.get("functions", []), "Function"),
|
|
415
|
+
(file_data.get("classes", []), "Class"),
|
|
416
|
+
(file_data.get("traits", []), "Trait"),
|
|
417
|
+
(file_data.get("variables", []), "Variable"),
|
|
418
|
+
(file_data.get("interfaces", []), "Interface"),
|
|
419
|
+
(file_data.get("macros", []), "Macro"),
|
|
420
|
+
(file_data.get("structs", []), "Struct"),
|
|
421
|
+
(file_data.get("enums", []), "Enum"),
|
|
422
|
+
(file_data.get("unions", []), "Union"),
|
|
423
|
+
(file_data.get("records", []), "Record"),
|
|
424
|
+
(file_data.get("properties", []), "Property"),
|
|
425
|
+
]
|
|
426
|
+
|
|
427
|
+
for item_list, label in item_mappings:
|
|
428
|
+
if not item_list:
|
|
429
|
+
continue
|
|
430
|
+
if label not in symbol_batches:
|
|
431
|
+
symbol_batches[label] = []
|
|
432
|
+
symbol_file_batches[label] = []
|
|
433
|
+
for item in item_list:
|
|
434
|
+
row = dict(item)
|
|
435
|
+
row["_file_path"] = file_path_str
|
|
436
|
+
if label == "Function" and "cyclomatic_complexity" not in row:
|
|
437
|
+
row["cyclomatic_complexity"] = 1
|
|
438
|
+
symbol_batches[label].append(sanitize_props(row))
|
|
439
|
+
symbol_file_batches[label].append({
|
|
440
|
+
"file_path": file_path_str,
|
|
441
|
+
"name": item["name"],
|
|
442
|
+
"line_number": item["line_number"],
|
|
443
|
+
})
|
|
444
|
+
|
|
445
|
+
if label == "Function":
|
|
446
|
+
for arg_name in item.get("args", []):
|
|
447
|
+
params_batch.append({
|
|
448
|
+
"func_name": item["name"],
|
|
449
|
+
"line_number": item["line_number"],
|
|
450
|
+
"arg_name": arg_name,
|
|
451
|
+
"file_path": file_path_str,
|
|
452
|
+
})
|
|
453
|
+
if item.get("class_context"):
|
|
454
|
+
class_fn_batch.append({
|
|
455
|
+
"class_name": item["class_context"],
|
|
456
|
+
"func_name": item["name"],
|
|
457
|
+
"func_line": item["line_number"],
|
|
458
|
+
"file_path": file_path_str,
|
|
459
|
+
})
|
|
460
|
+
if item.get("context_type") == "function_definition":
|
|
461
|
+
nested_fn_batch.append({
|
|
462
|
+
"outer": item["context"],
|
|
463
|
+
"inner_name": item["name"],
|
|
464
|
+
"inner_line": item["line_number"],
|
|
465
|
+
"file_path": file_path_str,
|
|
466
|
+
})
|
|
467
|
+
|
|
468
|
+
# Imports
|
|
469
|
+
for imp in file_data.get("imports", []):
|
|
470
|
+
if lang == "javascript":
|
|
471
|
+
module_name = imp.get("source")
|
|
472
|
+
if module_name:
|
|
473
|
+
js_imports_batch.append({
|
|
474
|
+
"module_name": module_name,
|
|
475
|
+
"imported_name": imp.get("name", "*"),
|
|
476
|
+
"alias": imp.get("alias"),
|
|
477
|
+
"line_number": imp.get("line_number"),
|
|
478
|
+
"file_path": file_path_str,
|
|
479
|
+
})
|
|
480
|
+
else:
|
|
481
|
+
other_imports_batch.append({
|
|
482
|
+
"name": imp.get("name", ""),
|
|
483
|
+
"alias": imp.get("alias"),
|
|
484
|
+
"full_import_name": imp.get("full_import_name", imp.get("name", "")),
|
|
485
|
+
"line_number": imp.get("line_number"),
|
|
486
|
+
"file_path": file_path_str,
|
|
487
|
+
})
|
|
488
|
+
|
|
489
|
+
# ── Execute batched queries ─────────────────────────────────────
|
|
490
|
+
if on_progress:
|
|
491
|
+
on_progress(0, total, "Writing files to graph...")
|
|
492
|
+
|
|
493
|
+
batch_size = 500
|
|
494
|
+
|
|
495
|
+
with self.driver.session() as session:
|
|
496
|
+
# 1. Files
|
|
497
|
+
for i in range(0, len(file_batch), batch_size):
|
|
498
|
+
session.run(
|
|
499
|
+
"UNWIND $batch AS row "
|
|
500
|
+
"MERGE (f:File {path: row.path}) "
|
|
501
|
+
"SET f.name = row.name, f.relative_path = row.relative_path, "
|
|
502
|
+
" f.is_dependency = row.is_dependency",
|
|
503
|
+
batch=file_batch[i:i + batch_size],
|
|
504
|
+
)
|
|
505
|
+
|
|
506
|
+
if on_progress:
|
|
507
|
+
on_progress(total // 4, total, "Writing directories...")
|
|
508
|
+
|
|
509
|
+
# 2. Directories (grouped by parent_label to avoid dynamic labels)
|
|
510
|
+
for plabel in ("Repository", "Directory"):
|
|
511
|
+
subset = [d for d in dir_batch if d["parent_label"] == plabel]
|
|
512
|
+
for i in range(0, len(subset), batch_size):
|
|
513
|
+
chunk = [{k: v for k, v in d.items() if k != "parent_label"} for d in subset[i:i + batch_size]]
|
|
514
|
+
session.run(
|
|
515
|
+
f"UNWIND $batch AS row "
|
|
516
|
+
f"MATCH (p:{plabel} {{path: row.parent_path}}) "
|
|
517
|
+
f"MERGE (d:Directory {{path: row.dir_path}}) "
|
|
518
|
+
f"SET d.name = row.dir_name "
|
|
519
|
+
f"MERGE (p)-[:CONTAINS]->(d)",
|
|
520
|
+
batch=chunk,
|
|
521
|
+
)
|
|
522
|
+
|
|
523
|
+
# 3. File-parent CONTAINS
|
|
524
|
+
for plabel in ("Repository", "Directory"):
|
|
525
|
+
subset = [d for d in file_parent_batch if d["parent_label"] == plabel]
|
|
526
|
+
for i in range(0, len(subset), batch_size):
|
|
527
|
+
chunk = [{k: v for k, v in d.items() if k != "parent_label"} for d in subset[i:i + batch_size]]
|
|
528
|
+
session.run(
|
|
529
|
+
f"UNWIND $batch AS row "
|
|
530
|
+
f"MATCH (p:{plabel} {{path: row.parent_path}}) "
|
|
531
|
+
f"MATCH (f:File {{path: row.file_path}}) "
|
|
532
|
+
f"MERGE (p)-[:CONTAINS]->(f)",
|
|
533
|
+
batch=chunk,
|
|
534
|
+
)
|
|
535
|
+
|
|
536
|
+
if on_progress:
|
|
537
|
+
on_progress(total // 2, total, "Writing symbols...")
|
|
538
|
+
|
|
539
|
+
# 4. Symbols (per label)
|
|
540
|
+
for label, batch in symbol_batches.items():
|
|
541
|
+
# Normalize types across batch
|
|
542
|
+
all_keys = set()
|
|
543
|
+
for b in batch:
|
|
544
|
+
all_keys.update(b.keys())
|
|
545
|
+
all_keys.discard("_file_path")
|
|
546
|
+
|
|
547
|
+
for k in all_keys:
|
|
548
|
+
counts: Dict[str, int] = {}
|
|
549
|
+
for b in batch:
|
|
550
|
+
v = b.get(k)
|
|
551
|
+
if v is not None:
|
|
552
|
+
tname = type(v).__name__
|
|
553
|
+
counts[tname] = counts.get(tname, 0) + 1
|
|
554
|
+
dominant = max(counts, key=counts.get) if counts else "str"
|
|
555
|
+
|
|
556
|
+
for b in batch:
|
|
557
|
+
v = b.get(k)
|
|
558
|
+
if dominant == "list":
|
|
559
|
+
if isinstance(v, list):
|
|
560
|
+
b[k] = [str(x) for x in v] if v else [""]
|
|
561
|
+
elif isinstance(v, str) and v:
|
|
562
|
+
try:
|
|
563
|
+
p = _json.loads(v)
|
|
564
|
+
b[k] = [str(x) for x in p] if isinstance(p, list) and p else [""]
|
|
565
|
+
except Exception:
|
|
566
|
+
b[k] = [v]
|
|
567
|
+
else:
|
|
568
|
+
b[k] = [""]
|
|
569
|
+
elif dominant == "int":
|
|
570
|
+
if v is None or v == "":
|
|
571
|
+
b[k] = 0
|
|
572
|
+
elif not isinstance(v, int):
|
|
573
|
+
try:
|
|
574
|
+
b[k] = int(v)
|
|
575
|
+
except Exception:
|
|
576
|
+
b[k] = 0
|
|
577
|
+
elif dominant == "bool":
|
|
578
|
+
b[k] = bool(v) if v is not None else False
|
|
579
|
+
else:
|
|
580
|
+
if v is None:
|
|
581
|
+
b[k] = ""
|
|
582
|
+
elif isinstance(v, list):
|
|
583
|
+
b[k] = _json.dumps(v)
|
|
584
|
+
elif not isinstance(v, str):
|
|
585
|
+
b[k] = str(v)
|
|
586
|
+
|
|
587
|
+
key_order = sorted(all_keys)
|
|
588
|
+
normalized = []
|
|
589
|
+
file_paths_for_batch = []
|
|
590
|
+
for b in batch:
|
|
591
|
+
fp = b.pop("_file_path", "")
|
|
592
|
+
normalized.append({k: b[k] for k in key_order if k in b})
|
|
593
|
+
file_paths_for_batch.append(fp)
|
|
594
|
+
|
|
595
|
+
# Deduplicate by (name, path, line_number) — minified JS can
|
|
596
|
+
# produce duplicates which violate KuzuDB primary key constraint
|
|
597
|
+
seen_keys = set()
|
|
598
|
+
deduped = []
|
|
599
|
+
deduped_fps = []
|
|
600
|
+
for row, fp in zip(normalized, file_paths_for_batch):
|
|
601
|
+
key = (row.get("name", ""), fp, row.get("line_number", 0))
|
|
602
|
+
if key not in seen_keys:
|
|
603
|
+
seen_keys.add(key)
|
|
604
|
+
deduped.append(row)
|
|
605
|
+
deduped_fps.append(fp)
|
|
606
|
+
normalized = deduped
|
|
607
|
+
file_paths_for_batch = deduped_fps
|
|
608
|
+
|
|
609
|
+
# Write symbols in chunks
|
|
610
|
+
for i in range(0, len(normalized), batch_size):
|
|
611
|
+
chunk = normalized[i:i + batch_size]
|
|
612
|
+
fps = file_paths_for_batch[i:i + batch_size]
|
|
613
|
+
# Add file_path back for MERGE key
|
|
614
|
+
for row, fp in zip(chunk, fps):
|
|
615
|
+
row["_fp"] = fp
|
|
616
|
+
|
|
617
|
+
session.run(
|
|
618
|
+
f"UNWIND $batch AS row "
|
|
619
|
+
f"MERGE (n:{label} {{name: row.name, path: row._fp, line_number: row.line_number}}) "
|
|
620
|
+
f"SET n += row",
|
|
621
|
+
batch=chunk,
|
|
622
|
+
)
|
|
623
|
+
session.run(
|
|
624
|
+
f"UNWIND $batch AS row "
|
|
625
|
+
f"MATCH (f:File {{path: row._fp}}) "
|
|
626
|
+
f"MATCH (n:{label} {{name: row.name, path: row._fp, line_number: row.line_number}}) "
|
|
627
|
+
f"MERGE (f)-[:CONTAINS]->(n)",
|
|
628
|
+
batch=chunk,
|
|
629
|
+
)
|
|
630
|
+
|
|
631
|
+
if on_progress:
|
|
632
|
+
on_progress(total * 3 // 4, total, "Writing relationships...")
|
|
633
|
+
|
|
634
|
+
# 5. Parameters
|
|
635
|
+
if params_batch:
|
|
636
|
+
for i in range(0, len(params_batch), batch_size):
|
|
637
|
+
session.run(
|
|
638
|
+
"UNWIND $batch AS row "
|
|
639
|
+
"MATCH (fn:Function {name: row.func_name, path: row.file_path, line_number: row.line_number}) "
|
|
640
|
+
"MERGE (p:Parameter {name: row.arg_name, path: row.file_path, function_line_number: row.line_number}) "
|
|
641
|
+
"MERGE (fn)-[:HAS_PARAMETER]->(p)",
|
|
642
|
+
batch=params_batch[i:i + batch_size],
|
|
643
|
+
)
|
|
644
|
+
|
|
645
|
+
# 6. Class-function containment
|
|
646
|
+
if class_fn_batch:
|
|
647
|
+
for i in range(0, len(class_fn_batch), batch_size):
|
|
648
|
+
session.run(
|
|
649
|
+
"UNWIND $batch AS row "
|
|
650
|
+
"MATCH (c:Class {name: row.class_name, path: row.file_path}) "
|
|
651
|
+
"MATCH (fn:Function {name: row.func_name, path: row.file_path, line_number: row.func_line}) "
|
|
652
|
+
"MERGE (c)-[:CONTAINS]->(fn)",
|
|
653
|
+
batch=class_fn_batch[i:i + batch_size],
|
|
654
|
+
)
|
|
655
|
+
|
|
656
|
+
# 7. Nested functions
|
|
657
|
+
if nested_fn_batch:
|
|
658
|
+
for i in range(0, len(nested_fn_batch), batch_size):
|
|
659
|
+
session.run(
|
|
660
|
+
"UNWIND $batch AS row "
|
|
661
|
+
"MATCH (outer:Function {name: row.outer, path: row.file_path}) "
|
|
662
|
+
"MATCH (inner:Function {name: row.inner_name, path: row.file_path, line_number: row.inner_line}) "
|
|
663
|
+
"MERGE (outer)-[:CONTAINS]->(inner)",
|
|
664
|
+
batch=nested_fn_batch[i:i + batch_size],
|
|
665
|
+
)
|
|
666
|
+
|
|
667
|
+
# 8. JS imports
|
|
668
|
+
if js_imports_batch:
|
|
669
|
+
for i in range(0, len(js_imports_batch), batch_size):
|
|
670
|
+
session.run(
|
|
671
|
+
"UNWIND $batch AS row "
|
|
672
|
+
"MATCH (f:File {path: row.file_path}) "
|
|
673
|
+
"MERGE (m:Module {name: row.module_name}) "
|
|
674
|
+
"MERGE (f)-[r:IMPORTS]->(m) "
|
|
675
|
+
"SET r.imported_name = row.imported_name, r.alias = row.alias, r.line_number = row.line_number",
|
|
676
|
+
batch=js_imports_batch[i:i + batch_size],
|
|
677
|
+
)
|
|
678
|
+
|
|
679
|
+
# 9. Other imports
|
|
680
|
+
if other_imports_batch:
|
|
681
|
+
for i in range(0, len(other_imports_batch), batch_size):
|
|
682
|
+
session.run(
|
|
683
|
+
"UNWIND $batch AS row "
|
|
684
|
+
"MATCH (f:File {path: row.file_path}) "
|
|
685
|
+
"MERGE (m:Module {name: row.name}) "
|
|
686
|
+
"SET m.alias = row.alias, m.full_import_name = coalesce(row.full_import_name, m.full_import_name) "
|
|
687
|
+
"MERGE (f)-[r:IMPORTS]->(m) "
|
|
688
|
+
"SET r.line_number = row.line_number, r.alias = row.alias",
|
|
689
|
+
batch=other_imports_batch[i:i + batch_size],
|
|
690
|
+
)
|
|
691
|
+
|
|
692
|
+
if on_progress:
|
|
693
|
+
on_progress(total, total, "Graph write complete")
|
|
694
|
+
|
|
695
|
+
return len(file_batch)
|
|
696
|
+
|
|
697
|
+
def add_minimal_file_node(
|
|
698
|
+
self, file_path: Path, repo_path: Path, is_dependency: bool = False
|
|
699
|
+
) -> None:
|
|
700
|
+
file_path_str = str(file_path.resolve())
|
|
701
|
+
file_name = file_path.name
|
|
702
|
+
repo_name = repo_path.name
|
|
703
|
+
repo_path_str = str(repo_path.resolve())
|
|
704
|
+
|
|
705
|
+
with self.driver.session() as session:
|
|
706
|
+
session.run(
|
|
707
|
+
"""
|
|
708
|
+
MERGE (r:Repository {path: $repo_path})
|
|
709
|
+
SET r.name = $repo_name
|
|
710
|
+
""",
|
|
711
|
+
repo_path=repo_path_str,
|
|
712
|
+
repo_name=repo_name,
|
|
713
|
+
)
|
|
714
|
+
|
|
715
|
+
session.run(
|
|
716
|
+
"""
|
|
717
|
+
MERGE (f:File {path: $file_path})
|
|
718
|
+
SET f.name = $file_name,
|
|
719
|
+
f.is_dependency = $is_dependency
|
|
720
|
+
""",
|
|
721
|
+
file_path=file_path_str,
|
|
722
|
+
file_name=file_name,
|
|
723
|
+
is_dependency=is_dependency,
|
|
724
|
+
)
|
|
725
|
+
|
|
726
|
+
file_path_obj = Path(file_path_str)
|
|
727
|
+
repo_path_obj = Path(repo_path_str)
|
|
728
|
+
try:
|
|
729
|
+
relative_path_to_file = file_path_obj.relative_to(repo_path_obj)
|
|
730
|
+
except ValueError:
|
|
731
|
+
relative_path_to_file = Path(file_path_obj.name)
|
|
732
|
+
|
|
733
|
+
parent_path = repo_path_str
|
|
734
|
+
parent_label = "Repository"
|
|
735
|
+
|
|
736
|
+
for part in relative_path_to_file.parts[:-1]:
|
|
737
|
+
current_path = Path(parent_path) / part
|
|
738
|
+
current_path_str = str(current_path)
|
|
739
|
+
|
|
740
|
+
session.run(
|
|
741
|
+
f"""
|
|
742
|
+
MATCH (p:{parent_label} {{path: $parent_path}})
|
|
743
|
+
MERGE (d:Directory {{path: $current_path}})
|
|
744
|
+
SET d.name = $part
|
|
745
|
+
MERGE (p)-[:CONTAINS]->(d)
|
|
746
|
+
""",
|
|
747
|
+
parent_path=parent_path,
|
|
748
|
+
current_path=current_path_str,
|
|
749
|
+
part=part,
|
|
750
|
+
)
|
|
751
|
+
|
|
752
|
+
parent_path = current_path_str
|
|
753
|
+
parent_label = "Directory"
|
|
754
|
+
|
|
755
|
+
session.run(
|
|
756
|
+
f"""
|
|
757
|
+
MATCH (p:{parent_label} {{path: $parent_path}})
|
|
758
|
+
MATCH (f:File {{path: $file_path}})
|
|
759
|
+
MERGE (p)-[:CONTAINS]->(f)
|
|
760
|
+
""",
|
|
761
|
+
parent_path=parent_path,
|
|
762
|
+
file_path=file_path_str,
|
|
763
|
+
)
|
|
764
|
+
|
|
765
|
+
def write_function_call_groups(
|
|
766
|
+
self,
|
|
767
|
+
fn_to_fn: List[Dict],
|
|
768
|
+
fn_to_cls: List[Dict],
|
|
769
|
+
cls_to_fn: List[Dict],
|
|
770
|
+
cls_to_cls: List[Dict],
|
|
771
|
+
file_to_fn: List[Dict],
|
|
772
|
+
file_to_cls: List[Dict],
|
|
773
|
+
) -> None:
|
|
774
|
+
batch_size = 1000
|
|
775
|
+
q_fn_to_fn = """
|
|
776
|
+
UNWIND $batch AS row
|
|
777
|
+
MATCH (caller:Function {name: row.caller_name, path: row.caller_file_path, line_number: row.caller_line_number})
|
|
778
|
+
MATCH (called:Function {name: row.called_name, path: row.called_file_path})
|
|
779
|
+
MERGE (caller)-[:CALLS {line_number: row.line_number, args: row.args, full_call_name: row.full_call_name}]->(called)
|
|
780
|
+
"""
|
|
781
|
+
q_fn_to_cls = """
|
|
782
|
+
UNWIND $batch AS row
|
|
783
|
+
MATCH (caller:Function {name: row.caller_name, path: row.caller_file_path, line_number: row.caller_line_number})
|
|
784
|
+
MATCH (called:Class {name: row.called_name, path: row.called_file_path})
|
|
785
|
+
MERGE (caller)-[:CALLS {line_number: row.line_number, args: row.args, full_call_name: row.full_call_name}]->(called)
|
|
786
|
+
"""
|
|
787
|
+
q_cls_to_fn = """
|
|
788
|
+
UNWIND $batch AS row
|
|
789
|
+
MATCH (caller:Class {name: row.caller_name, path: row.caller_file_path, line_number: row.caller_line_number})
|
|
790
|
+
MATCH (called:Function {name: row.called_name, path: row.called_file_path})
|
|
791
|
+
MERGE (caller)-[:CALLS {line_number: row.line_number, args: row.args, full_call_name: row.full_call_name}]->(called)
|
|
792
|
+
"""
|
|
793
|
+
q_cls_to_cls = """
|
|
794
|
+
UNWIND $batch AS row
|
|
795
|
+
MATCH (caller:Class {name: row.caller_name, path: row.caller_file_path, line_number: row.caller_line_number})
|
|
796
|
+
MATCH (called:Class {name: row.called_name, path: row.called_file_path})
|
|
797
|
+
MERGE (caller)-[:CALLS {line_number: row.line_number, args: row.args, full_call_name: row.full_call_name}]->(called)
|
|
798
|
+
"""
|
|
799
|
+
q_file_to_fn = """
|
|
800
|
+
UNWIND $batch AS row
|
|
801
|
+
MATCH (caller:File {path: row.caller_file_path})
|
|
802
|
+
MATCH (called:Function {name: row.called_name, path: row.called_file_path})
|
|
803
|
+
MERGE (caller)-[:CALLS {line_number: row.line_number, args: row.args, full_call_name: row.full_call_name}]->(called)
|
|
804
|
+
"""
|
|
805
|
+
q_file_to_cls = """
|
|
806
|
+
UNWIND $batch AS row
|
|
807
|
+
MATCH (caller:File {path: row.caller_file_path})
|
|
808
|
+
MATCH (called:Class {name: row.called_name, path: row.called_file_path})
|
|
809
|
+
MERGE (caller)-[:CALLS {line_number: row.line_number, args: row.args, full_call_name: row.full_call_name}]->(called)
|
|
810
|
+
"""
|
|
811
|
+
groups: List[Tuple[str, List[Dict], str]] = [
|
|
812
|
+
("fn→fn", fn_to_fn, q_fn_to_fn),
|
|
813
|
+
("fn→cls", fn_to_cls, q_fn_to_cls),
|
|
814
|
+
("cls→fn", cls_to_fn, q_cls_to_fn),
|
|
815
|
+
("cls→cls", cls_to_cls, q_cls_to_cls),
|
|
816
|
+
("file→fn", file_to_fn, q_file_to_fn),
|
|
817
|
+
("file→cls", file_to_cls, q_file_to_cls),
|
|
818
|
+
]
|
|
819
|
+
total_all = sum(len(g[1]) for g in groups)
|
|
820
|
+
with self.driver.session() as session:
|
|
821
|
+
for label, calls, query in groups:
|
|
822
|
+
if not calls:
|
|
823
|
+
info_logger(f"[CALLS] {label}: 0 (skipped)")
|
|
824
|
+
continue
|
|
825
|
+
t0 = time.time()
|
|
826
|
+
for i in range(0, len(calls), batch_size):
|
|
827
|
+
batch = calls[i : i + batch_size]
|
|
828
|
+
session.run(query, batch=batch)
|
|
829
|
+
written = min(i + batch_size, len(calls))
|
|
830
|
+
if written % 5000 < batch_size or written == len(calls):
|
|
831
|
+
elapsed = time.time() - t0
|
|
832
|
+
info_logger(f"[CALLS] {label}: {written}/{len(calls)} ({elapsed:.1f}s)")
|
|
833
|
+
elapsed = time.time() - t0
|
|
834
|
+
info_logger(f"[CALLS] {label} done: {len(calls)} in {elapsed:.1f}s")
|
|
835
|
+
info_logger(f"[CALLS] All complete: {total_all} CALLS relationships processed.")
|
|
836
|
+
|
|
837
|
+
def _create_csharp_inheritance_and_interfaces(
|
|
838
|
+
self, session: Any, file_data: Dict[str, Any], imports_map: dict
|
|
839
|
+
) -> None:
|
|
840
|
+
if file_data.get("lang") != "c_sharp":
|
|
841
|
+
return
|
|
842
|
+
|
|
843
|
+
caller_file_path = str(Path(file_data["path"]).resolve())
|
|
844
|
+
|
|
845
|
+
for type_list_name, type_label in [
|
|
846
|
+
("classes", "Class"),
|
|
847
|
+
("structs", "Struct"),
|
|
848
|
+
("records", "Record"),
|
|
849
|
+
("interfaces", "Interface"),
|
|
850
|
+
]:
|
|
851
|
+
for type_item in file_data.get(type_list_name, []):
|
|
852
|
+
if not type_item.get("bases"):
|
|
853
|
+
continue
|
|
854
|
+
|
|
855
|
+
for base_str in type_item["bases"]:
|
|
856
|
+
base_name = base_str.split("<")[0].strip()
|
|
857
|
+
|
|
858
|
+
is_interface = False
|
|
859
|
+
resolved_path = caller_file_path
|
|
860
|
+
|
|
861
|
+
for iface in file_data.get("interfaces", []):
|
|
862
|
+
if iface["name"] == base_name:
|
|
863
|
+
is_interface = True
|
|
864
|
+
break
|
|
865
|
+
|
|
866
|
+
if base_name in imports_map:
|
|
867
|
+
possible_paths = imports_map[base_name]
|
|
868
|
+
if len(possible_paths) > 0:
|
|
869
|
+
resolved_path = possible_paths[0]
|
|
870
|
+
|
|
871
|
+
base_index = type_item["bases"].index(base_str)
|
|
872
|
+
|
|
873
|
+
if is_interface or (base_index > 0 and type_label == "Class"):
|
|
874
|
+
session.run(
|
|
875
|
+
"""
|
|
876
|
+
MATCH (child {name: $child_name, path: $path})
|
|
877
|
+
WHERE child:Class OR child:Struct OR child:Record
|
|
878
|
+
MATCH (iface:Interface {name: $interface_name})
|
|
879
|
+
MERGE (child)-[:IMPLEMENTS]->(iface)
|
|
880
|
+
""",
|
|
881
|
+
child_name=type_item["name"],
|
|
882
|
+
path=caller_file_path,
|
|
883
|
+
interface_name=base_name,
|
|
884
|
+
)
|
|
885
|
+
else:
|
|
886
|
+
session.run(
|
|
887
|
+
"""
|
|
888
|
+
MATCH (child {name: $child_name, path: $path})
|
|
889
|
+
WHERE child:Class OR child:Record OR child:Interface
|
|
890
|
+
MATCH (parent {name: $parent_name})
|
|
891
|
+
WHERE parent:Class OR parent:Record OR parent:Interface
|
|
892
|
+
MERGE (child)-[:INHERITS]->(parent)
|
|
893
|
+
""",
|
|
894
|
+
child_name=type_item["name"],
|
|
895
|
+
path=caller_file_path,
|
|
896
|
+
parent_name=base_name,
|
|
897
|
+
)
|
|
898
|
+
|
|
899
|
+
def write_inheritance_links(
|
|
900
|
+
self,
|
|
901
|
+
inheritance_batch: List[Dict[str, Any]],
|
|
902
|
+
csharp_files: List[Dict[str, Any]],
|
|
903
|
+
imports_map: dict,
|
|
904
|
+
) -> None:
|
|
905
|
+
info_logger(
|
|
906
|
+
f"[INHERITS] Resolved {len(inheritance_batch)} inheritance links, "
|
|
907
|
+
f"{len(csharp_files)} C# files. Writing to Neo4j..."
|
|
908
|
+
)
|
|
909
|
+
batch_size = 500
|
|
910
|
+
with self.driver.session() as session:
|
|
911
|
+
for i in range(0, len(inheritance_batch), batch_size):
|
|
912
|
+
batch = inheritance_batch[i : i + batch_size]
|
|
913
|
+
session.run(
|
|
914
|
+
"""
|
|
915
|
+
UNWIND $batch AS row
|
|
916
|
+
MATCH (child:Class {name: row.child_name, path: row.path})
|
|
917
|
+
MATCH (parent:Class {name: row.parent_name, path: row.resolved_parent_file_path})
|
|
918
|
+
MERGE (child)-[:INHERITS]->(parent)
|
|
919
|
+
""",
|
|
920
|
+
batch=batch,
|
|
921
|
+
)
|
|
922
|
+
|
|
923
|
+
for file_data in csharp_files:
|
|
924
|
+
self._create_csharp_inheritance_and_interfaces(session, file_data, imports_map)
|
|
925
|
+
|
|
926
|
+
info_logger(f"[INHERITS] Complete: {len(inheritance_batch)} inheritance links processed.")
|
|
927
|
+
|
|
928
|
+
def write_scip_call_edges(
|
|
929
|
+
self, files_data: Dict[str, Any], name_from_symbol: Callable[[str], str]
|
|
930
|
+
) -> None:
|
|
931
|
+
with self.driver.session() as session:
|
|
932
|
+
for file_data in files_data.values():
|
|
933
|
+
for edge in file_data.get("function_calls_scip", []):
|
|
934
|
+
try:
|
|
935
|
+
session.run(
|
|
936
|
+
"""
|
|
937
|
+
MATCH (caller:Function {name: $caller_name, path: $caller_file, line_number: $caller_line})
|
|
938
|
+
MATCH (callee:Function {name: $callee_name, path: $callee_file, line_number: $callee_line})
|
|
939
|
+
MERGE (caller)-[:CALLS {line_number: $ref_line, source: 'scip'}]->(callee)
|
|
940
|
+
""",
|
|
941
|
+
caller_name=name_from_symbol(edge["caller_symbol"]),
|
|
942
|
+
caller_file=edge["caller_file"],
|
|
943
|
+
caller_line=edge["caller_line"],
|
|
944
|
+
callee_name=edge["callee_name"],
|
|
945
|
+
callee_file=edge["callee_file"],
|
|
946
|
+
callee_line=edge["callee_line"],
|
|
947
|
+
ref_line=edge["ref_line"],
|
|
948
|
+
)
|
|
949
|
+
except Exception as e:
|
|
950
|
+
warning_logger(f"Failed to write SCIP call edge: {e}")
|
|
951
|
+
|
|
952
|
+
def delete_file_from_graph(self, path: str) -> None:
|
|
953
|
+
file_path_str = str(Path(path).resolve())
|
|
954
|
+
with self.driver.session() as session:
|
|
955
|
+
parents_res = session.run(
|
|
956
|
+
"""
|
|
957
|
+
MATCH (f:File {path: $path})<-[:CONTAINS*]-(d:Directory)
|
|
958
|
+
RETURN d.path as path ORDER BY d.path DESC
|
|
959
|
+
""",
|
|
960
|
+
path=file_path_str,
|
|
961
|
+
)
|
|
962
|
+
parent_paths = [record["path"] for record in parents_res]
|
|
963
|
+
|
|
964
|
+
session.run(
|
|
965
|
+
"""
|
|
966
|
+
MATCH (f:File {path: $path})
|
|
967
|
+
OPTIONAL MATCH (f)-[:CONTAINS]->(element)
|
|
968
|
+
DETACH DELETE f, element
|
|
969
|
+
""",
|
|
970
|
+
path=file_path_str,
|
|
971
|
+
)
|
|
972
|
+
info_logger(f"Deleted file and its elements from graph: {file_path_str}")
|
|
973
|
+
|
|
974
|
+
for p in parent_paths:
|
|
975
|
+
session.run(
|
|
976
|
+
"""
|
|
977
|
+
MATCH (d:Directory {path: $path})
|
|
978
|
+
WHERE NOT (d)-[:CONTAINS]->()
|
|
979
|
+
DETACH DELETE d
|
|
980
|
+
""",
|
|
981
|
+
path=p,
|
|
982
|
+
)
|
|
983
|
+
|
|
984
|
+
def delete_repository_from_graph(self, repo_path: str) -> bool:
|
|
985
|
+
repo_path_str = str(Path(repo_path).resolve())
|
|
986
|
+
path_prefix = repo_path_str + "/"
|
|
987
|
+
with self.driver.session() as session:
|
|
988
|
+
result = session.run(
|
|
989
|
+
"MATCH (r:Repository {path: $path}) RETURN count(r) as cnt", path=repo_path_str
|
|
990
|
+
).single()
|
|
991
|
+
if not result or result["cnt"] == 0:
|
|
992
|
+
warning_logger(f"Attempted to delete non-existent repository: {repo_path_str}")
|
|
993
|
+
return False
|
|
994
|
+
|
|
995
|
+
for rel_type in ("CALLS", "INHERITS", "IMPORTS"):
|
|
996
|
+
while True:
|
|
997
|
+
with self.driver.session() as session:
|
|
998
|
+
result = session.run(
|
|
999
|
+
f"MATCH (a)-[r:{rel_type}]->(b) "
|
|
1000
|
+
"WHERE a.path STARTS WITH $prefix OR b.path STARTS WITH $prefix "
|
|
1001
|
+
"WITH r LIMIT 5000 DELETE r RETURN count(r) AS deleted",
|
|
1002
|
+
prefix=path_prefix,
|
|
1003
|
+
).single()
|
|
1004
|
+
deleted = result["deleted"] if result else 0
|
|
1005
|
+
if deleted == 0:
|
|
1006
|
+
break
|
|
1007
|
+
info_logger(f"[DELETE] Removed {deleted} {rel_type} rels for {repo_path_str}")
|
|
1008
|
+
|
|
1009
|
+
while True:
|
|
1010
|
+
with self.driver.session() as session:
|
|
1011
|
+
result = session.run(
|
|
1012
|
+
"MATCH (a)-[r:CONTAINS]->(b) "
|
|
1013
|
+
"WHERE a.path STARTS WITH $prefix OR a.path = $path "
|
|
1014
|
+
"WITH r LIMIT 10000 DELETE r RETURN count(r) AS deleted",
|
|
1015
|
+
prefix=path_prefix,
|
|
1016
|
+
path=repo_path_str,
|
|
1017
|
+
).single()
|
|
1018
|
+
deleted = result["deleted"] if result else 0
|
|
1019
|
+
if deleted == 0:
|
|
1020
|
+
break
|
|
1021
|
+
info_logger(f"[DELETE] Removed {deleted} CONTAINS rels for {repo_path_str}")
|
|
1022
|
+
|
|
1023
|
+
for label in ("Function", "Class", "File"):
|
|
1024
|
+
while True:
|
|
1025
|
+
with self.driver.session() as session:
|
|
1026
|
+
result = session.run(
|
|
1027
|
+
f"MATCH (n:{label}) WHERE n.path STARTS WITH $prefix "
|
|
1028
|
+
"WITH n LIMIT 10000 DETACH DELETE n RETURN count(n) AS deleted",
|
|
1029
|
+
prefix=path_prefix,
|
|
1030
|
+
).single()
|
|
1031
|
+
deleted = result["deleted"] if result else 0
|
|
1032
|
+
if deleted == 0:
|
|
1033
|
+
break
|
|
1034
|
+
info_logger(f"[DELETE] Removed {deleted} {label} nodes for {repo_path_str}")
|
|
1035
|
+
|
|
1036
|
+
with self.driver.session() as session:
|
|
1037
|
+
session.run("MATCH (r:Repository {path: $path}) DETACH DELETE r", path=repo_path_str)
|
|
1038
|
+
|
|
1039
|
+
info_logger(f"Deleted repository and its contents from graph: {repo_path_str}")
|
|
1040
|
+
return True
|
|
1041
|
+
|
|
1042
|
+
def get_caller_file_paths(self, file_path_str: str) -> set:
|
|
1043
|
+
with self.driver.session() as session:
|
|
1044
|
+
result = session.run(
|
|
1045
|
+
"MATCH (caller)-[:CALLS]->(callee) "
|
|
1046
|
+
"WHERE callee.path = $path "
|
|
1047
|
+
"RETURN DISTINCT coalesce(caller.path, '') AS p",
|
|
1048
|
+
path=file_path_str,
|
|
1049
|
+
)
|
|
1050
|
+
return {r["p"] for r in result if r["p"] and r["p"] != file_path_str}
|
|
1051
|
+
|
|
1052
|
+
def get_inheritance_neighbor_paths(self, file_path_str: str) -> set:
|
|
1053
|
+
with self.driver.session() as session:
|
|
1054
|
+
result = session.run(
|
|
1055
|
+
"MATCH (a)-[:INHERITS]->(b) "
|
|
1056
|
+
"WHERE a.path = $path OR b.path = $path "
|
|
1057
|
+
"RETURN DISTINCT CASE WHEN a.path = $path THEN b.path ELSE a.path END AS p",
|
|
1058
|
+
path=file_path_str,
|
|
1059
|
+
)
|
|
1060
|
+
return {r["p"] for r in result if r["p"] and r["p"] != file_path_str}
|
|
1061
|
+
|
|
1062
|
+
def delete_outgoing_calls_from_files(self, file_paths: List[str]) -> None:
|
|
1063
|
+
with self.driver.session() as session:
|
|
1064
|
+
result = session.run(
|
|
1065
|
+
"MATCH (a)-[r:CALLS]->(b) WHERE a.path IN $paths DELETE r RETURN count(r) AS cnt",
|
|
1066
|
+
paths=file_paths,
|
|
1067
|
+
).single()
|
|
1068
|
+
cnt = result["cnt"] if result else 0
|
|
1069
|
+
info_logger(f"[RELINK] Deleted {cnt} outgoing CALLS from {len(file_paths)} caller files")
|
|
1070
|
+
|
|
1071
|
+
def delete_inherits_for_files(self, file_paths: List[str]) -> None:
|
|
1072
|
+
with self.driver.session() as session:
|
|
1073
|
+
result = session.run(
|
|
1074
|
+
"MATCH (a)-[r:INHERITS]->(b) WHERE a.path IN $paths OR b.path IN $paths "
|
|
1075
|
+
"DELETE r RETURN count(r) AS cnt",
|
|
1076
|
+
paths=file_paths,
|
|
1077
|
+
).single()
|
|
1078
|
+
cnt = result["cnt"] if result else 0
|
|
1079
|
+
info_logger(f"[RELINK] Deleted {cnt} INHERITS for {len(file_paths)} affected files")
|
|
1080
|
+
|
|
1081
|
+
def get_repo_class_lookup(self, repo_path: Path) -> Dict[str, set]:
|
|
1082
|
+
prefix = str(repo_path.resolve()) + "/"
|
|
1083
|
+
result_map: Dict[str, set] = {}
|
|
1084
|
+
with self.driver.session() as session:
|
|
1085
|
+
result = session.run(
|
|
1086
|
+
"MATCH (c:Class) WHERE c.path STARTS WITH $prefix "
|
|
1087
|
+
"RETURN c.name AS name, c.path AS path",
|
|
1088
|
+
prefix=prefix,
|
|
1089
|
+
)
|
|
1090
|
+
for record in result:
|
|
1091
|
+
path = record["path"]
|
|
1092
|
+
if path not in result_map:
|
|
1093
|
+
result_map[path] = set()
|
|
1094
|
+
result_map[path].add(record["name"])
|
|
1095
|
+
return result_map
|
|
1096
|
+
|
|
1097
|
+
def delete_relationship_links(self, repo_path: Path) -> None:
|
|
1098
|
+
repo_path_str = str(repo_path.resolve()) + "/"
|
|
1099
|
+
with self.driver.session() as session:
|
|
1100
|
+
result = session.run(
|
|
1101
|
+
"MATCH (a)-[r:CALLS]->(b) WHERE a.path STARTS WITH $prefix DELETE r RETURN count(r) AS cnt",
|
|
1102
|
+
prefix=repo_path_str,
|
|
1103
|
+
).single()
|
|
1104
|
+
calls_deleted = result["cnt"] if result else 0
|
|
1105
|
+
|
|
1106
|
+
result = session.run(
|
|
1107
|
+
"MATCH (a)-[r:INHERITS]->(b) WHERE a.path STARTS WITH $prefix DELETE r RETURN count(r) AS cnt",
|
|
1108
|
+
prefix=repo_path_str,
|
|
1109
|
+
).single()
|
|
1110
|
+
inherits_deleted = result["cnt"] if result else 0
|
|
1111
|
+
|
|
1112
|
+
info_logger(
|
|
1113
|
+
f"[RELINK] Cleared {calls_deleted} CALLS and {inherits_deleted} INHERITS before re-linking: {repo_path}"
|
|
1114
|
+
)
|