raggiecode 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- Agent/__init__.py +0 -0
- Agent/agent.py +891 -0
- Agent/chat_history_db.py +1500 -0
- Agent/command.py +49 -0
- Agent/config.py +46 -0
- Agent/effort_levels.py +33 -0
- Agent/git_manager.py +727 -0
- Agent/tools.py +35 -0
- Commands/__init__.py +18 -0
- Commands/effort.py +42 -0
- Commands/global_todo.py +23 -0
- Commands/help.py +22 -0
- Commands/reasoning.py +24 -0
- Commands/redo.py +11 -0
- Commands/reindex.py +27 -0
- Commands/shell.py +28 -0
- Commands/stream.py +24 -0
- Commands/undo.py +13 -0
- Commands/unlimited_effort.py +8 -0
- Commands/window_size.py +29 -0
- RAG/__init__.py +0 -0
- RAG/document.py +119 -0
- RAG/find.py +408 -0
- RAG/graph.py +231 -0
- Tools/GetFileCodeStructure.py +43 -0
- Tools/GetSymbolSourceCode.py +27 -0
- Tools/__init__.py +39 -0
- Tools/ask_user.py +102 -0
- Tools/dispatch_subagent.py +215 -0
- Tools/document.py +35 -0
- Tools/edit_symbol.py +250 -0
- Tools/fuzzy_search.py +119 -0
- Tools/list_dir.py +51 -0
- Tools/read.py +49 -0
- Tools/read_image.py +75 -0
- Tools/remove.py +75 -0
- Tools/replace.py +305 -0
- Tools/search.py +41 -0
- Tools/shell.py +149 -0
- Tools/shell_kill.py +87 -0
- Tools/temp_background_service.py +113 -0
- Tools/todo_list.py +481 -0
- Tools/utils.py +116 -0
- Tools/view_changes.py +179 -0
- Tools/walk_call_tree.py +30 -0
- Tools/web_fetch.py +175 -0
- Tools/web_search.py +69 -0
- Tools/write.py +48 -0
- cli.py +111 -0
- config/__init__.py +0 -0
- config/coder_system_prompt.md +119 -0
- config/roles.json +43 -0
- config/tools.json +709 -0
- indexing/__init__.py +0 -0
- indexing/cli.py +128 -0
- indexing/code_index_sdk.py +832 -0
- indexing/code_indexer.py +1763 -0
- indexing/db_schema.py +396 -0
- indexing/export_to_json.py +346 -0
- indexing/extractors.py +189 -0
- indexing/file_utils.py +97 -0
- indexing/frontend/__init__.py +0 -0
- indexing/frontend/css_extractor.py +195 -0
- indexing/frontend/css_parser.py +387 -0
- indexing/frontend/css_selector_utils.py +226 -0
- indexing/frontend/edit_safety.py +573 -0
- indexing/frontend/graph.py +838 -0
- indexing/frontend/html_extractor.py +496 -0
- indexing/frontend/html_parser.py +314 -0
- indexing/frontend/jsx_extractor.py +1204 -0
- indexing/frontend/location_lookup.py +247 -0
- indexing/frontend/resolver.py +485 -0
- indexing/frontend/runtime_resolver.py +862 -0
- indexing/frontend/semantic_output.py +705 -0
- indexing/frontend/source_location.py +69 -0
- indexing/frontend_config.py +72 -0
- indexing/frontend_models.py +347 -0
- indexing/language_config.py +360 -0
- indexing/models.py +284 -0
- indexing/node_utils.py +1112 -0
- indexing/parse_worker.py +1082 -0
- indexing/queries.py +1542 -0
- indexing/sdk_examples.py +426 -0
- interactive.py +248 -0
- raggie.py +673 -0
- raggiecode-0.2.1.dist-info/METADATA +944 -0
- raggiecode-0.2.1.dist-info/RECORD +93 -0
- raggiecode-0.2.1.dist-info/WHEEL +5 -0
- raggiecode-0.2.1.dist-info/entry_points.txt +2 -0
- raggiecode-0.2.1.dist-info/top_level.txt +10 -0
- skills/__init__.py +3 -0
- skills/manager.py +114 -0
- skills/tool.py +121 -0
indexing/code_indexer.py
ADDED
|
@@ -0,0 +1,1763 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
Code Indexer using tree-sitter
|
|
4
|
+
Indexes files in a codebase and tracks functions, classes, variables, methods, and type definitions.
|
|
5
|
+
Supports multiple languages: Python, Go, C#, JavaScript, TypeScript, Rust, Zig, Elixir, C++, PHP
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
import time
|
|
11
|
+
import threading
|
|
12
|
+
import queue as queue_mod
|
|
13
|
+
from concurrent.futures import ProcessPoolExecutor, as_completed
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
import xxhash
|
|
16
|
+
|
|
17
|
+
from indexing.language_config import (
|
|
18
|
+
LANGUAGE_CONFIG,
|
|
19
|
+
)
|
|
20
|
+
from indexing.file_utils import (
|
|
21
|
+
detect_language,
|
|
22
|
+
collect_files_to_index,
|
|
23
|
+
read_file_content,
|
|
24
|
+
get_relative_path
|
|
25
|
+
)
|
|
26
|
+
from indexing.cli import parse_arguments, list_supported_languages, list_frontend_languages
|
|
27
|
+
from indexing.db_schema import init_database
|
|
28
|
+
from indexing.export_to_json import export_to_json
|
|
29
|
+
from indexing.code_index_sdk import CodeIndexSDK
|
|
30
|
+
from indexing.parse_worker import parse_file
|
|
31
|
+
from indexing.frontend.resolver import normalize_import_path, load_path_aliases, FrontendResolver
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
VAR_DATA = "this is to test the indexer"
|
|
35
|
+
|
|
36
|
+
class CodeIndexer:
|
|
37
|
+
def __init__(self, root_dir, languages=None, db_path=".code_index.raggie", force_reindex=False, verbose=False, frontend_enabled=True):
|
|
38
|
+
self.root_dir = Path(root_dir)
|
|
39
|
+
self.languages = languages if languages else list(LANGUAGE_CONFIG.keys())
|
|
40
|
+
self.frontend_enabled = frontend_enabled
|
|
41
|
+
if not frontend_enabled:
|
|
42
|
+
frontend_langs = {"html", "css", "tsx"}
|
|
43
|
+
self.languages = [l for l in self.languages if l not in frontend_langs]
|
|
44
|
+
self.parsers = {}
|
|
45
|
+
self.db_path = db_path
|
|
46
|
+
self.conn = None
|
|
47
|
+
self.cursor = None
|
|
48
|
+
self.current_file_id = None
|
|
49
|
+
self.current_class_id = None
|
|
50
|
+
self.force_reindex = force_reindex
|
|
51
|
+
self.batch_size = 50000
|
|
52
|
+
self.verbose = verbose
|
|
53
|
+
self.path_aliases = {} # loaded lazily on first frontend insert
|
|
54
|
+
|
|
55
|
+
# Statistics counters
|
|
56
|
+
self.stats = {
|
|
57
|
+
"total_functions": 0,
|
|
58
|
+
"total_macros": 0,
|
|
59
|
+
"total_classes": 0,
|
|
60
|
+
"total_variables": 0,
|
|
61
|
+
"total_methods": 0,
|
|
62
|
+
"total_type_defs": 0,
|
|
63
|
+
"total_structs": 0,
|
|
64
|
+
"total_interfaces": 0,
|
|
65
|
+
"total_enums": 0,
|
|
66
|
+
"total_namespaces": 0,
|
|
67
|
+
"skipped_files": 0
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
# Batch buffers for executemany()
|
|
71
|
+
self.batches = {
|
|
72
|
+
"files": [],
|
|
73
|
+
"functions": [],
|
|
74
|
+
"classes": [],
|
|
75
|
+
"variables": [],
|
|
76
|
+
"type_aliases": [],
|
|
77
|
+
"structs": [],
|
|
78
|
+
"interfaces": [],
|
|
79
|
+
"dependencies": []
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
self._initialize_database()
|
|
83
|
+
self._initialize_parsers()
|
|
84
|
+
|
|
85
|
+
def _flush_batches(self):
|
|
86
|
+
"""Flush all batch buffers using executemany()."""
|
|
87
|
+
if self.batches["variables"]:
|
|
88
|
+
self.cursor.executemany(
|
|
89
|
+
"""INSERT INTO variables
|
|
90
|
+
(file_id, parent_id, parent_type, name, type, location, field_type)
|
|
91
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)""",
|
|
92
|
+
self.batches["variables"]
|
|
93
|
+
)
|
|
94
|
+
self.batches["variables"] = []
|
|
95
|
+
|
|
96
|
+
if self.batches["type_aliases"]:
|
|
97
|
+
self.cursor.executemany(
|
|
98
|
+
"""INSERT INTO type_aliases
|
|
99
|
+
(file_id, name, location, type_definition)
|
|
100
|
+
VALUES (?, ?, ?, ?)""",
|
|
101
|
+
self.batches["type_aliases"]
|
|
102
|
+
)
|
|
103
|
+
self.batches["type_aliases"] = []
|
|
104
|
+
|
|
105
|
+
if self.batches["dependencies"]:
|
|
106
|
+
self.cursor.executemany(
|
|
107
|
+
"""INSERT INTO dependencies (file_id, dependency_type, name, source_function_id, target_function_id, target_class_id, temp_symbol_id, location, is_external)
|
|
108
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
|
109
|
+
self.batches["dependencies"]
|
|
110
|
+
)
|
|
111
|
+
self.batches["dependencies"] = []
|
|
112
|
+
self.conn.commit()
|
|
113
|
+
|
|
114
|
+
def _check_and_flush(self, batch_name):
|
|
115
|
+
"""Check if a batch has reached the size limit and flush if so."""
|
|
116
|
+
if len(self.batches[batch_name]) >= self.batch_size:
|
|
117
|
+
self._flush_batches()
|
|
118
|
+
|
|
119
|
+
def _initialize_database(self):
|
|
120
|
+
"""Initialize SQLite database."""
|
|
121
|
+
# init_database creates schema and returns a connection
|
|
122
|
+
init_database(self.db_path)
|
|
123
|
+
# Reopen with check_same_thread=False so the writer thread can use it
|
|
124
|
+
import sqlite3
|
|
125
|
+
self.conn = sqlite3.connect(self.db_path, check_same_thread=False)
|
|
126
|
+
# Optimized settings for batch indexing
|
|
127
|
+
self.conn.execute("PRAGMA journal_mode = MEMORY")
|
|
128
|
+
self.conn.execute("PRAGMA synchronous = OFF")
|
|
129
|
+
self.conn.execute("PRAGMA temp_store = MEMORY")
|
|
130
|
+
self.cursor = self.conn.cursor()
|
|
131
|
+
if self.verbose:
|
|
132
|
+
print(f"Database initialized: {self.db_path}")
|
|
133
|
+
|
|
134
|
+
def _initialize_parsers(self):
|
|
135
|
+
"""Initialize tree-sitter parsers for selected languages.
|
|
136
|
+
|
|
137
|
+
Parsers are only needed in the parent for the detect_language check
|
|
138
|
+
in index_directory. Actual parsing happens in worker processes
|
|
139
|
+
which have their own parser cache. Skip parser creation to save memory.
|
|
140
|
+
"""
|
|
141
|
+
# Only track which languages are configured (for the parser check in index_directory)
|
|
142
|
+
self._configured_languages = set()
|
|
143
|
+
for lang in self.languages:
|
|
144
|
+
if lang in LANGUAGE_CONFIG:
|
|
145
|
+
self._configured_languages.add(lang)
|
|
146
|
+
if self.verbose:
|
|
147
|
+
print(f"Configured parser for: {lang}")
|
|
148
|
+
|
|
149
|
+
def _get_dependent_files(self, file_id):
|
|
150
|
+
"""Get all files that have dependencies on symbols in the given file.
|
|
151
|
+
|
|
152
|
+
Checks both traditional dependency edges (function/class references) and
|
|
153
|
+
frontend dependency edges (render relationships, style imports, custom
|
|
154
|
+
property usages, event handler symbols, selector matches).
|
|
155
|
+
"""
|
|
156
|
+
cursor = self.conn.cursor()
|
|
157
|
+
|
|
158
|
+
# Get function IDs from this file
|
|
159
|
+
cursor.execute("SELECT id FROM functions WHERE file_id = ?", (file_id,))
|
|
160
|
+
function_ids = [row[0] for row in cursor.fetchall()]
|
|
161
|
+
|
|
162
|
+
# Get class IDs from this file
|
|
163
|
+
cursor.execute("SELECT id FROM classes WHERE file_id = ?", (file_id,))
|
|
164
|
+
class_ids = [row[0] for row in cursor.fetchall()]
|
|
165
|
+
|
|
166
|
+
# Get frontend component IDs from this file
|
|
167
|
+
cursor.execute("SELECT id FROM frontend_components WHERE file_id = ?", (file_id,))
|
|
168
|
+
component_ids = [row[0] for row in cursor.fetchall()]
|
|
169
|
+
|
|
170
|
+
# Get custom property IDs from this file
|
|
171
|
+
cursor.execute("SELECT id FROM style_custom_properties WHERE file_id = ?", (file_id,))
|
|
172
|
+
custom_property_ids = [row[0] for row in cursor.fetchall()]
|
|
173
|
+
|
|
174
|
+
# Get selector IDs from this file
|
|
175
|
+
cursor.execute("SELECT id FROM style_selectors WHERE file_id = ?", (file_id,))
|
|
176
|
+
selector_ids = [row[0] for row in cursor.fetchall()]
|
|
177
|
+
|
|
178
|
+
# Get markup element IDs from this file
|
|
179
|
+
cursor.execute("SELECT id FROM markup_elements WHERE file_id = ?", (file_id,))
|
|
180
|
+
element_ids = [row[0] for row in cursor.fetchall()]
|
|
181
|
+
|
|
182
|
+
dependent_file_ids = set()
|
|
183
|
+
|
|
184
|
+
# Find files that reference these functions
|
|
185
|
+
if function_ids:
|
|
186
|
+
placeholders = ','.join('?' * len(function_ids))
|
|
187
|
+
cursor.execute(
|
|
188
|
+
f"SELECT DISTINCT file_id FROM dependencies WHERE target_function_id IN ({placeholders})",
|
|
189
|
+
function_ids
|
|
190
|
+
)
|
|
191
|
+
dependent_file_ids.update(row[0] for row in cursor.fetchall())
|
|
192
|
+
|
|
193
|
+
# Find files that reference these classes
|
|
194
|
+
if class_ids:
|
|
195
|
+
placeholders = ','.join('?' * len(class_ids))
|
|
196
|
+
cursor.execute(
|
|
197
|
+
f"SELECT DISTINCT file_id FROM dependencies WHERE target_class_id IN ({placeholders})",
|
|
198
|
+
class_ids
|
|
199
|
+
)
|
|
200
|
+
dependent_file_ids.update(row[0] for row in cursor.fetchall())
|
|
201
|
+
|
|
202
|
+
# Frontend: render_relationships — files containing parent components
|
|
203
|
+
# that render this file's components as children
|
|
204
|
+
if component_ids:
|
|
205
|
+
placeholders = ','.join('?' * len(component_ids))
|
|
206
|
+
# Find parent component IDs that reference these child components
|
|
207
|
+
cursor.execute(
|
|
208
|
+
f"SELECT DISTINCT fc.file_id FROM render_relationships rr "
|
|
209
|
+
f"JOIN frontend_components fc ON rr.parent_component_id = fc.id "
|
|
210
|
+
f"WHERE rr.child_component_id IN ({placeholders})",
|
|
211
|
+
component_ids
|
|
212
|
+
)
|
|
213
|
+
dependent_file_ids.update(row[0] for row in cursor.fetchall())
|
|
214
|
+
|
|
215
|
+
# Frontend: style_imports — files that import this file via @import or <link>
|
|
216
|
+
cursor.execute(
|
|
217
|
+
"SELECT DISTINCT si.file_id FROM style_imports si WHERE si.resolved_file_id = ?",
|
|
218
|
+
(file_id,)
|
|
219
|
+
)
|
|
220
|
+
dependent_file_ids.update(row[0] for row in cursor.fetchall())
|
|
221
|
+
|
|
222
|
+
# Frontend: style_custom_property_usages — files that use this file's custom properties
|
|
223
|
+
if custom_property_ids:
|
|
224
|
+
placeholders = ','.join('?' * len(custom_property_ids))
|
|
225
|
+
cursor.execute(
|
|
226
|
+
f"SELECT DISTINCT file_id FROM style_custom_property_usages WHERE resolved_property_id IN ({placeholders})",
|
|
227
|
+
custom_property_ids
|
|
228
|
+
)
|
|
229
|
+
dependent_file_ids.update(row[0] for row in cursor.fetchall())
|
|
230
|
+
|
|
231
|
+
# Frontend: frontend_events — files whose events reference this file's handler functions
|
|
232
|
+
if function_ids:
|
|
233
|
+
placeholders = ','.join('?' * len(function_ids))
|
|
234
|
+
cursor.execute(
|
|
235
|
+
f"SELECT DISTINCT file_id FROM frontend_events WHERE handler_symbol_id IN ({placeholders})",
|
|
236
|
+
function_ids
|
|
237
|
+
)
|
|
238
|
+
dependent_file_ids.update(row[0] for row in cursor.fetchall())
|
|
239
|
+
|
|
240
|
+
# Frontend: style_selector_matches — bidirectional
|
|
241
|
+
# When a CSS file changes (selectors), files containing matched elements need re-resolution
|
|
242
|
+
if selector_ids:
|
|
243
|
+
placeholders = ','.join('?' * len(selector_ids))
|
|
244
|
+
cursor.execute(
|
|
245
|
+
f"SELECT DISTINCT me.file_id FROM style_selector_matches ssm "
|
|
246
|
+
f"JOIN markup_elements me ON ssm.element_id = me.id "
|
|
247
|
+
f"WHERE ssm.selector_id IN ({placeholders})",
|
|
248
|
+
selector_ids
|
|
249
|
+
)
|
|
250
|
+
dependent_file_ids.update(row[0] for row in cursor.fetchall())
|
|
251
|
+
|
|
252
|
+
# When a TSX/HTML file changes (elements), CSS files with matched selectors need re-resolution
|
|
253
|
+
if element_ids:
|
|
254
|
+
placeholders = ','.join('?' * len(element_ids))
|
|
255
|
+
cursor.execute(
|
|
256
|
+
f"SELECT DISTINCT ss.file_id FROM style_selector_matches ssm "
|
|
257
|
+
f"JOIN style_selectors ss ON ssm.selector_id = ss.id "
|
|
258
|
+
f"WHERE ssm.element_id IN ({placeholders})",
|
|
259
|
+
element_ids
|
|
260
|
+
)
|
|
261
|
+
dependent_file_ids.update(row[0] for row in cursor.fetchall())
|
|
262
|
+
|
|
263
|
+
# Don't include the file itself
|
|
264
|
+
dependent_file_ids.discard(file_id)
|
|
265
|
+
|
|
266
|
+
return list(dependent_file_ids)
|
|
267
|
+
|
|
268
|
+
def _delete_file_symbols(self, file_id):
|
|
269
|
+
"""Delete all symbol rows belonging to a file (for re-indexing)."""
|
|
270
|
+
cursor = self.conn.cursor()
|
|
271
|
+
|
|
272
|
+
# First, collect the IDs of symbols being deleted
|
|
273
|
+
cursor.execute("SELECT id FROM functions WHERE file_id = ?", (file_id,))
|
|
274
|
+
function_ids = [row[0] for row in cursor.fetchall()]
|
|
275
|
+
|
|
276
|
+
cursor.execute("SELECT id FROM classes WHERE file_id = ?", (file_id,))
|
|
277
|
+
class_ids = [row[0] for row in cursor.fetchall()]
|
|
278
|
+
|
|
279
|
+
# Update dependencies in OTHER files that reference these symbols to NULL their target IDs
|
|
280
|
+
# This prevents stale references when the file is re-indexed
|
|
281
|
+
if function_ids:
|
|
282
|
+
placeholders = ','.join('?' * len(function_ids))
|
|
283
|
+
cursor.execute(
|
|
284
|
+
f"UPDATE dependencies SET target_function_id = NULL WHERE target_function_id IN ({placeholders})",
|
|
285
|
+
function_ids
|
|
286
|
+
)
|
|
287
|
+
|
|
288
|
+
if class_ids:
|
|
289
|
+
placeholders = ','.join('?' * len(class_ids))
|
|
290
|
+
cursor.execute(
|
|
291
|
+
f"UPDATE dependencies SET target_class_id = NULL WHERE target_class_id IN ({placeholders})",
|
|
292
|
+
class_ids
|
|
293
|
+
)
|
|
294
|
+
|
|
295
|
+
# Collect frontend component IDs for cross-file FK cleanup
|
|
296
|
+
cursor.execute("SELECT id FROM frontend_components WHERE file_id = ?", (file_id,))
|
|
297
|
+
component_ids = [row[0] for row in cursor.fetchall()]
|
|
298
|
+
|
|
299
|
+
# Collect custom property IDs for cross-file FK cleanup
|
|
300
|
+
cursor.execute("SELECT id FROM style_custom_properties WHERE file_id = ?", (file_id,))
|
|
301
|
+
custom_property_ids = [row[0] for row in cursor.fetchall()]
|
|
302
|
+
|
|
303
|
+
# NULL out cross-file references to symbols being deleted (FK constraints not enforced)
|
|
304
|
+
if component_ids:
|
|
305
|
+
placeholders = ','.join('?' * len(component_ids))
|
|
306
|
+
# render_relationships in OTHER files referencing these components as child
|
|
307
|
+
cursor.execute(
|
|
308
|
+
f"UPDATE render_relationships SET child_component_id = NULL WHERE child_component_id IN ({placeholders})",
|
|
309
|
+
component_ids
|
|
310
|
+
)
|
|
311
|
+
# markup_elements in OTHER files referencing these components
|
|
312
|
+
cursor.execute(
|
|
313
|
+
f"UPDATE markup_elements SET component_id = NULL WHERE component_id IN ({placeholders})",
|
|
314
|
+
component_ids
|
|
315
|
+
)
|
|
316
|
+
# style_selectors in OTHER files referencing these components
|
|
317
|
+
cursor.execute(
|
|
318
|
+
f"UPDATE style_selectors SET component_id = NULL WHERE component_id IN ({placeholders})",
|
|
319
|
+
component_ids
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
if custom_property_ids:
|
|
323
|
+
placeholders = ','.join('?' * len(custom_property_ids))
|
|
324
|
+
# style_custom_property_usages in OTHER files referencing these properties
|
|
325
|
+
cursor.execute(
|
|
326
|
+
f"UPDATE style_custom_property_usages SET resolved_property_id = NULL WHERE resolved_property_id IN ({placeholders})",
|
|
327
|
+
custom_property_ids
|
|
328
|
+
)
|
|
329
|
+
|
|
330
|
+
# Now delete the file's own symbols
|
|
331
|
+
# Delete render_relationships FIRST (before frontend_components are deleted),
|
|
332
|
+
# since FK constraints are not enforced and the subquery would find no rows after.
|
|
333
|
+
cursor.execute(
|
|
334
|
+
"DELETE FROM render_relationships WHERE parent_component_id IN "
|
|
335
|
+
"(SELECT id FROM frontend_components WHERE file_id = ?)",
|
|
336
|
+
(file_id,)
|
|
337
|
+
)
|
|
338
|
+
cursor.execute(
|
|
339
|
+
"DELETE FROM style_selector_matches WHERE selector_id IN "
|
|
340
|
+
"(SELECT id FROM style_selectors WHERE file_id = ?) "
|
|
341
|
+
"OR element_id IN (SELECT id FROM markup_elements WHERE file_id = ?)",
|
|
342
|
+
(file_id, file_id)
|
|
343
|
+
)
|
|
344
|
+
# Tables with direct file_id column
|
|
345
|
+
for table in ("functions", "classes", "variables", "type_aliases", "structs", "interfaces", "dependencies", "temp_symbols", "namespaces",
|
|
346
|
+
"frontend_components", "markup_elements", "style_selectors", "style_custom_properties",
|
|
347
|
+
"style_custom_property_usages", "style_keyframes", "style_imports", "frontend_events",
|
|
348
|
+
"frontend_bindings", "frontend_diagnostics"):
|
|
349
|
+
cursor.execute(f"DELETE FROM {table} WHERE file_id = ?", (file_id,))
|
|
350
|
+
# Clean up temp_file_references where this file is the source
|
|
351
|
+
cursor.execute("DELETE FROM temp_file_references WHERE source_file_id = ?", (file_id,))
|
|
352
|
+
|
|
353
|
+
def _get_deleted_files(self, current_files):
|
|
354
|
+
"""Get list of file IDs for files that exist in database but not on filesystem."""
|
|
355
|
+
cursor = self.conn.cursor()
|
|
356
|
+
cursor.execute("SELECT id, absolute_path FROM files")
|
|
357
|
+
db_files = cursor.fetchall()
|
|
358
|
+
|
|
359
|
+
deleted_file_ids = []
|
|
360
|
+
current_file_paths = {str(f) for f in current_files}
|
|
361
|
+
|
|
362
|
+
for file_id, abs_path in db_files:
|
|
363
|
+
if abs_path not in current_file_paths:
|
|
364
|
+
deleted_file_ids.append(file_id)
|
|
365
|
+
|
|
366
|
+
return deleted_file_ids
|
|
367
|
+
|
|
368
|
+
def _remove_deleted_files(self, deleted_file_ids):
|
|
369
|
+
"""Remove deleted files and all their symbols from the database (bulk, chunked)."""
|
|
370
|
+
cursor = self.conn.cursor()
|
|
371
|
+
SQLITE_MAX_VARS = 999
|
|
372
|
+
|
|
373
|
+
for i in range(0, len(deleted_file_ids), SQLITE_MAX_VARS):
|
|
374
|
+
chunk = deleted_file_ids[i:i + SQLITE_MAX_VARS]
|
|
375
|
+
placeholders = ','.join('?' * len(chunk))
|
|
376
|
+
|
|
377
|
+
# Collect function/class IDs being deleted
|
|
378
|
+
cursor.execute(f"SELECT id FROM functions WHERE file_id IN ({placeholders})", chunk)
|
|
379
|
+
function_ids = [row[0] for row in cursor.fetchall()]
|
|
380
|
+
|
|
381
|
+
cursor.execute(f"SELECT id FROM classes WHERE file_id IN ({placeholders})", chunk)
|
|
382
|
+
class_ids = [row[0] for row in cursor.fetchall()]
|
|
383
|
+
|
|
384
|
+
# NULL out dependency references to deleted symbols
|
|
385
|
+
if function_ids:
|
|
386
|
+
fp = ','.join('?' * len(function_ids))
|
|
387
|
+
cursor.execute(f"UPDATE dependencies SET target_function_id = NULL WHERE target_function_id IN ({fp})", function_ids)
|
|
388
|
+
|
|
389
|
+
if class_ids:
|
|
390
|
+
cp = ','.join('?' * len(class_ids))
|
|
391
|
+
cursor.execute(f"UPDATE dependencies SET target_class_id = NULL WHERE target_class_id IN ({cp})", class_ids)
|
|
392
|
+
|
|
393
|
+
# Collect frontend component and custom property IDs for cross-file FK cleanup
|
|
394
|
+
cursor.execute(f"SELECT id FROM frontend_components WHERE file_id IN ({placeholders})", chunk)
|
|
395
|
+
component_ids = [row[0] for row in cursor.fetchall()]
|
|
396
|
+
|
|
397
|
+
cursor.execute(f"SELECT id FROM style_custom_properties WHERE file_id IN ({placeholders})", chunk)
|
|
398
|
+
custom_property_ids = [row[0] for row in cursor.fetchall()]
|
|
399
|
+
|
|
400
|
+
# NULL out cross-file references to symbols being deleted (FK constraints not enforced)
|
|
401
|
+
if component_ids:
|
|
402
|
+
fp = ','.join('?' * len(component_ids))
|
|
403
|
+
cursor.execute(f"UPDATE render_relationships SET child_component_id = NULL WHERE child_component_id IN ({fp})", component_ids)
|
|
404
|
+
cursor.execute(f"UPDATE markup_elements SET component_id = NULL WHERE component_id IN ({fp})", component_ids)
|
|
405
|
+
cursor.execute(f"UPDATE style_selectors SET component_id = NULL WHERE component_id IN ({fp})", component_ids)
|
|
406
|
+
|
|
407
|
+
if custom_property_ids:
|
|
408
|
+
cp2 = ','.join('?' * len(custom_property_ids))
|
|
409
|
+
cursor.execute(f"UPDATE style_custom_property_usages SET resolved_property_id = NULL WHERE resolved_property_id IN ({cp2})", custom_property_ids)
|
|
410
|
+
|
|
411
|
+
# NULL out style_imports.resolved_file_id in OTHER files pointing to deleted files
|
|
412
|
+
cursor.execute(f"UPDATE style_imports SET resolved_file_id = NULL WHERE resolved_file_id IN ({placeholders})", chunk)
|
|
413
|
+
|
|
414
|
+
# Delete render_relationships and style_selector_matches FIRST (before their parent
|
|
415
|
+
# tables are emptied), since FK constraints are not enforced and subqueries would
|
|
416
|
+
# find no rows after the parent table delete.
|
|
417
|
+
cursor.execute(
|
|
418
|
+
f"DELETE FROM render_relationships WHERE parent_component_id IN "
|
|
419
|
+
f"(SELECT id FROM frontend_components WHERE file_id IN ({placeholders}))",
|
|
420
|
+
chunk
|
|
421
|
+
)
|
|
422
|
+
cursor.execute(
|
|
423
|
+
f"DELETE FROM style_selector_matches WHERE selector_id IN "
|
|
424
|
+
f"(SELECT id FROM style_selectors WHERE file_id IN ({placeholders})) "
|
|
425
|
+
f"OR element_id IN (SELECT id FROM markup_elements WHERE file_id IN ({placeholders}))",
|
|
426
|
+
chunk + chunk
|
|
427
|
+
)
|
|
428
|
+
# Tables with direct file_id column
|
|
429
|
+
for table in ("functions", "classes", "variables", "type_aliases", "structs", "interfaces", "dependencies", "temp_symbols", "namespaces",
|
|
430
|
+
"frontend_components", "markup_elements", "style_selectors", "style_custom_properties",
|
|
431
|
+
"style_custom_property_usages", "style_keyframes", "style_imports", "frontend_events",
|
|
432
|
+
"frontend_bindings", "frontend_diagnostics"):
|
|
433
|
+
cursor.execute(f"DELETE FROM {table} WHERE file_id IN ({placeholders})", chunk)
|
|
434
|
+
|
|
435
|
+
# Clean up temp_file_references for deleted source files
|
|
436
|
+
cursor.execute(f"DELETE FROM temp_file_references WHERE source_file_id IN ({placeholders})", chunk)
|
|
437
|
+
|
|
438
|
+
# Delete file records
|
|
439
|
+
cursor.execute(f"DELETE FROM files WHERE id IN ({placeholders})", chunk)
|
|
440
|
+
|
|
441
|
+
self.conn.commit()
|
|
442
|
+
|
|
443
|
+
def _insert_file(self, file_path, language, content_hash, mtime):
|
|
444
|
+
"""Insert or update file record and return its ID. Returns None if file should be skipped."""
|
|
445
|
+
relative_path = get_relative_path(file_path, self.root_dir)
|
|
446
|
+
self.cursor.execute("SELECT id, content_hash, mtime FROM files WHERE path = ?", (relative_path,))
|
|
447
|
+
existing = self.cursor.fetchone()
|
|
448
|
+
if existing:
|
|
449
|
+
file_id, existing_hash, existing_mtime = existing
|
|
450
|
+
# Fast path: mtime unchanged => file hasn't been modified
|
|
451
|
+
if not self.force_reindex and existing_mtime == mtime:
|
|
452
|
+
return None # Skip reindexing
|
|
453
|
+
# Slow path: mtime changed, check content hash
|
|
454
|
+
if not self.force_reindex and content_hash is not None and existing_hash == content_hash:
|
|
455
|
+
# mtime changed but content is the same (e.g. touch), update mtime only
|
|
456
|
+
self.cursor.execute("UPDATE files SET mtime = ? WHERE id = ?", (mtime, file_id))
|
|
457
|
+
return None
|
|
458
|
+
# Content changed or force_reindex, reindex
|
|
459
|
+
self._delete_file_symbols(file_id)
|
|
460
|
+
self.cursor.execute(
|
|
461
|
+
"UPDATE files SET absolute_path = ?, language = ?, content_hash = ?, mtime = ? WHERE id = ?",
|
|
462
|
+
(str(file_path), language, content_hash, mtime, file_id)
|
|
463
|
+
)
|
|
464
|
+
return file_id
|
|
465
|
+
self.cursor.execute(
|
|
466
|
+
"INSERT INTO files (path, absolute_path, language, content_hash, mtime) VALUES (?, ?, ?, ?, ?)",
|
|
467
|
+
(relative_path, str(file_path), language, content_hash, mtime)
|
|
468
|
+
)
|
|
469
|
+
return self.cursor.lastrowid
|
|
470
|
+
|
|
471
|
+
def _update_file_hash(self, file_id, content_hash, mtime):
|
|
472
|
+
"""Update content hash and mtime after file has been read and parsed."""
|
|
473
|
+
self.cursor.execute(
|
|
474
|
+
"UPDATE files SET content_hash = ?, mtime = ? WHERE id = ?",
|
|
475
|
+
(content_hash, mtime, file_id)
|
|
476
|
+
)
|
|
477
|
+
|
|
478
|
+
def _insert_parsed_file(self, data):
|
|
479
|
+
"""Insert a file parsed by a worker process into the database.
|
|
480
|
+
|
|
481
|
+
Wraps the insertion in a SAVEPOINT so that a failed update does not
|
|
482
|
+
leave partially updated semantic state.
|
|
483
|
+
"""
|
|
484
|
+
file_path = Path(data['file_path'])
|
|
485
|
+
language = data['language']
|
|
486
|
+
content_hash = data['content_hash']
|
|
487
|
+
file_mtime = data['file_mtime']
|
|
488
|
+
|
|
489
|
+
# Insert/update file record
|
|
490
|
+
file_id = self._insert_file(file_path, language, content_hash, file_mtime)
|
|
491
|
+
if file_id is None:
|
|
492
|
+
self.stats["skipped_files"] += 1
|
|
493
|
+
return
|
|
494
|
+
|
|
495
|
+
self.current_file_id = file_id
|
|
496
|
+
|
|
497
|
+
try:
|
|
498
|
+
self._insert_parsed_file_inner(data, file_id)
|
|
499
|
+
except Exception as e:
|
|
500
|
+
if self.verbose:
|
|
501
|
+
print(f"Warning: Failed to insert {file_path}: {e}")
|
|
502
|
+
return
|
|
503
|
+
|
|
504
|
+
def _insert_parsed_file_inner(self, data, file_id):
|
|
505
|
+
"""Inner insertion logic — called within a SAVEPOINT."""
|
|
506
|
+
|
|
507
|
+
# Insert imports
|
|
508
|
+
for imp in data.get('imports', []):
|
|
509
|
+
self._insert_import(imp['name'], imp['location'], imp.get('is_external', True))
|
|
510
|
+
|
|
511
|
+
# Insert classes first (so methods can reference them) — batch
|
|
512
|
+
class_id_map = {} # temp_id -> real_id
|
|
513
|
+
class_names = []
|
|
514
|
+
class_temp_ids = []
|
|
515
|
+
class_rows = []
|
|
516
|
+
for cls in data.get('classes', []):
|
|
517
|
+
temp_id = cls.pop('_temp_id', None)
|
|
518
|
+
class_temp_ids.append(temp_id)
|
|
519
|
+
class_rows.append((
|
|
520
|
+
file_id, None, cls['name'],
|
|
521
|
+
json.dumps(cls['location']),
|
|
522
|
+
json.dumps(cls.get('base_classes', [])),
|
|
523
|
+
cls.get('docstring'),
|
|
524
|
+
cls.get('namespace'),
|
|
525
|
+
))
|
|
526
|
+
self.stats["total_classes"] += 1
|
|
527
|
+
class_names.append(cls['name'])
|
|
528
|
+
if class_rows:
|
|
529
|
+
self.cursor.executemany(
|
|
530
|
+
"""INSERT INTO classes (file_id, parent_id, name, location, base_classes, docstring, namespace)
|
|
531
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)""",
|
|
532
|
+
class_rows
|
|
533
|
+
)
|
|
534
|
+
# SELECT back IDs in insertion order to rebuild class_id_map
|
|
535
|
+
self.cursor.execute(
|
|
536
|
+
"SELECT id FROM classes WHERE file_id = ? ORDER BY id DESC LIMIT ?",
|
|
537
|
+
(file_id, len(class_rows))
|
|
538
|
+
)
|
|
539
|
+
real_ids = [r[0] for r in self.cursor.fetchall()]
|
|
540
|
+
real_ids.reverse() # back to insertion order
|
|
541
|
+
for temp_id, real_id in zip(class_temp_ids, real_ids):
|
|
542
|
+
if temp_id is not None:
|
|
543
|
+
class_id_map[temp_id] = real_id
|
|
544
|
+
|
|
545
|
+
# Insert functions/methods — batch
|
|
546
|
+
func_id_map = {}
|
|
547
|
+
func_names = []
|
|
548
|
+
func_rows = []
|
|
549
|
+
for i, func in enumerate(data.get('functions', [])):
|
|
550
|
+
parent_class_id = func.pop('parent_class_id', None)
|
|
551
|
+
if parent_class_id is not None:
|
|
552
|
+
parent_class_id = class_id_map.get(parent_class_id)
|
|
553
|
+
func_rows.append((
|
|
554
|
+
file_id,
|
|
555
|
+
parent_class_id,
|
|
556
|
+
'class' if parent_class_id else None,
|
|
557
|
+
func['name'],
|
|
558
|
+
func['type'],
|
|
559
|
+
json.dumps(func['location']),
|
|
560
|
+
json.dumps(func.get('parameters', [])),
|
|
561
|
+
func.get('return_type'),
|
|
562
|
+
func.get('docstring'),
|
|
563
|
+
func.get('receiver'),
|
|
564
|
+
func.get('branch_count', 0),
|
|
565
|
+
))
|
|
566
|
+
if parent_class_id is not None:
|
|
567
|
+
self.stats["total_methods"] += 1
|
|
568
|
+
elif func.get('type') == 'macro':
|
|
569
|
+
self.stats["total_macros"] += 1
|
|
570
|
+
else:
|
|
571
|
+
self.stats["total_functions"] += 1
|
|
572
|
+
func_names.append(func['name'])
|
|
573
|
+
if func_rows:
|
|
574
|
+
self.cursor.executemany(
|
|
575
|
+
"""INSERT INTO functions
|
|
576
|
+
(file_id, parent_id, parent_type, name, type, location, parameters, return_type, docstring, receiver, branch_count)
|
|
577
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
|
578
|
+
func_rows
|
|
579
|
+
)
|
|
580
|
+
# SELECT back IDs in insertion order to rebuild func_id_map
|
|
581
|
+
self.cursor.execute(
|
|
582
|
+
"SELECT id FROM functions WHERE file_id = ? ORDER BY id DESC LIMIT ?",
|
|
583
|
+
(file_id, len(func_rows))
|
|
584
|
+
)
|
|
585
|
+
real_ids = [r[0] for r in self.cursor.fetchall()]
|
|
586
|
+
real_ids.reverse()
|
|
587
|
+
for i, real_id in enumerate(real_ids):
|
|
588
|
+
func_id_map[i] = real_id
|
|
589
|
+
|
|
590
|
+
# Insert variables (batch)
|
|
591
|
+
var_names = []
|
|
592
|
+
var_rows = []
|
|
593
|
+
for var in data.get('variables', []):
|
|
594
|
+
parent_class_id = var.pop('parent_class_id', None)
|
|
595
|
+
if parent_class_id is not None:
|
|
596
|
+
parent_class_id = class_id_map.get(parent_class_id)
|
|
597
|
+
var_rows.append((
|
|
598
|
+
file_id,
|
|
599
|
+
parent_class_id,
|
|
600
|
+
'class' if parent_class_id else None,
|
|
601
|
+
var['name'],
|
|
602
|
+
var['type'],
|
|
603
|
+
json.dumps(var['location']),
|
|
604
|
+
var.get('field_type'),
|
|
605
|
+
))
|
|
606
|
+
self.stats["total_variables"] += 1
|
|
607
|
+
var_names.append(var['name'])
|
|
608
|
+
if var_rows:
|
|
609
|
+
self.cursor.executemany(
|
|
610
|
+
"""INSERT INTO variables (file_id, parent_id, parent_type, name, type, location, field_type)
|
|
611
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)""",
|
|
612
|
+
var_rows
|
|
613
|
+
)
|
|
614
|
+
|
|
615
|
+
# Dependencies are inserted as external with temp_symbols.
|
|
616
|
+
# Resolution happens in _resolve_all_dependencies after all files are indexed.
|
|
617
|
+
|
|
618
|
+
# Batch insert type aliases
|
|
619
|
+
type_alias_rows = []
|
|
620
|
+
type_alias_names = []
|
|
621
|
+
for alias in data.get('type_aliases', []):
|
|
622
|
+
type_alias_rows.append((
|
|
623
|
+
file_id, alias['name'],
|
|
624
|
+
json.dumps(alias['location']),
|
|
625
|
+
alias.get('type_definition'),
|
|
626
|
+
))
|
|
627
|
+
type_alias_names.append(alias['name'])
|
|
628
|
+
if type_alias_rows:
|
|
629
|
+
self.cursor.executemany(
|
|
630
|
+
"INSERT INTO type_aliases (file_id, name, location, type_definition) VALUES (?, ?, ?, ?)",
|
|
631
|
+
type_alias_rows
|
|
632
|
+
)
|
|
633
|
+
self.stats["total_type_defs"] += len(type_alias_rows)
|
|
634
|
+
|
|
635
|
+
# Batch insert structs
|
|
636
|
+
struct_rows = []
|
|
637
|
+
struct_names = []
|
|
638
|
+
for struct in data.get('structs', []):
|
|
639
|
+
struct_rows.append((file_id, struct['name'], json.dumps(struct['location'])))
|
|
640
|
+
struct_names.append(struct['name'])
|
|
641
|
+
if struct_rows:
|
|
642
|
+
self.cursor.executemany(
|
|
643
|
+
"INSERT INTO structs (file_id, name, location) VALUES (?, ?, ?)",
|
|
644
|
+
struct_rows
|
|
645
|
+
)
|
|
646
|
+
self.stats["total_structs"] += len(struct_rows)
|
|
647
|
+
|
|
648
|
+
# Batch insert interfaces
|
|
649
|
+
iface_rows = []
|
|
650
|
+
iface_names = []
|
|
651
|
+
for iface in data.get('interfaces', []):
|
|
652
|
+
iface_rows.append((file_id, iface['name'], json.dumps(iface['location'])))
|
|
653
|
+
iface_names.append(iface['name'])
|
|
654
|
+
if iface_rows:
|
|
655
|
+
self.cursor.executemany(
|
|
656
|
+
"INSERT INTO interfaces (file_id, name, location) VALUES (?, ?, ?)",
|
|
657
|
+
iface_rows
|
|
658
|
+
)
|
|
659
|
+
self.stats["total_interfaces"] += len(iface_rows)
|
|
660
|
+
|
|
661
|
+
# Batch insert enums
|
|
662
|
+
enum_rows = []
|
|
663
|
+
enum_names = []
|
|
664
|
+
for enum in data.get('enums', []):
|
|
665
|
+
enum_rows.append((file_id, enum['name'], json.dumps(enum['location'])))
|
|
666
|
+
enum_names.append(enum['name'])
|
|
667
|
+
if enum_rows:
|
|
668
|
+
self.cursor.executemany(
|
|
669
|
+
"INSERT INTO enums (file_id, name, location) VALUES (?, ?, ?)",
|
|
670
|
+
enum_rows
|
|
671
|
+
)
|
|
672
|
+
self.stats["total_enums"] += len(enum_rows)
|
|
673
|
+
|
|
674
|
+
# Batch insert namespaces
|
|
675
|
+
ns_rows = []
|
|
676
|
+
for ns in data.get('namespaces', []):
|
|
677
|
+
ns_rows.append((file_id, ns['name'], json.dumps(ns['location'])))
|
|
678
|
+
if ns_rows:
|
|
679
|
+
self.cursor.executemany(
|
|
680
|
+
"INSERT INTO namespaces (file_id, name, location) VALUES (?, ?, ?)",
|
|
681
|
+
ns_rows
|
|
682
|
+
)
|
|
683
|
+
self.stats["total_namespaces"] += len(ns_rows)
|
|
684
|
+
|
|
685
|
+
# Dependencies are inserted as external with temp_symbols.
|
|
686
|
+
# Resolution happens in _resolve_all_dependencies after all files are indexed.
|
|
687
|
+
|
|
688
|
+
# Insert dependencies — all as external with temp_symbols.
|
|
689
|
+
# Resolution happens in a single batch pass after all files are indexed.
|
|
690
|
+
# This avoids 7+ SELECT IN queries per file against tables that grow to millions of rows.
|
|
691
|
+
deps = data.get('dependencies', [])
|
|
692
|
+
if deps:
|
|
693
|
+
dep_rows = []
|
|
694
|
+
temp_pairs_set = set()
|
|
695
|
+
for dep in deps:
|
|
696
|
+
dep_type = dep['type']
|
|
697
|
+
dep_name = dep['name']
|
|
698
|
+
source_func_id = func_id_map.get(dep.pop('_func_index', None))
|
|
699
|
+
loc = json.dumps(dep.get('location')) if dep.get('location') else None
|
|
700
|
+
dep_rows.append((
|
|
701
|
+
file_id, dep_type, dep_name, source_func_id,
|
|
702
|
+
None, None, None, loc, 1,
|
|
703
|
+
))
|
|
704
|
+
temp_pairs_set.add((dep_name, dep_type))
|
|
705
|
+
|
|
706
|
+
# Batch insert dependencies
|
|
707
|
+
self.cursor.executemany(
|
|
708
|
+
"""INSERT INTO dependencies (file_id, dependency_type, name, source_function_id, target_function_id, target_class_id, temp_symbol_id, location, is_external)
|
|
709
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
|
710
|
+
dep_rows
|
|
711
|
+
)
|
|
712
|
+
|
|
713
|
+
# Batch create temp_symbols and link them to deps via SQL UPDATE
|
|
714
|
+
if temp_pairs_set:
|
|
715
|
+
temp_pairs = [(name, dtype, file_id) for name, dtype in temp_pairs_set]
|
|
716
|
+
self.cursor.executemany(
|
|
717
|
+
"INSERT OR IGNORE INTO temp_symbols (name, symbol_type, file_id) VALUES (?, ?, ?)",
|
|
718
|
+
temp_pairs
|
|
719
|
+
)
|
|
720
|
+
# Link temp_symbols to deps by matching (name, type) within this file
|
|
721
|
+
self.cursor.execute(
|
|
722
|
+
"""UPDATE dependencies SET temp_symbol_id = (
|
|
723
|
+
SELECT ts.id FROM temp_symbols ts
|
|
724
|
+
WHERE ts.name = dependencies.name
|
|
725
|
+
AND ts.symbol_type = dependencies.dependency_type
|
|
726
|
+
LIMIT 1
|
|
727
|
+
)
|
|
728
|
+
WHERE dependencies.file_id = ? AND dependencies.is_external = 1
|
|
729
|
+
AND dependencies.temp_symbol_id IS NULL""",
|
|
730
|
+
(file_id,)
|
|
731
|
+
)
|
|
732
|
+
|
|
733
|
+
# Insert frontend semantic data (HTML files)
|
|
734
|
+
self._insert_frontend_data(file_id, data)
|
|
735
|
+
|
|
736
|
+
def _insert_import(self, name, location, is_external):
|
|
737
|
+
"""Queue an import dependency for batch insertion."""
|
|
738
|
+
self.batches["dependencies"].append(
|
|
739
|
+
(
|
|
740
|
+
self.current_file_id,
|
|
741
|
+
'import',
|
|
742
|
+
name,
|
|
743
|
+
None,
|
|
744
|
+
None,
|
|
745
|
+
None,
|
|
746
|
+
None,
|
|
747
|
+
json.dumps(location) if location else None,
|
|
748
|
+
1 if is_external else 0
|
|
749
|
+
)
|
|
750
|
+
)
|
|
751
|
+
self._check_and_flush("dependencies")
|
|
752
|
+
|
|
753
|
+
def _insert_frontend_data(self, file_id, data):
|
|
754
|
+
"""Insert frontend semantic data (markup elements, events, selectors, etc.).
|
|
755
|
+
|
|
756
|
+
Also performs inline cross-file resolution using temp_symbols and temp_file_references,
|
|
757
|
+
consistent with the existing temp_symbols pattern for dependencies.
|
|
758
|
+
"""
|
|
759
|
+
# Lazily load path aliases on first frontend insert
|
|
760
|
+
if not hasattr(self, '_path_aliases_loaded'):
|
|
761
|
+
self.path_aliases = load_path_aliases(self.root_dir)
|
|
762
|
+
self._path_aliases_loaded = True
|
|
763
|
+
|
|
764
|
+
# Get the file path for import normalization
|
|
765
|
+
self.cursor.execute("SELECT absolute_path FROM files WHERE id = ?", (file_id,))
|
|
766
|
+
file_row = self.cursor.fetchone()
|
|
767
|
+
source_file_path = file_row[0] if file_row else ""
|
|
768
|
+
|
|
769
|
+
# Insert frontend components (JSX/TSX files)
|
|
770
|
+
component_id_map = {} # component_index -> db id
|
|
771
|
+
for i, comp in enumerate(data.get('frontend_components', [])):
|
|
772
|
+
# Try to link to already-inserted function/class by name
|
|
773
|
+
impl_func_id = None
|
|
774
|
+
impl_class_id = None
|
|
775
|
+
func_name = comp.get('impl_function_name')
|
|
776
|
+
class_name = comp.get('impl_class_name')
|
|
777
|
+
if func_name:
|
|
778
|
+
self.cursor.execute(
|
|
779
|
+
"SELECT id FROM functions WHERE file_id = ? AND name = ?",
|
|
780
|
+
(file_id, func_name)
|
|
781
|
+
)
|
|
782
|
+
row = self.cursor.fetchone()
|
|
783
|
+
if row:
|
|
784
|
+
impl_func_id = row[0]
|
|
785
|
+
if class_name:
|
|
786
|
+
self.cursor.execute(
|
|
787
|
+
"SELECT id FROM classes WHERE file_id = ? AND name = ?",
|
|
788
|
+
(file_id, class_name)
|
|
789
|
+
)
|
|
790
|
+
row = self.cursor.fetchone()
|
|
791
|
+
if row:
|
|
792
|
+
impl_class_id = row[0]
|
|
793
|
+
|
|
794
|
+
self.cursor.execute(
|
|
795
|
+
"""INSERT INTO frontend_components
|
|
796
|
+
(file_id, name, framework, source_range, is_exported, impl_function_id, impl_class_id)
|
|
797
|
+
VALUES (?, ?, 'react', ?, ?, ?, ?)""",
|
|
798
|
+
(
|
|
799
|
+
file_id,
|
|
800
|
+
comp['name'],
|
|
801
|
+
json.dumps(comp.get('source_range')) if comp.get('source_range') else None,
|
|
802
|
+
1 if comp.get('is_exported') else 0,
|
|
803
|
+
impl_func_id,
|
|
804
|
+
impl_class_id,
|
|
805
|
+
)
|
|
806
|
+
)
|
|
807
|
+
comp_db_id = self.cursor.lastrowid
|
|
808
|
+
component_id_map[i] = comp_db_id
|
|
809
|
+
|
|
810
|
+
# Resolve any pending temp_symbols for this component name
|
|
811
|
+
self._resolve_component_temp(comp['name'], comp_db_id)
|
|
812
|
+
|
|
813
|
+
# Insert markup elements with parent-child and component resolution
|
|
814
|
+
elem_id_map = {} # index -> db id
|
|
815
|
+
for i, elem in enumerate(data.get('markup_elements', [])):
|
|
816
|
+
parent_idx = elem.get('parent_index')
|
|
817
|
+
parent_db_id = elem_id_map.get(parent_idx) if parent_idx is not None else None
|
|
818
|
+
comp_idx = elem.get('component_index')
|
|
819
|
+
comp_db_id = component_id_map.get(comp_idx) if comp_idx is not None else None
|
|
820
|
+
self.cursor.execute(
|
|
821
|
+
"""INSERT INTO markup_elements
|
|
822
|
+
(file_id, component_id, parent_element_id, tag_name, element_type,
|
|
823
|
+
source_range, element_id_attr, static_classes, attributes,
|
|
824
|
+
is_conditional, is_repeated, conditional_expr, repeated_expr)
|
|
825
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
|
826
|
+
(
|
|
827
|
+
file_id,
|
|
828
|
+
comp_db_id,
|
|
829
|
+
parent_db_id,
|
|
830
|
+
elem['tag_name'],
|
|
831
|
+
elem['element_type'],
|
|
832
|
+
json.dumps(elem.get('source_range')) if elem.get('source_range') else None,
|
|
833
|
+
elem.get('element_id_attr'),
|
|
834
|
+
json.dumps(elem.get('static_classes')) if elem.get('static_classes') else None,
|
|
835
|
+
json.dumps(elem.get('attributes')) if elem.get('attributes') else None,
|
|
836
|
+
1 if elem.get('is_conditional') else 0,
|
|
837
|
+
1 if elem.get('is_repeated') else 0,
|
|
838
|
+
elem.get('conditional_expr'),
|
|
839
|
+
elem.get('repeated_expr'),
|
|
840
|
+
)
|
|
841
|
+
)
|
|
842
|
+
elem_id_map[i] = self.cursor.lastrowid
|
|
843
|
+
|
|
844
|
+
# Insert style selectors
|
|
845
|
+
selector_id_map = {} # index -> db id
|
|
846
|
+
for i, sel in enumerate(data.get('style_selectors', [])):
|
|
847
|
+
self.cursor.execute(
|
|
848
|
+
"""INSERT INTO style_selectors
|
|
849
|
+
(file_id, selector_text, normalized_selector, selector_type, source_range, is_scoped)
|
|
850
|
+
VALUES (?, ?, ?, ?, ?, ?)""",
|
|
851
|
+
(
|
|
852
|
+
file_id,
|
|
853
|
+
sel['selector_text'],
|
|
854
|
+
sel.get('normalized_selector'),
|
|
855
|
+
sel['selector_type'],
|
|
856
|
+
json.dumps(sel.get('source_range')) if sel.get('source_range') else None,
|
|
857
|
+
1 if sel.get('is_scoped') else 0,
|
|
858
|
+
)
|
|
859
|
+
)
|
|
860
|
+
selector_id_map[i] = self.cursor.lastrowid
|
|
861
|
+
|
|
862
|
+
# Insert style custom properties
|
|
863
|
+
for cp in data.get('style_custom_properties', []):
|
|
864
|
+
self.cursor.execute(
|
|
865
|
+
"""INSERT INTO style_custom_properties
|
|
866
|
+
(file_id, name, value, source_range, scope_selector)
|
|
867
|
+
VALUES (?, ?, ?, ?, ?)""",
|
|
868
|
+
(
|
|
869
|
+
file_id,
|
|
870
|
+
cp['name'],
|
|
871
|
+
cp.get('value'),
|
|
872
|
+
json.dumps(cp.get('source_range')) if cp.get('source_range') else None,
|
|
873
|
+
cp.get('scope_selector'),
|
|
874
|
+
)
|
|
875
|
+
)
|
|
876
|
+
cp_db_id = self.cursor.lastrowid
|
|
877
|
+
# Resolve any pending temp_symbols for this custom property name
|
|
878
|
+
self._resolve_custom_property_temp(cp['name'], cp_db_id)
|
|
879
|
+
|
|
880
|
+
# Insert style custom property usages with resolution
|
|
881
|
+
for pu in data.get('style_custom_property_usages', []):
|
|
882
|
+
prop_name = pu['property_name']
|
|
883
|
+
# Try to resolve to existing custom property definition
|
|
884
|
+
resolved_id = self._find_custom_property(prop_name)
|
|
885
|
+
self.cursor.execute(
|
|
886
|
+
"""INSERT INTO style_custom_property_usages
|
|
887
|
+
(file_id, property_name, source_range, selector_id, resolved_property_id)
|
|
888
|
+
VALUES (?, ?, ?, NULL, ?)""",
|
|
889
|
+
(
|
|
890
|
+
file_id,
|
|
891
|
+
prop_name,
|
|
892
|
+
json.dumps(pu.get('source_range')) if pu.get('source_range') else None,
|
|
893
|
+
resolved_id,
|
|
894
|
+
)
|
|
895
|
+
)
|
|
896
|
+
if resolved_id is None:
|
|
897
|
+
# Create temp symbol for unresolved custom property reference
|
|
898
|
+
self._create_temp_symbol(prop_name, 'custom_property_reference')
|
|
899
|
+
|
|
900
|
+
# Insert style imports with file-path resolution
|
|
901
|
+
for si in data.get('style_imports', []):
|
|
902
|
+
import_path = si['import_path']
|
|
903
|
+
is_external = si.get('is_external', False) or import_path.startswith(("http://", "https://", "//"))
|
|
904
|
+
|
|
905
|
+
self.cursor.execute(
|
|
906
|
+
"""INSERT INTO style_imports
|
|
907
|
+
(file_id, import_path, is_external, resolved_file_id, source_range)
|
|
908
|
+
VALUES (?, ?, ?, NULL, ?)""",
|
|
909
|
+
(
|
|
910
|
+
file_id,
|
|
911
|
+
import_path,
|
|
912
|
+
1 if is_external else 0,
|
|
913
|
+
json.dumps(si.get('source_range')) if si.get('source_range') else None,
|
|
914
|
+
)
|
|
915
|
+
)
|
|
916
|
+
si_row_id = self.cursor.lastrowid
|
|
917
|
+
|
|
918
|
+
if not is_external:
|
|
919
|
+
# Try to resolve the import path to an existing file
|
|
920
|
+
resolved_path = normalize_import_path(import_path, source_file_path, str(self.root_dir), self.path_aliases)
|
|
921
|
+
resolved_file_id = self._find_file_by_path(resolved_path)
|
|
922
|
+
if resolved_file_id:
|
|
923
|
+
self.cursor.execute(
|
|
924
|
+
"UPDATE style_imports SET resolved_file_id = ? WHERE id = ?",
|
|
925
|
+
(resolved_file_id, si_row_id)
|
|
926
|
+
)
|
|
927
|
+
else:
|
|
928
|
+
# Create temp_file_reference for later resolution
|
|
929
|
+
self._create_temp_file_reference(
|
|
930
|
+
resolved_path, 'style_import', file_id, 'style_imports', si_row_id
|
|
931
|
+
)
|
|
932
|
+
|
|
933
|
+
# Insert frontend events
|
|
934
|
+
for ev in data.get('frontend_events', []):
|
|
935
|
+
elem_idx = ev.get('element_index')
|
|
936
|
+
elem_db_id = elem_id_map.get(elem_idx) if elem_idx is not None else None
|
|
937
|
+
self.cursor.execute(
|
|
938
|
+
"""INSERT INTO frontend_events
|
|
939
|
+
(file_id, element_id, event_name, handler_type, handler_expression,
|
|
940
|
+
handler_symbol_id, resolution_status, source_range)
|
|
941
|
+
VALUES (?, ?, ?, ?, ?, NULL, ?, ?)""",
|
|
942
|
+
(
|
|
943
|
+
file_id,
|
|
944
|
+
elem_db_id,
|
|
945
|
+
ev['event_name'],
|
|
946
|
+
ev['handler_type'],
|
|
947
|
+
ev.get('handler_expression'),
|
|
948
|
+
ev.get('resolution_status', 'unresolved'),
|
|
949
|
+
json.dumps(ev.get('source_range')) if ev.get('source_range') else None,
|
|
950
|
+
)
|
|
951
|
+
)
|
|
952
|
+
|
|
953
|
+
# Insert style selector matches
|
|
954
|
+
for sm in data.get('style_selector_matches', []):
|
|
955
|
+
sel_idx = sm.get('selector_index')
|
|
956
|
+
elem_idx = sm.get('element_index')
|
|
957
|
+
sel_db_id = selector_id_map.get(sel_idx) if sel_idx is not None else None
|
|
958
|
+
elem_db_id = elem_id_map.get(elem_idx) if elem_idx is not None else None
|
|
959
|
+
if sel_db_id and elem_db_id:
|
|
960
|
+
self.cursor.execute(
|
|
961
|
+
"""INSERT INTO style_selector_matches
|
|
962
|
+
(selector_id, element_id, match_type, confidence, source_range)
|
|
963
|
+
VALUES (?, ?, ?, ?, ?)""",
|
|
964
|
+
(
|
|
965
|
+
sel_db_id,
|
|
966
|
+
elem_db_id,
|
|
967
|
+
sm['match_type'],
|
|
968
|
+
sm.get('confidence', 'high'),
|
|
969
|
+
json.dumps(sm.get('source_range')) if sm.get('source_range') else None,
|
|
970
|
+
)
|
|
971
|
+
)
|
|
972
|
+
|
|
973
|
+
# Insert frontend diagnostics
|
|
974
|
+
for diag in data.get('frontend_diagnostics', []):
|
|
975
|
+
self.cursor.execute(
|
|
976
|
+
"""INSERT INTO frontend_diagnostics
|
|
977
|
+
(file_id, diagnostic_type, severity, message, source_range)
|
|
978
|
+
VALUES (?, ?, ?, ?, ?)""",
|
|
979
|
+
(
|
|
980
|
+
file_id,
|
|
981
|
+
diag['diagnostic_type'],
|
|
982
|
+
diag['severity'],
|
|
983
|
+
diag.get('message'),
|
|
984
|
+
json.dumps(diag.get('source_range')) if diag.get('source_range') else None,
|
|
985
|
+
)
|
|
986
|
+
)
|
|
987
|
+
|
|
988
|
+
# Insert style keyframes (CSS files)
|
|
989
|
+
for kf in data.get('style_keyframes', []):
|
|
990
|
+
self.cursor.execute(
|
|
991
|
+
"""INSERT INTO style_keyframes
|
|
992
|
+
(file_id, name, source_range)
|
|
993
|
+
VALUES (?, ?, ?)""",
|
|
994
|
+
(
|
|
995
|
+
file_id,
|
|
996
|
+
kf['name'],
|
|
997
|
+
json.dumps(kf.get('source_range')) if kf.get('source_range') else None,
|
|
998
|
+
)
|
|
999
|
+
)
|
|
1000
|
+
|
|
1001
|
+
# Insert frontend bindings (JSX/TSX files)
|
|
1002
|
+
for b in data.get('frontend_bindings', []):
|
|
1003
|
+
elem_db_id = elem_id_map.get(b.get('element_index'))
|
|
1004
|
+
self.cursor.execute(
|
|
1005
|
+
"""INSERT INTO frontend_bindings
|
|
1006
|
+
(file_id, element_id, binding_type, binding_name, binding_expression,
|
|
1007
|
+
resolution_status, source_range)
|
|
1008
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)""",
|
|
1009
|
+
(
|
|
1010
|
+
file_id,
|
|
1011
|
+
elem_db_id,
|
|
1012
|
+
b['binding_type'],
|
|
1013
|
+
b.get('binding_name'),
|
|
1014
|
+
b.get('expression'),
|
|
1015
|
+
b.get('resolution_status', 'unresolved'),
|
|
1016
|
+
json.dumps(b.get('source_range')) if b.get('source_range') else None,
|
|
1017
|
+
)
|
|
1018
|
+
)
|
|
1019
|
+
|
|
1020
|
+
# Insert render relationships with component resolution
|
|
1021
|
+
for rr in data.get('render_relationships', []):
|
|
1022
|
+
parent_db_id = component_id_map.get(rr.get('parent_component_index'))
|
|
1023
|
+
child_elem_db_id = elem_id_map.get(rr.get('element_index'))
|
|
1024
|
+
child_name = rr.get('child_component_name')
|
|
1025
|
+
|
|
1026
|
+
# Try to resolve child component by name
|
|
1027
|
+
child_component_id = None
|
|
1028
|
+
if child_name:
|
|
1029
|
+
child_component_id = self._find_component_by_name(child_name)
|
|
1030
|
+
|
|
1031
|
+
self.cursor.execute(
|
|
1032
|
+
"""INSERT INTO render_relationships
|
|
1033
|
+
(parent_component_id, child_component_id, child_component_name, child_element_id, render_type, controlling_expr)
|
|
1034
|
+
VALUES (?, ?, ?, ?, ?, ?)""",
|
|
1035
|
+
(
|
|
1036
|
+
parent_db_id,
|
|
1037
|
+
child_component_id,
|
|
1038
|
+
child_name,
|
|
1039
|
+
child_elem_db_id,
|
|
1040
|
+
rr['render_type'],
|
|
1041
|
+
rr.get('controlling_expr'),
|
|
1042
|
+
)
|
|
1043
|
+
)
|
|
1044
|
+
|
|
1045
|
+
# If child component not found, create temp symbol for later resolution
|
|
1046
|
+
if child_name and child_component_id is None:
|
|
1047
|
+
self._create_temp_symbol(child_name, 'component_reference')
|
|
1048
|
+
|
|
1049
|
+
# After inserting all data for this file, resolve any pending temp_file_references
|
|
1050
|
+
# that point to this file
|
|
1051
|
+
relative_path = get_relative_path(source_file_path, self.root_dir)
|
|
1052
|
+
self._resolve_temp_file_references_for_file(file_id, relative_path)
|
|
1053
|
+
|
|
1054
|
+
def _find_component_by_name(self, name):
|
|
1055
|
+
"""Find a frontend_component by name. Returns its id or None."""
|
|
1056
|
+
self.cursor.execute(
|
|
1057
|
+
"SELECT id FROM frontend_components WHERE name = ? LIMIT 1",
|
|
1058
|
+
(name,)
|
|
1059
|
+
)
|
|
1060
|
+
row = self.cursor.fetchone()
|
|
1061
|
+
return row[0] if row else None
|
|
1062
|
+
|
|
1063
|
+
def _resolve_component_temp(self, name, component_id):
|
|
1064
|
+
"""When a component is defined, resolve pending component_reference temp_symbols.
|
|
1065
|
+
|
|
1066
|
+
Updates render_relationships.child_component_id for any rows that
|
|
1067
|
+
were waiting for this component.
|
|
1068
|
+
"""
|
|
1069
|
+
self.cursor.execute(
|
|
1070
|
+
"SELECT id FROM temp_symbols WHERE name = ? AND symbol_type = 'component_reference'",
|
|
1071
|
+
(name,)
|
|
1072
|
+
)
|
|
1073
|
+
temp_rows = self.cursor.fetchall()
|
|
1074
|
+
for temp_row in temp_rows:
|
|
1075
|
+
temp_id = temp_row[0]
|
|
1076
|
+
self.cursor.execute(
|
|
1077
|
+
"UPDATE render_relationships SET child_component_id = ? WHERE child_component_id IS NULL AND child_component_name = ?",
|
|
1078
|
+
(component_id, name)
|
|
1079
|
+
)
|
|
1080
|
+
self.cursor.execute("DELETE FROM temp_symbols WHERE id = ?", (temp_id,))
|
|
1081
|
+
|
|
1082
|
+
def _find_custom_property(self, name):
|
|
1083
|
+
"""Find a style_custom_property by name. Returns its id or None."""
|
|
1084
|
+
self.cursor.execute(
|
|
1085
|
+
"SELECT id FROM style_custom_properties WHERE name = ? LIMIT 1",
|
|
1086
|
+
(name,)
|
|
1087
|
+
)
|
|
1088
|
+
row = self.cursor.fetchone()
|
|
1089
|
+
return row[0] if row else None
|
|
1090
|
+
|
|
1091
|
+
def _resolve_custom_property_temp(self, name, property_id):
|
|
1092
|
+
"""When a custom property is defined, resolve pending custom_property_reference temp_symbols.
|
|
1093
|
+
|
|
1094
|
+
Updates style_custom_property_usages.resolved_property_id for any rows
|
|
1095
|
+
that were waiting for this property definition.
|
|
1096
|
+
"""
|
|
1097
|
+
self.cursor.execute(
|
|
1098
|
+
"SELECT id FROM temp_symbols WHERE name = ? AND symbol_type = 'custom_property_reference'",
|
|
1099
|
+
(name,)
|
|
1100
|
+
)
|
|
1101
|
+
temp_rows = self.cursor.fetchall()
|
|
1102
|
+
for temp_row in temp_rows:
|
|
1103
|
+
temp_id = temp_row[0]
|
|
1104
|
+
self.cursor.execute(
|
|
1105
|
+
"UPDATE style_custom_property_usages SET resolved_property_id = ? WHERE resolved_property_id IS NULL AND property_name = ?",
|
|
1106
|
+
(property_id, name)
|
|
1107
|
+
)
|
|
1108
|
+
self.cursor.execute("DELETE FROM temp_symbols WHERE id = ?", (temp_id,))
|
|
1109
|
+
|
|
1110
|
+
def _find_file_by_path(self, relative_path):
|
|
1111
|
+
"""Find a file by its project-relative path. Returns its id or None."""
|
|
1112
|
+
self.cursor.execute(
|
|
1113
|
+
"SELECT id FROM files WHERE path = ? LIMIT 1",
|
|
1114
|
+
(relative_path,)
|
|
1115
|
+
)
|
|
1116
|
+
row = self.cursor.fetchone()
|
|
1117
|
+
return row[0] if row else None
|
|
1118
|
+
|
|
1119
|
+
def _create_temp_file_reference(self, resolved_path, reference_type, source_file_id, target_table, target_row_id):
|
|
1120
|
+
"""Create a temp_file_reference for an unresolved file-path reference."""
|
|
1121
|
+
self.cursor.execute(
|
|
1122
|
+
"""INSERT INTO temp_file_references
|
|
1123
|
+
(resolved_path, reference_type, source_file_id, target_table, target_row_id)
|
|
1124
|
+
VALUES (?, ?, ?, ?, ?)""",
|
|
1125
|
+
(resolved_path, reference_type, source_file_id, target_table, target_row_id)
|
|
1126
|
+
)
|
|
1127
|
+
|
|
1128
|
+
def _resolve_temp_file_references_for_file(self, file_id, relative_path):
|
|
1129
|
+
"""When a file is indexed, resolve any temp_file_references pointing to it.
|
|
1130
|
+
|
|
1131
|
+
Updates the target table rows (e.g. style_imports.resolved_file_id)
|
|
1132
|
+
and deletes the resolved temp_file_reference entries.
|
|
1133
|
+
"""
|
|
1134
|
+
self.cursor.execute(
|
|
1135
|
+
"SELECT id, target_table, target_row_id FROM temp_file_references WHERE resolved_path = ?",
|
|
1136
|
+
(relative_path,)
|
|
1137
|
+
)
|
|
1138
|
+
refs = self.cursor.fetchall()
|
|
1139
|
+
for ref_id, target_table, target_row_id in refs:
|
|
1140
|
+
if target_table == 'style_imports':
|
|
1141
|
+
self.cursor.execute(
|
|
1142
|
+
"UPDATE style_imports SET resolved_file_id = ? WHERE id = ?",
|
|
1143
|
+
(file_id, target_row_id)
|
|
1144
|
+
)
|
|
1145
|
+
self.cursor.execute("DELETE FROM temp_file_references WHERE id = ?", (ref_id,))
|
|
1146
|
+
|
|
1147
|
+
def _resolve_dependency(self, name, dep_type, source_file_id=None):
|
|
1148
|
+
"""Try to resolve a dependency to an existing symbol.
|
|
1149
|
+
Returns (target_function_id, target_class_id, is_external) or (None, None, 1).
|
|
1150
|
+
|
|
1151
|
+
Args:
|
|
1152
|
+
name: The dependency name
|
|
1153
|
+
dep_type: The dependency type (function_call, method_call, etc.)
|
|
1154
|
+
source_file_id: Optional file ID of the source making the reference, for disambiguation
|
|
1155
|
+
"""
|
|
1156
|
+
# For method calls like self.init or obj.method, extract just the method name
|
|
1157
|
+
resolve_name = name
|
|
1158
|
+
if dep_type == 'method_call' and '.' in name:
|
|
1159
|
+
resolve_name = name.rsplit('.', 1)[-1]
|
|
1160
|
+
|
|
1161
|
+
if dep_type in ('function_call', 'method_call'):
|
|
1162
|
+
# Prefer functions in the same file if available
|
|
1163
|
+
if source_file_id:
|
|
1164
|
+
self.cursor.execute(
|
|
1165
|
+
"SELECT id, parent_type FROM functions WHERE name = ? AND file_id = ? LIMIT 1",
|
|
1166
|
+
(resolve_name, source_file_id)
|
|
1167
|
+
)
|
|
1168
|
+
row = self.cursor.fetchone()
|
|
1169
|
+
if row:
|
|
1170
|
+
if dep_type == 'method_call' and row[1] != 'class':
|
|
1171
|
+
pass # It's a standalone function, not a method
|
|
1172
|
+
else:
|
|
1173
|
+
return (row[0], None, 0)
|
|
1174
|
+
# Fall back to any function with that name
|
|
1175
|
+
self.cursor.execute(
|
|
1176
|
+
"SELECT id, parent_type FROM functions WHERE name = ? LIMIT 1",
|
|
1177
|
+
(resolve_name,)
|
|
1178
|
+
)
|
|
1179
|
+
row = self.cursor.fetchone()
|
|
1180
|
+
if row:
|
|
1181
|
+
if dep_type == 'method_call' and row[1] != 'class':
|
|
1182
|
+
pass # It's a standalone function, not a method
|
|
1183
|
+
else:
|
|
1184
|
+
return (row[0], None, 0)
|
|
1185
|
+
|
|
1186
|
+
if dep_type in ('function_call', 'method_call', 'class_reference'):
|
|
1187
|
+
# Prefer classes in the same file if available
|
|
1188
|
+
if source_file_id:
|
|
1189
|
+
self.cursor.execute(
|
|
1190
|
+
"SELECT id FROM classes WHERE name = ? AND file_id = ? LIMIT 1",
|
|
1191
|
+
(resolve_name, source_file_id)
|
|
1192
|
+
)
|
|
1193
|
+
row = self.cursor.fetchone()
|
|
1194
|
+
if row:
|
|
1195
|
+
return (None, row[0], 0)
|
|
1196
|
+
# Fall back to any class with that name
|
|
1197
|
+
self.cursor.execute(
|
|
1198
|
+
"SELECT id FROM classes WHERE name = ? LIMIT 1",
|
|
1199
|
+
(resolve_name,)
|
|
1200
|
+
)
|
|
1201
|
+
row = self.cursor.fetchone()
|
|
1202
|
+
if row:
|
|
1203
|
+
return (None, row[0], 0)
|
|
1204
|
+
|
|
1205
|
+
for table in ('structs', 'interfaces', 'enums', 'type_aliases'):
|
|
1206
|
+
self.cursor.execute(
|
|
1207
|
+
f"SELECT id FROM {table} WHERE name = ? LIMIT 1",
|
|
1208
|
+
(resolve_name,)
|
|
1209
|
+
)
|
|
1210
|
+
row = self.cursor.fetchone()
|
|
1211
|
+
if row:
|
|
1212
|
+
return (None, None, 0)
|
|
1213
|
+
|
|
1214
|
+
if dep_type == 'variable_reference':
|
|
1215
|
+
# Prefer variables in the same file if available
|
|
1216
|
+
if source_file_id:
|
|
1217
|
+
self.cursor.execute(
|
|
1218
|
+
"SELECT id FROM variables WHERE name = ? AND file_id = ? LIMIT 1",
|
|
1219
|
+
(name, source_file_id)
|
|
1220
|
+
)
|
|
1221
|
+
row = self.cursor.fetchone()
|
|
1222
|
+
if row:
|
|
1223
|
+
return (None, None, 0)
|
|
1224
|
+
# Fall back to any variable with that name
|
|
1225
|
+
self.cursor.execute(
|
|
1226
|
+
"SELECT id FROM variables WHERE name = ? LIMIT 1",
|
|
1227
|
+
(name,)
|
|
1228
|
+
)
|
|
1229
|
+
row = self.cursor.fetchone()
|
|
1230
|
+
if row:
|
|
1231
|
+
return (None, None, 0)
|
|
1232
|
+
|
|
1233
|
+
return (None, None, 1)
|
|
1234
|
+
|
|
1235
|
+
def _create_temp_symbol(self, name, dep_type):
|
|
1236
|
+
"""Create a temp_symbol for an unresolved dependency. Returns the temp_symbol_id."""
|
|
1237
|
+
self.cursor.execute(
|
|
1238
|
+
"INSERT OR IGNORE INTO temp_symbols (name, symbol_type, file_id) VALUES (?, ?, ?)",
|
|
1239
|
+
(name, dep_type, self.current_file_id)
|
|
1240
|
+
)
|
|
1241
|
+
if self.cursor.lastrowid:
|
|
1242
|
+
return self.cursor.lastrowid
|
|
1243
|
+
self.cursor.execute(
|
|
1244
|
+
"SELECT id FROM temp_symbols WHERE name = ? AND symbol_type = ?",
|
|
1245
|
+
(name, dep_type)
|
|
1246
|
+
)
|
|
1247
|
+
return self.cursor.fetchone()[0]
|
|
1248
|
+
|
|
1249
|
+
def _batch_resolve_temp_symbols(self, names, match_types):
|
|
1250
|
+
"""Batch-resolve temp_symbols for multiple symbol names at once.
|
|
1251
|
+
|
|
1252
|
+
Instead of calling _resolve_temp_for_symbol per symbol (which does a
|
|
1253
|
+
SELECT + UPDATE + DELETE per name), this does a single SELECT for all
|
|
1254
|
+
names and batches the UPDATE/DELETE.
|
|
1255
|
+
"""
|
|
1256
|
+
if not names or not match_types:
|
|
1257
|
+
return
|
|
1258
|
+
name_list = list(set(names)) # deduplicate
|
|
1259
|
+
type_list = list(match_types)
|
|
1260
|
+
SQLITE_MAX_VARS = 900
|
|
1261
|
+
all_temp_ids = []
|
|
1262
|
+
for i in range(0, len(name_list), SQLITE_MAX_VARS):
|
|
1263
|
+
chunk = name_list[i:i + SQLITE_MAX_VARS]
|
|
1264
|
+
name_ph = ','.join('?' * len(chunk))
|
|
1265
|
+
type_ph = ','.join('?' * len(type_list))
|
|
1266
|
+
self.cursor.execute(
|
|
1267
|
+
f"SELECT id FROM temp_symbols WHERE name IN ({name_ph}) AND symbol_type IN ({type_ph})",
|
|
1268
|
+
chunk + type_list
|
|
1269
|
+
)
|
|
1270
|
+
all_temp_ids.extend(r[0] for r in self.cursor.fetchall())
|
|
1271
|
+
if not all_temp_ids:
|
|
1272
|
+
return
|
|
1273
|
+
# Batch update + delete in chunks
|
|
1274
|
+
for i in range(0, len(all_temp_ids), SQLITE_MAX_VARS):
|
|
1275
|
+
chunk = all_temp_ids[i:i + SQLITE_MAX_VARS]
|
|
1276
|
+
temp_ph = ','.join('?' * len(chunk))
|
|
1277
|
+
self.cursor.execute(
|
|
1278
|
+
f"UPDATE dependencies SET is_external = 0, temp_symbol_id = NULL WHERE temp_symbol_id IN ({temp_ph})",
|
|
1279
|
+
chunk
|
|
1280
|
+
)
|
|
1281
|
+
self.cursor.execute(
|
|
1282
|
+
f"DELETE FROM temp_symbols WHERE id IN ({temp_ph})",
|
|
1283
|
+
chunk
|
|
1284
|
+
)
|
|
1285
|
+
|
|
1286
|
+
def _resolve_temp_for_symbol(self, name, match_types, target_function_id=None, target_class_id=None):
|
|
1287
|
+
"""When a real symbol is defined, resolve any temp_symbols waiting for it.
|
|
1288
|
+
|
|
1289
|
+
Args:
|
|
1290
|
+
name: Symbol name to match
|
|
1291
|
+
match_types: List of dependency_type values to match
|
|
1292
|
+
target_function_id: If set, update dependencies with this function ID
|
|
1293
|
+
target_class_id: If set, update dependencies with this class ID
|
|
1294
|
+
"""
|
|
1295
|
+
if not match_types:
|
|
1296
|
+
return
|
|
1297
|
+
|
|
1298
|
+
placeholders = ','.join('?' * len(match_types))
|
|
1299
|
+
self.cursor.execute(
|
|
1300
|
+
f"SELECT id, symbol_type FROM temp_symbols WHERE name = ? AND symbol_type IN ({placeholders})",
|
|
1301
|
+
[name] + list(match_types)
|
|
1302
|
+
)
|
|
1303
|
+
temp_rows = self.cursor.fetchall()
|
|
1304
|
+
|
|
1305
|
+
for temp_row in temp_rows:
|
|
1306
|
+
temp_id = temp_row[0]
|
|
1307
|
+
sym_type = temp_row[1]
|
|
1308
|
+
|
|
1309
|
+
if target_function_id is not None and sym_type in ('function_call', 'method_call'):
|
|
1310
|
+
self.cursor.execute(
|
|
1311
|
+
"UPDATE dependencies SET target_function_id = ?, is_external = 0, temp_symbol_id = NULL WHERE temp_symbol_id = ?",
|
|
1312
|
+
(target_function_id, temp_id)
|
|
1313
|
+
)
|
|
1314
|
+
elif target_class_id is not None and sym_type == 'class_reference':
|
|
1315
|
+
self.cursor.execute(
|
|
1316
|
+
"UPDATE dependencies SET target_class_id = ?, is_external = 0, temp_symbol_id = NULL WHERE temp_symbol_id = ?",
|
|
1317
|
+
(target_class_id, temp_id)
|
|
1318
|
+
)
|
|
1319
|
+
else:
|
|
1320
|
+
self.cursor.execute(
|
|
1321
|
+
"UPDATE dependencies SET is_external = 0, temp_symbol_id = NULL WHERE temp_symbol_id = ?",
|
|
1322
|
+
(temp_id,)
|
|
1323
|
+
)
|
|
1324
|
+
|
|
1325
|
+
self.cursor.execute("DELETE FROM temp_symbols WHERE id = ?", (temp_id,))
|
|
1326
|
+
|
|
1327
|
+
def _resolve_all_dependencies(self):
|
|
1328
|
+
"""Resolve all temp_symbols against the complete symbol tables.
|
|
1329
|
+
|
|
1330
|
+
Called once after all files are indexed. Uses bulk JOIN/UPDATE queries
|
|
1331
|
+
instead of per-file SELECT IN queries. This is the key optimization:
|
|
1332
|
+
5 SQL statements instead of 62K × 7.
|
|
1333
|
+
"""
|
|
1334
|
+
# 1. Resolve function_call / method_call temp_symbols → target_function_id
|
|
1335
|
+
self.cursor.execute(
|
|
1336
|
+
"""UPDATE dependencies SET
|
|
1337
|
+
target_function_id = (
|
|
1338
|
+
SELECT f.id FROM functions f
|
|
1339
|
+
WHERE f.name = temp_symbols.name LIMIT 1
|
|
1340
|
+
),
|
|
1341
|
+
is_external = CASE WHEN EXISTS (
|
|
1342
|
+
SELECT 1 FROM functions f WHERE f.name = temp_symbols.name
|
|
1343
|
+
) THEN 0 ELSE 1 END,
|
|
1344
|
+
temp_symbol_id = CASE WHEN EXISTS (
|
|
1345
|
+
SELECT 1 FROM functions f WHERE f.name = temp_symbols.name
|
|
1346
|
+
) THEN NULL ELSE temp_symbol_id END
|
|
1347
|
+
FROM temp_symbols
|
|
1348
|
+
WHERE dependencies.temp_symbol_id = temp_symbols.id
|
|
1349
|
+
AND temp_symbols.symbol_type IN ('function_call', 'method_call')
|
|
1350
|
+
AND dependencies.target_function_id IS NULL"""
|
|
1351
|
+
)
|
|
1352
|
+
|
|
1353
|
+
# 2. Resolve class_reference temp_symbols → target_class_id
|
|
1354
|
+
# Check classes, structs, enums, interfaces, type_aliases
|
|
1355
|
+
self.cursor.execute(
|
|
1356
|
+
"""UPDATE dependencies SET
|
|
1357
|
+
target_class_id = (
|
|
1358
|
+
SELECT c.id FROM classes c
|
|
1359
|
+
WHERE c.name = temp_symbols.name LIMIT 1
|
|
1360
|
+
),
|
|
1361
|
+
is_external = CASE WHEN EXISTS (
|
|
1362
|
+
SELECT 1 FROM classes c WHERE c.name = temp_symbols.name
|
|
1363
|
+
UNION SELECT 1 FROM structs s WHERE s.name = temp_symbols.name
|
|
1364
|
+
UNION SELECT 1 FROM enums e WHERE e.name = temp_symbols.name
|
|
1365
|
+
UNION SELECT 1 FROM interfaces i WHERE i.name = temp_symbols.name
|
|
1366
|
+
UNION SELECT 1 FROM type_aliases t WHERE t.name = temp_symbols.name
|
|
1367
|
+
) THEN 0 ELSE 1 END,
|
|
1368
|
+
temp_symbol_id = CASE WHEN EXISTS (
|
|
1369
|
+
SELECT 1 FROM classes c WHERE c.name = temp_symbols.name
|
|
1370
|
+
UNION SELECT 1 FROM structs s WHERE s.name = temp_symbols.name
|
|
1371
|
+
UNION SELECT 1 FROM enums e WHERE e.name = temp_symbols.name
|
|
1372
|
+
UNION SELECT 1 FROM interfaces i WHERE i.name = temp_symbols.name
|
|
1373
|
+
UNION SELECT 1 FROM type_aliases t WHERE t.name = temp_symbols.name
|
|
1374
|
+
) THEN NULL ELSE temp_symbol_id END
|
|
1375
|
+
FROM temp_symbols
|
|
1376
|
+
WHERE dependencies.temp_symbol_id = temp_symbols.id
|
|
1377
|
+
AND temp_symbols.symbol_type = 'class_reference'
|
|
1378
|
+
AND dependencies.target_class_id IS NULL"""
|
|
1379
|
+
)
|
|
1380
|
+
|
|
1381
|
+
# 3. Resolve variable_reference temp_symbols → is_external = 0
|
|
1382
|
+
self.cursor.execute(
|
|
1383
|
+
"""UPDATE dependencies SET
|
|
1384
|
+
is_external = 0,
|
|
1385
|
+
temp_symbol_id = NULL
|
|
1386
|
+
FROM temp_symbols
|
|
1387
|
+
WHERE dependencies.temp_symbol_id = temp_symbols.id
|
|
1388
|
+
AND temp_symbols.symbol_type = 'variable_reference'
|
|
1389
|
+
AND EXISTS (
|
|
1390
|
+
SELECT 1 FROM variables v WHERE v.name = temp_symbols.name
|
|
1391
|
+
)"""
|
|
1392
|
+
)
|
|
1393
|
+
|
|
1394
|
+
# 4. Delete resolved temp_symbols (those no longer referenced by any dependency)
|
|
1395
|
+
self.cursor.execute(
|
|
1396
|
+
"""DELETE FROM temp_symbols
|
|
1397
|
+
WHERE id NOT IN (SELECT DISTINCT temp_symbol_id FROM dependencies WHERE temp_symbol_id IS NOT NULL)"""
|
|
1398
|
+
)
|
|
1399
|
+
|
|
1400
|
+
self.conn.commit()
|
|
1401
|
+
|
|
1402
|
+
def _compute_content_hash(self, content):
|
|
1403
|
+
"""Compute xxhash of file content (faster than SHA-256)."""
|
|
1404
|
+
return xxhash.xxh64(content).hexdigest()
|
|
1405
|
+
|
|
1406
|
+
def index_directory(self):
|
|
1407
|
+
B = "\033[34m"
|
|
1408
|
+
R = "\033[0m"
|
|
1409
|
+
print(f"{B}analyzing...{R}", flush=True)
|
|
1410
|
+
"""Index all supported files in the directory recursively."""
|
|
1411
|
+
if self.verbose:
|
|
1412
|
+
print(f"Starting indexing of {self.root_dir}...")
|
|
1413
|
+
print(f"Languages: {', '.join(self.languages)}")
|
|
1414
|
+
|
|
1415
|
+
t_total_start = time.perf_counter()
|
|
1416
|
+
|
|
1417
|
+
t0 = time.perf_counter()
|
|
1418
|
+
files = collect_files_to_index(self.root_dir, self.languages)
|
|
1419
|
+
if self.verbose:
|
|
1420
|
+
print(f" [timing] file collection: {time.perf_counter() - t0:.3f}s ({len(files)} files)")
|
|
1421
|
+
|
|
1422
|
+
# First pass: identify changed files using mtime (fast, no file reads)
|
|
1423
|
+
# Batch-load all existing file metadata from the database in one query
|
|
1424
|
+
t0 = time.perf_counter()
|
|
1425
|
+
changed_files = []
|
|
1426
|
+
files_to_reindex = set()
|
|
1427
|
+
skipped_count = 0
|
|
1428
|
+
|
|
1429
|
+
# Load all file metadata in one query instead of per-file SELECT
|
|
1430
|
+
self.cursor.execute("SELECT path, id, content_hash, mtime FROM files")
|
|
1431
|
+
db_file_meta = {}
|
|
1432
|
+
for row in self.cursor.fetchall():
|
|
1433
|
+
db_file_meta[row[0]] = (row[1], row[2], row[3])
|
|
1434
|
+
|
|
1435
|
+
for file_path in files:
|
|
1436
|
+
language = detect_language(file_path)
|
|
1437
|
+
if not language or language not in self._configured_languages:
|
|
1438
|
+
continue
|
|
1439
|
+
|
|
1440
|
+
relative_path = get_relative_path(file_path, self.root_dir)
|
|
1441
|
+
file_mtime = os.stat(file_path).st_mtime
|
|
1442
|
+
|
|
1443
|
+
existing = db_file_meta.get(relative_path)
|
|
1444
|
+
|
|
1445
|
+
if existing:
|
|
1446
|
+
file_id, existing_hash, existing_mtime = existing
|
|
1447
|
+
if self.force_reindex or existing_mtime != file_mtime:
|
|
1448
|
+
# mtime changed - verify with content hash
|
|
1449
|
+
source_bytes = read_file_content(file_path)
|
|
1450
|
+
content_hash = self._compute_content_hash(source_bytes)
|
|
1451
|
+
if self.force_reindex or existing_hash != content_hash:
|
|
1452
|
+
changed_files.append((file_path, file_id))
|
|
1453
|
+
files_to_reindex.add(file_path)
|
|
1454
|
+
else:
|
|
1455
|
+
skipped_count += 1
|
|
1456
|
+
else:
|
|
1457
|
+
skipped_count += 1
|
|
1458
|
+
else:
|
|
1459
|
+
# New file
|
|
1460
|
+
files_to_reindex.add(file_path)
|
|
1461
|
+
|
|
1462
|
+
self.stats["skipped_files"] = skipped_count
|
|
1463
|
+
if self.verbose:
|
|
1464
|
+
print(f" [timing] changed-file detection: {time.perf_counter() - t0:.3f}s ({len(files_to_reindex)} to reindex, {skipped_count} skipped)")
|
|
1465
|
+
|
|
1466
|
+
# Cascade: find all files that depend on changed files
|
|
1467
|
+
for file_path, file_id in changed_files:
|
|
1468
|
+
dependent_file_ids = self._get_dependent_files(file_id)
|
|
1469
|
+
for dep_file_id in dependent_file_ids:
|
|
1470
|
+
self.cursor.execute("SELECT absolute_path FROM files WHERE id = ?", (dep_file_id,))
|
|
1471
|
+
result = self.cursor.fetchone()
|
|
1472
|
+
if result:
|
|
1473
|
+
dep_file_path = Path(result[0])
|
|
1474
|
+
if dep_file_path.exists(): # Only reindex if file still exists
|
|
1475
|
+
files_to_reindex.add(dep_file_path)
|
|
1476
|
+
#print(f"Cascade re-index: {dep_file_path} depends on changed file {file_path}")
|
|
1477
|
+
|
|
1478
|
+
# Remove deleted files from the database
|
|
1479
|
+
deleted_file_ids = self._get_deleted_files(files)
|
|
1480
|
+
if deleted_file_ids:
|
|
1481
|
+
if self.verbose:
|
|
1482
|
+
print(f"Found {len(deleted_file_ids)} deleted file(s) to remove from index")
|
|
1483
|
+
self._remove_deleted_files(deleted_file_ids)
|
|
1484
|
+
|
|
1485
|
+
# Second pass: parse files in parallel with sliding window,
|
|
1486
|
+
# while a dedicated writer thread handles DB inserts.
|
|
1487
|
+
# This eliminates the sawtooth utilization pattern where workers
|
|
1488
|
+
# sit idle while the main process does serial DB inserts.
|
|
1489
|
+
if files_to_reindex:
|
|
1490
|
+
t_parse_start = time.perf_counter()
|
|
1491
|
+
worker_args = [(str(fp), str(self.root_dir), self.frontend_enabled) for fp in files_to_reindex]
|
|
1492
|
+
max_workers = min(os.cpu_count() or 4, len(worker_args), 16)
|
|
1493
|
+
window_size = max_workers * 4 # Keep workers fed with a sliding window
|
|
1494
|
+
|
|
1495
|
+
# Writer thread: consumes parsed results from queue, inserts into DB
|
|
1496
|
+
write_queue = queue_mod.Queue(maxsize=window_size * 2)
|
|
1497
|
+
insert_errors = []
|
|
1498
|
+
|
|
1499
|
+
def _writer_loop():
|
|
1500
|
+
"""Dedicated writer thread — pulls parsed results and inserts into DB."""
|
|
1501
|
+
while True:
|
|
1502
|
+
result = write_queue.get()
|
|
1503
|
+
if result is None: # sentinel — we're done
|
|
1504
|
+
write_queue.task_done()
|
|
1505
|
+
break
|
|
1506
|
+
if 'error' in result:
|
|
1507
|
+
if self.verbose:
|
|
1508
|
+
print(f"Error parsing {result.get('file_path', '?')}: {result['error']}")
|
|
1509
|
+
write_queue.task_done()
|
|
1510
|
+
continue
|
|
1511
|
+
try:
|
|
1512
|
+
self._insert_parsed_file(result)
|
|
1513
|
+
except Exception as e:
|
|
1514
|
+
insert_errors.append(e)
|
|
1515
|
+
if self.verbose:
|
|
1516
|
+
print(f"Error inserting {result.get('file_path', '?')}: {e}")
|
|
1517
|
+
write_queue.task_done()
|
|
1518
|
+
# Final flush after all files inserted
|
|
1519
|
+
self._flush_batches()
|
|
1520
|
+
|
|
1521
|
+
writer_thread = threading.Thread(target=_writer_loop, daemon=True)
|
|
1522
|
+
writer_thread.start()
|
|
1523
|
+
|
|
1524
|
+
# Sliding window: maintain window_size futures in flight at all times
|
|
1525
|
+
with ProcessPoolExecutor(max_workers=max_workers) as executor:
|
|
1526
|
+
# Submit initial window
|
|
1527
|
+
futures = {}
|
|
1528
|
+
arg_iter = iter(worker_args)
|
|
1529
|
+
for _ in range(min(window_size, len(worker_args))):
|
|
1530
|
+
arg = next(arg_iter)
|
|
1531
|
+
futures[executor.submit(parse_file, arg)] = arg
|
|
1532
|
+
|
|
1533
|
+
while futures:
|
|
1534
|
+
# Wait for any one to complete
|
|
1535
|
+
done = next(as_completed(futures))
|
|
1536
|
+
del futures[done]
|
|
1537
|
+
result = done.result()
|
|
1538
|
+
# Feed to writer thread (blocks if queue is full — backpressure)
|
|
1539
|
+
write_queue.put(result)
|
|
1540
|
+
# Immediately submit next file to keep workers fed
|
|
1541
|
+
try:
|
|
1542
|
+
arg = next(arg_iter)
|
|
1543
|
+
futures[executor.submit(parse_file, arg)] = arg
|
|
1544
|
+
except StopIteration:
|
|
1545
|
+
pass
|
|
1546
|
+
|
|
1547
|
+
# Wait for writer to finish all remaining inserts
|
|
1548
|
+
write_queue.put(None)
|
|
1549
|
+
writer_thread.join()
|
|
1550
|
+
|
|
1551
|
+
if insert_errors and self.verbose:
|
|
1552
|
+
print(f" {len(insert_errors)} insert errors during indexing")
|
|
1553
|
+
|
|
1554
|
+
if self.verbose and files_to_reindex:
|
|
1555
|
+
print(f" [timing] parse + insert: {time.perf_counter() - t_parse_start:.3f}s ({len(files_to_reindex)} files)")
|
|
1556
|
+
|
|
1557
|
+
# Post-indexing dependency resolution pass
|
|
1558
|
+
# Resolve all temp_symbols against the now-complete symbol tables.
|
|
1559
|
+
# This is much faster than per-file resolution because we do a few large
|
|
1560
|
+
# JOIN/UPDATE queries instead of 62K × 7 SELECT IN queries.
|
|
1561
|
+
if files_to_reindex:
|
|
1562
|
+
t_resolve_start = time.perf_counter()
|
|
1563
|
+
self._resolve_all_dependencies()
|
|
1564
|
+
if self.verbose:
|
|
1565
|
+
print(f" [timing] dependency resolution: {time.perf_counter() - t_resolve_start:.3f}s")
|
|
1566
|
+
|
|
1567
|
+
# Run cross-file frontend resolution pass to refresh relationships
|
|
1568
|
+
# (render_relationships, style_imports, custom_property_usages, event handlers, selector matches)
|
|
1569
|
+
# Skip when no files changed or frontend is disabled to avoid unconditional global work
|
|
1570
|
+
if files_to_reindex and self.frontend_enabled:
|
|
1571
|
+
t_resolve_start = time.perf_counter()
|
|
1572
|
+
try:
|
|
1573
|
+
resolver = FrontendResolver(self.conn, str(self.root_dir))
|
|
1574
|
+
resolver.resolve_all()
|
|
1575
|
+
except Exception as e:
|
|
1576
|
+
if self.verbose:
|
|
1577
|
+
print(f"Warning: Frontend resolution pass failed: {e}")
|
|
1578
|
+
|
|
1579
|
+
if self.verbose:
|
|
1580
|
+
print(f" [timing] frontend resolution: {time.perf_counter() - t_resolve_start:.3f}s")
|
|
1581
|
+
|
|
1582
|
+
# Clean up unresolved temp_symbols and their dependencies
|
|
1583
|
+
# (these are external/builtin symbols that were never defined in the codebase)
|
|
1584
|
+
self.cursor.execute("SELECT COUNT(*) FROM temp_symbols")
|
|
1585
|
+
remaining = self.cursor.fetchone()[0]
|
|
1586
|
+
if remaining > 0:
|
|
1587
|
+
# Clean up traditional dependency temp_symbols
|
|
1588
|
+
self.cursor.execute("DELETE FROM dependencies WHERE temp_symbol_id IS NOT NULL")
|
|
1589
|
+
# Clean up all remaining temp_symbols (including frontend types: component_reference,
|
|
1590
|
+
# custom_property_reference, css_module_class_reference — these are external/missing)
|
|
1591
|
+
self.cursor.execute("DELETE FROM temp_symbols")
|
|
1592
|
+
self.conn.commit()
|
|
1593
|
+
|
|
1594
|
+
# Clean up unresolved temp_file_references (external URLs or missing files)
|
|
1595
|
+
self.cursor.execute("SELECT COUNT(*) FROM temp_file_references")
|
|
1596
|
+
remaining_refs = self.cursor.fetchone()[0]
|
|
1597
|
+
if remaining_refs > 0:
|
|
1598
|
+
self.cursor.execute("DELETE FROM temp_file_references")
|
|
1599
|
+
self.conn.commit()
|
|
1600
|
+
|
|
1601
|
+
if self.verbose:
|
|
1602
|
+
print(f" [timing] total index_directory: {time.perf_counter() - t_total_start:.3f}s")
|
|
1603
|
+
self._print_summary()
|
|
1604
|
+
|
|
1605
|
+
def _print_summary(self):
|
|
1606
|
+
"""Print indexing summary statistics."""
|
|
1607
|
+
cursor = self.conn.cursor()
|
|
1608
|
+
cursor.execute("SELECT COUNT(*) FROM files")
|
|
1609
|
+
file_count = cursor.fetchone()[0]
|
|
1610
|
+
cursor.execute("SELECT COUNT(*) FROM functions WHERE parent_id IS NULL AND type != 'macro'")
|
|
1611
|
+
func_count = cursor.fetchone()[0]
|
|
1612
|
+
cursor.execute("SELECT COUNT(*) FROM functions WHERE type = 'macro'")
|
|
1613
|
+
macro_count = cursor.fetchone()[0]
|
|
1614
|
+
cursor.execute("SELECT COUNT(*) FROM functions WHERE parent_id IS NOT NULL")
|
|
1615
|
+
method_count = cursor.fetchone()[0]
|
|
1616
|
+
cursor.execute("SELECT COUNT(*) FROM classes")
|
|
1617
|
+
class_count = cursor.fetchone()[0]
|
|
1618
|
+
cursor.execute("SELECT COUNT(*) FROM structs")
|
|
1619
|
+
struct_count = cursor.fetchone()[0]
|
|
1620
|
+
cursor.execute("SELECT COUNT(*) FROM interfaces")
|
|
1621
|
+
iface_count = cursor.fetchone()[0]
|
|
1622
|
+
cursor.execute("SELECT COUNT(*) FROM enums")
|
|
1623
|
+
enum_count = cursor.fetchone()[0]
|
|
1624
|
+
cursor.execute("SELECT COUNT(*) FROM namespaces")
|
|
1625
|
+
ns_count = cursor.fetchone()[0]
|
|
1626
|
+
cursor.execute("SELECT COUNT(*) FROM variables")
|
|
1627
|
+
var_count = cursor.fetchone()[0]
|
|
1628
|
+
cursor.execute("SELECT COUNT(*) FROM type_aliases")
|
|
1629
|
+
alias_count = cursor.fetchone()[0]
|
|
1630
|
+
|
|
1631
|
+
indexed = file_count - self.stats['skipped_files']
|
|
1632
|
+
|
|
1633
|
+
print(f"\nIndexing complete!")
|
|
1634
|
+
print(f"Total files in database: {file_count}")
|
|
1635
|
+
print(f"Files indexed this run: {indexed}")
|
|
1636
|
+
print(f"Files skipped (unchanged): {self.stats['skipped_files']}")
|
|
1637
|
+
if indexed > 0:
|
|
1638
|
+
print(f" Functions indexed: {self.stats['total_functions']}")
|
|
1639
|
+
print(f" Macros indexed: {self.stats['total_macros']}")
|
|
1640
|
+
print(f" Methods indexed: {self.stats['total_methods']}")
|
|
1641
|
+
print(f" Classes indexed: {self.stats['total_classes']}")
|
|
1642
|
+
print(f" Structs indexed: {self.stats['total_structs']}")
|
|
1643
|
+
print(f" Interfaces indexed:{self.stats['total_interfaces']}")
|
|
1644
|
+
print(f" Enums indexed: {self.stats['total_enums']}")
|
|
1645
|
+
print(f" Namespaces indexed:{self.stats['total_namespaces']}")
|
|
1646
|
+
print(f" Variables indexed: {self.stats['total_variables']}")
|
|
1647
|
+
print(f" Type aliases idx: {self.stats['total_type_defs']}")
|
|
1648
|
+
print(f"Total functions: {func_count}")
|
|
1649
|
+
print(f"Total macros: {macro_count}")
|
|
1650
|
+
print(f"Total methods: {method_count}")
|
|
1651
|
+
print(f"Total classes: {class_count}")
|
|
1652
|
+
print(f"Total structs: {struct_count}")
|
|
1653
|
+
print(f"Total interfaces: {iface_count}")
|
|
1654
|
+
print(f"Total enums: {enum_count}")
|
|
1655
|
+
print(f"Total namespaces: {ns_count}")
|
|
1656
|
+
print(f"Total variables: {var_count}")
|
|
1657
|
+
print(f"Total type aliases: {alias_count}")
|
|
1658
|
+
|
|
1659
|
+
# Frontend table counts
|
|
1660
|
+
for table_name in ("frontend_components", "markup_elements", "style_selectors",
|
|
1661
|
+
"style_custom_properties", "style_custom_property_usages",
|
|
1662
|
+
"style_keyframes", "style_imports", "frontend_events",
|
|
1663
|
+
"frontend_bindings", "render_relationships",
|
|
1664
|
+
"style_selector_matches", "frontend_diagnostics"):
|
|
1665
|
+
try:
|
|
1666
|
+
cursor.execute(f"SELECT COUNT(*) FROM {table_name}")
|
|
1667
|
+
count = cursor.fetchone()[0]
|
|
1668
|
+
if count > 0:
|
|
1669
|
+
print(f" {table_name}: {count}")
|
|
1670
|
+
except Exception:
|
|
1671
|
+
pass
|
|
1672
|
+
|
|
1673
|
+
def save_index(self, output_file=None):
|
|
1674
|
+
"""Commit and optionally close the database connection. Data is already saved during indexing."""
|
|
1675
|
+
if self.conn:
|
|
1676
|
+
self.conn.commit()
|
|
1677
|
+
if self.verbose:
|
|
1678
|
+
print(f"\nDatabase saved to {self.db_path}")
|
|
1679
|
+
|
|
1680
|
+
def close(self):
|
|
1681
|
+
"""Close the database connection."""
|
|
1682
|
+
if self.conn:
|
|
1683
|
+
self.conn.close()
|
|
1684
|
+
|
|
1685
|
+
|
|
1686
|
+
def main():
|
|
1687
|
+
import sys
|
|
1688
|
+
import traceback
|
|
1689
|
+
|
|
1690
|
+
args = parse_arguments()
|
|
1691
|
+
|
|
1692
|
+
if args.list_languages:
|
|
1693
|
+
list_supported_languages()
|
|
1694
|
+
return
|
|
1695
|
+
|
|
1696
|
+
if args.list_frontend_languages:
|
|
1697
|
+
list_frontend_languages()
|
|
1698
|
+
return
|
|
1699
|
+
|
|
1700
|
+
try:
|
|
1701
|
+
# Handle view graph mode
|
|
1702
|
+
if args.graph:
|
|
1703
|
+
# Determine database file
|
|
1704
|
+
db_file = args.output
|
|
1705
|
+
if db_file.endswith('.json'):
|
|
1706
|
+
db_file = db_file.rsplit('.', 1)[0] + '.db'
|
|
1707
|
+
|
|
1708
|
+
if not Path(db_file).exists():
|
|
1709
|
+
print(f"Error: Database file not found: {db_file}")
|
|
1710
|
+
print("Please run indexing first to create the database.")
|
|
1711
|
+
sys.exit(1)
|
|
1712
|
+
|
|
1713
|
+
sdk = CodeIndexSDK(db_file)
|
|
1714
|
+
graph = sdk.get_dependency_graph(args.graph)
|
|
1715
|
+
print(graph)
|
|
1716
|
+
sdk.close()
|
|
1717
|
+
return
|
|
1718
|
+
|
|
1719
|
+
# Handle JSON export mode
|
|
1720
|
+
if args.export_json:
|
|
1721
|
+
# Determine database file
|
|
1722
|
+
db_file = args.output
|
|
1723
|
+
if db_file.endswith('.json'):
|
|
1724
|
+
db_file = db_file.rsplit('.', 1)[0] + '.db'
|
|
1725
|
+
|
|
1726
|
+
if not Path(db_file).exists():
|
|
1727
|
+
print(f"Error: Database file not found: {db_file}")
|
|
1728
|
+
print("Please run indexing first to create the database.")
|
|
1729
|
+
sys.exit(1)
|
|
1730
|
+
|
|
1731
|
+
export_to_json(db_file, args.export_json)
|
|
1732
|
+
return
|
|
1733
|
+
|
|
1734
|
+
# Handle indexing mode - directory is required
|
|
1735
|
+
directory = args.directory
|
|
1736
|
+
if not directory:
|
|
1737
|
+
print("Error: directory argument is required for indexing mode")
|
|
1738
|
+
print("Usage: python code_indexer.py <directory> [options]")
|
|
1739
|
+
sys.exit(1)
|
|
1740
|
+
|
|
1741
|
+
# Determine output file extension
|
|
1742
|
+
output_file = args.output
|
|
1743
|
+
if output_file.endswith('.json'):
|
|
1744
|
+
# Convert to .db for SQLite
|
|
1745
|
+
output_file = output_file.rsplit('.', 1)[0] + '.db'
|
|
1746
|
+
print(f"Note: Output file changed to {output_file} (SQLite database)")
|
|
1747
|
+
|
|
1748
|
+
# Parse languages if provided
|
|
1749
|
+
languages = None
|
|
1750
|
+
if args.languages:
|
|
1751
|
+
languages = [lang.strip() for lang in args.languages.split(',')]
|
|
1752
|
+
|
|
1753
|
+
indexer = CodeIndexer(directory, languages, output_file, args.force, verbose=args.verbose, frontend_enabled=args.frontend)
|
|
1754
|
+
indexer.index_directory()
|
|
1755
|
+
indexer.save_index()
|
|
1756
|
+
indexer.close()
|
|
1757
|
+
except Exception as e:
|
|
1758
|
+
print(f"Error: {e}")
|
|
1759
|
+
traceback.print_exc()
|
|
1760
|
+
|
|
1761
|
+
|
|
1762
|
+
if __name__ == "__main__":
|
|
1763
|
+
main()
|