raggiecode 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. Agent/__init__.py +0 -0
  2. Agent/agent.py +891 -0
  3. Agent/chat_history_db.py +1500 -0
  4. Agent/command.py +49 -0
  5. Agent/config.py +46 -0
  6. Agent/effort_levels.py +33 -0
  7. Agent/git_manager.py +727 -0
  8. Agent/tools.py +35 -0
  9. Commands/__init__.py +18 -0
  10. Commands/effort.py +42 -0
  11. Commands/global_todo.py +23 -0
  12. Commands/help.py +22 -0
  13. Commands/reasoning.py +24 -0
  14. Commands/redo.py +11 -0
  15. Commands/reindex.py +27 -0
  16. Commands/shell.py +28 -0
  17. Commands/stream.py +24 -0
  18. Commands/undo.py +13 -0
  19. Commands/unlimited_effort.py +8 -0
  20. Commands/window_size.py +29 -0
  21. RAG/__init__.py +0 -0
  22. RAG/document.py +119 -0
  23. RAG/find.py +408 -0
  24. RAG/graph.py +231 -0
  25. Tools/GetFileCodeStructure.py +43 -0
  26. Tools/GetSymbolSourceCode.py +27 -0
  27. Tools/__init__.py +39 -0
  28. Tools/ask_user.py +102 -0
  29. Tools/dispatch_subagent.py +215 -0
  30. Tools/document.py +35 -0
  31. Tools/edit_symbol.py +250 -0
  32. Tools/fuzzy_search.py +119 -0
  33. Tools/list_dir.py +51 -0
  34. Tools/read.py +49 -0
  35. Tools/read_image.py +75 -0
  36. Tools/remove.py +75 -0
  37. Tools/replace.py +305 -0
  38. Tools/search.py +41 -0
  39. Tools/shell.py +149 -0
  40. Tools/shell_kill.py +87 -0
  41. Tools/temp_background_service.py +113 -0
  42. Tools/todo_list.py +481 -0
  43. Tools/utils.py +116 -0
  44. Tools/view_changes.py +179 -0
  45. Tools/walk_call_tree.py +30 -0
  46. Tools/web_fetch.py +175 -0
  47. Tools/web_search.py +69 -0
  48. Tools/write.py +48 -0
  49. cli.py +111 -0
  50. config/__init__.py +0 -0
  51. config/coder_system_prompt.md +119 -0
  52. config/roles.json +43 -0
  53. config/tools.json +709 -0
  54. indexing/__init__.py +0 -0
  55. indexing/cli.py +128 -0
  56. indexing/code_index_sdk.py +832 -0
  57. indexing/code_indexer.py +1763 -0
  58. indexing/db_schema.py +396 -0
  59. indexing/export_to_json.py +346 -0
  60. indexing/extractors.py +189 -0
  61. indexing/file_utils.py +97 -0
  62. indexing/frontend/__init__.py +0 -0
  63. indexing/frontend/css_extractor.py +195 -0
  64. indexing/frontend/css_parser.py +387 -0
  65. indexing/frontend/css_selector_utils.py +226 -0
  66. indexing/frontend/edit_safety.py +573 -0
  67. indexing/frontend/graph.py +838 -0
  68. indexing/frontend/html_extractor.py +496 -0
  69. indexing/frontend/html_parser.py +314 -0
  70. indexing/frontend/jsx_extractor.py +1204 -0
  71. indexing/frontend/location_lookup.py +247 -0
  72. indexing/frontend/resolver.py +485 -0
  73. indexing/frontend/runtime_resolver.py +862 -0
  74. indexing/frontend/semantic_output.py +705 -0
  75. indexing/frontend/source_location.py +69 -0
  76. indexing/frontend_config.py +72 -0
  77. indexing/frontend_models.py +347 -0
  78. indexing/language_config.py +360 -0
  79. indexing/models.py +284 -0
  80. indexing/node_utils.py +1112 -0
  81. indexing/parse_worker.py +1082 -0
  82. indexing/queries.py +1542 -0
  83. indexing/sdk_examples.py +426 -0
  84. interactive.py +248 -0
  85. raggie.py +673 -0
  86. raggiecode-0.2.1.dist-info/METADATA +944 -0
  87. raggiecode-0.2.1.dist-info/RECORD +93 -0
  88. raggiecode-0.2.1.dist-info/WHEEL +5 -0
  89. raggiecode-0.2.1.dist-info/entry_points.txt +2 -0
  90. raggiecode-0.2.1.dist-info/top_level.txt +10 -0
  91. skills/__init__.py +3 -0
  92. skills/manager.py +114 -0
  93. skills/tool.py +121 -0
@@ -0,0 +1,1763 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ Code Indexer using tree-sitter
4
+ Indexes files in a codebase and tracks functions, classes, variables, methods, and type definitions.
5
+ Supports multiple languages: Python, Go, C#, JavaScript, TypeScript, Rust, Zig, Elixir, C++, PHP
6
+ """
7
+
8
+ import json
9
+ import os
10
+ import time
11
+ import threading
12
+ import queue as queue_mod
13
+ from concurrent.futures import ProcessPoolExecutor, as_completed
14
+ from pathlib import Path
15
+ import xxhash
16
+
17
+ from indexing.language_config import (
18
+ LANGUAGE_CONFIG,
19
+ )
20
+ from indexing.file_utils import (
21
+ detect_language,
22
+ collect_files_to_index,
23
+ read_file_content,
24
+ get_relative_path
25
+ )
26
+ from indexing.cli import parse_arguments, list_supported_languages, list_frontend_languages
27
+ from indexing.db_schema import init_database
28
+ from indexing.export_to_json import export_to_json
29
+ from indexing.code_index_sdk import CodeIndexSDK
30
+ from indexing.parse_worker import parse_file
31
+ from indexing.frontend.resolver import normalize_import_path, load_path_aliases, FrontendResolver
32
+
33
+
34
+ VAR_DATA = "this is to test the indexer"
35
+
36
+ class CodeIndexer:
37
+ def __init__(self, root_dir, languages=None, db_path=".code_index.raggie", force_reindex=False, verbose=False, frontend_enabled=True):
38
+ self.root_dir = Path(root_dir)
39
+ self.languages = languages if languages else list(LANGUAGE_CONFIG.keys())
40
+ self.frontend_enabled = frontend_enabled
41
+ if not frontend_enabled:
42
+ frontend_langs = {"html", "css", "tsx"}
43
+ self.languages = [l for l in self.languages if l not in frontend_langs]
44
+ self.parsers = {}
45
+ self.db_path = db_path
46
+ self.conn = None
47
+ self.cursor = None
48
+ self.current_file_id = None
49
+ self.current_class_id = None
50
+ self.force_reindex = force_reindex
51
+ self.batch_size = 50000
52
+ self.verbose = verbose
53
+ self.path_aliases = {} # loaded lazily on first frontend insert
54
+
55
+ # Statistics counters
56
+ self.stats = {
57
+ "total_functions": 0,
58
+ "total_macros": 0,
59
+ "total_classes": 0,
60
+ "total_variables": 0,
61
+ "total_methods": 0,
62
+ "total_type_defs": 0,
63
+ "total_structs": 0,
64
+ "total_interfaces": 0,
65
+ "total_enums": 0,
66
+ "total_namespaces": 0,
67
+ "skipped_files": 0
68
+ }
69
+
70
+ # Batch buffers for executemany()
71
+ self.batches = {
72
+ "files": [],
73
+ "functions": [],
74
+ "classes": [],
75
+ "variables": [],
76
+ "type_aliases": [],
77
+ "structs": [],
78
+ "interfaces": [],
79
+ "dependencies": []
80
+ }
81
+
82
+ self._initialize_database()
83
+ self._initialize_parsers()
84
+
85
+ def _flush_batches(self):
86
+ """Flush all batch buffers using executemany()."""
87
+ if self.batches["variables"]:
88
+ self.cursor.executemany(
89
+ """INSERT INTO variables
90
+ (file_id, parent_id, parent_type, name, type, location, field_type)
91
+ VALUES (?, ?, ?, ?, ?, ?, ?)""",
92
+ self.batches["variables"]
93
+ )
94
+ self.batches["variables"] = []
95
+
96
+ if self.batches["type_aliases"]:
97
+ self.cursor.executemany(
98
+ """INSERT INTO type_aliases
99
+ (file_id, name, location, type_definition)
100
+ VALUES (?, ?, ?, ?)""",
101
+ self.batches["type_aliases"]
102
+ )
103
+ self.batches["type_aliases"] = []
104
+
105
+ if self.batches["dependencies"]:
106
+ self.cursor.executemany(
107
+ """INSERT INTO dependencies (file_id, dependency_type, name, source_function_id, target_function_id, target_class_id, temp_symbol_id, location, is_external)
108
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
109
+ self.batches["dependencies"]
110
+ )
111
+ self.batches["dependencies"] = []
112
+ self.conn.commit()
113
+
114
+ def _check_and_flush(self, batch_name):
115
+ """Check if a batch has reached the size limit and flush if so."""
116
+ if len(self.batches[batch_name]) >= self.batch_size:
117
+ self._flush_batches()
118
+
119
+ def _initialize_database(self):
120
+ """Initialize SQLite database."""
121
+ # init_database creates schema and returns a connection
122
+ init_database(self.db_path)
123
+ # Reopen with check_same_thread=False so the writer thread can use it
124
+ import sqlite3
125
+ self.conn = sqlite3.connect(self.db_path, check_same_thread=False)
126
+ # Optimized settings for batch indexing
127
+ self.conn.execute("PRAGMA journal_mode = MEMORY")
128
+ self.conn.execute("PRAGMA synchronous = OFF")
129
+ self.conn.execute("PRAGMA temp_store = MEMORY")
130
+ self.cursor = self.conn.cursor()
131
+ if self.verbose:
132
+ print(f"Database initialized: {self.db_path}")
133
+
134
+ def _initialize_parsers(self):
135
+ """Initialize tree-sitter parsers for selected languages.
136
+
137
+ Parsers are only needed in the parent for the detect_language check
138
+ in index_directory. Actual parsing happens in worker processes
139
+ which have their own parser cache. Skip parser creation to save memory.
140
+ """
141
+ # Only track which languages are configured (for the parser check in index_directory)
142
+ self._configured_languages = set()
143
+ for lang in self.languages:
144
+ if lang in LANGUAGE_CONFIG:
145
+ self._configured_languages.add(lang)
146
+ if self.verbose:
147
+ print(f"Configured parser for: {lang}")
148
+
149
+ def _get_dependent_files(self, file_id):
150
+ """Get all files that have dependencies on symbols in the given file.
151
+
152
+ Checks both traditional dependency edges (function/class references) and
153
+ frontend dependency edges (render relationships, style imports, custom
154
+ property usages, event handler symbols, selector matches).
155
+ """
156
+ cursor = self.conn.cursor()
157
+
158
+ # Get function IDs from this file
159
+ cursor.execute("SELECT id FROM functions WHERE file_id = ?", (file_id,))
160
+ function_ids = [row[0] for row in cursor.fetchall()]
161
+
162
+ # Get class IDs from this file
163
+ cursor.execute("SELECT id FROM classes WHERE file_id = ?", (file_id,))
164
+ class_ids = [row[0] for row in cursor.fetchall()]
165
+
166
+ # Get frontend component IDs from this file
167
+ cursor.execute("SELECT id FROM frontend_components WHERE file_id = ?", (file_id,))
168
+ component_ids = [row[0] for row in cursor.fetchall()]
169
+
170
+ # Get custom property IDs from this file
171
+ cursor.execute("SELECT id FROM style_custom_properties WHERE file_id = ?", (file_id,))
172
+ custom_property_ids = [row[0] for row in cursor.fetchall()]
173
+
174
+ # Get selector IDs from this file
175
+ cursor.execute("SELECT id FROM style_selectors WHERE file_id = ?", (file_id,))
176
+ selector_ids = [row[0] for row in cursor.fetchall()]
177
+
178
+ # Get markup element IDs from this file
179
+ cursor.execute("SELECT id FROM markup_elements WHERE file_id = ?", (file_id,))
180
+ element_ids = [row[0] for row in cursor.fetchall()]
181
+
182
+ dependent_file_ids = set()
183
+
184
+ # Find files that reference these functions
185
+ if function_ids:
186
+ placeholders = ','.join('?' * len(function_ids))
187
+ cursor.execute(
188
+ f"SELECT DISTINCT file_id FROM dependencies WHERE target_function_id IN ({placeholders})",
189
+ function_ids
190
+ )
191
+ dependent_file_ids.update(row[0] for row in cursor.fetchall())
192
+
193
+ # Find files that reference these classes
194
+ if class_ids:
195
+ placeholders = ','.join('?' * len(class_ids))
196
+ cursor.execute(
197
+ f"SELECT DISTINCT file_id FROM dependencies WHERE target_class_id IN ({placeholders})",
198
+ class_ids
199
+ )
200
+ dependent_file_ids.update(row[0] for row in cursor.fetchall())
201
+
202
+ # Frontend: render_relationships — files containing parent components
203
+ # that render this file's components as children
204
+ if component_ids:
205
+ placeholders = ','.join('?' * len(component_ids))
206
+ # Find parent component IDs that reference these child components
207
+ cursor.execute(
208
+ f"SELECT DISTINCT fc.file_id FROM render_relationships rr "
209
+ f"JOIN frontend_components fc ON rr.parent_component_id = fc.id "
210
+ f"WHERE rr.child_component_id IN ({placeholders})",
211
+ component_ids
212
+ )
213
+ dependent_file_ids.update(row[0] for row in cursor.fetchall())
214
+
215
+ # Frontend: style_imports — files that import this file via @import or <link>
216
+ cursor.execute(
217
+ "SELECT DISTINCT si.file_id FROM style_imports si WHERE si.resolved_file_id = ?",
218
+ (file_id,)
219
+ )
220
+ dependent_file_ids.update(row[0] for row in cursor.fetchall())
221
+
222
+ # Frontend: style_custom_property_usages — files that use this file's custom properties
223
+ if custom_property_ids:
224
+ placeholders = ','.join('?' * len(custom_property_ids))
225
+ cursor.execute(
226
+ f"SELECT DISTINCT file_id FROM style_custom_property_usages WHERE resolved_property_id IN ({placeholders})",
227
+ custom_property_ids
228
+ )
229
+ dependent_file_ids.update(row[0] for row in cursor.fetchall())
230
+
231
+ # Frontend: frontend_events — files whose events reference this file's handler functions
232
+ if function_ids:
233
+ placeholders = ','.join('?' * len(function_ids))
234
+ cursor.execute(
235
+ f"SELECT DISTINCT file_id FROM frontend_events WHERE handler_symbol_id IN ({placeholders})",
236
+ function_ids
237
+ )
238
+ dependent_file_ids.update(row[0] for row in cursor.fetchall())
239
+
240
+ # Frontend: style_selector_matches — bidirectional
241
+ # When a CSS file changes (selectors), files containing matched elements need re-resolution
242
+ if selector_ids:
243
+ placeholders = ','.join('?' * len(selector_ids))
244
+ cursor.execute(
245
+ f"SELECT DISTINCT me.file_id FROM style_selector_matches ssm "
246
+ f"JOIN markup_elements me ON ssm.element_id = me.id "
247
+ f"WHERE ssm.selector_id IN ({placeholders})",
248
+ selector_ids
249
+ )
250
+ dependent_file_ids.update(row[0] for row in cursor.fetchall())
251
+
252
+ # When a TSX/HTML file changes (elements), CSS files with matched selectors need re-resolution
253
+ if element_ids:
254
+ placeholders = ','.join('?' * len(element_ids))
255
+ cursor.execute(
256
+ f"SELECT DISTINCT ss.file_id FROM style_selector_matches ssm "
257
+ f"JOIN style_selectors ss ON ssm.selector_id = ss.id "
258
+ f"WHERE ssm.element_id IN ({placeholders})",
259
+ element_ids
260
+ )
261
+ dependent_file_ids.update(row[0] for row in cursor.fetchall())
262
+
263
+ # Don't include the file itself
264
+ dependent_file_ids.discard(file_id)
265
+
266
+ return list(dependent_file_ids)
267
+
268
+ def _delete_file_symbols(self, file_id):
269
+ """Delete all symbol rows belonging to a file (for re-indexing)."""
270
+ cursor = self.conn.cursor()
271
+
272
+ # First, collect the IDs of symbols being deleted
273
+ cursor.execute("SELECT id FROM functions WHERE file_id = ?", (file_id,))
274
+ function_ids = [row[0] for row in cursor.fetchall()]
275
+
276
+ cursor.execute("SELECT id FROM classes WHERE file_id = ?", (file_id,))
277
+ class_ids = [row[0] for row in cursor.fetchall()]
278
+
279
+ # Update dependencies in OTHER files that reference these symbols to NULL their target IDs
280
+ # This prevents stale references when the file is re-indexed
281
+ if function_ids:
282
+ placeholders = ','.join('?' * len(function_ids))
283
+ cursor.execute(
284
+ f"UPDATE dependencies SET target_function_id = NULL WHERE target_function_id IN ({placeholders})",
285
+ function_ids
286
+ )
287
+
288
+ if class_ids:
289
+ placeholders = ','.join('?' * len(class_ids))
290
+ cursor.execute(
291
+ f"UPDATE dependencies SET target_class_id = NULL WHERE target_class_id IN ({placeholders})",
292
+ class_ids
293
+ )
294
+
295
+ # Collect frontend component IDs for cross-file FK cleanup
296
+ cursor.execute("SELECT id FROM frontend_components WHERE file_id = ?", (file_id,))
297
+ component_ids = [row[0] for row in cursor.fetchall()]
298
+
299
+ # Collect custom property IDs for cross-file FK cleanup
300
+ cursor.execute("SELECT id FROM style_custom_properties WHERE file_id = ?", (file_id,))
301
+ custom_property_ids = [row[0] for row in cursor.fetchall()]
302
+
303
+ # NULL out cross-file references to symbols being deleted (FK constraints not enforced)
304
+ if component_ids:
305
+ placeholders = ','.join('?' * len(component_ids))
306
+ # render_relationships in OTHER files referencing these components as child
307
+ cursor.execute(
308
+ f"UPDATE render_relationships SET child_component_id = NULL WHERE child_component_id IN ({placeholders})",
309
+ component_ids
310
+ )
311
+ # markup_elements in OTHER files referencing these components
312
+ cursor.execute(
313
+ f"UPDATE markup_elements SET component_id = NULL WHERE component_id IN ({placeholders})",
314
+ component_ids
315
+ )
316
+ # style_selectors in OTHER files referencing these components
317
+ cursor.execute(
318
+ f"UPDATE style_selectors SET component_id = NULL WHERE component_id IN ({placeholders})",
319
+ component_ids
320
+ )
321
+
322
+ if custom_property_ids:
323
+ placeholders = ','.join('?' * len(custom_property_ids))
324
+ # style_custom_property_usages in OTHER files referencing these properties
325
+ cursor.execute(
326
+ f"UPDATE style_custom_property_usages SET resolved_property_id = NULL WHERE resolved_property_id IN ({placeholders})",
327
+ custom_property_ids
328
+ )
329
+
330
+ # Now delete the file's own symbols
331
+ # Delete render_relationships FIRST (before frontend_components are deleted),
332
+ # since FK constraints are not enforced and the subquery would find no rows after.
333
+ cursor.execute(
334
+ "DELETE FROM render_relationships WHERE parent_component_id IN "
335
+ "(SELECT id FROM frontend_components WHERE file_id = ?)",
336
+ (file_id,)
337
+ )
338
+ cursor.execute(
339
+ "DELETE FROM style_selector_matches WHERE selector_id IN "
340
+ "(SELECT id FROM style_selectors WHERE file_id = ?) "
341
+ "OR element_id IN (SELECT id FROM markup_elements WHERE file_id = ?)",
342
+ (file_id, file_id)
343
+ )
344
+ # Tables with direct file_id column
345
+ for table in ("functions", "classes", "variables", "type_aliases", "structs", "interfaces", "dependencies", "temp_symbols", "namespaces",
346
+ "frontend_components", "markup_elements", "style_selectors", "style_custom_properties",
347
+ "style_custom_property_usages", "style_keyframes", "style_imports", "frontend_events",
348
+ "frontend_bindings", "frontend_diagnostics"):
349
+ cursor.execute(f"DELETE FROM {table} WHERE file_id = ?", (file_id,))
350
+ # Clean up temp_file_references where this file is the source
351
+ cursor.execute("DELETE FROM temp_file_references WHERE source_file_id = ?", (file_id,))
352
+
353
+ def _get_deleted_files(self, current_files):
354
+ """Get list of file IDs for files that exist in database but not on filesystem."""
355
+ cursor = self.conn.cursor()
356
+ cursor.execute("SELECT id, absolute_path FROM files")
357
+ db_files = cursor.fetchall()
358
+
359
+ deleted_file_ids = []
360
+ current_file_paths = {str(f) for f in current_files}
361
+
362
+ for file_id, abs_path in db_files:
363
+ if abs_path not in current_file_paths:
364
+ deleted_file_ids.append(file_id)
365
+
366
+ return deleted_file_ids
367
+
368
+ def _remove_deleted_files(self, deleted_file_ids):
369
+ """Remove deleted files and all their symbols from the database (bulk, chunked)."""
370
+ cursor = self.conn.cursor()
371
+ SQLITE_MAX_VARS = 999
372
+
373
+ for i in range(0, len(deleted_file_ids), SQLITE_MAX_VARS):
374
+ chunk = deleted_file_ids[i:i + SQLITE_MAX_VARS]
375
+ placeholders = ','.join('?' * len(chunk))
376
+
377
+ # Collect function/class IDs being deleted
378
+ cursor.execute(f"SELECT id FROM functions WHERE file_id IN ({placeholders})", chunk)
379
+ function_ids = [row[0] for row in cursor.fetchall()]
380
+
381
+ cursor.execute(f"SELECT id FROM classes WHERE file_id IN ({placeholders})", chunk)
382
+ class_ids = [row[0] for row in cursor.fetchall()]
383
+
384
+ # NULL out dependency references to deleted symbols
385
+ if function_ids:
386
+ fp = ','.join('?' * len(function_ids))
387
+ cursor.execute(f"UPDATE dependencies SET target_function_id = NULL WHERE target_function_id IN ({fp})", function_ids)
388
+
389
+ if class_ids:
390
+ cp = ','.join('?' * len(class_ids))
391
+ cursor.execute(f"UPDATE dependencies SET target_class_id = NULL WHERE target_class_id IN ({cp})", class_ids)
392
+
393
+ # Collect frontend component and custom property IDs for cross-file FK cleanup
394
+ cursor.execute(f"SELECT id FROM frontend_components WHERE file_id IN ({placeholders})", chunk)
395
+ component_ids = [row[0] for row in cursor.fetchall()]
396
+
397
+ cursor.execute(f"SELECT id FROM style_custom_properties WHERE file_id IN ({placeholders})", chunk)
398
+ custom_property_ids = [row[0] for row in cursor.fetchall()]
399
+
400
+ # NULL out cross-file references to symbols being deleted (FK constraints not enforced)
401
+ if component_ids:
402
+ fp = ','.join('?' * len(component_ids))
403
+ cursor.execute(f"UPDATE render_relationships SET child_component_id = NULL WHERE child_component_id IN ({fp})", component_ids)
404
+ cursor.execute(f"UPDATE markup_elements SET component_id = NULL WHERE component_id IN ({fp})", component_ids)
405
+ cursor.execute(f"UPDATE style_selectors SET component_id = NULL WHERE component_id IN ({fp})", component_ids)
406
+
407
+ if custom_property_ids:
408
+ cp2 = ','.join('?' * len(custom_property_ids))
409
+ cursor.execute(f"UPDATE style_custom_property_usages SET resolved_property_id = NULL WHERE resolved_property_id IN ({cp2})", custom_property_ids)
410
+
411
+ # NULL out style_imports.resolved_file_id in OTHER files pointing to deleted files
412
+ cursor.execute(f"UPDATE style_imports SET resolved_file_id = NULL WHERE resolved_file_id IN ({placeholders})", chunk)
413
+
414
+ # Delete render_relationships and style_selector_matches FIRST (before their parent
415
+ # tables are emptied), since FK constraints are not enforced and subqueries would
416
+ # find no rows after the parent table delete.
417
+ cursor.execute(
418
+ f"DELETE FROM render_relationships WHERE parent_component_id IN "
419
+ f"(SELECT id FROM frontend_components WHERE file_id IN ({placeholders}))",
420
+ chunk
421
+ )
422
+ cursor.execute(
423
+ f"DELETE FROM style_selector_matches WHERE selector_id IN "
424
+ f"(SELECT id FROM style_selectors WHERE file_id IN ({placeholders})) "
425
+ f"OR element_id IN (SELECT id FROM markup_elements WHERE file_id IN ({placeholders}))",
426
+ chunk + chunk
427
+ )
428
+ # Tables with direct file_id column
429
+ for table in ("functions", "classes", "variables", "type_aliases", "structs", "interfaces", "dependencies", "temp_symbols", "namespaces",
430
+ "frontend_components", "markup_elements", "style_selectors", "style_custom_properties",
431
+ "style_custom_property_usages", "style_keyframes", "style_imports", "frontend_events",
432
+ "frontend_bindings", "frontend_diagnostics"):
433
+ cursor.execute(f"DELETE FROM {table} WHERE file_id IN ({placeholders})", chunk)
434
+
435
+ # Clean up temp_file_references for deleted source files
436
+ cursor.execute(f"DELETE FROM temp_file_references WHERE source_file_id IN ({placeholders})", chunk)
437
+
438
+ # Delete file records
439
+ cursor.execute(f"DELETE FROM files WHERE id IN ({placeholders})", chunk)
440
+
441
+ self.conn.commit()
442
+
443
+ def _insert_file(self, file_path, language, content_hash, mtime):
444
+ """Insert or update file record and return its ID. Returns None if file should be skipped."""
445
+ relative_path = get_relative_path(file_path, self.root_dir)
446
+ self.cursor.execute("SELECT id, content_hash, mtime FROM files WHERE path = ?", (relative_path,))
447
+ existing = self.cursor.fetchone()
448
+ if existing:
449
+ file_id, existing_hash, existing_mtime = existing
450
+ # Fast path: mtime unchanged => file hasn't been modified
451
+ if not self.force_reindex and existing_mtime == mtime:
452
+ return None # Skip reindexing
453
+ # Slow path: mtime changed, check content hash
454
+ if not self.force_reindex and content_hash is not None and existing_hash == content_hash:
455
+ # mtime changed but content is the same (e.g. touch), update mtime only
456
+ self.cursor.execute("UPDATE files SET mtime = ? WHERE id = ?", (mtime, file_id))
457
+ return None
458
+ # Content changed or force_reindex, reindex
459
+ self._delete_file_symbols(file_id)
460
+ self.cursor.execute(
461
+ "UPDATE files SET absolute_path = ?, language = ?, content_hash = ?, mtime = ? WHERE id = ?",
462
+ (str(file_path), language, content_hash, mtime, file_id)
463
+ )
464
+ return file_id
465
+ self.cursor.execute(
466
+ "INSERT INTO files (path, absolute_path, language, content_hash, mtime) VALUES (?, ?, ?, ?, ?)",
467
+ (relative_path, str(file_path), language, content_hash, mtime)
468
+ )
469
+ return self.cursor.lastrowid
470
+
471
+ def _update_file_hash(self, file_id, content_hash, mtime):
472
+ """Update content hash and mtime after file has been read and parsed."""
473
+ self.cursor.execute(
474
+ "UPDATE files SET content_hash = ?, mtime = ? WHERE id = ?",
475
+ (content_hash, mtime, file_id)
476
+ )
477
+
478
+ def _insert_parsed_file(self, data):
479
+ """Insert a file parsed by a worker process into the database.
480
+
481
+ Wraps the insertion in a SAVEPOINT so that a failed update does not
482
+ leave partially updated semantic state.
483
+ """
484
+ file_path = Path(data['file_path'])
485
+ language = data['language']
486
+ content_hash = data['content_hash']
487
+ file_mtime = data['file_mtime']
488
+
489
+ # Insert/update file record
490
+ file_id = self._insert_file(file_path, language, content_hash, file_mtime)
491
+ if file_id is None:
492
+ self.stats["skipped_files"] += 1
493
+ return
494
+
495
+ self.current_file_id = file_id
496
+
497
+ try:
498
+ self._insert_parsed_file_inner(data, file_id)
499
+ except Exception as e:
500
+ if self.verbose:
501
+ print(f"Warning: Failed to insert {file_path}: {e}")
502
+ return
503
+
504
+ def _insert_parsed_file_inner(self, data, file_id):
505
+ """Inner insertion logic — called within a SAVEPOINT."""
506
+
507
+ # Insert imports
508
+ for imp in data.get('imports', []):
509
+ self._insert_import(imp['name'], imp['location'], imp.get('is_external', True))
510
+
511
+ # Insert classes first (so methods can reference them) — batch
512
+ class_id_map = {} # temp_id -> real_id
513
+ class_names = []
514
+ class_temp_ids = []
515
+ class_rows = []
516
+ for cls in data.get('classes', []):
517
+ temp_id = cls.pop('_temp_id', None)
518
+ class_temp_ids.append(temp_id)
519
+ class_rows.append((
520
+ file_id, None, cls['name'],
521
+ json.dumps(cls['location']),
522
+ json.dumps(cls.get('base_classes', [])),
523
+ cls.get('docstring'),
524
+ cls.get('namespace'),
525
+ ))
526
+ self.stats["total_classes"] += 1
527
+ class_names.append(cls['name'])
528
+ if class_rows:
529
+ self.cursor.executemany(
530
+ """INSERT INTO classes (file_id, parent_id, name, location, base_classes, docstring, namespace)
531
+ VALUES (?, ?, ?, ?, ?, ?, ?)""",
532
+ class_rows
533
+ )
534
+ # SELECT back IDs in insertion order to rebuild class_id_map
535
+ self.cursor.execute(
536
+ "SELECT id FROM classes WHERE file_id = ? ORDER BY id DESC LIMIT ?",
537
+ (file_id, len(class_rows))
538
+ )
539
+ real_ids = [r[0] for r in self.cursor.fetchall()]
540
+ real_ids.reverse() # back to insertion order
541
+ for temp_id, real_id in zip(class_temp_ids, real_ids):
542
+ if temp_id is not None:
543
+ class_id_map[temp_id] = real_id
544
+
545
+ # Insert functions/methods — batch
546
+ func_id_map = {}
547
+ func_names = []
548
+ func_rows = []
549
+ for i, func in enumerate(data.get('functions', [])):
550
+ parent_class_id = func.pop('parent_class_id', None)
551
+ if parent_class_id is not None:
552
+ parent_class_id = class_id_map.get(parent_class_id)
553
+ func_rows.append((
554
+ file_id,
555
+ parent_class_id,
556
+ 'class' if parent_class_id else None,
557
+ func['name'],
558
+ func['type'],
559
+ json.dumps(func['location']),
560
+ json.dumps(func.get('parameters', [])),
561
+ func.get('return_type'),
562
+ func.get('docstring'),
563
+ func.get('receiver'),
564
+ func.get('branch_count', 0),
565
+ ))
566
+ if parent_class_id is not None:
567
+ self.stats["total_methods"] += 1
568
+ elif func.get('type') == 'macro':
569
+ self.stats["total_macros"] += 1
570
+ else:
571
+ self.stats["total_functions"] += 1
572
+ func_names.append(func['name'])
573
+ if func_rows:
574
+ self.cursor.executemany(
575
+ """INSERT INTO functions
576
+ (file_id, parent_id, parent_type, name, type, location, parameters, return_type, docstring, receiver, branch_count)
577
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
578
+ func_rows
579
+ )
580
+ # SELECT back IDs in insertion order to rebuild func_id_map
581
+ self.cursor.execute(
582
+ "SELECT id FROM functions WHERE file_id = ? ORDER BY id DESC LIMIT ?",
583
+ (file_id, len(func_rows))
584
+ )
585
+ real_ids = [r[0] for r in self.cursor.fetchall()]
586
+ real_ids.reverse()
587
+ for i, real_id in enumerate(real_ids):
588
+ func_id_map[i] = real_id
589
+
590
+ # Insert variables (batch)
591
+ var_names = []
592
+ var_rows = []
593
+ for var in data.get('variables', []):
594
+ parent_class_id = var.pop('parent_class_id', None)
595
+ if parent_class_id is not None:
596
+ parent_class_id = class_id_map.get(parent_class_id)
597
+ var_rows.append((
598
+ file_id,
599
+ parent_class_id,
600
+ 'class' if parent_class_id else None,
601
+ var['name'],
602
+ var['type'],
603
+ json.dumps(var['location']),
604
+ var.get('field_type'),
605
+ ))
606
+ self.stats["total_variables"] += 1
607
+ var_names.append(var['name'])
608
+ if var_rows:
609
+ self.cursor.executemany(
610
+ """INSERT INTO variables (file_id, parent_id, parent_type, name, type, location, field_type)
611
+ VALUES (?, ?, ?, ?, ?, ?, ?)""",
612
+ var_rows
613
+ )
614
+
615
+ # Dependencies are inserted as external with temp_symbols.
616
+ # Resolution happens in _resolve_all_dependencies after all files are indexed.
617
+
618
+ # Batch insert type aliases
619
+ type_alias_rows = []
620
+ type_alias_names = []
621
+ for alias in data.get('type_aliases', []):
622
+ type_alias_rows.append((
623
+ file_id, alias['name'],
624
+ json.dumps(alias['location']),
625
+ alias.get('type_definition'),
626
+ ))
627
+ type_alias_names.append(alias['name'])
628
+ if type_alias_rows:
629
+ self.cursor.executemany(
630
+ "INSERT INTO type_aliases (file_id, name, location, type_definition) VALUES (?, ?, ?, ?)",
631
+ type_alias_rows
632
+ )
633
+ self.stats["total_type_defs"] += len(type_alias_rows)
634
+
635
+ # Batch insert structs
636
+ struct_rows = []
637
+ struct_names = []
638
+ for struct in data.get('structs', []):
639
+ struct_rows.append((file_id, struct['name'], json.dumps(struct['location'])))
640
+ struct_names.append(struct['name'])
641
+ if struct_rows:
642
+ self.cursor.executemany(
643
+ "INSERT INTO structs (file_id, name, location) VALUES (?, ?, ?)",
644
+ struct_rows
645
+ )
646
+ self.stats["total_structs"] += len(struct_rows)
647
+
648
+ # Batch insert interfaces
649
+ iface_rows = []
650
+ iface_names = []
651
+ for iface in data.get('interfaces', []):
652
+ iface_rows.append((file_id, iface['name'], json.dumps(iface['location'])))
653
+ iface_names.append(iface['name'])
654
+ if iface_rows:
655
+ self.cursor.executemany(
656
+ "INSERT INTO interfaces (file_id, name, location) VALUES (?, ?, ?)",
657
+ iface_rows
658
+ )
659
+ self.stats["total_interfaces"] += len(iface_rows)
660
+
661
+ # Batch insert enums
662
+ enum_rows = []
663
+ enum_names = []
664
+ for enum in data.get('enums', []):
665
+ enum_rows.append((file_id, enum['name'], json.dumps(enum['location'])))
666
+ enum_names.append(enum['name'])
667
+ if enum_rows:
668
+ self.cursor.executemany(
669
+ "INSERT INTO enums (file_id, name, location) VALUES (?, ?, ?)",
670
+ enum_rows
671
+ )
672
+ self.stats["total_enums"] += len(enum_rows)
673
+
674
+ # Batch insert namespaces
675
+ ns_rows = []
676
+ for ns in data.get('namespaces', []):
677
+ ns_rows.append((file_id, ns['name'], json.dumps(ns['location'])))
678
+ if ns_rows:
679
+ self.cursor.executemany(
680
+ "INSERT INTO namespaces (file_id, name, location) VALUES (?, ?, ?)",
681
+ ns_rows
682
+ )
683
+ self.stats["total_namespaces"] += len(ns_rows)
684
+
685
+ # Dependencies are inserted as external with temp_symbols.
686
+ # Resolution happens in _resolve_all_dependencies after all files are indexed.
687
+
688
+ # Insert dependencies — all as external with temp_symbols.
689
+ # Resolution happens in a single batch pass after all files are indexed.
690
+ # This avoids 7+ SELECT IN queries per file against tables that grow to millions of rows.
691
+ deps = data.get('dependencies', [])
692
+ if deps:
693
+ dep_rows = []
694
+ temp_pairs_set = set()
695
+ for dep in deps:
696
+ dep_type = dep['type']
697
+ dep_name = dep['name']
698
+ source_func_id = func_id_map.get(dep.pop('_func_index', None))
699
+ loc = json.dumps(dep.get('location')) if dep.get('location') else None
700
+ dep_rows.append((
701
+ file_id, dep_type, dep_name, source_func_id,
702
+ None, None, None, loc, 1,
703
+ ))
704
+ temp_pairs_set.add((dep_name, dep_type))
705
+
706
+ # Batch insert dependencies
707
+ self.cursor.executemany(
708
+ """INSERT INTO dependencies (file_id, dependency_type, name, source_function_id, target_function_id, target_class_id, temp_symbol_id, location, is_external)
709
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
710
+ dep_rows
711
+ )
712
+
713
+ # Batch create temp_symbols and link them to deps via SQL UPDATE
714
+ if temp_pairs_set:
715
+ temp_pairs = [(name, dtype, file_id) for name, dtype in temp_pairs_set]
716
+ self.cursor.executemany(
717
+ "INSERT OR IGNORE INTO temp_symbols (name, symbol_type, file_id) VALUES (?, ?, ?)",
718
+ temp_pairs
719
+ )
720
+ # Link temp_symbols to deps by matching (name, type) within this file
721
+ self.cursor.execute(
722
+ """UPDATE dependencies SET temp_symbol_id = (
723
+ SELECT ts.id FROM temp_symbols ts
724
+ WHERE ts.name = dependencies.name
725
+ AND ts.symbol_type = dependencies.dependency_type
726
+ LIMIT 1
727
+ )
728
+ WHERE dependencies.file_id = ? AND dependencies.is_external = 1
729
+ AND dependencies.temp_symbol_id IS NULL""",
730
+ (file_id,)
731
+ )
732
+
733
+ # Insert frontend semantic data (HTML files)
734
+ self._insert_frontend_data(file_id, data)
735
+
736
+ def _insert_import(self, name, location, is_external):
737
+ """Queue an import dependency for batch insertion."""
738
+ self.batches["dependencies"].append(
739
+ (
740
+ self.current_file_id,
741
+ 'import',
742
+ name,
743
+ None,
744
+ None,
745
+ None,
746
+ None,
747
+ json.dumps(location) if location else None,
748
+ 1 if is_external else 0
749
+ )
750
+ )
751
+ self._check_and_flush("dependencies")
752
+
753
+ def _insert_frontend_data(self, file_id, data):
754
+ """Insert frontend semantic data (markup elements, events, selectors, etc.).
755
+
756
+ Also performs inline cross-file resolution using temp_symbols and temp_file_references,
757
+ consistent with the existing temp_symbols pattern for dependencies.
758
+ """
759
+ # Lazily load path aliases on first frontend insert
760
+ if not hasattr(self, '_path_aliases_loaded'):
761
+ self.path_aliases = load_path_aliases(self.root_dir)
762
+ self._path_aliases_loaded = True
763
+
764
+ # Get the file path for import normalization
765
+ self.cursor.execute("SELECT absolute_path FROM files WHERE id = ?", (file_id,))
766
+ file_row = self.cursor.fetchone()
767
+ source_file_path = file_row[0] if file_row else ""
768
+
769
+ # Insert frontend components (JSX/TSX files)
770
+ component_id_map = {} # component_index -> db id
771
+ for i, comp in enumerate(data.get('frontend_components', [])):
772
+ # Try to link to already-inserted function/class by name
773
+ impl_func_id = None
774
+ impl_class_id = None
775
+ func_name = comp.get('impl_function_name')
776
+ class_name = comp.get('impl_class_name')
777
+ if func_name:
778
+ self.cursor.execute(
779
+ "SELECT id FROM functions WHERE file_id = ? AND name = ?",
780
+ (file_id, func_name)
781
+ )
782
+ row = self.cursor.fetchone()
783
+ if row:
784
+ impl_func_id = row[0]
785
+ if class_name:
786
+ self.cursor.execute(
787
+ "SELECT id FROM classes WHERE file_id = ? AND name = ?",
788
+ (file_id, class_name)
789
+ )
790
+ row = self.cursor.fetchone()
791
+ if row:
792
+ impl_class_id = row[0]
793
+
794
+ self.cursor.execute(
795
+ """INSERT INTO frontend_components
796
+ (file_id, name, framework, source_range, is_exported, impl_function_id, impl_class_id)
797
+ VALUES (?, ?, 'react', ?, ?, ?, ?)""",
798
+ (
799
+ file_id,
800
+ comp['name'],
801
+ json.dumps(comp.get('source_range')) if comp.get('source_range') else None,
802
+ 1 if comp.get('is_exported') else 0,
803
+ impl_func_id,
804
+ impl_class_id,
805
+ )
806
+ )
807
+ comp_db_id = self.cursor.lastrowid
808
+ component_id_map[i] = comp_db_id
809
+
810
+ # Resolve any pending temp_symbols for this component name
811
+ self._resolve_component_temp(comp['name'], comp_db_id)
812
+
813
+ # Insert markup elements with parent-child and component resolution
814
+ elem_id_map = {} # index -> db id
815
+ for i, elem in enumerate(data.get('markup_elements', [])):
816
+ parent_idx = elem.get('parent_index')
817
+ parent_db_id = elem_id_map.get(parent_idx) if parent_idx is not None else None
818
+ comp_idx = elem.get('component_index')
819
+ comp_db_id = component_id_map.get(comp_idx) if comp_idx is not None else None
820
+ self.cursor.execute(
821
+ """INSERT INTO markup_elements
822
+ (file_id, component_id, parent_element_id, tag_name, element_type,
823
+ source_range, element_id_attr, static_classes, attributes,
824
+ is_conditional, is_repeated, conditional_expr, repeated_expr)
825
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
826
+ (
827
+ file_id,
828
+ comp_db_id,
829
+ parent_db_id,
830
+ elem['tag_name'],
831
+ elem['element_type'],
832
+ json.dumps(elem.get('source_range')) if elem.get('source_range') else None,
833
+ elem.get('element_id_attr'),
834
+ json.dumps(elem.get('static_classes')) if elem.get('static_classes') else None,
835
+ json.dumps(elem.get('attributes')) if elem.get('attributes') else None,
836
+ 1 if elem.get('is_conditional') else 0,
837
+ 1 if elem.get('is_repeated') else 0,
838
+ elem.get('conditional_expr'),
839
+ elem.get('repeated_expr'),
840
+ )
841
+ )
842
+ elem_id_map[i] = self.cursor.lastrowid
843
+
844
+ # Insert style selectors
845
+ selector_id_map = {} # index -> db id
846
+ for i, sel in enumerate(data.get('style_selectors', [])):
847
+ self.cursor.execute(
848
+ """INSERT INTO style_selectors
849
+ (file_id, selector_text, normalized_selector, selector_type, source_range, is_scoped)
850
+ VALUES (?, ?, ?, ?, ?, ?)""",
851
+ (
852
+ file_id,
853
+ sel['selector_text'],
854
+ sel.get('normalized_selector'),
855
+ sel['selector_type'],
856
+ json.dumps(sel.get('source_range')) if sel.get('source_range') else None,
857
+ 1 if sel.get('is_scoped') else 0,
858
+ )
859
+ )
860
+ selector_id_map[i] = self.cursor.lastrowid
861
+
862
+ # Insert style custom properties
863
+ for cp in data.get('style_custom_properties', []):
864
+ self.cursor.execute(
865
+ """INSERT INTO style_custom_properties
866
+ (file_id, name, value, source_range, scope_selector)
867
+ VALUES (?, ?, ?, ?, ?)""",
868
+ (
869
+ file_id,
870
+ cp['name'],
871
+ cp.get('value'),
872
+ json.dumps(cp.get('source_range')) if cp.get('source_range') else None,
873
+ cp.get('scope_selector'),
874
+ )
875
+ )
876
+ cp_db_id = self.cursor.lastrowid
877
+ # Resolve any pending temp_symbols for this custom property name
878
+ self._resolve_custom_property_temp(cp['name'], cp_db_id)
879
+
880
+ # Insert style custom property usages with resolution
881
+ for pu in data.get('style_custom_property_usages', []):
882
+ prop_name = pu['property_name']
883
+ # Try to resolve to existing custom property definition
884
+ resolved_id = self._find_custom_property(prop_name)
885
+ self.cursor.execute(
886
+ """INSERT INTO style_custom_property_usages
887
+ (file_id, property_name, source_range, selector_id, resolved_property_id)
888
+ VALUES (?, ?, ?, NULL, ?)""",
889
+ (
890
+ file_id,
891
+ prop_name,
892
+ json.dumps(pu.get('source_range')) if pu.get('source_range') else None,
893
+ resolved_id,
894
+ )
895
+ )
896
+ if resolved_id is None:
897
+ # Create temp symbol for unresolved custom property reference
898
+ self._create_temp_symbol(prop_name, 'custom_property_reference')
899
+
900
+ # Insert style imports with file-path resolution
901
+ for si in data.get('style_imports', []):
902
+ import_path = si['import_path']
903
+ is_external = si.get('is_external', False) or import_path.startswith(("http://", "https://", "//"))
904
+
905
+ self.cursor.execute(
906
+ """INSERT INTO style_imports
907
+ (file_id, import_path, is_external, resolved_file_id, source_range)
908
+ VALUES (?, ?, ?, NULL, ?)""",
909
+ (
910
+ file_id,
911
+ import_path,
912
+ 1 if is_external else 0,
913
+ json.dumps(si.get('source_range')) if si.get('source_range') else None,
914
+ )
915
+ )
916
+ si_row_id = self.cursor.lastrowid
917
+
918
+ if not is_external:
919
+ # Try to resolve the import path to an existing file
920
+ resolved_path = normalize_import_path(import_path, source_file_path, str(self.root_dir), self.path_aliases)
921
+ resolved_file_id = self._find_file_by_path(resolved_path)
922
+ if resolved_file_id:
923
+ self.cursor.execute(
924
+ "UPDATE style_imports SET resolved_file_id = ? WHERE id = ?",
925
+ (resolved_file_id, si_row_id)
926
+ )
927
+ else:
928
+ # Create temp_file_reference for later resolution
929
+ self._create_temp_file_reference(
930
+ resolved_path, 'style_import', file_id, 'style_imports', si_row_id
931
+ )
932
+
933
+ # Insert frontend events
934
+ for ev in data.get('frontend_events', []):
935
+ elem_idx = ev.get('element_index')
936
+ elem_db_id = elem_id_map.get(elem_idx) if elem_idx is not None else None
937
+ self.cursor.execute(
938
+ """INSERT INTO frontend_events
939
+ (file_id, element_id, event_name, handler_type, handler_expression,
940
+ handler_symbol_id, resolution_status, source_range)
941
+ VALUES (?, ?, ?, ?, ?, NULL, ?, ?)""",
942
+ (
943
+ file_id,
944
+ elem_db_id,
945
+ ev['event_name'],
946
+ ev['handler_type'],
947
+ ev.get('handler_expression'),
948
+ ev.get('resolution_status', 'unresolved'),
949
+ json.dumps(ev.get('source_range')) if ev.get('source_range') else None,
950
+ )
951
+ )
952
+
953
+ # Insert style selector matches
954
+ for sm in data.get('style_selector_matches', []):
955
+ sel_idx = sm.get('selector_index')
956
+ elem_idx = sm.get('element_index')
957
+ sel_db_id = selector_id_map.get(sel_idx) if sel_idx is not None else None
958
+ elem_db_id = elem_id_map.get(elem_idx) if elem_idx is not None else None
959
+ if sel_db_id and elem_db_id:
960
+ self.cursor.execute(
961
+ """INSERT INTO style_selector_matches
962
+ (selector_id, element_id, match_type, confidence, source_range)
963
+ VALUES (?, ?, ?, ?, ?)""",
964
+ (
965
+ sel_db_id,
966
+ elem_db_id,
967
+ sm['match_type'],
968
+ sm.get('confidence', 'high'),
969
+ json.dumps(sm.get('source_range')) if sm.get('source_range') else None,
970
+ )
971
+ )
972
+
973
+ # Insert frontend diagnostics
974
+ for diag in data.get('frontend_diagnostics', []):
975
+ self.cursor.execute(
976
+ """INSERT INTO frontend_diagnostics
977
+ (file_id, diagnostic_type, severity, message, source_range)
978
+ VALUES (?, ?, ?, ?, ?)""",
979
+ (
980
+ file_id,
981
+ diag['diagnostic_type'],
982
+ diag['severity'],
983
+ diag.get('message'),
984
+ json.dumps(diag.get('source_range')) if diag.get('source_range') else None,
985
+ )
986
+ )
987
+
988
+ # Insert style keyframes (CSS files)
989
+ for kf in data.get('style_keyframes', []):
990
+ self.cursor.execute(
991
+ """INSERT INTO style_keyframes
992
+ (file_id, name, source_range)
993
+ VALUES (?, ?, ?)""",
994
+ (
995
+ file_id,
996
+ kf['name'],
997
+ json.dumps(kf.get('source_range')) if kf.get('source_range') else None,
998
+ )
999
+ )
1000
+
1001
+ # Insert frontend bindings (JSX/TSX files)
1002
+ for b in data.get('frontend_bindings', []):
1003
+ elem_db_id = elem_id_map.get(b.get('element_index'))
1004
+ self.cursor.execute(
1005
+ """INSERT INTO frontend_bindings
1006
+ (file_id, element_id, binding_type, binding_name, binding_expression,
1007
+ resolution_status, source_range)
1008
+ VALUES (?, ?, ?, ?, ?, ?, ?)""",
1009
+ (
1010
+ file_id,
1011
+ elem_db_id,
1012
+ b['binding_type'],
1013
+ b.get('binding_name'),
1014
+ b.get('expression'),
1015
+ b.get('resolution_status', 'unresolved'),
1016
+ json.dumps(b.get('source_range')) if b.get('source_range') else None,
1017
+ )
1018
+ )
1019
+
1020
+ # Insert render relationships with component resolution
1021
+ for rr in data.get('render_relationships', []):
1022
+ parent_db_id = component_id_map.get(rr.get('parent_component_index'))
1023
+ child_elem_db_id = elem_id_map.get(rr.get('element_index'))
1024
+ child_name = rr.get('child_component_name')
1025
+
1026
+ # Try to resolve child component by name
1027
+ child_component_id = None
1028
+ if child_name:
1029
+ child_component_id = self._find_component_by_name(child_name)
1030
+
1031
+ self.cursor.execute(
1032
+ """INSERT INTO render_relationships
1033
+ (parent_component_id, child_component_id, child_component_name, child_element_id, render_type, controlling_expr)
1034
+ VALUES (?, ?, ?, ?, ?, ?)""",
1035
+ (
1036
+ parent_db_id,
1037
+ child_component_id,
1038
+ child_name,
1039
+ child_elem_db_id,
1040
+ rr['render_type'],
1041
+ rr.get('controlling_expr'),
1042
+ )
1043
+ )
1044
+
1045
+ # If child component not found, create temp symbol for later resolution
1046
+ if child_name and child_component_id is None:
1047
+ self._create_temp_symbol(child_name, 'component_reference')
1048
+
1049
+ # After inserting all data for this file, resolve any pending temp_file_references
1050
+ # that point to this file
1051
+ relative_path = get_relative_path(source_file_path, self.root_dir)
1052
+ self._resolve_temp_file_references_for_file(file_id, relative_path)
1053
+
1054
+ def _find_component_by_name(self, name):
1055
+ """Find a frontend_component by name. Returns its id or None."""
1056
+ self.cursor.execute(
1057
+ "SELECT id FROM frontend_components WHERE name = ? LIMIT 1",
1058
+ (name,)
1059
+ )
1060
+ row = self.cursor.fetchone()
1061
+ return row[0] if row else None
1062
+
1063
+ def _resolve_component_temp(self, name, component_id):
1064
+ """When a component is defined, resolve pending component_reference temp_symbols.
1065
+
1066
+ Updates render_relationships.child_component_id for any rows that
1067
+ were waiting for this component.
1068
+ """
1069
+ self.cursor.execute(
1070
+ "SELECT id FROM temp_symbols WHERE name = ? AND symbol_type = 'component_reference'",
1071
+ (name,)
1072
+ )
1073
+ temp_rows = self.cursor.fetchall()
1074
+ for temp_row in temp_rows:
1075
+ temp_id = temp_row[0]
1076
+ self.cursor.execute(
1077
+ "UPDATE render_relationships SET child_component_id = ? WHERE child_component_id IS NULL AND child_component_name = ?",
1078
+ (component_id, name)
1079
+ )
1080
+ self.cursor.execute("DELETE FROM temp_symbols WHERE id = ?", (temp_id,))
1081
+
1082
+ def _find_custom_property(self, name):
1083
+ """Find a style_custom_property by name. Returns its id or None."""
1084
+ self.cursor.execute(
1085
+ "SELECT id FROM style_custom_properties WHERE name = ? LIMIT 1",
1086
+ (name,)
1087
+ )
1088
+ row = self.cursor.fetchone()
1089
+ return row[0] if row else None
1090
+
1091
+ def _resolve_custom_property_temp(self, name, property_id):
1092
+ """When a custom property is defined, resolve pending custom_property_reference temp_symbols.
1093
+
1094
+ Updates style_custom_property_usages.resolved_property_id for any rows
1095
+ that were waiting for this property definition.
1096
+ """
1097
+ self.cursor.execute(
1098
+ "SELECT id FROM temp_symbols WHERE name = ? AND symbol_type = 'custom_property_reference'",
1099
+ (name,)
1100
+ )
1101
+ temp_rows = self.cursor.fetchall()
1102
+ for temp_row in temp_rows:
1103
+ temp_id = temp_row[0]
1104
+ self.cursor.execute(
1105
+ "UPDATE style_custom_property_usages SET resolved_property_id = ? WHERE resolved_property_id IS NULL AND property_name = ?",
1106
+ (property_id, name)
1107
+ )
1108
+ self.cursor.execute("DELETE FROM temp_symbols WHERE id = ?", (temp_id,))
1109
+
1110
+ def _find_file_by_path(self, relative_path):
1111
+ """Find a file by its project-relative path. Returns its id or None."""
1112
+ self.cursor.execute(
1113
+ "SELECT id FROM files WHERE path = ? LIMIT 1",
1114
+ (relative_path,)
1115
+ )
1116
+ row = self.cursor.fetchone()
1117
+ return row[0] if row else None
1118
+
1119
+ def _create_temp_file_reference(self, resolved_path, reference_type, source_file_id, target_table, target_row_id):
1120
+ """Create a temp_file_reference for an unresolved file-path reference."""
1121
+ self.cursor.execute(
1122
+ """INSERT INTO temp_file_references
1123
+ (resolved_path, reference_type, source_file_id, target_table, target_row_id)
1124
+ VALUES (?, ?, ?, ?, ?)""",
1125
+ (resolved_path, reference_type, source_file_id, target_table, target_row_id)
1126
+ )
1127
+
1128
+ def _resolve_temp_file_references_for_file(self, file_id, relative_path):
1129
+ """When a file is indexed, resolve any temp_file_references pointing to it.
1130
+
1131
+ Updates the target table rows (e.g. style_imports.resolved_file_id)
1132
+ and deletes the resolved temp_file_reference entries.
1133
+ """
1134
+ self.cursor.execute(
1135
+ "SELECT id, target_table, target_row_id FROM temp_file_references WHERE resolved_path = ?",
1136
+ (relative_path,)
1137
+ )
1138
+ refs = self.cursor.fetchall()
1139
+ for ref_id, target_table, target_row_id in refs:
1140
+ if target_table == 'style_imports':
1141
+ self.cursor.execute(
1142
+ "UPDATE style_imports SET resolved_file_id = ? WHERE id = ?",
1143
+ (file_id, target_row_id)
1144
+ )
1145
+ self.cursor.execute("DELETE FROM temp_file_references WHERE id = ?", (ref_id,))
1146
+
1147
+ def _resolve_dependency(self, name, dep_type, source_file_id=None):
1148
+ """Try to resolve a dependency to an existing symbol.
1149
+ Returns (target_function_id, target_class_id, is_external) or (None, None, 1).
1150
+
1151
+ Args:
1152
+ name: The dependency name
1153
+ dep_type: The dependency type (function_call, method_call, etc.)
1154
+ source_file_id: Optional file ID of the source making the reference, for disambiguation
1155
+ """
1156
+ # For method calls like self.init or obj.method, extract just the method name
1157
+ resolve_name = name
1158
+ if dep_type == 'method_call' and '.' in name:
1159
+ resolve_name = name.rsplit('.', 1)[-1]
1160
+
1161
+ if dep_type in ('function_call', 'method_call'):
1162
+ # Prefer functions in the same file if available
1163
+ if source_file_id:
1164
+ self.cursor.execute(
1165
+ "SELECT id, parent_type FROM functions WHERE name = ? AND file_id = ? LIMIT 1",
1166
+ (resolve_name, source_file_id)
1167
+ )
1168
+ row = self.cursor.fetchone()
1169
+ if row:
1170
+ if dep_type == 'method_call' and row[1] != 'class':
1171
+ pass # It's a standalone function, not a method
1172
+ else:
1173
+ return (row[0], None, 0)
1174
+ # Fall back to any function with that name
1175
+ self.cursor.execute(
1176
+ "SELECT id, parent_type FROM functions WHERE name = ? LIMIT 1",
1177
+ (resolve_name,)
1178
+ )
1179
+ row = self.cursor.fetchone()
1180
+ if row:
1181
+ if dep_type == 'method_call' and row[1] != 'class':
1182
+ pass # It's a standalone function, not a method
1183
+ else:
1184
+ return (row[0], None, 0)
1185
+
1186
+ if dep_type in ('function_call', 'method_call', 'class_reference'):
1187
+ # Prefer classes in the same file if available
1188
+ if source_file_id:
1189
+ self.cursor.execute(
1190
+ "SELECT id FROM classes WHERE name = ? AND file_id = ? LIMIT 1",
1191
+ (resolve_name, source_file_id)
1192
+ )
1193
+ row = self.cursor.fetchone()
1194
+ if row:
1195
+ return (None, row[0], 0)
1196
+ # Fall back to any class with that name
1197
+ self.cursor.execute(
1198
+ "SELECT id FROM classes WHERE name = ? LIMIT 1",
1199
+ (resolve_name,)
1200
+ )
1201
+ row = self.cursor.fetchone()
1202
+ if row:
1203
+ return (None, row[0], 0)
1204
+
1205
+ for table in ('structs', 'interfaces', 'enums', 'type_aliases'):
1206
+ self.cursor.execute(
1207
+ f"SELECT id FROM {table} WHERE name = ? LIMIT 1",
1208
+ (resolve_name,)
1209
+ )
1210
+ row = self.cursor.fetchone()
1211
+ if row:
1212
+ return (None, None, 0)
1213
+
1214
+ if dep_type == 'variable_reference':
1215
+ # Prefer variables in the same file if available
1216
+ if source_file_id:
1217
+ self.cursor.execute(
1218
+ "SELECT id FROM variables WHERE name = ? AND file_id = ? LIMIT 1",
1219
+ (name, source_file_id)
1220
+ )
1221
+ row = self.cursor.fetchone()
1222
+ if row:
1223
+ return (None, None, 0)
1224
+ # Fall back to any variable with that name
1225
+ self.cursor.execute(
1226
+ "SELECT id FROM variables WHERE name = ? LIMIT 1",
1227
+ (name,)
1228
+ )
1229
+ row = self.cursor.fetchone()
1230
+ if row:
1231
+ return (None, None, 0)
1232
+
1233
+ return (None, None, 1)
1234
+
1235
+ def _create_temp_symbol(self, name, dep_type):
1236
+ """Create a temp_symbol for an unresolved dependency. Returns the temp_symbol_id."""
1237
+ self.cursor.execute(
1238
+ "INSERT OR IGNORE INTO temp_symbols (name, symbol_type, file_id) VALUES (?, ?, ?)",
1239
+ (name, dep_type, self.current_file_id)
1240
+ )
1241
+ if self.cursor.lastrowid:
1242
+ return self.cursor.lastrowid
1243
+ self.cursor.execute(
1244
+ "SELECT id FROM temp_symbols WHERE name = ? AND symbol_type = ?",
1245
+ (name, dep_type)
1246
+ )
1247
+ return self.cursor.fetchone()[0]
1248
+
1249
+ def _batch_resolve_temp_symbols(self, names, match_types):
1250
+ """Batch-resolve temp_symbols for multiple symbol names at once.
1251
+
1252
+ Instead of calling _resolve_temp_for_symbol per symbol (which does a
1253
+ SELECT + UPDATE + DELETE per name), this does a single SELECT for all
1254
+ names and batches the UPDATE/DELETE.
1255
+ """
1256
+ if not names or not match_types:
1257
+ return
1258
+ name_list = list(set(names)) # deduplicate
1259
+ type_list = list(match_types)
1260
+ SQLITE_MAX_VARS = 900
1261
+ all_temp_ids = []
1262
+ for i in range(0, len(name_list), SQLITE_MAX_VARS):
1263
+ chunk = name_list[i:i + SQLITE_MAX_VARS]
1264
+ name_ph = ','.join('?' * len(chunk))
1265
+ type_ph = ','.join('?' * len(type_list))
1266
+ self.cursor.execute(
1267
+ f"SELECT id FROM temp_symbols WHERE name IN ({name_ph}) AND symbol_type IN ({type_ph})",
1268
+ chunk + type_list
1269
+ )
1270
+ all_temp_ids.extend(r[0] for r in self.cursor.fetchall())
1271
+ if not all_temp_ids:
1272
+ return
1273
+ # Batch update + delete in chunks
1274
+ for i in range(0, len(all_temp_ids), SQLITE_MAX_VARS):
1275
+ chunk = all_temp_ids[i:i + SQLITE_MAX_VARS]
1276
+ temp_ph = ','.join('?' * len(chunk))
1277
+ self.cursor.execute(
1278
+ f"UPDATE dependencies SET is_external = 0, temp_symbol_id = NULL WHERE temp_symbol_id IN ({temp_ph})",
1279
+ chunk
1280
+ )
1281
+ self.cursor.execute(
1282
+ f"DELETE FROM temp_symbols WHERE id IN ({temp_ph})",
1283
+ chunk
1284
+ )
1285
+
1286
+ def _resolve_temp_for_symbol(self, name, match_types, target_function_id=None, target_class_id=None):
1287
+ """When a real symbol is defined, resolve any temp_symbols waiting for it.
1288
+
1289
+ Args:
1290
+ name: Symbol name to match
1291
+ match_types: List of dependency_type values to match
1292
+ target_function_id: If set, update dependencies with this function ID
1293
+ target_class_id: If set, update dependencies with this class ID
1294
+ """
1295
+ if not match_types:
1296
+ return
1297
+
1298
+ placeholders = ','.join('?' * len(match_types))
1299
+ self.cursor.execute(
1300
+ f"SELECT id, symbol_type FROM temp_symbols WHERE name = ? AND symbol_type IN ({placeholders})",
1301
+ [name] + list(match_types)
1302
+ )
1303
+ temp_rows = self.cursor.fetchall()
1304
+
1305
+ for temp_row in temp_rows:
1306
+ temp_id = temp_row[0]
1307
+ sym_type = temp_row[1]
1308
+
1309
+ if target_function_id is not None and sym_type in ('function_call', 'method_call'):
1310
+ self.cursor.execute(
1311
+ "UPDATE dependencies SET target_function_id = ?, is_external = 0, temp_symbol_id = NULL WHERE temp_symbol_id = ?",
1312
+ (target_function_id, temp_id)
1313
+ )
1314
+ elif target_class_id is not None and sym_type == 'class_reference':
1315
+ self.cursor.execute(
1316
+ "UPDATE dependencies SET target_class_id = ?, is_external = 0, temp_symbol_id = NULL WHERE temp_symbol_id = ?",
1317
+ (target_class_id, temp_id)
1318
+ )
1319
+ else:
1320
+ self.cursor.execute(
1321
+ "UPDATE dependencies SET is_external = 0, temp_symbol_id = NULL WHERE temp_symbol_id = ?",
1322
+ (temp_id,)
1323
+ )
1324
+
1325
+ self.cursor.execute("DELETE FROM temp_symbols WHERE id = ?", (temp_id,))
1326
+
1327
+ def _resolve_all_dependencies(self):
1328
+ """Resolve all temp_symbols against the complete symbol tables.
1329
+
1330
+ Called once after all files are indexed. Uses bulk JOIN/UPDATE queries
1331
+ instead of per-file SELECT IN queries. This is the key optimization:
1332
+ 5 SQL statements instead of 62K × 7.
1333
+ """
1334
+ # 1. Resolve function_call / method_call temp_symbols → target_function_id
1335
+ self.cursor.execute(
1336
+ """UPDATE dependencies SET
1337
+ target_function_id = (
1338
+ SELECT f.id FROM functions f
1339
+ WHERE f.name = temp_symbols.name LIMIT 1
1340
+ ),
1341
+ is_external = CASE WHEN EXISTS (
1342
+ SELECT 1 FROM functions f WHERE f.name = temp_symbols.name
1343
+ ) THEN 0 ELSE 1 END,
1344
+ temp_symbol_id = CASE WHEN EXISTS (
1345
+ SELECT 1 FROM functions f WHERE f.name = temp_symbols.name
1346
+ ) THEN NULL ELSE temp_symbol_id END
1347
+ FROM temp_symbols
1348
+ WHERE dependencies.temp_symbol_id = temp_symbols.id
1349
+ AND temp_symbols.symbol_type IN ('function_call', 'method_call')
1350
+ AND dependencies.target_function_id IS NULL"""
1351
+ )
1352
+
1353
+ # 2. Resolve class_reference temp_symbols → target_class_id
1354
+ # Check classes, structs, enums, interfaces, type_aliases
1355
+ self.cursor.execute(
1356
+ """UPDATE dependencies SET
1357
+ target_class_id = (
1358
+ SELECT c.id FROM classes c
1359
+ WHERE c.name = temp_symbols.name LIMIT 1
1360
+ ),
1361
+ is_external = CASE WHEN EXISTS (
1362
+ SELECT 1 FROM classes c WHERE c.name = temp_symbols.name
1363
+ UNION SELECT 1 FROM structs s WHERE s.name = temp_symbols.name
1364
+ UNION SELECT 1 FROM enums e WHERE e.name = temp_symbols.name
1365
+ UNION SELECT 1 FROM interfaces i WHERE i.name = temp_symbols.name
1366
+ UNION SELECT 1 FROM type_aliases t WHERE t.name = temp_symbols.name
1367
+ ) THEN 0 ELSE 1 END,
1368
+ temp_symbol_id = CASE WHEN EXISTS (
1369
+ SELECT 1 FROM classes c WHERE c.name = temp_symbols.name
1370
+ UNION SELECT 1 FROM structs s WHERE s.name = temp_symbols.name
1371
+ UNION SELECT 1 FROM enums e WHERE e.name = temp_symbols.name
1372
+ UNION SELECT 1 FROM interfaces i WHERE i.name = temp_symbols.name
1373
+ UNION SELECT 1 FROM type_aliases t WHERE t.name = temp_symbols.name
1374
+ ) THEN NULL ELSE temp_symbol_id END
1375
+ FROM temp_symbols
1376
+ WHERE dependencies.temp_symbol_id = temp_symbols.id
1377
+ AND temp_symbols.symbol_type = 'class_reference'
1378
+ AND dependencies.target_class_id IS NULL"""
1379
+ )
1380
+
1381
+ # 3. Resolve variable_reference temp_symbols → is_external = 0
1382
+ self.cursor.execute(
1383
+ """UPDATE dependencies SET
1384
+ is_external = 0,
1385
+ temp_symbol_id = NULL
1386
+ FROM temp_symbols
1387
+ WHERE dependencies.temp_symbol_id = temp_symbols.id
1388
+ AND temp_symbols.symbol_type = 'variable_reference'
1389
+ AND EXISTS (
1390
+ SELECT 1 FROM variables v WHERE v.name = temp_symbols.name
1391
+ )"""
1392
+ )
1393
+
1394
+ # 4. Delete resolved temp_symbols (those no longer referenced by any dependency)
1395
+ self.cursor.execute(
1396
+ """DELETE FROM temp_symbols
1397
+ WHERE id NOT IN (SELECT DISTINCT temp_symbol_id FROM dependencies WHERE temp_symbol_id IS NOT NULL)"""
1398
+ )
1399
+
1400
+ self.conn.commit()
1401
+
1402
+ def _compute_content_hash(self, content):
1403
+ """Compute xxhash of file content (faster than SHA-256)."""
1404
+ return xxhash.xxh64(content).hexdigest()
1405
+
1406
+ def index_directory(self):
1407
+ B = "\033[34m"
1408
+ R = "\033[0m"
1409
+ print(f"{B}analyzing...{R}", flush=True)
1410
+ """Index all supported files in the directory recursively."""
1411
+ if self.verbose:
1412
+ print(f"Starting indexing of {self.root_dir}...")
1413
+ print(f"Languages: {', '.join(self.languages)}")
1414
+
1415
+ t_total_start = time.perf_counter()
1416
+
1417
+ t0 = time.perf_counter()
1418
+ files = collect_files_to_index(self.root_dir, self.languages)
1419
+ if self.verbose:
1420
+ print(f" [timing] file collection: {time.perf_counter() - t0:.3f}s ({len(files)} files)")
1421
+
1422
+ # First pass: identify changed files using mtime (fast, no file reads)
1423
+ # Batch-load all existing file metadata from the database in one query
1424
+ t0 = time.perf_counter()
1425
+ changed_files = []
1426
+ files_to_reindex = set()
1427
+ skipped_count = 0
1428
+
1429
+ # Load all file metadata in one query instead of per-file SELECT
1430
+ self.cursor.execute("SELECT path, id, content_hash, mtime FROM files")
1431
+ db_file_meta = {}
1432
+ for row in self.cursor.fetchall():
1433
+ db_file_meta[row[0]] = (row[1], row[2], row[3])
1434
+
1435
+ for file_path in files:
1436
+ language = detect_language(file_path)
1437
+ if not language or language not in self._configured_languages:
1438
+ continue
1439
+
1440
+ relative_path = get_relative_path(file_path, self.root_dir)
1441
+ file_mtime = os.stat(file_path).st_mtime
1442
+
1443
+ existing = db_file_meta.get(relative_path)
1444
+
1445
+ if existing:
1446
+ file_id, existing_hash, existing_mtime = existing
1447
+ if self.force_reindex or existing_mtime != file_mtime:
1448
+ # mtime changed - verify with content hash
1449
+ source_bytes = read_file_content(file_path)
1450
+ content_hash = self._compute_content_hash(source_bytes)
1451
+ if self.force_reindex or existing_hash != content_hash:
1452
+ changed_files.append((file_path, file_id))
1453
+ files_to_reindex.add(file_path)
1454
+ else:
1455
+ skipped_count += 1
1456
+ else:
1457
+ skipped_count += 1
1458
+ else:
1459
+ # New file
1460
+ files_to_reindex.add(file_path)
1461
+
1462
+ self.stats["skipped_files"] = skipped_count
1463
+ if self.verbose:
1464
+ print(f" [timing] changed-file detection: {time.perf_counter() - t0:.3f}s ({len(files_to_reindex)} to reindex, {skipped_count} skipped)")
1465
+
1466
+ # Cascade: find all files that depend on changed files
1467
+ for file_path, file_id in changed_files:
1468
+ dependent_file_ids = self._get_dependent_files(file_id)
1469
+ for dep_file_id in dependent_file_ids:
1470
+ self.cursor.execute("SELECT absolute_path FROM files WHERE id = ?", (dep_file_id,))
1471
+ result = self.cursor.fetchone()
1472
+ if result:
1473
+ dep_file_path = Path(result[0])
1474
+ if dep_file_path.exists(): # Only reindex if file still exists
1475
+ files_to_reindex.add(dep_file_path)
1476
+ #print(f"Cascade re-index: {dep_file_path} depends on changed file {file_path}")
1477
+
1478
+ # Remove deleted files from the database
1479
+ deleted_file_ids = self._get_deleted_files(files)
1480
+ if deleted_file_ids:
1481
+ if self.verbose:
1482
+ print(f"Found {len(deleted_file_ids)} deleted file(s) to remove from index")
1483
+ self._remove_deleted_files(deleted_file_ids)
1484
+
1485
+ # Second pass: parse files in parallel with sliding window,
1486
+ # while a dedicated writer thread handles DB inserts.
1487
+ # This eliminates the sawtooth utilization pattern where workers
1488
+ # sit idle while the main process does serial DB inserts.
1489
+ if files_to_reindex:
1490
+ t_parse_start = time.perf_counter()
1491
+ worker_args = [(str(fp), str(self.root_dir), self.frontend_enabled) for fp in files_to_reindex]
1492
+ max_workers = min(os.cpu_count() or 4, len(worker_args), 16)
1493
+ window_size = max_workers * 4 # Keep workers fed with a sliding window
1494
+
1495
+ # Writer thread: consumes parsed results from queue, inserts into DB
1496
+ write_queue = queue_mod.Queue(maxsize=window_size * 2)
1497
+ insert_errors = []
1498
+
1499
+ def _writer_loop():
1500
+ """Dedicated writer thread — pulls parsed results and inserts into DB."""
1501
+ while True:
1502
+ result = write_queue.get()
1503
+ if result is None: # sentinel — we're done
1504
+ write_queue.task_done()
1505
+ break
1506
+ if 'error' in result:
1507
+ if self.verbose:
1508
+ print(f"Error parsing {result.get('file_path', '?')}: {result['error']}")
1509
+ write_queue.task_done()
1510
+ continue
1511
+ try:
1512
+ self._insert_parsed_file(result)
1513
+ except Exception as e:
1514
+ insert_errors.append(e)
1515
+ if self.verbose:
1516
+ print(f"Error inserting {result.get('file_path', '?')}: {e}")
1517
+ write_queue.task_done()
1518
+ # Final flush after all files inserted
1519
+ self._flush_batches()
1520
+
1521
+ writer_thread = threading.Thread(target=_writer_loop, daemon=True)
1522
+ writer_thread.start()
1523
+
1524
+ # Sliding window: maintain window_size futures in flight at all times
1525
+ with ProcessPoolExecutor(max_workers=max_workers) as executor:
1526
+ # Submit initial window
1527
+ futures = {}
1528
+ arg_iter = iter(worker_args)
1529
+ for _ in range(min(window_size, len(worker_args))):
1530
+ arg = next(arg_iter)
1531
+ futures[executor.submit(parse_file, arg)] = arg
1532
+
1533
+ while futures:
1534
+ # Wait for any one to complete
1535
+ done = next(as_completed(futures))
1536
+ del futures[done]
1537
+ result = done.result()
1538
+ # Feed to writer thread (blocks if queue is full — backpressure)
1539
+ write_queue.put(result)
1540
+ # Immediately submit next file to keep workers fed
1541
+ try:
1542
+ arg = next(arg_iter)
1543
+ futures[executor.submit(parse_file, arg)] = arg
1544
+ except StopIteration:
1545
+ pass
1546
+
1547
+ # Wait for writer to finish all remaining inserts
1548
+ write_queue.put(None)
1549
+ writer_thread.join()
1550
+
1551
+ if insert_errors and self.verbose:
1552
+ print(f" {len(insert_errors)} insert errors during indexing")
1553
+
1554
+ if self.verbose and files_to_reindex:
1555
+ print(f" [timing] parse + insert: {time.perf_counter() - t_parse_start:.3f}s ({len(files_to_reindex)} files)")
1556
+
1557
+ # Post-indexing dependency resolution pass
1558
+ # Resolve all temp_symbols against the now-complete symbol tables.
1559
+ # This is much faster than per-file resolution because we do a few large
1560
+ # JOIN/UPDATE queries instead of 62K × 7 SELECT IN queries.
1561
+ if files_to_reindex:
1562
+ t_resolve_start = time.perf_counter()
1563
+ self._resolve_all_dependencies()
1564
+ if self.verbose:
1565
+ print(f" [timing] dependency resolution: {time.perf_counter() - t_resolve_start:.3f}s")
1566
+
1567
+ # Run cross-file frontend resolution pass to refresh relationships
1568
+ # (render_relationships, style_imports, custom_property_usages, event handlers, selector matches)
1569
+ # Skip when no files changed or frontend is disabled to avoid unconditional global work
1570
+ if files_to_reindex and self.frontend_enabled:
1571
+ t_resolve_start = time.perf_counter()
1572
+ try:
1573
+ resolver = FrontendResolver(self.conn, str(self.root_dir))
1574
+ resolver.resolve_all()
1575
+ except Exception as e:
1576
+ if self.verbose:
1577
+ print(f"Warning: Frontend resolution pass failed: {e}")
1578
+
1579
+ if self.verbose:
1580
+ print(f" [timing] frontend resolution: {time.perf_counter() - t_resolve_start:.3f}s")
1581
+
1582
+ # Clean up unresolved temp_symbols and their dependencies
1583
+ # (these are external/builtin symbols that were never defined in the codebase)
1584
+ self.cursor.execute("SELECT COUNT(*) FROM temp_symbols")
1585
+ remaining = self.cursor.fetchone()[0]
1586
+ if remaining > 0:
1587
+ # Clean up traditional dependency temp_symbols
1588
+ self.cursor.execute("DELETE FROM dependencies WHERE temp_symbol_id IS NOT NULL")
1589
+ # Clean up all remaining temp_symbols (including frontend types: component_reference,
1590
+ # custom_property_reference, css_module_class_reference — these are external/missing)
1591
+ self.cursor.execute("DELETE FROM temp_symbols")
1592
+ self.conn.commit()
1593
+
1594
+ # Clean up unresolved temp_file_references (external URLs or missing files)
1595
+ self.cursor.execute("SELECT COUNT(*) FROM temp_file_references")
1596
+ remaining_refs = self.cursor.fetchone()[0]
1597
+ if remaining_refs > 0:
1598
+ self.cursor.execute("DELETE FROM temp_file_references")
1599
+ self.conn.commit()
1600
+
1601
+ if self.verbose:
1602
+ print(f" [timing] total index_directory: {time.perf_counter() - t_total_start:.3f}s")
1603
+ self._print_summary()
1604
+
1605
+ def _print_summary(self):
1606
+ """Print indexing summary statistics."""
1607
+ cursor = self.conn.cursor()
1608
+ cursor.execute("SELECT COUNT(*) FROM files")
1609
+ file_count = cursor.fetchone()[0]
1610
+ cursor.execute("SELECT COUNT(*) FROM functions WHERE parent_id IS NULL AND type != 'macro'")
1611
+ func_count = cursor.fetchone()[0]
1612
+ cursor.execute("SELECT COUNT(*) FROM functions WHERE type = 'macro'")
1613
+ macro_count = cursor.fetchone()[0]
1614
+ cursor.execute("SELECT COUNT(*) FROM functions WHERE parent_id IS NOT NULL")
1615
+ method_count = cursor.fetchone()[0]
1616
+ cursor.execute("SELECT COUNT(*) FROM classes")
1617
+ class_count = cursor.fetchone()[0]
1618
+ cursor.execute("SELECT COUNT(*) FROM structs")
1619
+ struct_count = cursor.fetchone()[0]
1620
+ cursor.execute("SELECT COUNT(*) FROM interfaces")
1621
+ iface_count = cursor.fetchone()[0]
1622
+ cursor.execute("SELECT COUNT(*) FROM enums")
1623
+ enum_count = cursor.fetchone()[0]
1624
+ cursor.execute("SELECT COUNT(*) FROM namespaces")
1625
+ ns_count = cursor.fetchone()[0]
1626
+ cursor.execute("SELECT COUNT(*) FROM variables")
1627
+ var_count = cursor.fetchone()[0]
1628
+ cursor.execute("SELECT COUNT(*) FROM type_aliases")
1629
+ alias_count = cursor.fetchone()[0]
1630
+
1631
+ indexed = file_count - self.stats['skipped_files']
1632
+
1633
+ print(f"\nIndexing complete!")
1634
+ print(f"Total files in database: {file_count}")
1635
+ print(f"Files indexed this run: {indexed}")
1636
+ print(f"Files skipped (unchanged): {self.stats['skipped_files']}")
1637
+ if indexed > 0:
1638
+ print(f" Functions indexed: {self.stats['total_functions']}")
1639
+ print(f" Macros indexed: {self.stats['total_macros']}")
1640
+ print(f" Methods indexed: {self.stats['total_methods']}")
1641
+ print(f" Classes indexed: {self.stats['total_classes']}")
1642
+ print(f" Structs indexed: {self.stats['total_structs']}")
1643
+ print(f" Interfaces indexed:{self.stats['total_interfaces']}")
1644
+ print(f" Enums indexed: {self.stats['total_enums']}")
1645
+ print(f" Namespaces indexed:{self.stats['total_namespaces']}")
1646
+ print(f" Variables indexed: {self.stats['total_variables']}")
1647
+ print(f" Type aliases idx: {self.stats['total_type_defs']}")
1648
+ print(f"Total functions: {func_count}")
1649
+ print(f"Total macros: {macro_count}")
1650
+ print(f"Total methods: {method_count}")
1651
+ print(f"Total classes: {class_count}")
1652
+ print(f"Total structs: {struct_count}")
1653
+ print(f"Total interfaces: {iface_count}")
1654
+ print(f"Total enums: {enum_count}")
1655
+ print(f"Total namespaces: {ns_count}")
1656
+ print(f"Total variables: {var_count}")
1657
+ print(f"Total type aliases: {alias_count}")
1658
+
1659
+ # Frontend table counts
1660
+ for table_name in ("frontend_components", "markup_elements", "style_selectors",
1661
+ "style_custom_properties", "style_custom_property_usages",
1662
+ "style_keyframes", "style_imports", "frontend_events",
1663
+ "frontend_bindings", "render_relationships",
1664
+ "style_selector_matches", "frontend_diagnostics"):
1665
+ try:
1666
+ cursor.execute(f"SELECT COUNT(*) FROM {table_name}")
1667
+ count = cursor.fetchone()[0]
1668
+ if count > 0:
1669
+ print(f" {table_name}: {count}")
1670
+ except Exception:
1671
+ pass
1672
+
1673
+ def save_index(self, output_file=None):
1674
+ """Commit and optionally close the database connection. Data is already saved during indexing."""
1675
+ if self.conn:
1676
+ self.conn.commit()
1677
+ if self.verbose:
1678
+ print(f"\nDatabase saved to {self.db_path}")
1679
+
1680
+ def close(self):
1681
+ """Close the database connection."""
1682
+ if self.conn:
1683
+ self.conn.close()
1684
+
1685
+
1686
+ def main():
1687
+ import sys
1688
+ import traceback
1689
+
1690
+ args = parse_arguments()
1691
+
1692
+ if args.list_languages:
1693
+ list_supported_languages()
1694
+ return
1695
+
1696
+ if args.list_frontend_languages:
1697
+ list_frontend_languages()
1698
+ return
1699
+
1700
+ try:
1701
+ # Handle view graph mode
1702
+ if args.graph:
1703
+ # Determine database file
1704
+ db_file = args.output
1705
+ if db_file.endswith('.json'):
1706
+ db_file = db_file.rsplit('.', 1)[0] + '.db'
1707
+
1708
+ if not Path(db_file).exists():
1709
+ print(f"Error: Database file not found: {db_file}")
1710
+ print("Please run indexing first to create the database.")
1711
+ sys.exit(1)
1712
+
1713
+ sdk = CodeIndexSDK(db_file)
1714
+ graph = sdk.get_dependency_graph(args.graph)
1715
+ print(graph)
1716
+ sdk.close()
1717
+ return
1718
+
1719
+ # Handle JSON export mode
1720
+ if args.export_json:
1721
+ # Determine database file
1722
+ db_file = args.output
1723
+ if db_file.endswith('.json'):
1724
+ db_file = db_file.rsplit('.', 1)[0] + '.db'
1725
+
1726
+ if not Path(db_file).exists():
1727
+ print(f"Error: Database file not found: {db_file}")
1728
+ print("Please run indexing first to create the database.")
1729
+ sys.exit(1)
1730
+
1731
+ export_to_json(db_file, args.export_json)
1732
+ return
1733
+
1734
+ # Handle indexing mode - directory is required
1735
+ directory = args.directory
1736
+ if not directory:
1737
+ print("Error: directory argument is required for indexing mode")
1738
+ print("Usage: python code_indexer.py <directory> [options]")
1739
+ sys.exit(1)
1740
+
1741
+ # Determine output file extension
1742
+ output_file = args.output
1743
+ if output_file.endswith('.json'):
1744
+ # Convert to .db for SQLite
1745
+ output_file = output_file.rsplit('.', 1)[0] + '.db'
1746
+ print(f"Note: Output file changed to {output_file} (SQLite database)")
1747
+
1748
+ # Parse languages if provided
1749
+ languages = None
1750
+ if args.languages:
1751
+ languages = [lang.strip() for lang in args.languages.split(',')]
1752
+
1753
+ indexer = CodeIndexer(directory, languages, output_file, args.force, verbose=args.verbose, frontend_enabled=args.frontend)
1754
+ indexer.index_directory()
1755
+ indexer.save_index()
1756
+ indexer.close()
1757
+ except Exception as e:
1758
+ print(f"Error: {e}")
1759
+ traceback.print_exc()
1760
+
1761
+
1762
+ if __name__ == "__main__":
1763
+ main()