codegraph-engine 3.0.0__2-py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. codegraph/__init__.py +37 -0
  2. codegraph/agent.py +26 -0
  3. codegraph/agent_brain.py +2212 -0
  4. codegraph/agent_capabilities.py +4058 -0
  5. codegraph/agent_rules.py +531 -0
  6. codegraph/api_drift.py +455 -0
  7. codegraph/architecture.py +371 -0
  8. codegraph/async_queue/__init__.py +50 -0
  9. codegraph/async_queue/detectors/__init__.py +48 -0
  10. codegraph/async_queue/detectors/celery.py +160 -0
  11. codegraph/async_queue/detectors/kafka.py +168 -0
  12. codegraph/async_queue/detectors/rabbitmq.py +116 -0
  13. codegraph/async_queue/detectors/redis.py +130 -0
  14. codegraph/async_queue/detectors/sqs.py +135 -0
  15. codegraph/async_queue/models.py +161 -0
  16. codegraph/async_queue/otel_models.py +74 -0
  17. codegraph/async_queue/otel_parser.py +337 -0
  18. codegraph/async_queue/reconciler.py +427 -0
  19. codegraph/async_queue/scanner.py +291 -0
  20. codegraph/async_queue/traversal.py +447 -0
  21. codegraph/audit.py +106 -0
  22. codegraph/binding_resolver.py +616 -0
  23. codegraph/branch_comparison.py +197 -0
  24. codegraph/cache.py +95 -0
  25. codegraph/change_impact.py +547 -0
  26. codegraph/cli.py +2436 -0
  27. codegraph/cli_output.py +205 -0
  28. codegraph/config.py +45 -0
  29. codegraph/constraints.py +238 -0
  30. codegraph/context.py +2182 -0
  31. codegraph/context_freshness.py +351 -0
  32. codegraph/database/__init__.py +72 -0
  33. codegraph/database/extractor.py +3651 -0
  34. codegraph/database/interrogation.py +1494 -0
  35. codegraph/database/models.py +310 -0
  36. codegraph/database/schema_drift.py +862 -0
  37. codegraph/epistemic.py +200 -0
  38. codegraph/errors.py +282 -0
  39. codegraph/evidence/__init__.py +15 -0
  40. codegraph/evidence/citations.py +418 -0
  41. codegraph/evidence_contract.py +486 -0
  42. codegraph/frameworks.py +960 -0
  43. codegraph/frameworks_nestjs.py +502 -0
  44. codegraph/frameworks_nextjs.py +337 -0
  45. codegraph/frameworks_react.py +457 -0
  46. codegraph/freshness.py +335 -0
  47. codegraph/frontend_html.py +109 -0
  48. codegraph/git.py +362 -0
  49. codegraph/git_state.py +545 -0
  50. codegraph/graph/__init__.py +46 -0
  51. codegraph/graph/models.py +56 -0
  52. codegraph/graph/traversal.py +1993 -0
  53. codegraph/indexing/__init__.py +24 -0
  54. codegraph/indexing/classifier.py +564 -0
  55. codegraph/indexing/indexer.py +2578 -0
  56. codegraph/indexing/models.py +427 -0
  57. codegraph/indexing/parser.py +3404 -0
  58. codegraph/indexing/pool.py +136 -0
  59. codegraph/indexing/scanner.py +211 -0
  60. codegraph/indexing/telemetry.py +326 -0
  61. codegraph/indexing/templates.py +418 -0
  62. codegraph/indexing/test_framework.py +116 -0
  63. codegraph/installer.py +1702 -0
  64. codegraph/interrogation.py +2355 -0
  65. codegraph/llm/__init__.py +3 -0
  66. codegraph/llm/base.py +15 -0
  67. codegraph/llm/context.py +20 -0
  68. codegraph/mcp/__init__.py +3 -0
  69. codegraph/mcp/server.py +1476 -0
  70. codegraph/mcp_diagnostics.py +882 -0
  71. codegraph/memory/__init__.py +3 -0
  72. codegraph/memory/store.py +46 -0
  73. codegraph/models.py +310 -0
  74. codegraph/monorepo.py +925 -0
  75. codegraph/observability.py +262 -0
  76. codegraph/optimizer.py +1014 -0
  77. codegraph/planner.py +417 -0
  78. codegraph/process_lifecycle.py +1153 -0
  79. codegraph/py.typed +1 -0
  80. codegraph/query_expansion.py +199 -0
  81. codegraph/ranking.py +408 -0
  82. codegraph/refactor/__init__.py +36 -0
  83. codegraph/refactor/models.py +114 -0
  84. codegraph/refactor/renamer.py +844 -0
  85. codegraph/refactor/transactions.py +414 -0
  86. codegraph/resolver.py +3968 -0
  87. codegraph/resources/__init__.py +46 -0
  88. codegraph/resources/cache.py +117 -0
  89. codegraph/resources/coalescer.py +83 -0
  90. codegraph/resources/debouncer.py +98 -0
  91. codegraph/resources/governor.py +219 -0
  92. codegraph/resources/monitor.py +102 -0
  93. codegraph/resources/policy.py +123 -0
  94. codegraph/retrieval_policy.py +320 -0
  95. codegraph/roblox.py +2174 -0
  96. codegraph/route_composer.py +363 -0
  97. codegraph/route_topology.py +172 -0
  98. codegraph/runtime/__init__.py +51 -0
  99. codegraph/runtime/collector.py +155 -0
  100. codegraph/runtime/db_watcher.py +191 -0
  101. codegraph/runtime/ingestor.py +692 -0
  102. codegraph/runtime/models.py +130 -0
  103. codegraph/runtime/port_inspector.py +304 -0
  104. codegraph/runtime/reconciliation.py +492 -0
  105. codegraph/runtime/runner.py +353 -0
  106. codegraph/runtime/trace_linker.py +175 -0
  107. codegraph/search/__init__.py +25 -0
  108. codegraph/search/hybrid.py +846 -0
  109. codegraph/search/semantic.py +28 -0
  110. codegraph/security/__init__.py +51 -0
  111. codegraph/security/paths.py +189 -0
  112. codegraph/security/redaction.py +633 -0
  113. codegraph/semantic_conflicts.py +270 -0
  114. codegraph/semantic_decorators.py +372 -0
  115. codegraph/structural_diff.py +542 -0
  116. codegraph/styles.py +144 -0
  117. codegraph/symbol_history.py +277 -0
  118. codegraph/target_resolver.py +348 -0
  119. codegraph/task.py +637 -0
  120. codegraph/tool_selection_eval.py +1158 -0
  121. codegraph/tsconfig.py +236 -0
  122. codegraph/ui/__init__.py +6 -0
  123. codegraph/ui/server.py +755 -0
  124. codegraph/watcher.py +323 -0
  125. codegraph_engine-3.0.0.dist-info/METADATA +868 -0
  126. codegraph_engine-3.0.0.dist-info/RECORD +130 -0
  127. codegraph_engine-3.0.0.dist-info/WHEEL +6 -0
  128. codegraph_engine-3.0.0.dist-info/entry_points.txt +2 -0
  129. codegraph_engine-3.0.0.dist-info/licenses/LICENSE +21 -0
  130. codegraph_engine-3.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,2578 @@
1
+ """Incremental SQLite indexer with atomic per-file transactions and reference resolution.
2
+
3
+ Architectural Invariants:
4
+ 1. Normalized SQLite storage with PRAGMA foreign_keys = ON and ON DELETE CASCADE.
5
+ 2. Explicit FTS5 synchronization on every chunk insertion and deletion.
6
+ 3. Parse-failure safety: if a file has syntax errors, its last-known-good index state is
7
+ retained, marked 'parse_failed', and reported in freshness without corrupting the DB.
8
+ 4. Reference resolution runs globally after file parsing, turning raw source facts into
9
+ verified references, graph edges, and evidence.
10
+ 5. Parser version and schema version participate in cache and index validity.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import hashlib
15
+ import json
16
+ import sqlite3
17
+ import threading
18
+ import time
19
+ from collections.abc import Callable, Iterable, Iterator
20
+ from contextlib import contextmanager
21
+ from pathlib import Path
22
+ from typing import Any
23
+
24
+ from codegraph.config import Settings
25
+ from codegraph.evidence_contract import RelationshipRecord
26
+ from codegraph.frameworks import RouteDetection
27
+ from codegraph.resolver import (
28
+ DEFAULT_MAX_REEXPORT_DEPTH,
29
+ DEFAULT_MAX_WILDCARD_EXPANSIONS,
30
+ ReferenceResolver,
31
+ )
32
+ from codegraph.resources import (
33
+ ResourceGovernor,
34
+ TaskPriority,
35
+ get_global_governor,
36
+ get_graph_cache,
37
+ get_parse_cache,
38
+ )
39
+
40
+ from .classifier import classify_file_detailed, make_compat_row
41
+ from .models import (
42
+ BindingRef,
43
+ CallRef,
44
+ ImportRef,
45
+ InheritanceRef,
46
+ Symbol,
47
+ )
48
+ from .parser import PARSER_VERSION, parse
49
+ from .pool import SQLiteConnectionPool
50
+ from .scanner import LANGUAGES, scan
51
+ from .telemetry import IndexingTelemetry, get_process_rss_mb
52
+
53
+ SCHEMA_VERSION = 8
54
+
55
+ SCHEMA = """
56
+ CREATE TABLE IF NOT EXISTS metadata (
57
+ key TEXT PRIMARY KEY,
58
+ value TEXT NOT NULL
59
+ );
60
+
61
+ CREATE TABLE IF NOT EXISTS codegraph_meta (
62
+ key TEXT PRIMARY KEY,
63
+ value TEXT NOT NULL
64
+ );
65
+
66
+ CREATE TABLE IF NOT EXISTS files (
67
+ path TEXT PRIMARY KEY,
68
+ hash TEXT NOT NULL,
69
+ language TEXT NOT NULL,
70
+ indexed_at INTEGER NOT NULL DEFAULT 0,
71
+ status TEXT NOT NULL DEFAULT 'ok',
72
+ parse_error TEXT,
73
+ last_valid_hash TEXT,
74
+ category TEXT NOT NULL DEFAULT 'SOURCE'
75
+ );
76
+
77
+ CREATE TABLE IF NOT EXISTS chunks (
78
+ id INTEGER PRIMARY KEY,
79
+ path TEXT NOT NULL,
80
+ language TEXT NOT NULL,
81
+ symbol TEXT,
82
+ symbol_type TEXT,
83
+ start_line INTEGER NOT NULL,
84
+ end_line INTEGER NOT NULL,
85
+ content TEXT NOT NULL,
86
+ hash TEXT NOT NULL,
87
+ FOREIGN KEY(path) REFERENCES files(path) ON DELETE CASCADE
88
+ );
89
+
90
+ CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
91
+ content, path UNINDEXED, symbol UNINDEXED
92
+ );
93
+
94
+ CREATE TABLE IF NOT EXISTS symbols (
95
+ name TEXT NOT NULL,
96
+ qualified_name TEXT NOT NULL,
97
+ kind TEXT NOT NULL,
98
+ path TEXT NOT NULL,
99
+ start_line INTEGER NOT NULL,
100
+ end_line INTEGER NOT NULL,
101
+ decorators TEXT NOT NULL DEFAULT '',
102
+ id TEXT NOT NULL DEFAULT '',
103
+ canonical_id TEXT NOT NULL DEFAULT '',
104
+ language TEXT NOT NULL DEFAULT 'python',
105
+ module TEXT NOT NULL DEFAULT '',
106
+ scope TEXT NOT NULL DEFAULT '',
107
+ signature TEXT NOT NULL DEFAULT '',
108
+ content_hash TEXT NOT NULL DEFAULT '',
109
+ parent_symbol_id TEXT,
110
+ visibility TEXT NOT NULL DEFAULT 'public',
111
+ return_type TEXT,
112
+ parameter_count INTEGER,
113
+ documentation TEXT,
114
+ FOREIGN KEY(path) REFERENCES files(path) ON DELETE CASCADE
115
+ );
116
+
117
+ CREATE INDEX IF NOT EXISTS idx_symbols_name ON symbols (name);
118
+ CREATE INDEX IF NOT EXISTS idx_symbols_qualified ON symbols (qualified_name);
119
+ CREATE INDEX IF NOT EXISTS idx_symbols_canonical ON symbols (canonical_id);
120
+ CREATE INDEX IF NOT EXISTS idx_symbols_path ON symbols (path);
121
+ CREATE INDEX IF NOT EXISTS idx_symbols_parent ON symbols (parent_symbol_id);
122
+ CREATE INDEX IF NOT EXISTS idx_chunks_path ON chunks (path);
123
+ CREATE INDEX IF NOT EXISTS idx_chunks_symbol ON chunks (symbol);
124
+ CREATE INDEX IF NOT EXISTS idx_chunks_path_symbol ON chunks (path, symbol);
125
+
126
+ CREATE TABLE IF NOT EXISTS imports (
127
+ source_path TEXT NOT NULL,
128
+ module TEXT NOT NULL,
129
+ name TEXT,
130
+ alias TEXT,
131
+ full_name TEXT,
132
+ line INTEGER NOT NULL DEFAULT 1,
133
+ source_module TEXT NOT NULL DEFAULT '',
134
+ imported_module TEXT NOT NULL DEFAULT '',
135
+ imported_name TEXT,
136
+ local_name TEXT NOT NULL DEFAULT '',
137
+ import_type TEXT NOT NULL DEFAULT 'module',
138
+ resolved_path TEXT,
139
+ resolved_module TEXT,
140
+ FOREIGN KEY(source_path) REFERENCES files(path) ON DELETE CASCADE
141
+ );
142
+
143
+ CREATE INDEX IF NOT EXISTS idx_imports_source ON imports (source_path);
144
+ CREATE INDEX IF NOT EXISTS idx_imports_source_line ON imports (source_path, line);
145
+ CREATE INDEX IF NOT EXISTS idx_imports_module ON imports (module);
146
+ CREATE INDEX IF NOT EXISTS idx_imports_imported_module ON imports (imported_module);
147
+
148
+ CREATE TABLE IF NOT EXISTS calls (
149
+ source_path TEXT NOT NULL,
150
+ callee TEXT NOT NULL,
151
+ qualified_callee TEXT,
152
+ line INTEGER NOT NULL DEFAULT 1,
153
+ confidence TEXT NOT NULL DEFAULT 'LOW',
154
+ source_symbol_id TEXT,
155
+ resolved_symbol_id TEXT,
156
+ FOREIGN KEY(source_path) REFERENCES files(path) ON DELETE CASCADE
157
+ );
158
+
159
+ CREATE INDEX IF NOT EXISTS idx_calls_source ON calls (source_path);
160
+ CREATE INDEX IF NOT EXISTS idx_calls_source_line ON calls (source_path, line, callee);
161
+ CREATE INDEX IF NOT EXISTS idx_calls_callee ON calls (callee);
162
+ CREATE INDEX IF NOT EXISTS idx_calls_resolved ON calls (resolved_symbol_id);
163
+ CREATE INDEX IF NOT EXISTS idx_calls_callee_source ON calls (callee, source_path);
164
+
165
+ CREATE TABLE IF NOT EXISTS inheritance (
166
+ source_symbol TEXT NOT NULL,
167
+ base_name TEXT NOT NULL,
168
+ relationship TEXT NOT NULL,
169
+ source_file TEXT NOT NULL,
170
+ line INTEGER NOT NULL DEFAULT 1,
171
+ source_canonical_id TEXT NOT NULL DEFAULT '',
172
+ FOREIGN KEY(source_file) REFERENCES files(path) ON DELETE CASCADE
173
+ );
174
+
175
+ CREATE INDEX IF NOT EXISTS idx_inh_source ON inheritance (source_file);
176
+
177
+ CREATE TABLE IF NOT EXISTS raw_framework_routes (
178
+ endpoint_id TEXT PRIMARY KEY,
179
+ framework TEXT NOT NULL,
180
+ http_method TEXT NOT NULL,
181
+ route_path TEXT NOT NULL,
182
+ normalized_route TEXT NOT NULL,
183
+ handler_name TEXT NOT NULL,
184
+ handler_canonical_id TEXT NOT NULL,
185
+ file_path TEXT NOT NULL,
186
+ line INTEGER NOT NULL,
187
+ evidence TEXT NOT NULL,
188
+ confidence TEXT NOT NULL DEFAULT 'HIGH',
189
+ router_name TEXT NOT NULL DEFAULT '',
190
+ FOREIGN KEY(file_path) REFERENCES files(path) ON DELETE CASCADE
191
+ );
192
+
193
+ CREATE INDEX IF NOT EXISTS idx_raw_routes_file ON raw_framework_routes (file_path);
194
+ CREATE INDEX IF NOT EXISTS idx_raw_routes_path ON raw_framework_routes (route_path);
195
+
196
+ CREATE TABLE IF NOT EXISTS router_mounts (
197
+ mount_id TEXT PRIMARY KEY,
198
+ framework TEXT NOT NULL,
199
+ parent_router TEXT NOT NULL,
200
+ child_router TEXT NOT NULL,
201
+ prefix TEXT NOT NULL,
202
+ file_path TEXT NOT NULL,
203
+ line INTEGER NOT NULL,
204
+ evidence TEXT NOT NULL,
205
+ confidence TEXT NOT NULL DEFAULT 'HIGH',
206
+ FOREIGN KEY(file_path) REFERENCES files(path) ON DELETE CASCADE
207
+ );
208
+
209
+ CREATE INDEX IF NOT EXISTS idx_mounts_file ON router_mounts (file_path);
210
+
211
+ CREATE TABLE IF NOT EXISTS router_definitions (
212
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
213
+ framework TEXT NOT NULL,
214
+ router_name TEXT NOT NULL,
215
+ prefix TEXT NOT NULL,
216
+ file_path TEXT NOT NULL,
217
+ line INTEGER NOT NULL,
218
+ evidence TEXT NOT NULL,
219
+ FOREIGN KEY(file_path) REFERENCES files(path) ON DELETE CASCADE
220
+ );
221
+
222
+ CREATE INDEX IF NOT EXISTS idx_router_defs_file ON router_definitions (file_path);
223
+
224
+ CREATE TABLE IF NOT EXISTS local_bindings (
225
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
226
+ target_name TEXT NOT NULL,
227
+ file_path TEXT NOT NULL,
228
+ line INTEGER NOT NULL,
229
+ column INTEGER,
230
+ scope TEXT NOT NULL DEFAULT '',
231
+ expr_kind TEXT NOT NULL DEFAULT 'IDENTIFIER',
232
+ source_expr TEXT NOT NULL DEFAULT '',
233
+ is_conditional INTEGER NOT NULL DEFAULT 0,
234
+ base_expr TEXT NOT NULL DEFAULT '',
235
+ attr_name TEXT NOT NULL DEFAULT '',
236
+ dict_entries_json TEXT NOT NULL DEFAULT '[]',
237
+ list_entries_json TEXT NOT NULL DEFAULT '[]',
238
+ subscript_target TEXT,
239
+ subscript_key TEXT,
240
+ subscript_index INTEGER,
241
+ FOREIGN KEY(file_path) REFERENCES files(path) ON DELETE CASCADE
242
+ );
243
+
244
+ CREATE INDEX IF NOT EXISTS idx_bindings_file ON local_bindings (file_path);
245
+ CREATE INDEX IF NOT EXISTS idx_bindings_target ON local_bindings (target_name);
246
+ CREATE INDEX IF NOT EXISTS idx_bindings_file_target ON local_bindings (file_path, target_name);
247
+
248
+ CREATE TABLE IF NOT EXISTS framework_routes (
249
+ endpoint_id TEXT PRIMARY KEY,
250
+ framework TEXT NOT NULL,
251
+ http_method TEXT NOT NULL,
252
+ route_path TEXT NOT NULL,
253
+ normalized_route TEXT NOT NULL,
254
+ handler_name TEXT NOT NULL,
255
+ handler_canonical_id TEXT NOT NULL,
256
+ file_path TEXT NOT NULL,
257
+ line INTEGER NOT NULL,
258
+ evidence TEXT NOT NULL,
259
+ confidence TEXT NOT NULL DEFAULT 'HIGH',
260
+ FOREIGN KEY(file_path) REFERENCES files(path) ON DELETE CASCADE
261
+ );
262
+
263
+ CREATE INDEX IF NOT EXISTS idx_routes_file ON framework_routes (file_path);
264
+ CREATE INDEX IF NOT EXISTS idx_routes_handler ON framework_routes (handler_canonical_id);
265
+ CREATE INDEX IF NOT EXISTS idx_routes_path ON framework_routes (route_path);
266
+
267
+ CREATE TABLE IF NOT EXISTS "references" (
268
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
269
+ source_symbol_id TEXT NOT NULL,
270
+ target_symbol_id TEXT,
271
+ relationship TEXT NOT NULL,
272
+ confidence TEXT NOT NULL,
273
+ path TEXT NOT NULL,
274
+ start_line INTEGER NOT NULL,
275
+ end_line INTEGER NOT NULL,
276
+ evidence TEXT NOT NULL DEFAULT '',
277
+ source_hash TEXT NOT NULL DEFAULT '',
278
+ indexed_commit TEXT,
279
+ evidence_status TEXT NOT NULL DEFAULT 'current',
280
+ FOREIGN KEY(path) REFERENCES files(path) ON DELETE CASCADE
281
+ );
282
+
283
+ CREATE INDEX IF NOT EXISTS idx_refs_source ON "references" (source_symbol_id);
284
+ CREATE INDEX IF NOT EXISTS idx_refs_target ON "references" (target_symbol_id);
285
+ CREATE INDEX IF NOT EXISTS idx_refs_path ON "references" (path);
286
+ CREATE INDEX IF NOT EXISTS idx_refs_rel ON "references" (relationship);
287
+
288
+ CREATE TABLE IF NOT EXISTS graph_edges (
289
+ source TEXT NOT NULL,
290
+ target TEXT NOT NULL,
291
+ relationship TEXT NOT NULL,
292
+ confidence TEXT NOT NULL,
293
+ file TEXT NOT NULL,
294
+ start_line INTEGER NOT NULL,
295
+ end_line INTEGER NOT NULL,
296
+ evidence TEXT NOT NULL DEFAULT '',
297
+ evidence_id TEXT NOT NULL DEFAULT '',
298
+ source_hash TEXT NOT NULL DEFAULT '',
299
+ indexed_commit TEXT,
300
+ evidence_class TEXT NOT NULL DEFAULT 'AST_VERIFIED',
301
+ reason TEXT,
302
+ FOREIGN KEY(file) REFERENCES files(path) ON DELETE CASCADE
303
+ );
304
+
305
+ CREATE INDEX IF NOT EXISTS idx_edges_source ON graph_edges (source);
306
+ CREATE INDEX IF NOT EXISTS idx_edges_target ON graph_edges (target);
307
+ CREATE INDEX IF NOT EXISTS idx_edges_rel ON graph_edges (relationship);
308
+ CREATE INDEX IF NOT EXISTS idx_edges_file ON graph_edges (file);
309
+ CREATE INDEX IF NOT EXISTS idx_edges_evidence ON graph_edges (evidence_id);
310
+ CREATE INDEX IF NOT EXISTS idx_edges_target_rel ON graph_edges (target, relationship);
311
+ CREATE INDEX IF NOT EXISTS idx_edges_source_rel ON graph_edges (source, relationship);
312
+
313
+ CREATE TABLE IF NOT EXISTS context_cache (
314
+ cache_key TEXT PRIMARY KEY,
315
+ data TEXT NOT NULL,
316
+ created_at INTEGER NOT NULL
317
+ );
318
+
319
+ CREATE TABLE IF NOT EXISTS db_entities (
320
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
321
+ canonical_id TEXT NOT NULL,
322
+ kind TEXT NOT NULL,
323
+ name TEXT NOT NULL,
324
+ dialect TEXT NOT NULL DEFAULT 'UNKNOWN',
325
+ schema_name TEXT NOT NULL DEFAULT 'UNKNOWN',
326
+ table_name TEXT NOT NULL DEFAULT '',
327
+ column_name TEXT NOT NULL DEFAULT '',
328
+ data_type TEXT NOT NULL DEFAULT '',
329
+ nullable INTEGER NOT NULL DEFAULT 1,
330
+ default_value TEXT,
331
+ is_primary_key INTEGER NOT NULL DEFAULT 0,
332
+ is_foreign_key INTEGER NOT NULL DEFAULT 0,
333
+ is_unique INTEGER NOT NULL DEFAULT 0,
334
+ is_indexed INTEGER NOT NULL DEFAULT 0,
335
+ target_table TEXT,
336
+ target_column TEXT,
337
+ orm_model_id TEXT,
338
+ framework TEXT NOT NULL DEFAULT '',
339
+ file_path TEXT NOT NULL,
340
+ start_line INTEGER NOT NULL DEFAULT 1,
341
+ end_line INTEGER NOT NULL DEFAULT 1,
342
+ evidence TEXT NOT NULL DEFAULT '',
343
+ confidence TEXT NOT NULL DEFAULT 'HIGH',
344
+ evidence_class TEXT NOT NULL DEFAULT 'AST_VERIFIED',
345
+ status TEXT NOT NULL DEFAULT 'FACT',
346
+ metadata_json TEXT NOT NULL DEFAULT '{}',
347
+ FOREIGN KEY(file_path) REFERENCES files(path) ON DELETE CASCADE
348
+ );
349
+
350
+ CREATE INDEX IF NOT EXISTS idx_db_entities_canonical ON db_entities (canonical_id);
351
+ CREATE INDEX IF NOT EXISTS idx_db_entities_kind ON db_entities (kind);
352
+ CREATE INDEX IF NOT EXISTS idx_db_entities_table ON db_entities (table_name);
353
+ CREATE INDEX IF NOT EXISTS idx_db_entities_file ON db_entities (file_path);
354
+
355
+ CREATE TABLE IF NOT EXISTS db_queries (
356
+ query_id TEXT PRIMARY KEY,
357
+ caller_symbol_id TEXT NOT NULL,
358
+ operation TEXT NOT NULL,
359
+ relationship TEXT NOT NULL,
360
+ table_name TEXT NOT NULL,
361
+ table_canonical_id TEXT NOT NULL,
362
+ columns_json TEXT NOT NULL DEFAULT '[]',
363
+ framework TEXT NOT NULL DEFAULT 'raw_sql',
364
+ normalized_sql TEXT NOT NULL DEFAULT '',
365
+ file_path TEXT NOT NULL,
366
+ start_line INTEGER NOT NULL DEFAULT 1,
367
+ end_line INTEGER NOT NULL DEFAULT 1,
368
+ evidence TEXT NOT NULL DEFAULT '',
369
+ confidence TEXT NOT NULL DEFAULT 'HIGH',
370
+ evidence_class TEXT NOT NULL DEFAULT 'AST_VERIFIED',
371
+ status TEXT NOT NULL DEFAULT 'FACT',
372
+ FOREIGN KEY(file_path) REFERENCES files(path) ON DELETE CASCADE
373
+ );
374
+
375
+ CREATE INDEX IF NOT EXISTS idx_db_queries_caller ON db_queries (caller_symbol_id);
376
+ CREATE INDEX IF NOT EXISTS idx_db_queries_table ON db_queries (table_name);
377
+ CREATE INDEX IF NOT EXISTS idx_db_queries_op ON db_queries (operation);
378
+ CREATE INDEX IF NOT EXISTS idx_db_queries_file ON db_queries (file_path);
379
+
380
+ CREATE TABLE IF NOT EXISTS db_migrations (
381
+ migration_id TEXT PRIMARY KEY,
382
+ canonical_id TEXT NOT NULL,
383
+ system TEXT NOT NULL,
384
+ revision TEXT NOT NULL,
385
+ down_revision TEXT,
386
+ file_path TEXT NOT NULL,
387
+ start_line INTEGER NOT NULL DEFAULT 1,
388
+ end_line INTEGER NOT NULL DEFAULT 1,
389
+ created_tables_json TEXT NOT NULL DEFAULT '[]',
390
+ dropped_tables_json TEXT NOT NULL DEFAULT '[]',
391
+ added_columns_json TEXT NOT NULL DEFAULT '[]',
392
+ removed_columns_json TEXT NOT NULL DEFAULT '[]',
393
+ renamed_columns_json TEXT NOT NULL DEFAULT '[]',
394
+ indexes_json TEXT NOT NULL DEFAULT '[]',
395
+ foreign_keys_json TEXT NOT NULL DEFAULT '[]',
396
+ constraints_json TEXT NOT NULL DEFAULT '[]',
397
+ affected_tables_json TEXT NOT NULL DEFAULT '[]',
398
+ evidence TEXT NOT NULL DEFAULT '',
399
+ confidence TEXT NOT NULL DEFAULT 'HIGH',
400
+ evidence_class TEXT NOT NULL DEFAULT 'AST_VERIFIED',
401
+ FOREIGN KEY(file_path) REFERENCES files(path) ON DELETE CASCADE
402
+ );
403
+
404
+ CREATE INDEX IF NOT EXISTS idx_db_migrations_file ON db_migrations (file_path);
405
+ CREATE INDEX IF NOT EXISTS idx_db_migrations_rev ON db_migrations (revision);
406
+
407
+ CREATE TABLE IF NOT EXISTS runtime_observations (
408
+ event_id TEXT PRIMARY KEY,
409
+ trace_id TEXT NOT NULL,
410
+ span_id TEXT NOT NULL,
411
+ parent_span_id TEXT NOT NULL DEFAULT '',
412
+ timestamp TEXT NOT NULL DEFAULT '',
413
+ source_format TEXT NOT NULL DEFAULT 'json',
414
+ http_method TEXT NOT NULL DEFAULT '',
415
+ route_path TEXT NOT NULL DEFAULT '',
416
+ handler_symbol TEXT NOT NULL DEFAULT '',
417
+ caller_symbol TEXT NOT NULL DEFAULT '',
418
+ callee_symbol TEXT NOT NULL DEFAULT '',
419
+ service_name TEXT NOT NULL DEFAULT '',
420
+ db_operation TEXT NOT NULL DEFAULT '',
421
+ db_table TEXT NOT NULL DEFAULT '',
422
+ db_columns_json TEXT NOT NULL DEFAULT '[]',
423
+ normalized_sql TEXT NOT NULL DEFAULT '',
424
+ status_code INTEGER,
425
+ exception_type TEXT NOT NULL DEFAULT '',
426
+ duration_ms REAL NOT NULL DEFAULT 0.0,
427
+ evidence_class TEXT NOT NULL DEFAULT 'RUNTIME_OBSERVED',
428
+ runtime_generation INTEGER NOT NULL DEFAULT 1
429
+ );
430
+
431
+ CREATE INDEX IF NOT EXISTS idx_rt_obs_trace ON runtime_observations (trace_id);
432
+ CREATE INDEX IF NOT EXISTS idx_rt_obs_route ON runtime_observations (route_path);
433
+ CREATE INDEX IF NOT EXISTS idx_rt_obs_caller ON runtime_observations (caller_symbol);
434
+ CREATE INDEX IF NOT EXISTS idx_rt_obs_table ON runtime_observations (db_table);
435
+
436
+ CREATE TABLE IF NOT EXISTS runtime_edges (
437
+ edge_id TEXT PRIMARY KEY,
438
+ source TEXT NOT NULL,
439
+ target TEXT NOT NULL,
440
+ relationship TEXT NOT NULL,
441
+ operation TEXT NOT NULL DEFAULT '',
442
+ evidence_class TEXT NOT NULL DEFAULT 'RUNTIME_OBSERVED',
443
+ observation_count INTEGER NOT NULL DEFAULT 1,
444
+ first_seen TEXT NOT NULL DEFAULT '',
445
+ last_seen TEXT NOT NULL DEFAULT '',
446
+ avg_duration_ms REAL NOT NULL DEFAULT 0.0,
447
+ max_duration_ms REAL NOT NULL DEFAULT 0.0,
448
+ sample_trace_id TEXT NOT NULL DEFAULT '',
449
+ sample_span_id TEXT NOT NULL DEFAULT '',
450
+ normalized_sql TEXT NOT NULL DEFAULT '',
451
+ status_code INTEGER,
452
+ exception_type TEXT NOT NULL DEFAULT '',
453
+ runtime_generation INTEGER NOT NULL DEFAULT 1
454
+ );
455
+
456
+ CREATE INDEX IF NOT EXISTS idx_rt_edges_source ON runtime_edges (source);
457
+ CREATE INDEX IF NOT EXISTS idx_rt_edges_target ON runtime_edges (target);
458
+ CREATE INDEX IF NOT EXISTS idx_rt_edges_rel ON runtime_edges (relationship);
459
+ """
460
+
461
+
462
+ _DB_EDGE_RELATIONSHIPS = (
463
+ "MAPS_TO_TABLE",
464
+ "FOREIGN_KEY_TO",
465
+ "READS_TABLE",
466
+ "WRITES_TABLE",
467
+ "UPDATES_TABLE",
468
+ "DELETES_FROM_TABLE",
469
+ "REFERENCES_COLUMN",
470
+ "MIGRATES_TABLE",
471
+ )
472
+
473
+
474
+ class Indexer:
475
+ def __init__(
476
+ self,
477
+ repository: Path,
478
+ settings: Settings | None = None,
479
+ governor: ResourceGovernor | None = None,
480
+ *,
481
+ commit_batch_size: int = 500,
482
+ max_reexport_depth: int = DEFAULT_MAX_REEXPORT_DEPTH,
483
+ max_wildcard_expansions: int = DEFAULT_MAX_WILDCARD_EXPANSIONS,
484
+ ) -> None:
485
+ self.repository = repository.resolve(strict=True)
486
+ self.settings = settings or Settings()
487
+ self.db_path = self.settings.db_path or self.repository / ".codegraph.sqlite3"
488
+ self.governor = governor or get_global_governor()
489
+ self.commit_batch_size = max(1, commit_batch_size)
490
+ self.max_reexport_depth = max_reexport_depth
491
+ self.max_wildcard_expansions = max_wildcard_expansions
492
+ self.last_epistemic_findings: list[dict[str, object]] = []
493
+ self.last_telemetry: IndexingTelemetry | None = None
494
+ self._pool: SQLiteConnectionPool | None = None
495
+ self._schema_lock = threading.Lock()
496
+ self._schema_verified = False
497
+
498
+ @property
499
+ def pool(self) -> SQLiteConnectionPool:
500
+ if self._pool is None:
501
+ self._pool = SQLiteConnectionPool(
502
+ self.db_path,
503
+ max_size=16,
504
+ timeout=30.0,
505
+ initializer=self._ensure_schema,
506
+ )
507
+ return self._pool
508
+
509
+ def connect(self) -> sqlite3.Connection:
510
+ """Create a dedicated unpooled connection (for indexing or long-lived writes)."""
511
+ con = sqlite3.connect(
512
+ self.db_path,
513
+ check_same_thread=False,
514
+ timeout=30.0,
515
+ )
516
+ con.row_factory = make_compat_row
517
+ con.execute("PRAGMA journal_mode = WAL")
518
+ con.execute("PRAGMA synchronous = NORMAL")
519
+ con.execute("PRAGMA foreign_keys = ON")
520
+ con.execute("PRAGMA busy_timeout = 30000")
521
+ con.execute("PRAGMA cache_size = -64000")
522
+ con.execute("PRAGMA temp_store = MEMORY")
523
+ self._ensure_schema(con)
524
+ return con
525
+
526
+ @contextmanager
527
+ def session(self) -> Iterator[sqlite3.Connection]:
528
+ """Borrow a pooled connection; commit on success, rollback on failure, return to pool."""
529
+ con, is_owner = self.pool.acquire()
530
+ try:
531
+ yield con
532
+ self.pool.release(con, is_owner=is_owner, rollback=False)
533
+ except BaseException:
534
+ self.pool.release(con, is_owner=is_owner, rollback=True)
535
+ raise
536
+
537
+ def reset_pool(self) -> None:
538
+ """Close pooled connections and clear schema verification flag."""
539
+ if self._pool is not None:
540
+ self._pool.close_all()
541
+ self._pool = None
542
+ self._schema_verified = False
543
+
544
+ def close(self) -> None:
545
+ """Dispose of resources and close pooled connections."""
546
+ self.reset_pool()
547
+
548
+ def __del__(self) -> None:
549
+ try:
550
+ self.close()
551
+ except Exception:
552
+ pass
553
+
554
+ def _ensure_schema(self, con: sqlite3.Connection) -> None:
555
+ """Migrate schema via PRAGMA user_version non-destructively."""
556
+ if self._schema_verified:
557
+ return
558
+ with self._schema_lock:
559
+ if self._schema_verified:
560
+ return
561
+ current_version = con.execute("PRAGMA user_version").fetchone()[0]
562
+
563
+ if current_version == 0:
564
+ try:
565
+ row = con.execute("SELECT version FROM schema_version").fetchone()
566
+ if row:
567
+ current_version = int(row[0])
568
+ except sqlite3.OperationalError:
569
+ pass
570
+
571
+ target_version = SCHEMA_VERSION
572
+
573
+ if current_version < target_version:
574
+ if current_version > 0:
575
+ self._migrate_schema(con, current_version, target_version)
576
+ else:
577
+ con.executescript(SCHEMA)
578
+
579
+ con.execute(f"PRAGMA user_version = {target_version}")
580
+ try:
581
+ con.execute("DROP TABLE IF EXISTS schema_version")
582
+ except sqlite3.OperationalError:
583
+ pass
584
+ con.commit()
585
+ else:
586
+ existing_core = con.execute(
587
+ "SELECT count(*) FROM sqlite_master WHERE type='table' "
588
+ "AND name IN ('files', 'symbols', 'graph_edges', 'db_tables', 'runtime_traces')"
589
+ ).fetchone()[0]
590
+ if existing_core < 5:
591
+ con.executescript(SCHEMA)
592
+ con.commit()
593
+
594
+ from codegraph.async_queue import ensure_async_tables
595
+
596
+ ensure_async_tables(con)
597
+ self._schema_verified = True
598
+
599
+
600
+ def _migrate_schema(
601
+ self, con: sqlite3.Connection, from_version: int, to_version: int
602
+ ) -> None:
603
+ """Non-destructive schema migration using ALTER TABLE."""
604
+ del from_version, to_version
605
+ # Ensure new tables and indexes are created
606
+ con.executescript(SCHEMA)
607
+
608
+ # Add missing columns to existing tables
609
+ def existing_columns(tbl: str) -> set[str]:
610
+ try:
611
+ return {
612
+ str(r["name"])
613
+ for r in con.execute(f"PRAGMA table_info({tbl})").fetchall()
614
+ }
615
+ except sqlite3.OperationalError:
616
+ return set()
617
+
618
+ files_cols = existing_columns("files")
619
+ if "status" not in files_cols:
620
+ con.execute("ALTER TABLE files ADD COLUMN status TEXT NOT NULL DEFAULT 'ok'")
621
+ if "parse_error" not in files_cols:
622
+ con.execute("ALTER TABLE files ADD COLUMN parse_error TEXT")
623
+ if "last_valid_hash" not in files_cols:
624
+ con.execute("ALTER TABLE files ADD COLUMN last_valid_hash TEXT")
625
+ if "category" not in files_cols:
626
+ con.execute("ALTER TABLE files ADD COLUMN category TEXT NOT NULL DEFAULT 'SOURCE'")
627
+
628
+ symbols_cols = existing_columns("symbols")
629
+ for col_def in (
630
+ ("id", "TEXT NOT NULL DEFAULT ''"),
631
+ ("canonical_id", "TEXT NOT NULL DEFAULT ''"),
632
+ ("language", "TEXT NOT NULL DEFAULT 'python'"),
633
+ ("module", "TEXT NOT NULL DEFAULT ''"),
634
+ ("scope", "TEXT NOT NULL DEFAULT ''"),
635
+ ("signature", "TEXT NOT NULL DEFAULT ''"),
636
+ ("content_hash", "TEXT NOT NULL DEFAULT ''"),
637
+ ("parent_symbol_id", "TEXT"),
638
+ ("visibility", "TEXT NOT NULL DEFAULT 'public'"),
639
+ ("return_type", "TEXT"),
640
+ ("parameter_count", "INTEGER"),
641
+ ("documentation", "TEXT"),
642
+ ):
643
+ if col_def[0] not in symbols_cols:
644
+ con.execute(f"ALTER TABLE symbols ADD COLUMN {col_def[0]} {col_def[1]}")
645
+
646
+ imports_cols = existing_columns("imports")
647
+ for col_def in (
648
+ ("source_module", "TEXT NOT NULL DEFAULT ''"),
649
+ ("imported_module", "TEXT NOT NULL DEFAULT ''"),
650
+ ("imported_name", "TEXT"),
651
+ ("local_name", "TEXT NOT NULL DEFAULT ''"),
652
+ ("import_type", "TEXT NOT NULL DEFAULT 'module'"),
653
+ ("resolved_path", "TEXT"),
654
+ ("resolved_module", "TEXT"),
655
+ ):
656
+ if col_def[0] not in imports_cols:
657
+ con.execute(f"ALTER TABLE imports ADD COLUMN {col_def[0]} {col_def[1]}")
658
+
659
+ files_cols = existing_columns("files")
660
+ if "category" not in files_cols:
661
+ con.execute("ALTER TABLE files ADD COLUMN category TEXT NOT NULL DEFAULT 'SOURCE'")
662
+
663
+ calls_cols = existing_columns("calls")
664
+ for col_def in (
665
+ ("source_symbol_id", "TEXT"),
666
+ ("resolved_symbol_id", "TEXT"),
667
+ ):
668
+ if col_def[0] not in calls_cols:
669
+ con.execute(f"ALTER TABLE calls ADD COLUMN {col_def[0]} {col_def[1]}")
670
+
671
+ edges_cols = existing_columns("graph_edges")
672
+ if "evidence_class" not in edges_cols:
673
+ con.execute("ALTER TABLE graph_edges ADD COLUMN evidence_class TEXT NOT NULL DEFAULT 'AST_VERIFIED'")
674
+ if "reason" not in edges_cols:
675
+ con.execute("ALTER TABLE graph_edges ADD COLUMN reason TEXT")
676
+
677
+ try:
678
+ con.execute(
679
+ "INSERT OR IGNORE INTO raw_framework_routes("
680
+ "endpoint_id, framework, http_method, route_path, normalized_route, "
681
+ "handler_name, handler_canonical_id, file_path, line, evidence, confidence) "
682
+ "SELECT endpoint_id, framework, http_method, route_path, normalized_route, "
683
+ "handler_name, handler_canonical_id, file_path, line, evidence, confidence "
684
+ "FROM framework_routes"
685
+ )
686
+ except sqlite3.OperationalError:
687
+ pass
688
+
689
+ try:
690
+ con.execute(
691
+ "CREATE TABLE IF NOT EXISTS local_bindings ("
692
+ "id INTEGER PRIMARY KEY AUTOINCREMENT, "
693
+ "target_name TEXT NOT NULL, "
694
+ "file_path TEXT NOT NULL, "
695
+ "line INTEGER NOT NULL, "
696
+ "column INTEGER, "
697
+ "scope TEXT NOT NULL DEFAULT '', "
698
+ "expr_kind TEXT NOT NULL DEFAULT 'IDENTIFIER', "
699
+ "source_expr TEXT NOT NULL DEFAULT '', "
700
+ "is_conditional INTEGER NOT NULL DEFAULT 0, "
701
+ "base_expr TEXT NOT NULL DEFAULT '', "
702
+ "attr_name TEXT NOT NULL DEFAULT '', "
703
+ "dict_entries_json TEXT NOT NULL DEFAULT '[]', "
704
+ "list_entries_json TEXT NOT NULL DEFAULT '[]', "
705
+ "subscript_target TEXT, "
706
+ "subscript_key TEXT, "
707
+ "subscript_index INTEGER, "
708
+ "FOREIGN KEY(file_path) REFERENCES files(path) ON DELETE CASCADE)"
709
+ )
710
+ con.execute("CREATE INDEX IF NOT EXISTS idx_bindings_file ON local_bindings (file_path)")
711
+ con.execute("CREATE INDEX IF NOT EXISTS idx_bindings_target ON local_bindings (target_name)")
712
+ con.execute("CREATE INDEX IF NOT EXISTS idx_bindings_file_target ON local_bindings (file_path, target_name)")
713
+ except sqlite3.OperationalError:
714
+ pass
715
+
716
+ def index(
717
+ self,
718
+ *,
719
+ progress_callback: Callable[[str, int, int, IndexingTelemetry], None] | None = None,
720
+ catch_interrupt: bool = False,
721
+ ) -> dict[str, int]:
722
+ """Perform an incremental indexing pass, followed by global reference resolution."""
723
+ indexed = unchanged = parse_failed_count = 0
724
+ stale: list[str] = []
725
+ dirty_files: set[str] = set()
726
+ files = []
727
+ telemetry = IndexingTelemetry()
728
+ self.last_telemetry = telemetry
729
+ from codegraph.freshness import current_commit, save_commit
730
+
731
+ with self.governor.task_scope(TaskPriority.BACKGROUND):
732
+ self.governor.set_indexing_active(True)
733
+ try:
734
+ with telemetry.phase("repository_scan") as p_scan:
735
+ files = scan(self.repository, self.settings.max_file_size, self.settings.exclude)
736
+ p_scan.files_processed = len(files)
737
+ p_scan.items_processed = len(files)
738
+ total_files = len(files)
739
+ if progress_callback is not None:
740
+ progress_callback("repository_scan", total_files, total_files, telemetry)
741
+
742
+ seen = {item.relative_path.as_posix() for item in files}
743
+ batch_limit = self.governor.policy.max_files_per_incremental_batch
744
+ uncommitted_in_batch = 0
745
+
746
+ con = self.connect()
747
+ try:
748
+ # Check parser version invalidation & previous interrupted state
749
+ last_parser_ver = None
750
+ resolution_dirty = False
751
+ try:
752
+ for mrow in con.execute(
753
+ "SELECT key, value FROM metadata WHERE key IN ('parser_version', 'resolution_dirty')"
754
+ ).fetchall():
755
+ mkey = str(mrow["key"])
756
+ if mkey == "parser_version":
757
+ last_parser_ver = str(mrow["value"])
758
+ elif mkey == "resolution_dirty" and str(mrow["value"]) == "1":
759
+ resolution_dirty = True
760
+ except sqlite3.OperationalError:
761
+ pass
762
+
763
+ force_reparse = (last_parser_ver != PARSER_VERSION)
764
+
765
+ # Pre-load existing file hashes and statuses in a single query
766
+ existing_files: dict[str, tuple[str, str]] = {
767
+ str(r["path"]): (str(r["hash"]), str(r["status"]))
768
+ for r in con.execute("SELECT path, hash, status FROM files").fetchall()
769
+ }
770
+
771
+ for idx, item in enumerate(files):
772
+ if idx > 0 and idx % batch_limit == 0:
773
+ self.governor.yield_if_needed(TaskPriority.BACKGROUND)
774
+
775
+ relative = item.relative_path.as_posix()
776
+ t_read = time.perf_counter()
777
+ try:
778
+ content = item.path.read_text(encoding="utf-8", errors="replace")
779
+ except OSError:
780
+ telemetry.record("file_reading", time.perf_counter() - t_read, errors_skips=1)
781
+ continue
782
+ digest = hashlib.sha256(content.encode()).hexdigest()
783
+ telemetry.record(
784
+ "file_reading",
785
+ time.perf_counter() - t_read,
786
+ files_processed=1,
787
+ items_processed=len(content),
788
+ )
789
+
790
+ old_state = existing_files.get(relative)
791
+ if old_state is not None and old_state[0] == digest and not force_reparse:
792
+ if old_state[1] == "parse_failed":
793
+ parse_failed_count += 1
794
+ else:
795
+ unchanged += 1
796
+ telemetry.phases["file_reading"].errors_skips += 1
797
+ continue
798
+
799
+ t_cls = time.perf_counter()
800
+ cat_info = classify_file_detailed(relative, content)
801
+ cat_str = cat_info.category.value
802
+ telemetry.record(
803
+ "file_classification",
804
+ time.perf_counter() - t_cls,
805
+ files_processed=1,
806
+ items_processed=1,
807
+ )
808
+
809
+ success = self._replace_file(
810
+ con,
811
+ relative,
812
+ item.language,
813
+ content,
814
+ digest,
815
+ category=cat_str,
816
+ is_existing=(old_state is not None),
817
+ telemetry=telemetry,
818
+ )
819
+ if success:
820
+ indexed += 1
821
+ dirty_files.add(relative)
822
+ existing_files[relative] = (digest, "ok")
823
+ else:
824
+ parse_failed_count += 1
825
+ existing_files[relative] = (digest, "parse_failed")
826
+
827
+ uncommitted_in_batch += 1
828
+ if uncommitted_in_batch >= self.commit_batch_size:
829
+ t_commit = time.perf_counter()
830
+ con.execute(
831
+ "INSERT INTO metadata(key, value) VALUES('resolution_dirty', '1') "
832
+ "ON CONFLICT(key) DO UPDATE SET value='1'"
833
+ )
834
+ con.execute(
835
+ "INSERT INTO metadata(key, value) VALUES('index_status', 'in_progress') "
836
+ "ON CONFLICT(key) DO UPDATE SET value='in_progress'"
837
+ )
838
+ con.commit()
839
+ telemetry.batch_commits += 1
840
+ telemetry.observe_wal(self.db_path)
841
+ try:
842
+ con.execute("PRAGMA wal_checkpoint(TRUNCATE)")
843
+ telemetry.wal_checkpoints += 1
844
+ except sqlite3.OperationalError:
845
+ pass
846
+ telemetry.observe_wal(self.db_path)
847
+ telemetry.record(
848
+ "sqlite_writes",
849
+ time.perf_counter() - t_commit,
850
+ db_writes=1,
851
+ )
852
+ uncommitted_in_batch = 0
853
+ if progress_callback is not None:
854
+ progress_callback("ast_parsing", idx + 1, total_files, telemetry)
855
+
856
+ telemetry.post_parse_rss_mb = get_process_rss_mb()
857
+ telemetry.peak_rss_mb = max(telemetry.peak_rss_mb, telemetry.post_parse_rss_mb)
858
+
859
+ stale = [
860
+ p for p in existing_files
861
+ if p not in seen
862
+ ]
863
+ if stale:
864
+ t_del = time.perf_counter()
865
+ for path in stale:
866
+ self._delete_file(con, path, telemetry=telemetry)
867
+ telemetry.record(
868
+ "sqlite_writes",
869
+ time.perf_counter() - t_del,
870
+ files_processed=len(stale),
871
+ db_writes=len(stale),
872
+ )
873
+
874
+ # Commit file indexing phase before global post-processing so completed file work is durable
875
+ head_commit = current_commit(self.repository)
876
+ now_ts = int(time.time())
877
+ need_resolution = (indexed > 0 or len(stale) > 0 or force_reparse or resolution_dirty)
878
+
879
+ if uncommitted_in_batch > 0 or stale:
880
+ t_commit = time.perf_counter()
881
+ if need_resolution:
882
+ con.execute(
883
+ "INSERT INTO metadata(key, value) VALUES('resolution_dirty', '1') "
884
+ "ON CONFLICT(key) DO UPDATE SET value='1'"
885
+ )
886
+ con.commit()
887
+ telemetry.batch_commits += 1
888
+ telemetry.observe_wal(self.db_path)
889
+ try:
890
+ con.execute("PRAGMA wal_checkpoint(TRUNCATE)")
891
+ telemetry.wal_checkpoints += 1
892
+ except sqlite3.OperationalError:
893
+ pass
894
+ telemetry.observe_wal(self.db_path)
895
+ telemetry.record("sqlite_writes", time.perf_counter() - t_commit, db_writes=1)
896
+ uncommitted_in_batch = 0
897
+
898
+ # Run global reference resolution pass if any files were changed, removed, or pending from interrupt
899
+ if need_resolution:
900
+ with telemetry.phase("post_processing") as p_post:
901
+ if progress_callback is not None:
902
+ progress_callback("post_processing", total_files, total_files, telemetry)
903
+ self._run_global_resolution(
904
+ con,
905
+ head_commit,
906
+ dirty_files=dirty_files if not (force_reparse or resolution_dirty) else None,
907
+ removed_files=set(stale) if not (force_reparse or resolution_dirty) else None,
908
+ telemetry=telemetry,
909
+ progress_callback=progress_callback,
910
+ )
911
+ from codegraph.async_queue import AsyncQueueScanner
912
+
913
+ AsyncQueueScanner(self.repository).scan_repository(con)
914
+ p_post.files_processed = indexed + len(stale)
915
+ p_post.items_processed = indexed + len(stale)
916
+ # Invalidate context cache on index update
917
+ t_rank = time.perf_counter()
918
+ try:
919
+ con.execute("DELETE FROM context_cache")
920
+ except sqlite3.OperationalError:
921
+ pass
922
+ telemetry.record("ranking_index_building", time.perf_counter() - t_rank, db_writes=1)
923
+
924
+ # Final commit & WAL checkpoint
925
+ with telemetry.phase("final_commit_checkpoint") as p_final:
926
+ save_commit(con, head_commit, now_ts)
927
+ current_gen = 0
928
+ try:
929
+ grow = con.execute("SELECT value FROM metadata WHERE key='index_generation'").fetchone()
930
+ if grow and grow[0]:
931
+ current_gen = int(grow[0])
932
+ except (sqlite3.OperationalError, ValueError):
933
+ pass
934
+ next_gen = current_gen + 1
935
+
936
+ con.executemany(
937
+ "INSERT INTO metadata(key, value) VALUES(?,?) "
938
+ "ON CONFLICT(key) DO UPDATE SET value=excluded.value",
939
+ [
940
+ ("repository", str(self.repository)),
941
+ ("parser_version", PARSER_VERSION),
942
+ ("schema_version", str(SCHEMA_VERSION)),
943
+ ("index_generation", str(next_gen)),
944
+ ("index_timestamp", str(now_ts)),
945
+ ("resolution_dirty", "0"),
946
+ ("index_status", "ok"),
947
+ ],
948
+ )
949
+ con.commit()
950
+ telemetry.batch_commits += 1
951
+ telemetry.observe_wal(self.db_path)
952
+ try:
953
+ con.execute("PRAGMA wal_checkpoint(TRUNCATE)")
954
+ telemetry.wal_checkpoints += 1
955
+ except sqlite3.OperationalError:
956
+ pass
957
+ telemetry.observe_wal(self.db_path)
958
+ p_final.files_processed = total_files
959
+ p_final.items_processed = indexed
960
+ p_final.db_writes = 8
961
+ if progress_callback is not None:
962
+ progress_callback("final_commit_checkpoint", total_files, total_files, telemetry)
963
+
964
+ except KeyboardInterrupt:
965
+ # Safe cancellation: roll back only the active uncommitted batch, preserve committed batches
966
+ telemetry.interrupted = True
967
+ telemetry.scanned_files = len(files)
968
+ telemetry.indexed_files = indexed
969
+ telemetry.unchanged_files = unchanged
970
+ telemetry.removed_files = len(stale)
971
+ telemetry.parse_failed_files = parse_failed_count
972
+ try:
973
+ con.rollback()
974
+ con.executemany(
975
+ "INSERT INTO metadata(key, value) VALUES(?,?) "
976
+ "ON CONFLICT(key) DO UPDATE SET value=excluded.value",
977
+ [
978
+ ("resolution_dirty", "1"),
979
+ ("index_status", "interrupted"),
980
+ ("parser_version", PARSER_VERSION),
981
+ ],
982
+ )
983
+ con.commit()
984
+ con.execute("PRAGMA wal_checkpoint(PASSIVE)")
985
+ telemetry.wal_checkpoints += 1
986
+ telemetry.observe_wal(self.db_path)
987
+ except sqlite3.OperationalError:
988
+ pass
989
+ telemetry.finish(self.db_path)
990
+ if catch_interrupt:
991
+ return {
992
+ "scanned": len(files),
993
+ "indexed": indexed,
994
+ "unchanged": unchanged,
995
+ "removed": len(stale),
996
+ "interrupted": 1,
997
+ }
998
+ raise
999
+ except BaseException:
1000
+ con.rollback()
1001
+ raise
1002
+ finally:
1003
+ con.close()
1004
+ finally:
1005
+ if not telemetry.interrupted:
1006
+ telemetry.scanned_files = len(files)
1007
+ telemetry.indexed_files = indexed
1008
+ telemetry.unchanged_files = unchanged
1009
+ telemetry.removed_files = len(stale)
1010
+ telemetry.parse_failed_files = parse_failed_count
1011
+ telemetry.finish(self.db_path)
1012
+ self.governor.set_indexing_active(False)
1013
+
1014
+ result: dict[str, int] = {
1015
+ "scanned": len(files),
1016
+ "indexed": indexed,
1017
+ "unchanged": unchanged,
1018
+ "removed": len(stale),
1019
+ }
1020
+ if parse_failed_count > 0:
1021
+ result["parse_failed"] = parse_failed_count
1022
+ self.reset_pool()
1023
+ return result
1024
+
1025
+ def reindex_paths(
1026
+ self,
1027
+ paths: Iterable[str | Path],
1028
+ ) -> dict[str, Any]:
1029
+ """Incrementally reindex a set of changed or deleted file paths in sub-15ms.
1030
+
1031
+ Bypasses full repository filesystem scanning and only touches the specified files,
1032
+ updating their AST facts, re-running affected reference resolution, busting in-memory
1033
+ caches, and advancing index_generation atomically.
1034
+ """
1035
+ t0 = time.perf_counter()
1036
+ from codegraph.freshness import current_commit, save_commit
1037
+ from codegraph.watcher import _is_path_ignored
1038
+
1039
+ # 1. Normalize and deduplicate paths relative to self.repository
1040
+ normalized_paths: set[str] = set()
1041
+ for p in paths:
1042
+ try:
1043
+ p_path = Path(p)
1044
+ if p_path.is_absolute():
1045
+ rel = str(p_path.resolve().relative_to(self.repository)).replace("\\", "/")
1046
+ else:
1047
+ rel = str(p_path).replace("\\", "/").lstrip("./")
1048
+ except Exception:
1049
+ continue
1050
+
1051
+ if not rel or _is_path_ignored(rel):
1052
+ continue
1053
+ normalized_paths.add(rel)
1054
+
1055
+ if not normalized_paths:
1056
+ return {
1057
+ "status": "ok",
1058
+ "scanned": 0,
1059
+ "indexed": 0,
1060
+ "removed": 0,
1061
+ "reindexed": [],
1062
+ "deleted": [],
1063
+ "skipped": [],
1064
+ "parse_failed": [],
1065
+ "generation": 0,
1066
+ "elapsed_ms": round((time.perf_counter() - t0) * 1000.0, 2),
1067
+ }
1068
+
1069
+ # 2. Inspect filesystem vs SQLite to determine what to delete, update, or skip
1070
+ con = self.connect()
1071
+ try:
1072
+ to_delete: list[str] = []
1073
+ to_update: list[tuple[str, str, str, str, bool]] = []
1074
+ skipped: list[str] = []
1075
+
1076
+ for rel_path in sorted(normalized_paths):
1077
+ full_path = self.repository / rel_path
1078
+ existing_row = con.execute("SELECT hash, status FROM files WHERE path=?", (rel_path,)).fetchone()
1079
+
1080
+ if not full_path.is_file():
1081
+ # File was deleted from disk
1082
+ if existing_row is not None:
1083
+ to_delete.append(rel_path)
1084
+ else:
1085
+ skipped.append(rel_path)
1086
+ continue
1087
+
1088
+ # File exists on disk
1089
+ suffix = full_path.suffix.lower()
1090
+ lang = LANGUAGES.get(suffix)
1091
+ if not lang:
1092
+ skipped.append(rel_path)
1093
+ continue
1094
+
1095
+ try:
1096
+ content = full_path.read_text(encoding="utf-8", errors="replace")
1097
+ except Exception:
1098
+ skipped.append(rel_path)
1099
+ continue
1100
+
1101
+ digest = hashlib.sha256(content.encode("utf-8")).hexdigest()
1102
+
1103
+ if existing_row is not None and existing_row[0] == digest and existing_row[1] == "ok":
1104
+ # Identical digest, no AST or file modification
1105
+ skipped.append(rel_path)
1106
+ else:
1107
+ to_update.append((rel_path, lang, content, digest, existing_row is not None))
1108
+
1109
+ # If nothing changed, return early
1110
+ grow = con.execute("SELECT value FROM metadata WHERE key='index_generation'").fetchone()
1111
+ current_gen = int(grow[0]) if (grow and grow[0]) else 0
1112
+
1113
+ if not to_delete and not to_update:
1114
+ return {
1115
+ "status": "ok",
1116
+ "scanned": len(normalized_paths),
1117
+ "indexed": 0,
1118
+ "removed": 0,
1119
+ "reindexed": [],
1120
+ "deleted": [],
1121
+ "skipped": skipped,
1122
+ "parse_failed": [],
1123
+ "generation": current_gen,
1124
+ "elapsed_ms": round((time.perf_counter() - t0) * 1000.0, 2),
1125
+ }
1126
+
1127
+ # 3. Apply atomic updates
1128
+ self.governor.set_indexing_active(True)
1129
+ reindexed: list[str] = []
1130
+ parse_failed: list[str] = []
1131
+
1132
+ for del_path in to_delete:
1133
+ self._delete_file(con, del_path)
1134
+
1135
+ for rel_path, lang, content, digest, is_existing in to_update:
1136
+ cat_info = classify_file_detailed(Path(rel_path), content)
1137
+ success = self._replace_file(
1138
+ con,
1139
+ rel_path,
1140
+ lang,
1141
+ content,
1142
+ digest,
1143
+ category=cat_info.category.value,
1144
+ is_existing=is_existing,
1145
+ )
1146
+ if success:
1147
+ reindexed.append(rel_path)
1148
+ else:
1149
+ parse_failed.append(rel_path)
1150
+
1151
+ # 4. Run reference resolution pass
1152
+ head_commit = current_commit(self.repository)
1153
+ now_ts = int(time.time())
1154
+
1155
+ dirty_set = set(reindexed)
1156
+ removed_set = set(to_delete)
1157
+
1158
+ self._run_global_resolution(
1159
+ con,
1160
+ head_commit,
1161
+ dirty_files=dirty_set,
1162
+ removed_files=removed_set,
1163
+ )
1164
+
1165
+ from codegraph.async_queue import AsyncQueueScanner
1166
+
1167
+ AsyncQueueScanner(self.repository).scan_repository(con)
1168
+
1169
+ # Invalidate context cache on index update
1170
+ try:
1171
+ con.execute("DELETE FROM context_cache")
1172
+ except sqlite3.OperationalError:
1173
+ pass
1174
+
1175
+ # 5. Advance generation and update metadata
1176
+ next_gen = current_gen + 1
1177
+ con.executemany(
1178
+ "INSERT INTO metadata(key, value) VALUES(?,?) "
1179
+ "ON CONFLICT(key) DO UPDATE SET value=excluded.value",
1180
+ [
1181
+ ("repository", str(self.repository)),
1182
+ ("parser_version", PARSER_VERSION),
1183
+ ("schema_version", str(SCHEMA_VERSION)),
1184
+ ("index_generation", str(next_gen)),
1185
+ ("index_timestamp", str(now_ts)),
1186
+ ("resolution_dirty", "0"),
1187
+ ("index_status", "ok"),
1188
+ ],
1189
+ )
1190
+ save_commit(con, head_commit, now_ts)
1191
+ con.commit()
1192
+ try:
1193
+ con.execute("PRAGMA wal_checkpoint(PASSIVE)")
1194
+ except sqlite3.OperationalError:
1195
+ pass
1196
+ except Exception:
1197
+ con.rollback()
1198
+ raise
1199
+ finally:
1200
+ self.governor.set_indexing_active(False)
1201
+ con.close()
1202
+
1203
+ # Invalidate in-memory caches and reset pool
1204
+ get_parse_cache().clear()
1205
+ get_graph_cache().clear()
1206
+ self.reset_pool()
1207
+
1208
+ elapsed_ms = round((time.perf_counter() - t0) * 1000.0, 2)
1209
+ return {
1210
+ "status": "ok",
1211
+ "scanned": len(normalized_paths),
1212
+ "indexed": len(reindexed),
1213
+ "removed": len(to_delete),
1214
+ "reindexed": reindexed,
1215
+ "deleted": to_delete,
1216
+ "skipped": skipped,
1217
+ "parse_failed": parse_failed,
1218
+ "generation": next_gen,
1219
+ "elapsed_ms": elapsed_ms,
1220
+ }
1221
+
1222
+ def _delete_file(
1223
+ self,
1224
+ con: sqlite3.Connection,
1225
+ path: str,
1226
+ *,
1227
+ telemetry: IndexingTelemetry | None = None,
1228
+ ) -> None:
1229
+ """Atomically delete all indexed facts for a file, maintaining FTS synchronization."""
1230
+ ids = [r[0] for r in con.execute("SELECT id FROM chunks WHERE path=?", (path,))]
1231
+ if ids:
1232
+ t_fts = time.perf_counter()
1233
+ con.executemany("DELETE FROM chunks_fts WHERE rowid=?", [(chunk_id,) for chunk_id in ids])
1234
+ if telemetry is not None:
1235
+ telemetry.record(
1236
+ "fts_updates",
1237
+ time.perf_counter() - t_fts,
1238
+ items_processed=len(ids),
1239
+ db_writes=len(ids),
1240
+ )
1241
+ con.execute("DELETE FROM chunks WHERE path=?", (path,))
1242
+ con.execute("DELETE FROM symbols WHERE path=?", (path,))
1243
+ con.execute("DELETE FROM imports WHERE source_path=?", (path,))
1244
+ con.execute("DELETE FROM calls WHERE source_path=?", (path,))
1245
+ con.execute("DELETE FROM inheritance WHERE source_file=?", (path,))
1246
+ con.execute("DELETE FROM framework_routes WHERE file_path=?", (path,))
1247
+ con.execute("DELETE FROM raw_framework_routes WHERE file_path=?", (path,))
1248
+ con.execute("DELETE FROM router_mounts WHERE file_path=?", (path,))
1249
+ con.execute("DELETE FROM router_definitions WHERE file_path=?", (path,))
1250
+ con.execute("DELETE FROM local_bindings WHERE file_path=?", (path,))
1251
+ con.execute("DELETE FROM 'references' WHERE path=?", (path,))
1252
+ con.execute("DELETE FROM graph_edges WHERE file=?", (path,))
1253
+ con.execute("DELETE FROM db_entities WHERE file_path=?", (path,))
1254
+ con.execute("DELETE FROM db_queries WHERE file_path=?", (path,))
1255
+ con.execute("DELETE FROM db_migrations WHERE file_path=?", (path,))
1256
+ con.execute("DELETE FROM files WHERE path=?", (path,))
1257
+
1258
+ def _replace_file(
1259
+ self,
1260
+ con: sqlite3.Connection,
1261
+ path: str,
1262
+ language: str,
1263
+ content: str,
1264
+ digest: str,
1265
+ category: str = "SOURCE",
1266
+ *,
1267
+ is_existing: bool = True,
1268
+ telemetry: IndexingTelemetry | None = None,
1269
+ ) -> bool:
1270
+ """Parse and insert a file. If parse fails, preserves last-known-good state."""
1271
+ from codegraph.security.redaction import redact_secrets
1272
+
1273
+ t_parse = time.perf_counter()
1274
+ parse_cache = get_parse_cache()
1275
+ cached = parse_cache.get((digest, PARSER_VERSION, language))
1276
+ if cached is not None:
1277
+ result = cached
1278
+ else:
1279
+ result = parse(content, language, path)
1280
+ parse_cache.put((digest, PARSER_VERSION, language), result)
1281
+ parse_elapsed = time.perf_counter() - t_parse
1282
+
1283
+ if telemetry is not None:
1284
+ telemetry.record(
1285
+ "ast_parsing",
1286
+ parse_elapsed,
1287
+ files_processed=1,
1288
+ items_processed=len(result.symbols) + len(result.imports) + len(result.calls),
1289
+ errors_skips=1 if result.parse_failed else 0,
1290
+ )
1291
+ telemetry.record(
1292
+ "symbol_extraction",
1293
+ 0.0,
1294
+ files_processed=1,
1295
+ items_processed=len(result.symbols),
1296
+ )
1297
+ telemetry.record(
1298
+ "import_extraction",
1299
+ 0.0,
1300
+ files_processed=1,
1301
+ items_processed=len(result.imports),
1302
+ )
1303
+ if result.routes or result.mounts or result.router_definitions:
1304
+ telemetry.record(
1305
+ "framework_analysis",
1306
+ 0.0,
1307
+ files_processed=1,
1308
+ items_processed=len(result.routes) + len(result.mounts) + len(result.router_definitions),
1309
+ )
1310
+
1311
+ now = int(time.time())
1312
+
1313
+ if result.parse_failed:
1314
+ t_sql = time.perf_counter()
1315
+ # Parse failure safety: check if file previously existed
1316
+ old = con.execute("SELECT hash, status FROM files WHERE path=?", (path,)).fetchone() if is_existing else None
1317
+ if old:
1318
+ # Retain last-known-good index records, mark file as parse_failed
1319
+ con.execute(
1320
+ "UPDATE files SET hash=?, status='parse_failed', parse_error=?, indexed_at=?, category=? WHERE path=?",
1321
+ (digest, redact_secrets(result.parse_error or ""), now, category, path),
1322
+ )
1323
+ else:
1324
+ con.execute(
1325
+ "INSERT INTO files(path, hash, language, indexed_at, status, parse_error, category) VALUES (?, ?, ?, ?, ?, ?, ?)",
1326
+ (path, digest, language, now, "parse_failed", redact_secrets(result.parse_error or ""), category),
1327
+ )
1328
+ if telemetry is not None:
1329
+ telemetry.record("sqlite_writes", time.perf_counter() - t_sql, files_processed=1, db_writes=1)
1330
+ return False
1331
+
1332
+ t_sql = time.perf_counter()
1333
+ db_write_count = 0
1334
+ fts_elapsed = 0.0
1335
+ fts_write_count = 0
1336
+
1337
+ # Successful parse: clean up old file state only if file previously existed in DB
1338
+ if is_existing:
1339
+ self._delete_file(con, path, telemetry=telemetry)
1340
+ db_write_count += 16
1341
+
1342
+ con.execute(
1343
+ "INSERT INTO files(path, hash, language, indexed_at, status, parse_error, last_valid_hash, category) "
1344
+ "VALUES (?, ?, ?, ?, 'ok', NULL, ?, ?)",
1345
+ (path, digest, language, now, digest, category),
1346
+ )
1347
+ db_write_count += 1
1348
+
1349
+ if category in ("MINIFIED", "BUNDLE", "BINARY"):
1350
+ if telemetry is not None:
1351
+ telemetry.record("sqlite_writes", time.perf_counter() - t_sql, files_processed=1, db_writes=db_write_count)
1352
+ return True
1353
+
1354
+ lines = content.splitlines()
1355
+
1356
+ if result.symbols:
1357
+ symbol_rows = []
1358
+ for s in result.symbols:
1359
+ dec_str = ",".join(s.decorators) if s.decorators else ""
1360
+ safe_doc = redact_secrets(s.documentation) if s.documentation else None
1361
+ symbol_rows.append(
1362
+ (
1363
+ s.name,
1364
+ s.qualified_name,
1365
+ s.kind,
1366
+ path,
1367
+ s.start_line,
1368
+ s.end_line,
1369
+ dec_str,
1370
+ s.id,
1371
+ s.canonical_id,
1372
+ s.language,
1373
+ s.module,
1374
+ s.scope,
1375
+ redact_secrets(s.signature),
1376
+ s.content_hash,
1377
+ s.parent_symbol_id,
1378
+ s.visibility,
1379
+ s.return_type,
1380
+ s.parameter_count,
1381
+ safe_doc,
1382
+ )
1383
+ )
1384
+ body = redact_secrets("\n".join(lines[s.start_line - 1 : s.end_line]))
1385
+ cur = con.execute(
1386
+ "INSERT INTO chunks(path, language, symbol, symbol_type, start_line, end_line, content, hash) "
1387
+ "VALUES (?,?,?,?,?,?,?,?)",
1388
+ (
1389
+ path,
1390
+ language,
1391
+ s.qualified_name,
1392
+ s.kind,
1393
+ s.start_line,
1394
+ s.end_line,
1395
+ body,
1396
+ digest,
1397
+ ),
1398
+ )
1399
+ db_write_count += 1
1400
+ t_fts = time.perf_counter()
1401
+ con.execute(
1402
+ "INSERT INTO chunks_fts(rowid, content, path, symbol) VALUES (?,?,?,?)",
1403
+ (cur.lastrowid, body, path, s.qualified_name),
1404
+ )
1405
+ fts_elapsed += time.perf_counter() - t_fts
1406
+ fts_write_count += 1
1407
+
1408
+ con.executemany(
1409
+ "INSERT INTO symbols("
1410
+ "name, qualified_name, kind, path, start_line, end_line, decorators, "
1411
+ "id, canonical_id, language, module, scope, signature, content_hash, "
1412
+ "parent_symbol_id, visibility, return_type, parameter_count, documentation) "
1413
+ "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
1414
+ symbol_rows,
1415
+ )
1416
+ db_write_count += len(symbol_rows)
1417
+ else:
1418
+ safe_content = redact_secrets(content)
1419
+ cur = con.execute(
1420
+ "INSERT INTO chunks(path, language, symbol, symbol_type, start_line, end_line, content, hash) "
1421
+ "VALUES (?,?,?,?,?,?,?,?)",
1422
+ (path, language, None, "module", 1, max(1, len(lines)), safe_content, digest),
1423
+ )
1424
+ db_write_count += 1
1425
+ t_fts = time.perf_counter()
1426
+ con.execute(
1427
+ "INSERT INTO chunks_fts(rowid, content, path, symbol) VALUES (?,?,?,?)",
1428
+ (cur.lastrowid, safe_content, path, None),
1429
+ )
1430
+ fts_elapsed += time.perf_counter() - t_fts
1431
+ fts_write_count += 1
1432
+
1433
+ if result.imports:
1434
+ con.executemany(
1435
+ "INSERT INTO imports(source_path, module, name, alias, full_name, line, "
1436
+ "source_module, imported_module, imported_name, local_name, import_type) "
1437
+ "VALUES (?,?,?,?,?,?,?,?,?,?,?)",
1438
+ [
1439
+ (
1440
+ path,
1441
+ imp.module,
1442
+ imp.name,
1443
+ imp.alias,
1444
+ imp.full,
1445
+ imp.line,
1446
+ imp.source_module,
1447
+ imp.imported_module,
1448
+ imp.imported_name,
1449
+ imp.local_name,
1450
+ imp.import_type,
1451
+ )
1452
+ for imp in result.imports
1453
+ ],
1454
+ )
1455
+ db_write_count += len(result.imports)
1456
+
1457
+ if result.calls:
1458
+ con.executemany(
1459
+ "INSERT INTO calls(source_path, callee, qualified_callee, line, confidence, source_symbol_id) "
1460
+ "VALUES (?,?,?,?,?,?)",
1461
+ [
1462
+ (
1463
+ path,
1464
+ c.callee,
1465
+ c.qualified_callee,
1466
+ c.line,
1467
+ c.confidence,
1468
+ c.caller_canonical_id,
1469
+ )
1470
+ for c in result.calls
1471
+ ],
1472
+ )
1473
+ db_write_count += len(result.calls)
1474
+
1475
+ if result.inheritance:
1476
+ con.executemany(
1477
+ "INSERT INTO inheritance(source_symbol, base_name, relationship, source_file, line, source_canonical_id) "
1478
+ "VALUES (?,?,?,?,?,?)",
1479
+ [
1480
+ (
1481
+ inh.source_symbol,
1482
+ inh.base_name,
1483
+ inh.relationship,
1484
+ path,
1485
+ inh.line,
1486
+ inh.source_canonical_id,
1487
+ )
1488
+ for inh in result.inheritance
1489
+ ],
1490
+ )
1491
+ db_write_count += len(result.inheritance)
1492
+
1493
+ if result.routes:
1494
+ con.executemany(
1495
+ "INSERT INTO raw_framework_routes(endpoint_id, framework, http_method, route_path, "
1496
+ "normalized_route, handler_name, handler_canonical_id, file_path, line, evidence, confidence, router_name) "
1497
+ "VALUES (?,?,?,?,?,?,?,?,?,?,?,?) "
1498
+ "ON CONFLICT(endpoint_id) DO UPDATE SET "
1499
+ "framework=excluded.framework, "
1500
+ "http_method=excluded.http_method, "
1501
+ "route_path=excluded.route_path, "
1502
+ "normalized_route=excluded.normalized_route, "
1503
+ "handler_name=excluded.handler_name, "
1504
+ "handler_canonical_id=excluded.handler_canonical_id, "
1505
+ "file_path=excluded.file_path, "
1506
+ "line=excluded.line, "
1507
+ "evidence=excluded.evidence, "
1508
+ "confidence=excluded.confidence, "
1509
+ "router_name=excluded.router_name",
1510
+ [
1511
+ (
1512
+ r.endpoint_id,
1513
+ r.framework,
1514
+ r.http_method,
1515
+ r.route_path,
1516
+ r.normalized_route,
1517
+ r.handler_name,
1518
+ r.handler_canonical_id,
1519
+ path,
1520
+ r.line,
1521
+ redact_secrets(r.evidence),
1522
+ r.confidence,
1523
+ r.router_name,
1524
+ )
1525
+ for r in result.routes
1526
+ ],
1527
+ )
1528
+ con.executemany(
1529
+ "INSERT INTO framework_routes(endpoint_id, framework, http_method, route_path, "
1530
+ "normalized_route, handler_name, handler_canonical_id, file_path, line, evidence, confidence) "
1531
+ "VALUES (?,?,?,?,?,?,?,?,?,?,?) "
1532
+ "ON CONFLICT(endpoint_id) DO UPDATE SET "
1533
+ "framework=excluded.framework, "
1534
+ "http_method=excluded.http_method, "
1535
+ "route_path=excluded.route_path, "
1536
+ "normalized_route=excluded.normalized_route, "
1537
+ "handler_name=excluded.handler_name, "
1538
+ "handler_canonical_id=excluded.handler_canonical_id, "
1539
+ "file_path=excluded.file_path, "
1540
+ "line=excluded.line, "
1541
+ "evidence=excluded.evidence, "
1542
+ "confidence=excluded.confidence",
1543
+ [
1544
+ (
1545
+ r.endpoint_id,
1546
+ r.framework,
1547
+ r.http_method,
1548
+ r.route_path,
1549
+ r.normalized_route,
1550
+ r.handler_name,
1551
+ r.handler_canonical_id,
1552
+ path,
1553
+ r.line,
1554
+ redact_secrets(r.evidence),
1555
+ r.confidence,
1556
+ )
1557
+ for r in result.routes
1558
+ ],
1559
+ )
1560
+ db_write_count += len(result.routes) * 2
1561
+
1562
+ if result.mounts:
1563
+ con.executemany(
1564
+ "INSERT INTO router_mounts(mount_id, framework, parent_router, child_router, prefix, file_path, line, evidence, confidence) "
1565
+ "VALUES (?,?,?,?,?,?,?,?,?) "
1566
+ "ON CONFLICT(mount_id) DO UPDATE SET "
1567
+ "prefix=excluded.prefix, evidence=excluded.evidence, confidence=excluded.confidence",
1568
+ [
1569
+ (
1570
+ f"MOUNT:{m.framework}:{path}:{m.line}:{m.parent_router}->{m.child_router}",
1571
+ m.framework,
1572
+ m.parent_router,
1573
+ m.child_router,
1574
+ m.prefix,
1575
+ path,
1576
+ m.line,
1577
+ redact_secrets(m.evidence),
1578
+ m.confidence,
1579
+ )
1580
+ for m in result.mounts
1581
+ ],
1582
+ )
1583
+ db_write_count += len(result.mounts)
1584
+
1585
+ if result.router_definitions:
1586
+ con.executemany(
1587
+ "INSERT INTO router_definitions(framework, router_name, prefix, file_path, line, evidence) "
1588
+ "VALUES (?,?,?,?,?,?)",
1589
+ [
1590
+ (
1591
+ d.framework,
1592
+ d.router_name,
1593
+ d.prefix,
1594
+ path,
1595
+ d.line,
1596
+ redact_secrets(d.evidence),
1597
+ )
1598
+ for d in result.router_definitions
1599
+ ],
1600
+ )
1601
+ db_write_count += len(result.router_definitions)
1602
+
1603
+ if result.bindings:
1604
+ con.executemany(
1605
+ "INSERT INTO local_bindings("
1606
+ "target_name, file_path, line, column, scope, expr_kind, source_expr, "
1607
+ "is_conditional, base_expr, attr_name, dict_entries_json, list_entries_json, "
1608
+ "subscript_target, subscript_key, subscript_index) "
1609
+ "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
1610
+ [
1611
+ (
1612
+ b.target_name,
1613
+ path,
1614
+ b.line,
1615
+ b.column,
1616
+ b.scope,
1617
+ b.expr_kind,
1618
+ redact_secrets(b.source_expr),
1619
+ 1 if b.is_conditional else 0,
1620
+ b.base_expr,
1621
+ b.attr_name,
1622
+ json.dumps(b.dict_entries),
1623
+ json.dumps(b.list_entries),
1624
+ b.subscript_target,
1625
+ b.subscript_key,
1626
+ b.subscript_index,
1627
+ )
1628
+ for b in result.bindings
1629
+ ],
1630
+ )
1631
+ db_write_count += len(result.bindings)
1632
+
1633
+ total_sql_elapsed = max(0.0, (time.perf_counter() - t_sql) - fts_elapsed)
1634
+ if telemetry is not None:
1635
+ telemetry.record(
1636
+ "sqlite_writes",
1637
+ total_sql_elapsed,
1638
+ files_processed=1,
1639
+ items_processed=db_write_count,
1640
+ db_writes=db_write_count,
1641
+ )
1642
+ if fts_write_count > 0:
1643
+ telemetry.record(
1644
+ "fts_updates",
1645
+ fts_elapsed,
1646
+ files_processed=1,
1647
+ items_processed=fts_write_count,
1648
+ db_writes=fts_write_count,
1649
+ )
1650
+
1651
+ return True
1652
+
1653
+ def _run_global_resolution(
1654
+ self,
1655
+ con: sqlite3.Connection,
1656
+ indexed_commit: str | None,
1657
+ *,
1658
+ dirty_files: set[str] | None = None,
1659
+ removed_files: set[str] | None = None,
1660
+ telemetry: IndexingTelemetry | None = None,
1661
+ progress_callback: Callable[[str, int, int, IndexingTelemetry], None] | None = None,
1662
+ ) -> None:
1663
+ """Run repository-wide reference resolution, updating 'references' and 'graph_edges'."""
1664
+ from codegraph.security.redaction import redact_secrets
1665
+
1666
+ t_prep = time.perf_counter()
1667
+ # 1. Load all files
1668
+ file_rows = con.execute("SELECT path, hash, language FROM files WHERE status='ok'").fetchall()
1669
+ known_files = {str(r["path"]) for r in file_rows}
1670
+ file_hashes = {str(r["path"]): str(r["hash"]) for r in file_rows}
1671
+ file_langs = {str(r["path"]): str(r["language"]) for r in file_rows}
1672
+
1673
+ # 2. Load symbols
1674
+ sym_rows = con.execute(
1675
+ "SELECT name, qualified_name, kind, path, start_line, end_line, decorators, "
1676
+ "id, canonical_id, language, module, scope, signature, content_hash, "
1677
+ "parent_symbol_id, visibility, return_type, parameter_count, documentation "
1678
+ "FROM symbols"
1679
+ ).fetchall()
1680
+ symbols = [
1681
+ Symbol(
1682
+ id=str(r["id"]),
1683
+ canonical_id=str(r["canonical_id"]),
1684
+ name=str(r["name"]),
1685
+ qualified_name=str(r["qualified_name"]),
1686
+ kind=str(r["kind"]),
1687
+ start_line=int(r["start_line"]),
1688
+ end_line=int(r["end_line"]),
1689
+ file_path=str(r["path"]),
1690
+ decorators=[d for d in str(r["decorators"]).split(",") if d],
1691
+ language=str(r["language"]),
1692
+ module=str(r["module"]),
1693
+ path=str(r["path"]),
1694
+ scope=str(r["scope"]),
1695
+ signature=str(r["signature"]),
1696
+ content_hash=str(r["content_hash"]),
1697
+ parent_symbol_id=r["parent_symbol_id"],
1698
+ visibility=str(r["visibility"]),
1699
+ return_type=r["return_type"],
1700
+ parameter_count=r["parameter_count"],
1701
+ documentation=r["documentation"],
1702
+ )
1703
+ for r in sym_rows
1704
+ ]
1705
+
1706
+ # 3. Load imports
1707
+ imp_rows = con.execute(
1708
+ "SELECT source_path, module, name, alias, full_name, line, "
1709
+ "source_module, imported_module, imported_name, local_name, import_type "
1710
+ "FROM imports"
1711
+ ).fetchall()
1712
+ imports = [
1713
+ ImportRef(
1714
+ module=str(r["module"]),
1715
+ source_file=str(r["source_path"]),
1716
+ name=r["name"],
1717
+ alias=r["alias"],
1718
+ full=r["full_name"],
1719
+ line=int(r["line"]),
1720
+ source_module=str(r["source_module"]),
1721
+ imported_module=str(r["imported_module"]),
1722
+ imported_name=r["imported_name"],
1723
+ local_name=str(r["local_name"]),
1724
+ import_type=str(r["import_type"]),
1725
+ is_reexport=(str(r["import_type"]) == "reexport"),
1726
+ )
1727
+ for r in imp_rows
1728
+ ]
1729
+
1730
+ # 4. Load calls
1731
+ call_rows = con.execute(
1732
+ "SELECT source_path, callee, qualified_callee, line, confidence, source_symbol_id "
1733
+ "FROM calls"
1734
+ ).fetchall()
1735
+ calls = [
1736
+ CallRef(
1737
+ callee=str(r["callee"]),
1738
+ source_file=str(r["source_path"]),
1739
+ confidence=str(r["confidence"]),
1740
+ qualified_callee=r["qualified_callee"],
1741
+ line=int(r["line"]),
1742
+ caller_canonical_id=r["source_symbol_id"],
1743
+ )
1744
+ for r in call_rows
1745
+ ]
1746
+
1747
+ # 5. Load inheritance
1748
+ inh_rows = con.execute(
1749
+ "SELECT source_symbol, base_name, relationship, source_file, line, source_canonical_id "
1750
+ "FROM inheritance"
1751
+ ).fetchall()
1752
+ inheritance = [
1753
+ InheritanceRef(
1754
+ source_symbol=str(r["source_symbol"]),
1755
+ base_name=str(r["base_name"]),
1756
+ relationship=str(r["relationship"]),
1757
+ source_file=str(r["source_file"]),
1758
+ line=int(r["line"]),
1759
+ source_canonical_id=str(r["source_canonical_id"]),
1760
+ )
1761
+ for r in inh_rows
1762
+ ]
1763
+ if telemetry is not None:
1764
+ telemetry.record(
1765
+ "ranking_index_building",
1766
+ time.perf_counter() - t_prep,
1767
+ files_processed=len(known_files),
1768
+ items_processed=len(symbols) + len(imports) + len(calls),
1769
+ )
1770
+
1771
+ # 6. Load raw framework routes, router mounts, and router definitions for composition
1772
+ t_fw = time.perf_counter()
1773
+ from codegraph.frameworks import RouterDefinition, RouterMountDetection
1774
+ from codegraph.route_composer import compose_routes
1775
+
1776
+ raw_rt_rows = con.execute(
1777
+ "SELECT endpoint_id, framework, http_method, route_path, normalized_route, "
1778
+ "handler_name, handler_canonical_id, file_path, line, evidence, confidence, router_name "
1779
+ "FROM raw_framework_routes"
1780
+ ).fetchall()
1781
+ raw_routes = [
1782
+ RouteDetection(
1783
+ framework=str(r["framework"]),
1784
+ http_method=str(r["http_method"]),
1785
+ route_path=str(r["route_path"]),
1786
+ normalized_route=str(r["normalized_route"]),
1787
+ handler_name=str(r["handler_name"]),
1788
+ handler_canonical_id=str(r["handler_canonical_id"]),
1789
+ file_path=str(r["file_path"]),
1790
+ line=int(r["line"]),
1791
+ evidence=str(r["evidence"]),
1792
+ confidence=str(r["confidence"]),
1793
+ endpoint_id_override=str(r["endpoint_id"]),
1794
+ router_name=str(r["router_name"]) if "router_name" in r.keys() else "",
1795
+ )
1796
+ for r in raw_rt_rows
1797
+ ]
1798
+
1799
+ mount_rows = con.execute(
1800
+ "SELECT framework, parent_router, child_router, prefix, file_path, line, evidence, confidence "
1801
+ "FROM router_mounts"
1802
+ ).fetchall()
1803
+ mount_detections = [
1804
+ RouterMountDetection(
1805
+ framework=str(r["framework"]),
1806
+ parent_router=str(r["parent_router"]),
1807
+ child_router=str(r["child_router"]),
1808
+ prefix=str(r["prefix"]),
1809
+ file_path=str(r["file_path"]),
1810
+ line=int(r["line"]),
1811
+ evidence=str(r["evidence"]),
1812
+ confidence=str(r["confidence"]),
1813
+ )
1814
+ for r in mount_rows
1815
+ ]
1816
+
1817
+ rdef_rows = con.execute(
1818
+ "SELECT framework, router_name, prefix, file_path, line, evidence "
1819
+ "FROM router_definitions"
1820
+ ).fetchall()
1821
+ router_defs = [
1822
+ RouterDefinition(
1823
+ framework=str(r["framework"]),
1824
+ router_name=str(r["router_name"]),
1825
+ prefix=str(r["prefix"]),
1826
+ file_path=str(r["file_path"]),
1827
+ line=int(r["line"]),
1828
+ evidence=str(r["evidence"]),
1829
+ )
1830
+ for r in rdef_rows
1831
+ ]
1832
+
1833
+ # Compose routes across hierarchical router mounts
1834
+ composed_routes, verified_mount_edges = compose_routes(
1835
+ raw_routes=raw_routes,
1836
+ mounts=mount_detections,
1837
+ router_defs=router_defs,
1838
+ imports=imports,
1839
+ known_files=known_files,
1840
+ )
1841
+ if telemetry is not None:
1842
+ telemetry.record(
1843
+ "framework_analysis",
1844
+ time.perf_counter() - t_fw,
1845
+ files_processed=len(known_files),
1846
+ items_processed=len(composed_routes) + len(verified_mount_edges),
1847
+ )
1848
+
1849
+ # Repopulate framework_routes with the composed routes
1850
+ t_sql = time.perf_counter()
1851
+ con.execute("DELETE FROM framework_routes")
1852
+ if composed_routes:
1853
+ con.executemany(
1854
+ "INSERT INTO framework_routes(endpoint_id, framework, http_method, route_path, "
1855
+ "normalized_route, handler_name, handler_canonical_id, file_path, line, evidence, confidence) "
1856
+ "VALUES (?,?,?,?,?,?,?,?,?,?,?)",
1857
+ [
1858
+ (
1859
+ r.endpoint_id,
1860
+ r.framework,
1861
+ r.http_method,
1862
+ r.route_path,
1863
+ r.normalized_route,
1864
+ r.handler_name,
1865
+ r.handler_canonical_id,
1866
+ r.file_path,
1867
+ r.line,
1868
+ redact_secrets(r.evidence),
1869
+ r.confidence,
1870
+ )
1871
+ for r in composed_routes
1872
+ ],
1873
+ )
1874
+ if telemetry is not None:
1875
+ telemetry.record(
1876
+ "sqlite_writes",
1877
+ time.perf_counter() - t_sql,
1878
+ items_processed=len(composed_routes),
1879
+ db_writes=len(composed_routes) + 1,
1880
+ )
1881
+
1882
+ # 7. Load local bindings for data-flow and alias resolution
1883
+ t_sym_res = time.perf_counter()
1884
+ binding_rows = con.execute(
1885
+ "SELECT target_name, file_path, line, column, scope, expr_kind, source_expr, "
1886
+ "is_conditional, base_expr, attr_name, dict_entries_json, list_entries_json, "
1887
+ "subscript_target, subscript_key, subscript_index "
1888
+ "FROM local_bindings"
1889
+ ).fetchall()
1890
+ bindings = [
1891
+ BindingRef(
1892
+ target_name=str(r["target_name"]),
1893
+ file_path=str(r["file_path"]),
1894
+ line=int(r["line"]),
1895
+ column=r["column"],
1896
+ scope=str(r["scope"]),
1897
+ expr_kind=str(r["expr_kind"]),
1898
+ source_expr=str(r["source_expr"]),
1899
+ is_conditional=bool(r["is_conditional"]),
1900
+ base_expr=str(r["base_expr"]),
1901
+ attr_name=str(r["attr_name"]),
1902
+ dict_entries=tuple(tuple(x) for x in json.loads(r["dict_entries_json"])),
1903
+ list_entries=tuple(json.loads(r["list_entries_json"])),
1904
+ subscript_target=r["subscript_target"],
1905
+ subscript_key=r["subscript_key"],
1906
+ subscript_index=r["subscript_index"],
1907
+ )
1908
+ for r in binding_rows
1909
+ ]
1910
+
1911
+ from codegraph.monorepo import detect_workspace
1912
+ workspace = detect_workspace(self.repository, con=con)
1913
+
1914
+ from codegraph.tsconfig import TsConfigResolver
1915
+ tsconfig_resolver = TsConfigResolver.load_from_repo(self.repository)
1916
+
1917
+ # Run resolution engine with composed routes, mount edges, bindings, workspace, and tsconfig
1918
+ resolver = ReferenceResolver(
1919
+ symbols=symbols,
1920
+ imports=imports,
1921
+ calls=calls,
1922
+ inheritance=inheritance,
1923
+ routes=composed_routes,
1924
+ mount_edges=verified_mount_edges,
1925
+ known_files=known_files,
1926
+ file_hashes=file_hashes,
1927
+ indexed_commit=indexed_commit,
1928
+ bindings=bindings,
1929
+ workspace=workspace,
1930
+ tsconfig_resolver=tsconfig_resolver,
1931
+ repository=self.repository,
1932
+ max_reexport_depth=self.max_reexport_depth,
1933
+ max_wildcard_expansions=self.max_wildcard_expansions,
1934
+ )
1935
+ if telemetry is not None:
1936
+ telemetry.record(
1937
+ "symbol_resolution",
1938
+ time.perf_counter() - t_sym_res,
1939
+ files_processed=len(known_files),
1940
+ items_processed=len(symbols) + len(bindings),
1941
+ )
1942
+ if progress_callback is not None:
1943
+ progress_callback("symbol_resolution", len(known_files), len(known_files), telemetry)
1944
+
1945
+ t_rel_res = time.perf_counter()
1946
+ output = resolver.resolve_all()
1947
+ self.last_epistemic_findings = list(output.findings)
1948
+
1949
+ # Update framework_routes handler_canonical_id if resolved via local bindings / imports
1950
+ route_updates: list[tuple[str, str]] = []
1951
+ for rt in composed_routes:
1952
+ handler_sym, _conf, _reason = resolver.resolve_identifier_in_file(
1953
+ rt.handler_name, rt.file_path, use_line=rt.line
1954
+ )
1955
+ if handler_sym and handler_sym.canonical_id != rt.handler_canonical_id:
1956
+ route_updates.append((handler_sym.canonical_id, rt.endpoint_id))
1957
+ if route_updates:
1958
+ con.executemany(
1959
+ "UPDATE framework_routes SET handler_canonical_id=? WHERE endpoint_id=?",
1960
+ route_updates,
1961
+ )
1962
+
1963
+ num_symbols = len(symbols)
1964
+ # Release in-memory IR lists and resolver caches before DB pass & SQL serialization
1965
+ symbols.clear()
1966
+ imports.clear()
1967
+ calls.clear()
1968
+ inheritance.clear()
1969
+ bindings.clear()
1970
+ del resolver
1971
+
1972
+ if telemetry is not None:
1973
+ budget_skips = sum(
1974
+ 1 for f in output.findings if f.get("reason") == "resolution_budget_exceeded"
1975
+ )
1976
+ telemetry.record(
1977
+ "relationship_resolution",
1978
+ time.perf_counter() - t_rel_res,
1979
+ files_processed=len(known_files),
1980
+ items_processed=len(output.references) + len(output.resolved_imports) + len(output.resolved_calls),
1981
+ errors_skips=budget_skips,
1982
+ db_writes=len(route_updates),
1983
+ )
1984
+ if progress_callback is not None:
1985
+ progress_callback("relationship_resolution", len(known_files), len(known_files), telemetry)
1986
+
1987
+ # Run repository-wide Database Deep Intelligence extraction (Phases 1-8)
1988
+ with (telemetry.phase("database_analysis") if telemetry is not None else contextmanager(lambda: iter([None]))()) as p_db:
1989
+ db_edges = self._run_database_intelligence_pass(
1990
+ con,
1991
+ sorted(known_files),
1992
+ file_langs,
1993
+ dirty_files=dirty_files,
1994
+ removed_files=removed_files,
1995
+ )
1996
+ if p_db is not None:
1997
+ p_db.files_processed = len(dirty_files) if dirty_files is not None else len(known_files)
1998
+ p_db.items_processed = len(db_edges)
1999
+ if telemetry is not None and progress_callback is not None:
2000
+ progress_callback("database_analysis", len(known_files), len(known_files), telemetry)
2001
+
2002
+ t_edges = time.perf_counter()
2003
+ all_edges: list[Any] = [*output.edges, *db_edges]
2004
+ output.edges.clear()
2005
+ db_edges.clear()
2006
+ total_edges = len(all_edges)
2007
+ total_refs = len(output.references)
2008
+ if telemetry is not None:
2009
+ telemetry.symbols_count = num_symbols
2010
+ telemetry.edges_count = total_edges
2011
+ telemetry.record(
2012
+ "graph_edge_creation",
2013
+ time.perf_counter() - t_edges,
2014
+ files_processed=len(known_files),
2015
+ items_processed=total_edges,
2016
+ )
2017
+
2018
+ # Batch write references, graph_edges, and resolved imports/calls with bounded WAL checkpoints
2019
+ t_sql_post = time.perf_counter()
2020
+ post_chunk_size = 4000
2021
+
2022
+ def _checkpoint_post() -> None:
2023
+ con.commit()
2024
+ if telemetry is not None:
2025
+ telemetry.batch_commits += 1
2026
+ telemetry.observe_wal(self.db_path)
2027
+ try:
2028
+ con.execute("PRAGMA wal_checkpoint(TRUNCATE)")
2029
+ if telemetry is not None:
2030
+ telemetry.wal_checkpoints += 1
2031
+ except sqlite3.OperationalError:
2032
+ pass
2033
+ if telemetry is not None:
2034
+ telemetry.observe_wal(self.db_path)
2035
+
2036
+ con.execute("DELETE FROM 'references'")
2037
+ for i in range(0, total_refs, post_chunk_size):
2038
+ batch_refs = output.references[i : i + post_chunk_size]
2039
+ con.executemany(
2040
+ "INSERT INTO 'references'("
2041
+ "source_symbol_id, target_symbol_id, relationship, confidence, path, "
2042
+ "start_line, end_line, evidence, source_hash, indexed_commit, evidence_status) "
2043
+ "VALUES (?,?,?,?,?,?,?,?,?,?,?)",
2044
+ [
2045
+ (
2046
+ ref.source_symbol_id,
2047
+ ref.target_symbol_id,
2048
+ ref.relationship,
2049
+ ref.confidence,
2050
+ ref.path,
2051
+ ref.start_line,
2052
+ ref.end_line,
2053
+ redact_secrets(ref.evidence),
2054
+ ref.source_hash,
2055
+ ref.indexed_commit,
2056
+ ref.evidence_status,
2057
+ )
2058
+ for ref in batch_refs
2059
+ ],
2060
+ )
2061
+ if total_refs > post_chunk_size:
2062
+ _checkpoint_post()
2063
+ output.references.clear()
2064
+
2065
+ con.execute("DELETE FROM graph_edges")
2066
+ for i in range(0, total_edges, post_chunk_size):
2067
+ batch_edges = all_edges[i : i + post_chunk_size]
2068
+ con.executemany(
2069
+ "INSERT INTO graph_edges("
2070
+ "source, target, relationship, confidence, file, start_line, end_line, "
2071
+ "evidence, evidence_id, source_hash, indexed_commit, evidence_class, reason) "
2072
+ "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)",
2073
+ [
2074
+ (
2075
+ e.source,
2076
+ e.target,
2077
+ e.relationship,
2078
+ e.confidence,
2079
+ e.file,
2080
+ e.start_line,
2081
+ e.end_line,
2082
+ redact_secrets(e.evidence),
2083
+ f"edge_{i + offset}",
2084
+ file_hashes.get(e.file, ""),
2085
+ indexed_commit,
2086
+ getattr(e, "evidence_class", "AST_VERIFIED"),
2087
+ getattr(e, "reason", None),
2088
+ )
2089
+ for offset, e in enumerate(batch_edges)
2090
+ ],
2091
+ )
2092
+ if total_edges > post_chunk_size:
2093
+ _checkpoint_post()
2094
+ all_edges.clear()
2095
+
2096
+ # Batch update resolved fields on imports and calls tables using composite indexes
2097
+ num_res_imports = len(output.resolved_imports)
2098
+ if output.resolved_imports:
2099
+ imp_updates = [
2100
+ (tgt_path, tgt_mod, src_path, ln, loc_name, loc_name, loc_name)
2101
+ for (src_path, ln, loc_name), (tgt_path, tgt_mod) in output.resolved_imports.items()
2102
+ ]
2103
+ output.resolved_imports.clear()
2104
+ for i in range(0, len(imp_updates), post_chunk_size):
2105
+ con.executemany(
2106
+ "UPDATE imports SET resolved_path=?, resolved_module=? "
2107
+ "WHERE source_path=? AND line=? AND (local_name=? OR alias=? OR name=?)",
2108
+ imp_updates[i : i + post_chunk_size],
2109
+ )
2110
+ if len(imp_updates) > post_chunk_size:
2111
+ _checkpoint_post()
2112
+
2113
+ num_res_calls = len(output.resolved_calls)
2114
+ if output.resolved_calls:
2115
+ call_updates = [
2116
+ (resolved_id, conf, src_path, ln, callee_nm)
2117
+ for (src_path, ln, callee_nm), (resolved_id, conf) in output.resolved_calls.items()
2118
+ ]
2119
+ output.resolved_calls.clear()
2120
+ for i in range(0, len(call_updates), post_chunk_size):
2121
+ con.executemany(
2122
+ "UPDATE calls SET resolved_symbol_id=?, confidence=? "
2123
+ "WHERE source_path=? AND line=? AND callee=?",
2124
+ call_updates[i : i + post_chunk_size],
2125
+ )
2126
+ if len(call_updates) > post_chunk_size:
2127
+ _checkpoint_post()
2128
+
2129
+ if telemetry is not None:
2130
+ post_writes = (
2131
+ total_refs
2132
+ + total_edges
2133
+ + num_res_imports
2134
+ + num_res_calls
2135
+ )
2136
+ telemetry.record(
2137
+ "sqlite_writes",
2138
+ time.perf_counter() - t_sql_post,
2139
+ items_processed=post_writes,
2140
+ db_writes=post_writes,
2141
+ )
2142
+
2143
+ def _run_database_intelligence_pass(
2144
+ self,
2145
+ con: sqlite3.Connection,
2146
+ sorted_files: list[str],
2147
+ file_langs: dict[str, str],
2148
+ *,
2149
+ dirty_files: set[str] | None = None,
2150
+ removed_files: set[str] | None = None,
2151
+ ) -> list[RelationshipRecord]:
2152
+ """Extract ORM models, tables, columns, queries, migrations, and database edges across the repo.
2153
+
2154
+ Streams file contents instead of holding all repository files in memory at once,
2155
+ and supports incremental dirty-file extraction when the repository dialect and
2156
+ cross-file ORM model->table mappings are unchanged.
2157
+ """
2158
+ from codegraph.database import (
2159
+ DatabaseEntity,
2160
+ DatabaseQueryFact,
2161
+ MigrationFact,
2162
+ detect_file_dialects,
2163
+ extract_database_from_file,
2164
+ has_potential_database_activity,
2165
+ has_potential_orm_models,
2166
+ )
2167
+ from codegraph.security.redaction import redact_secrets
2168
+
2169
+ def _compute_repo_dialect(dialects_map: dict[str, list[str]]) -> str:
2170
+ union_dialects: set[str] = set()
2171
+ for dlist in dialects_map.values():
2172
+ union_dialects.update(dlist)
2173
+ if len(union_dialects) == 1:
2174
+ return next(iter(union_dialects))
2175
+ if len(union_dialects) > 1:
2176
+ return "AMBIGUOUS"
2177
+ return "UNKNOWN"
2178
+
2179
+ def _flatten_model_tables(models_map: dict[str, dict[str, str]]) -> dict[str, str]:
2180
+ flat: dict[str, str] = {}
2181
+ for fpath in sorted(models_map):
2182
+ flat.update(models_map[fpath])
2183
+ return flat
2184
+
2185
+ def _insert_db_rows(
2186
+ entities: list[DatabaseEntity],
2187
+ queries: list[DatabaseQueryFact],
2188
+ migrations: list[MigrationFact],
2189
+ ) -> None:
2190
+ if entities:
2191
+ con.executemany(
2192
+ "INSERT INTO db_entities("
2193
+ "canonical_id, kind, name, dialect, schema_name, table_name, column_name, "
2194
+ "data_type, nullable, default_value, is_primary_key, is_foreign_key, is_unique, "
2195
+ "is_indexed, target_table, target_column, orm_model_id, framework, file_path, "
2196
+ "start_line, end_line, evidence, confidence, evidence_class, status, metadata_json) "
2197
+ "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
2198
+ [
2199
+ (
2200
+ e.canonical_id,
2201
+ e.kind,
2202
+ e.name,
2203
+ e.dialect,
2204
+ e.schema_name,
2205
+ e.table_name,
2206
+ e.column_name,
2207
+ e.data_type,
2208
+ 1 if e.nullable else 0,
2209
+ redact_secrets(e.default_value) if e.default_value is not None else None,
2210
+ 1 if e.is_primary_key else 0,
2211
+ 1 if e.is_foreign_key else 0,
2212
+ 1 if e.is_unique else 0,
2213
+ 1 if e.is_indexed else 0,
2214
+ e.target_table,
2215
+ e.target_column,
2216
+ e.orm_model_id,
2217
+ e.framework,
2218
+ e.file_path,
2219
+ e.start_line,
2220
+ e.end_line,
2221
+ redact_secrets(e.evidence),
2222
+ e.confidence,
2223
+ e.evidence_class,
2224
+ e.status,
2225
+ json.dumps(e.metadata),
2226
+ )
2227
+ for e in entities
2228
+ ],
2229
+ )
2230
+ if queries:
2231
+ con.executemany(
2232
+ "INSERT OR REPLACE INTO db_queries("
2233
+ "query_id, caller_symbol_id, operation, relationship, table_name, table_canonical_id, "
2234
+ "columns_json, framework, normalized_sql, file_path, start_line, end_line, "
2235
+ "evidence, confidence, evidence_class, status) "
2236
+ "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
2237
+ [
2238
+ (
2239
+ q.query_id,
2240
+ q.caller_symbol_id,
2241
+ q.operation,
2242
+ q.relationship,
2243
+ q.table_name,
2244
+ q.table_canonical_id,
2245
+ json.dumps(list(q.columns)),
2246
+ q.framework,
2247
+ redact_secrets(q.normalized_sql),
2248
+ q.file_path,
2249
+ q.start_line,
2250
+ q.end_line,
2251
+ redact_secrets(q.evidence),
2252
+ q.confidence,
2253
+ q.evidence_class,
2254
+ q.status,
2255
+ )
2256
+ for q in queries
2257
+ ],
2258
+ )
2259
+ if migrations:
2260
+ con.executemany(
2261
+ "INSERT OR REPLACE INTO db_migrations("
2262
+ "migration_id, canonical_id, system, revision, down_revision, file_path, "
2263
+ "start_line, end_line, created_tables_json, dropped_tables_json, added_columns_json, "
2264
+ "removed_columns_json, renamed_columns_json, indexes_json, foreign_keys_json, "
2265
+ "constraints_json, affected_tables_json, evidence, confidence, evidence_class) "
2266
+ "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
2267
+ [
2268
+ (
2269
+ m.migration_id,
2270
+ m.canonical_id,
2271
+ m.system,
2272
+ m.revision,
2273
+ m.down_revision,
2274
+ m.file_path,
2275
+ m.start_line,
2276
+ m.end_line,
2277
+ json.dumps(list(m.created_tables)),
2278
+ json.dumps(list(m.dropped_tables)),
2279
+ json.dumps(list(m.added_columns)),
2280
+ json.dumps(list(m.removed_columns)),
2281
+ json.dumps(list(m.renamed_columns)),
2282
+ json.dumps(list(m.indexes)),
2283
+ json.dumps(list(m.foreign_keys)),
2284
+ json.dumps(list(m.constraints)),
2285
+ json.dumps(list(m.affected_tables)),
2286
+ redact_secrets(m.evidence),
2287
+ m.confidence,
2288
+ m.evidence_class,
2289
+ )
2290
+ for m in migrations
2291
+ ],
2292
+ )
2293
+
2294
+ # Attempt fast incremental DB extraction when dirty_files is a small subset
2295
+ can_try_incremental = (
2296
+ dirty_files is not None
2297
+ and len(sorted_files) > 0
2298
+ and 0 < len(dirty_files) + len(removed_files or ()) <= max(64, len(sorted_files) // 5)
2299
+ )
2300
+ if can_try_incremental and dirty_files is not None:
2301
+ prev_dialect_row = con.execute(
2302
+ "SELECT value FROM metadata WHERE key='db_repo_dialect'"
2303
+ ).fetchone()
2304
+ prev_dialects_row = con.execute(
2305
+ "SELECT value FROM metadata WHERE key='db_dialects_by_file_json'"
2306
+ ).fetchone()
2307
+ prev_models_row = con.execute(
2308
+ "SELECT value FROM metadata WHERE key='db_models_by_file_json'"
2309
+ ).fetchone()
2310
+ if prev_dialect_row and prev_dialects_row and prev_models_row:
2311
+ try:
2312
+ prev_repo_dialect = str(prev_dialect_row[0])
2313
+ dialects_by_file: dict[str, list[str]] = json.loads(str(prev_dialects_row[0]))
2314
+ models_by_file: dict[str, dict[str, str]] = json.loads(str(prev_models_row[0]))
2315
+ prev_known_model_tables = _flatten_model_tables(models_by_file)
2316
+
2317
+ changed_or_removed = set(dirty_files) | set(removed_files or ())
2318
+ for rem in changed_or_removed:
2319
+ dialects_by_file.pop(rem, None)
2320
+ models_by_file.pop(rem, None)
2321
+
2322
+ dirty_texts: dict[str, str] = {}
2323
+ for fpath in sorted(dirty_files):
2324
+ abs_p = self.repository / fpath
2325
+ try:
2326
+ text = abs_p.read_text(encoding="utf-8", errors="replace")
2327
+ except OSError:
2328
+ continue
2329
+ dirty_texts[fpath] = text
2330
+ f_dialects = sorted(detect_file_dialects(text, fpath))
2331
+ if f_dialects:
2332
+ dialects_by_file[fpath] = f_dialects
2333
+
2334
+ new_repo_dialect = _compute_repo_dialect(dialects_by_file)
2335
+ for fpath, text in dirty_texts.items():
2336
+ pass1 = extract_database_from_file(
2337
+ text,
2338
+ fpath,
2339
+ language=file_langs.get(fpath, "python"),
2340
+ repo_dialect=new_repo_dialect,
2341
+ )
2342
+ f_models: dict[str, str] = {}
2343
+ for ent in pass1.entities:
2344
+ if ent.kind == "ORMModel" and ent.table_name:
2345
+ f_models[ent.name] = ent.table_name
2346
+ f_models[ent.canonical_id] = ent.table_name
2347
+ if f_models:
2348
+ models_by_file[fpath] = f_models
2349
+
2350
+ new_known_model_tables = _flatten_model_tables(models_by_file)
2351
+ if (
2352
+ new_repo_dialect == prev_repo_dialect
2353
+ and new_known_model_tables == prev_known_model_tables
2354
+ ):
2355
+ # Dialect and ORM model->table mappings are unchanged!
2356
+ # Only update DB records for dirty_files and preserve unchanged files' DB edges.
2357
+ for fpath in changed_or_removed:
2358
+ con.execute("DELETE FROM db_entities WHERE file_path=?", (fpath,))
2359
+ con.execute("DELETE FROM db_queries WHERE file_path=?", (fpath,))
2360
+ con.execute("DELETE FROM db_migrations WHERE file_path=?", (fpath,))
2361
+
2362
+ inc_entities: list[DatabaseEntity] = []
2363
+ inc_queries: list[DatabaseQueryFact] = []
2364
+ inc_migrations: list[MigrationFact] = []
2365
+ inc_edges: list[RelationshipRecord] = []
2366
+
2367
+ for fpath in sorted(dirty_texts):
2368
+ pass2 = extract_database_from_file(
2369
+ dirty_texts[fpath],
2370
+ fpath,
2371
+ language=file_langs.get(fpath, "python"),
2372
+ repo_dialect=new_repo_dialect,
2373
+ known_model_tables=new_known_model_tables,
2374
+ )
2375
+ inc_entities.extend(pass2.entities)
2376
+ inc_queries.extend(pass2.queries)
2377
+ inc_migrations.extend(pass2.migrations)
2378
+ inc_edges.extend(pass2.edges)
2379
+
2380
+ _insert_db_rows(inc_entities, inc_queries, inc_migrations)
2381
+
2382
+ placeholders = ",".join("?" for _ in _DB_EDGE_RELATIONSHIPS)
2383
+ existing_db_edge_rows = con.execute(
2384
+ f"SELECT source, target, relationship, confidence, file, "
2385
+ f"start_line, end_line, evidence, evidence_class, reason "
2386
+ f"FROM graph_edges WHERE relationship IN ({placeholders})",
2387
+ _DB_EDGE_RELATIONSHIPS,
2388
+ ).fetchall()
2389
+ preserved_edges = [
2390
+ RelationshipRecord(
2391
+ source=str(r["source"]),
2392
+ target=str(r["target"]),
2393
+ relationship=str(r["relationship"]),
2394
+ file=str(r["file"]),
2395
+ start_line=int(r["start_line"]),
2396
+ end_line=int(r["end_line"]),
2397
+ evidence=str(r["evidence"]),
2398
+ evidence_class=str(r["evidence_class"]),
2399
+ confidence=str(r["confidence"]),
2400
+ reason=r["reason"],
2401
+ )
2402
+ for r in existing_db_edge_rows
2403
+ if str(r["file"]) not in changed_or_removed
2404
+ ]
2405
+ con.executemany(
2406
+ "INSERT INTO metadata(key, value) VALUES(?,?) "
2407
+ "ON CONFLICT(key) DO UPDATE SET value=excluded.value",
2408
+ [
2409
+ ("db_dialects_by_file_json", json.dumps(dialects_by_file)),
2410
+ ("db_models_by_file_json", json.dumps(models_by_file)),
2411
+ ],
2412
+ )
2413
+ return [*preserved_edges, *inc_edges]
2414
+ except (sqlite3.OperationalError, ValueError, KeyError, TypeError):
2415
+ pass
2416
+
2417
+ # Full streaming pass: never hold all file_contents in RAM at once
2418
+ dialects_by_file_full: dict[str, list[str]] = {}
2419
+ models_by_file_full: dict[str, dict[str, str]] = {}
2420
+ db_candidate_files: list[str] = []
2421
+
2422
+ # Pass 1: stream files to detect dialect and ORM model -> table mappings
2423
+ for fpath in sorted_files:
2424
+ abs_p = self.repository / fpath
2425
+ try:
2426
+ text = abs_p.read_text(encoding="utf-8", errors="replace")
2427
+ except OSError:
2428
+ continue
2429
+ if not text:
2430
+ continue
2431
+ lang = file_langs.get(fpath, "python")
2432
+ f_dialects = sorted(detect_file_dialects(text, fpath))
2433
+ if f_dialects:
2434
+ dialects_by_file_full[fpath] = f_dialects
2435
+
2436
+ if has_potential_database_activity(text, fpath, lang):
2437
+ db_candidate_files.append(fpath)
2438
+
2439
+ if has_potential_orm_models(text, fpath, lang):
2440
+ pass1 = extract_database_from_file(
2441
+ text,
2442
+ fpath,
2443
+ language=lang,
2444
+ repo_dialect="UNKNOWN",
2445
+ )
2446
+ f_models_full: dict[str, str] = {}
2447
+ for ent in pass1.entities:
2448
+ if ent.kind == "ORMModel" and ent.table_name:
2449
+ f_models_full[ent.name] = ent.table_name
2450
+ f_models_full[ent.canonical_id] = ent.table_name
2451
+ if f_models_full:
2452
+ models_by_file_full[fpath] = f_models_full
2453
+
2454
+ repo_dialect = _compute_repo_dialect(dialects_by_file_full)
2455
+ known_model_tables = _flatten_model_tables(models_by_file_full)
2456
+
2457
+ # Pass 2: stream only db_candidate_files (files that actually contain DB constructs)
2458
+ all_entities: list[DatabaseEntity] = []
2459
+ all_queries: list[DatabaseQueryFact] = []
2460
+ all_migrations: list[MigrationFact] = []
2461
+ all_edges: list[RelationshipRecord] = []
2462
+
2463
+ for fpath in db_candidate_files:
2464
+ abs_p = self.repository / fpath
2465
+ try:
2466
+ content = abs_p.read_text(encoding="utf-8", errors="replace")
2467
+ except OSError:
2468
+ continue
2469
+ if not content:
2470
+ continue
2471
+ pass2 = extract_database_from_file(
2472
+ content,
2473
+ fpath,
2474
+ language=file_langs.get(fpath, "python"),
2475
+ repo_dialect=repo_dialect,
2476
+ known_model_tables=known_model_tables,
2477
+ )
2478
+ all_entities.extend(pass2.entities)
2479
+ all_queries.extend(pass2.queries)
2480
+ all_migrations.extend(pass2.migrations)
2481
+ all_edges.extend(pass2.edges)
2482
+
2483
+ con.execute("DELETE FROM db_entities")
2484
+ con.execute("DELETE FROM db_queries")
2485
+ con.execute("DELETE FROM db_migrations")
2486
+ _insert_db_rows(all_entities, all_queries, all_migrations)
2487
+
2488
+ con.executemany(
2489
+ "INSERT INTO metadata(key, value) VALUES(?,?) "
2490
+ "ON CONFLICT(key) DO UPDATE SET value=excluded.value",
2491
+ [
2492
+ ("db_repo_dialect", repo_dialect),
2493
+ ("db_dialects_by_file_json", json.dumps(dialects_by_file_full)),
2494
+ ("db_models_by_file_json", json.dumps(models_by_file_full)),
2495
+ ],
2496
+ )
2497
+
2498
+ return list(all_edges)
2499
+
2500
+
2501
+ def check_database_health(con: sqlite3.Connection, repository: Path | None = None) -> dict[str, object]:
2502
+ """Inspect database integrity, FTS synchronization, orphan records, and consistency."""
2503
+ issues: list[str] = []
2504
+
2505
+ # 1. Foreign keys
2506
+ fk_enabled = con.execute("PRAGMA foreign_keys").fetchone()[0]
2507
+ if not fk_enabled:
2508
+ issues.append("Foreign keys are disabled in SQLite connection.")
2509
+
2510
+ # 2. Schema and user version
2511
+ user_ver = con.execute("PRAGMA user_version").fetchone()[0]
2512
+ if user_ver < SCHEMA_VERSION:
2513
+ issues.append(f"Schema version mismatch: database is v{user_ver}, expected v{SCHEMA_VERSION}.")
2514
+
2515
+ # 3. FTS synchronization check
2516
+ chunks_count = con.execute("SELECT count(*) FROM chunks").fetchone()[0]
2517
+ fts_count = con.execute("SELECT count(*) FROM chunks_fts").fetchone()[0]
2518
+ if chunks_count != fts_count:
2519
+ issues.append(f"FTS desynchronization: chunks table has {chunks_count} rows, chunks_fts has {fts_count} rows.")
2520
+
2521
+ # 4. Orphan symbols
2522
+ orphan_syms = con.execute(
2523
+ "SELECT count(*) FROM symbols s LEFT JOIN files f ON s.path = f.path WHERE f.path IS NULL"
2524
+ ).fetchone()[0]
2525
+ if orphan_syms > 0:
2526
+ issues.append(f"Found {orphan_syms} orphan symbol(s) referencing non-existent files.")
2527
+
2528
+ # 5. Orphan references
2529
+ orphan_refs = con.execute(
2530
+ "SELECT count(*) FROM 'references' r LEFT JOIN files f ON r.path = f.path WHERE f.path IS NULL"
2531
+ ).fetchone()[0]
2532
+ if orphan_refs > 0:
2533
+ issues.append(f"Found {orphan_refs} orphan reference(s) referencing non-existent files.")
2534
+
2535
+ # 6. Duplicate canonical IDs within same file
2536
+ dups = con.execute(
2537
+ "SELECT canonical_id, path, count(*) FROM symbols "
2538
+ "GROUP BY canonical_id, path HAVING count(*) > 1"
2539
+ ).fetchall()
2540
+ if dups:
2541
+ issues.append(f"Found {len(dups)} duplicate canonical symbol ID(s) in same file.")
2542
+
2543
+ # 7. Check parser version
2544
+ parser_ver = None
2545
+ try:
2546
+ prow = con.execute("SELECT value FROM metadata WHERE key='parser_version'").fetchone()
2547
+ if prow:
2548
+ parser_ver = str(prow[0])
2549
+ except sqlite3.OperationalError:
2550
+ pass
2551
+ if parser_ver and parser_ver != PARSER_VERSION:
2552
+ issues.append(f"Parser version mismatch: indexed with {parser_ver}, active parser is {PARSER_VERSION}.")
2553
+
2554
+ # 8. Orphan framework routes
2555
+ orphan_routes = con.execute(
2556
+ "SELECT count(*) FROM framework_routes r LEFT JOIN files f ON r.file_path = f.path WHERE f.path IS NULL"
2557
+ ).fetchone()[0]
2558
+ if orphan_routes > 0:
2559
+ issues.append(f"Found {orphan_routes} orphan route(s) referencing non-existent files.")
2560
+
2561
+ # 9. Duplicate endpoint IDs
2562
+ dup_routes = con.execute(
2563
+ "SELECT endpoint_id, count(*) FROM framework_routes GROUP BY endpoint_id HAVING count(*) > 1"
2564
+ ).fetchall()
2565
+ if dup_routes:
2566
+ issues.append(f"Found {len(dup_routes)} duplicate route endpoint ID(s).")
2567
+
2568
+ status = "OK" if not issues else "ISSUES_FOUND"
2569
+ return {
2570
+ "status": status,
2571
+ "database_version": user_ver,
2572
+ "expected_version": SCHEMA_VERSION,
2573
+ "parser_version": parser_ver or PARSER_VERSION,
2574
+ "chunks_count": chunks_count,
2575
+ "fts_count": fts_count,
2576
+ "foreign_keys": bool(fk_enabled),
2577
+ "issues": issues,
2578
+ }