codegraph-engine 3.0.0__2-py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codegraph/__init__.py +37 -0
- codegraph/agent.py +26 -0
- codegraph/agent_brain.py +2212 -0
- codegraph/agent_capabilities.py +4058 -0
- codegraph/agent_rules.py +531 -0
- codegraph/api_drift.py +455 -0
- codegraph/architecture.py +371 -0
- codegraph/async_queue/__init__.py +50 -0
- codegraph/async_queue/detectors/__init__.py +48 -0
- codegraph/async_queue/detectors/celery.py +160 -0
- codegraph/async_queue/detectors/kafka.py +168 -0
- codegraph/async_queue/detectors/rabbitmq.py +116 -0
- codegraph/async_queue/detectors/redis.py +130 -0
- codegraph/async_queue/detectors/sqs.py +135 -0
- codegraph/async_queue/models.py +161 -0
- codegraph/async_queue/otel_models.py +74 -0
- codegraph/async_queue/otel_parser.py +337 -0
- codegraph/async_queue/reconciler.py +427 -0
- codegraph/async_queue/scanner.py +291 -0
- codegraph/async_queue/traversal.py +447 -0
- codegraph/audit.py +106 -0
- codegraph/binding_resolver.py +616 -0
- codegraph/branch_comparison.py +197 -0
- codegraph/cache.py +95 -0
- codegraph/change_impact.py +547 -0
- codegraph/cli.py +2436 -0
- codegraph/cli_output.py +205 -0
- codegraph/config.py +45 -0
- codegraph/constraints.py +238 -0
- codegraph/context.py +2182 -0
- codegraph/context_freshness.py +351 -0
- codegraph/database/__init__.py +72 -0
- codegraph/database/extractor.py +3651 -0
- codegraph/database/interrogation.py +1494 -0
- codegraph/database/models.py +310 -0
- codegraph/database/schema_drift.py +862 -0
- codegraph/epistemic.py +200 -0
- codegraph/errors.py +282 -0
- codegraph/evidence/__init__.py +15 -0
- codegraph/evidence/citations.py +418 -0
- codegraph/evidence_contract.py +486 -0
- codegraph/frameworks.py +960 -0
- codegraph/frameworks_nestjs.py +502 -0
- codegraph/frameworks_nextjs.py +337 -0
- codegraph/frameworks_react.py +457 -0
- codegraph/freshness.py +335 -0
- codegraph/frontend_html.py +109 -0
- codegraph/git.py +362 -0
- codegraph/git_state.py +545 -0
- codegraph/graph/__init__.py +46 -0
- codegraph/graph/models.py +56 -0
- codegraph/graph/traversal.py +1993 -0
- codegraph/indexing/__init__.py +24 -0
- codegraph/indexing/classifier.py +564 -0
- codegraph/indexing/indexer.py +2578 -0
- codegraph/indexing/models.py +427 -0
- codegraph/indexing/parser.py +3404 -0
- codegraph/indexing/pool.py +136 -0
- codegraph/indexing/scanner.py +211 -0
- codegraph/indexing/telemetry.py +326 -0
- codegraph/indexing/templates.py +418 -0
- codegraph/indexing/test_framework.py +116 -0
- codegraph/installer.py +1702 -0
- codegraph/interrogation.py +2355 -0
- codegraph/llm/__init__.py +3 -0
- codegraph/llm/base.py +15 -0
- codegraph/llm/context.py +20 -0
- codegraph/mcp/__init__.py +3 -0
- codegraph/mcp/server.py +1476 -0
- codegraph/mcp_diagnostics.py +882 -0
- codegraph/memory/__init__.py +3 -0
- codegraph/memory/store.py +46 -0
- codegraph/models.py +310 -0
- codegraph/monorepo.py +925 -0
- codegraph/observability.py +262 -0
- codegraph/optimizer.py +1014 -0
- codegraph/planner.py +417 -0
- codegraph/process_lifecycle.py +1153 -0
- codegraph/py.typed +1 -0
- codegraph/query_expansion.py +199 -0
- codegraph/ranking.py +408 -0
- codegraph/refactor/__init__.py +36 -0
- codegraph/refactor/models.py +114 -0
- codegraph/refactor/renamer.py +844 -0
- codegraph/refactor/transactions.py +414 -0
- codegraph/resolver.py +3968 -0
- codegraph/resources/__init__.py +46 -0
- codegraph/resources/cache.py +117 -0
- codegraph/resources/coalescer.py +83 -0
- codegraph/resources/debouncer.py +98 -0
- codegraph/resources/governor.py +219 -0
- codegraph/resources/monitor.py +102 -0
- codegraph/resources/policy.py +123 -0
- codegraph/retrieval_policy.py +320 -0
- codegraph/roblox.py +2174 -0
- codegraph/route_composer.py +363 -0
- codegraph/route_topology.py +172 -0
- codegraph/runtime/__init__.py +51 -0
- codegraph/runtime/collector.py +155 -0
- codegraph/runtime/db_watcher.py +191 -0
- codegraph/runtime/ingestor.py +692 -0
- codegraph/runtime/models.py +130 -0
- codegraph/runtime/port_inspector.py +304 -0
- codegraph/runtime/reconciliation.py +492 -0
- codegraph/runtime/runner.py +353 -0
- codegraph/runtime/trace_linker.py +175 -0
- codegraph/search/__init__.py +25 -0
- codegraph/search/hybrid.py +846 -0
- codegraph/search/semantic.py +28 -0
- codegraph/security/__init__.py +51 -0
- codegraph/security/paths.py +189 -0
- codegraph/security/redaction.py +633 -0
- codegraph/semantic_conflicts.py +270 -0
- codegraph/semantic_decorators.py +372 -0
- codegraph/structural_diff.py +542 -0
- codegraph/styles.py +144 -0
- codegraph/symbol_history.py +277 -0
- codegraph/target_resolver.py +348 -0
- codegraph/task.py +637 -0
- codegraph/tool_selection_eval.py +1158 -0
- codegraph/tsconfig.py +236 -0
- codegraph/ui/__init__.py +6 -0
- codegraph/ui/server.py +755 -0
- codegraph/watcher.py +323 -0
- codegraph_engine-3.0.0.dist-info/METADATA +868 -0
- codegraph_engine-3.0.0.dist-info/RECORD +130 -0
- codegraph_engine-3.0.0.dist-info/WHEEL +6 -0
- codegraph_engine-3.0.0.dist-info/entry_points.txt +2 -0
- codegraph_engine-3.0.0.dist-info/licenses/LICENSE +21 -0
- codegraph_engine-3.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,2578 @@
|
|
|
1
|
+
"""Incremental SQLite indexer with atomic per-file transactions and reference resolution.
|
|
2
|
+
|
|
3
|
+
Architectural Invariants:
|
|
4
|
+
1. Normalized SQLite storage with PRAGMA foreign_keys = ON and ON DELETE CASCADE.
|
|
5
|
+
2. Explicit FTS5 synchronization on every chunk insertion and deletion.
|
|
6
|
+
3. Parse-failure safety: if a file has syntax errors, its last-known-good index state is
|
|
7
|
+
retained, marked 'parse_failed', and reported in freshness without corrupting the DB.
|
|
8
|
+
4. Reference resolution runs globally after file parsing, turning raw source facts into
|
|
9
|
+
verified references, graph edges, and evidence.
|
|
10
|
+
5. Parser version and schema version participate in cache and index validity.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import hashlib
|
|
15
|
+
import json
|
|
16
|
+
import sqlite3
|
|
17
|
+
import threading
|
|
18
|
+
import time
|
|
19
|
+
from collections.abc import Callable, Iterable, Iterator
|
|
20
|
+
from contextlib import contextmanager
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
24
|
+
from codegraph.config import Settings
|
|
25
|
+
from codegraph.evidence_contract import RelationshipRecord
|
|
26
|
+
from codegraph.frameworks import RouteDetection
|
|
27
|
+
from codegraph.resolver import (
|
|
28
|
+
DEFAULT_MAX_REEXPORT_DEPTH,
|
|
29
|
+
DEFAULT_MAX_WILDCARD_EXPANSIONS,
|
|
30
|
+
ReferenceResolver,
|
|
31
|
+
)
|
|
32
|
+
from codegraph.resources import (
|
|
33
|
+
ResourceGovernor,
|
|
34
|
+
TaskPriority,
|
|
35
|
+
get_global_governor,
|
|
36
|
+
get_graph_cache,
|
|
37
|
+
get_parse_cache,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
from .classifier import classify_file_detailed, make_compat_row
|
|
41
|
+
from .models import (
|
|
42
|
+
BindingRef,
|
|
43
|
+
CallRef,
|
|
44
|
+
ImportRef,
|
|
45
|
+
InheritanceRef,
|
|
46
|
+
Symbol,
|
|
47
|
+
)
|
|
48
|
+
from .parser import PARSER_VERSION, parse
|
|
49
|
+
from .pool import SQLiteConnectionPool
|
|
50
|
+
from .scanner import LANGUAGES, scan
|
|
51
|
+
from .telemetry import IndexingTelemetry, get_process_rss_mb
|
|
52
|
+
|
|
53
|
+
SCHEMA_VERSION = 8
|
|
54
|
+
|
|
55
|
+
SCHEMA = """
|
|
56
|
+
CREATE TABLE IF NOT EXISTS metadata (
|
|
57
|
+
key TEXT PRIMARY KEY,
|
|
58
|
+
value TEXT NOT NULL
|
|
59
|
+
);
|
|
60
|
+
|
|
61
|
+
CREATE TABLE IF NOT EXISTS codegraph_meta (
|
|
62
|
+
key TEXT PRIMARY KEY,
|
|
63
|
+
value TEXT NOT NULL
|
|
64
|
+
);
|
|
65
|
+
|
|
66
|
+
CREATE TABLE IF NOT EXISTS files (
|
|
67
|
+
path TEXT PRIMARY KEY,
|
|
68
|
+
hash TEXT NOT NULL,
|
|
69
|
+
language TEXT NOT NULL,
|
|
70
|
+
indexed_at INTEGER NOT NULL DEFAULT 0,
|
|
71
|
+
status TEXT NOT NULL DEFAULT 'ok',
|
|
72
|
+
parse_error TEXT,
|
|
73
|
+
last_valid_hash TEXT,
|
|
74
|
+
category TEXT NOT NULL DEFAULT 'SOURCE'
|
|
75
|
+
);
|
|
76
|
+
|
|
77
|
+
CREATE TABLE IF NOT EXISTS chunks (
|
|
78
|
+
id INTEGER PRIMARY KEY,
|
|
79
|
+
path TEXT NOT NULL,
|
|
80
|
+
language TEXT NOT NULL,
|
|
81
|
+
symbol TEXT,
|
|
82
|
+
symbol_type TEXT,
|
|
83
|
+
start_line INTEGER NOT NULL,
|
|
84
|
+
end_line INTEGER NOT NULL,
|
|
85
|
+
content TEXT NOT NULL,
|
|
86
|
+
hash TEXT NOT NULL,
|
|
87
|
+
FOREIGN KEY(path) REFERENCES files(path) ON DELETE CASCADE
|
|
88
|
+
);
|
|
89
|
+
|
|
90
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
91
|
+
content, path UNINDEXED, symbol UNINDEXED
|
|
92
|
+
);
|
|
93
|
+
|
|
94
|
+
CREATE TABLE IF NOT EXISTS symbols (
|
|
95
|
+
name TEXT NOT NULL,
|
|
96
|
+
qualified_name TEXT NOT NULL,
|
|
97
|
+
kind TEXT NOT NULL,
|
|
98
|
+
path TEXT NOT NULL,
|
|
99
|
+
start_line INTEGER NOT NULL,
|
|
100
|
+
end_line INTEGER NOT NULL,
|
|
101
|
+
decorators TEXT NOT NULL DEFAULT '',
|
|
102
|
+
id TEXT NOT NULL DEFAULT '',
|
|
103
|
+
canonical_id TEXT NOT NULL DEFAULT '',
|
|
104
|
+
language TEXT NOT NULL DEFAULT 'python',
|
|
105
|
+
module TEXT NOT NULL DEFAULT '',
|
|
106
|
+
scope TEXT NOT NULL DEFAULT '',
|
|
107
|
+
signature TEXT NOT NULL DEFAULT '',
|
|
108
|
+
content_hash TEXT NOT NULL DEFAULT '',
|
|
109
|
+
parent_symbol_id TEXT,
|
|
110
|
+
visibility TEXT NOT NULL DEFAULT 'public',
|
|
111
|
+
return_type TEXT,
|
|
112
|
+
parameter_count INTEGER,
|
|
113
|
+
documentation TEXT,
|
|
114
|
+
FOREIGN KEY(path) REFERENCES files(path) ON DELETE CASCADE
|
|
115
|
+
);
|
|
116
|
+
|
|
117
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_name ON symbols (name);
|
|
118
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_qualified ON symbols (qualified_name);
|
|
119
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_canonical ON symbols (canonical_id);
|
|
120
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_path ON symbols (path);
|
|
121
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_parent ON symbols (parent_symbol_id);
|
|
122
|
+
CREATE INDEX IF NOT EXISTS idx_chunks_path ON chunks (path);
|
|
123
|
+
CREATE INDEX IF NOT EXISTS idx_chunks_symbol ON chunks (symbol);
|
|
124
|
+
CREATE INDEX IF NOT EXISTS idx_chunks_path_symbol ON chunks (path, symbol);
|
|
125
|
+
|
|
126
|
+
CREATE TABLE IF NOT EXISTS imports (
|
|
127
|
+
source_path TEXT NOT NULL,
|
|
128
|
+
module TEXT NOT NULL,
|
|
129
|
+
name TEXT,
|
|
130
|
+
alias TEXT,
|
|
131
|
+
full_name TEXT,
|
|
132
|
+
line INTEGER NOT NULL DEFAULT 1,
|
|
133
|
+
source_module TEXT NOT NULL DEFAULT '',
|
|
134
|
+
imported_module TEXT NOT NULL DEFAULT '',
|
|
135
|
+
imported_name TEXT,
|
|
136
|
+
local_name TEXT NOT NULL DEFAULT '',
|
|
137
|
+
import_type TEXT NOT NULL DEFAULT 'module',
|
|
138
|
+
resolved_path TEXT,
|
|
139
|
+
resolved_module TEXT,
|
|
140
|
+
FOREIGN KEY(source_path) REFERENCES files(path) ON DELETE CASCADE
|
|
141
|
+
);
|
|
142
|
+
|
|
143
|
+
CREATE INDEX IF NOT EXISTS idx_imports_source ON imports (source_path);
|
|
144
|
+
CREATE INDEX IF NOT EXISTS idx_imports_source_line ON imports (source_path, line);
|
|
145
|
+
CREATE INDEX IF NOT EXISTS idx_imports_module ON imports (module);
|
|
146
|
+
CREATE INDEX IF NOT EXISTS idx_imports_imported_module ON imports (imported_module);
|
|
147
|
+
|
|
148
|
+
CREATE TABLE IF NOT EXISTS calls (
|
|
149
|
+
source_path TEXT NOT NULL,
|
|
150
|
+
callee TEXT NOT NULL,
|
|
151
|
+
qualified_callee TEXT,
|
|
152
|
+
line INTEGER NOT NULL DEFAULT 1,
|
|
153
|
+
confidence TEXT NOT NULL DEFAULT 'LOW',
|
|
154
|
+
source_symbol_id TEXT,
|
|
155
|
+
resolved_symbol_id TEXT,
|
|
156
|
+
FOREIGN KEY(source_path) REFERENCES files(path) ON DELETE CASCADE
|
|
157
|
+
);
|
|
158
|
+
|
|
159
|
+
CREATE INDEX IF NOT EXISTS idx_calls_source ON calls (source_path);
|
|
160
|
+
CREATE INDEX IF NOT EXISTS idx_calls_source_line ON calls (source_path, line, callee);
|
|
161
|
+
CREATE INDEX IF NOT EXISTS idx_calls_callee ON calls (callee);
|
|
162
|
+
CREATE INDEX IF NOT EXISTS idx_calls_resolved ON calls (resolved_symbol_id);
|
|
163
|
+
CREATE INDEX IF NOT EXISTS idx_calls_callee_source ON calls (callee, source_path);
|
|
164
|
+
|
|
165
|
+
CREATE TABLE IF NOT EXISTS inheritance (
|
|
166
|
+
source_symbol TEXT NOT NULL,
|
|
167
|
+
base_name TEXT NOT NULL,
|
|
168
|
+
relationship TEXT NOT NULL,
|
|
169
|
+
source_file TEXT NOT NULL,
|
|
170
|
+
line INTEGER NOT NULL DEFAULT 1,
|
|
171
|
+
source_canonical_id TEXT NOT NULL DEFAULT '',
|
|
172
|
+
FOREIGN KEY(source_file) REFERENCES files(path) ON DELETE CASCADE
|
|
173
|
+
);
|
|
174
|
+
|
|
175
|
+
CREATE INDEX IF NOT EXISTS idx_inh_source ON inheritance (source_file);
|
|
176
|
+
|
|
177
|
+
CREATE TABLE IF NOT EXISTS raw_framework_routes (
|
|
178
|
+
endpoint_id TEXT PRIMARY KEY,
|
|
179
|
+
framework TEXT NOT NULL,
|
|
180
|
+
http_method TEXT NOT NULL,
|
|
181
|
+
route_path TEXT NOT NULL,
|
|
182
|
+
normalized_route TEXT NOT NULL,
|
|
183
|
+
handler_name TEXT NOT NULL,
|
|
184
|
+
handler_canonical_id TEXT NOT NULL,
|
|
185
|
+
file_path TEXT NOT NULL,
|
|
186
|
+
line INTEGER NOT NULL,
|
|
187
|
+
evidence TEXT NOT NULL,
|
|
188
|
+
confidence TEXT NOT NULL DEFAULT 'HIGH',
|
|
189
|
+
router_name TEXT NOT NULL DEFAULT '',
|
|
190
|
+
FOREIGN KEY(file_path) REFERENCES files(path) ON DELETE CASCADE
|
|
191
|
+
);
|
|
192
|
+
|
|
193
|
+
CREATE INDEX IF NOT EXISTS idx_raw_routes_file ON raw_framework_routes (file_path);
|
|
194
|
+
CREATE INDEX IF NOT EXISTS idx_raw_routes_path ON raw_framework_routes (route_path);
|
|
195
|
+
|
|
196
|
+
CREATE TABLE IF NOT EXISTS router_mounts (
|
|
197
|
+
mount_id TEXT PRIMARY KEY,
|
|
198
|
+
framework TEXT NOT NULL,
|
|
199
|
+
parent_router TEXT NOT NULL,
|
|
200
|
+
child_router TEXT NOT NULL,
|
|
201
|
+
prefix TEXT NOT NULL,
|
|
202
|
+
file_path TEXT NOT NULL,
|
|
203
|
+
line INTEGER NOT NULL,
|
|
204
|
+
evidence TEXT NOT NULL,
|
|
205
|
+
confidence TEXT NOT NULL DEFAULT 'HIGH',
|
|
206
|
+
FOREIGN KEY(file_path) REFERENCES files(path) ON DELETE CASCADE
|
|
207
|
+
);
|
|
208
|
+
|
|
209
|
+
CREATE INDEX IF NOT EXISTS idx_mounts_file ON router_mounts (file_path);
|
|
210
|
+
|
|
211
|
+
CREATE TABLE IF NOT EXISTS router_definitions (
|
|
212
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
213
|
+
framework TEXT NOT NULL,
|
|
214
|
+
router_name TEXT NOT NULL,
|
|
215
|
+
prefix TEXT NOT NULL,
|
|
216
|
+
file_path TEXT NOT NULL,
|
|
217
|
+
line INTEGER NOT NULL,
|
|
218
|
+
evidence TEXT NOT NULL,
|
|
219
|
+
FOREIGN KEY(file_path) REFERENCES files(path) ON DELETE CASCADE
|
|
220
|
+
);
|
|
221
|
+
|
|
222
|
+
CREATE INDEX IF NOT EXISTS idx_router_defs_file ON router_definitions (file_path);
|
|
223
|
+
|
|
224
|
+
CREATE TABLE IF NOT EXISTS local_bindings (
|
|
225
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
226
|
+
target_name TEXT NOT NULL,
|
|
227
|
+
file_path TEXT NOT NULL,
|
|
228
|
+
line INTEGER NOT NULL,
|
|
229
|
+
column INTEGER,
|
|
230
|
+
scope TEXT NOT NULL DEFAULT '',
|
|
231
|
+
expr_kind TEXT NOT NULL DEFAULT 'IDENTIFIER',
|
|
232
|
+
source_expr TEXT NOT NULL DEFAULT '',
|
|
233
|
+
is_conditional INTEGER NOT NULL DEFAULT 0,
|
|
234
|
+
base_expr TEXT NOT NULL DEFAULT '',
|
|
235
|
+
attr_name TEXT NOT NULL DEFAULT '',
|
|
236
|
+
dict_entries_json TEXT NOT NULL DEFAULT '[]',
|
|
237
|
+
list_entries_json TEXT NOT NULL DEFAULT '[]',
|
|
238
|
+
subscript_target TEXT,
|
|
239
|
+
subscript_key TEXT,
|
|
240
|
+
subscript_index INTEGER,
|
|
241
|
+
FOREIGN KEY(file_path) REFERENCES files(path) ON DELETE CASCADE
|
|
242
|
+
);
|
|
243
|
+
|
|
244
|
+
CREATE INDEX IF NOT EXISTS idx_bindings_file ON local_bindings (file_path);
|
|
245
|
+
CREATE INDEX IF NOT EXISTS idx_bindings_target ON local_bindings (target_name);
|
|
246
|
+
CREATE INDEX IF NOT EXISTS idx_bindings_file_target ON local_bindings (file_path, target_name);
|
|
247
|
+
|
|
248
|
+
CREATE TABLE IF NOT EXISTS framework_routes (
|
|
249
|
+
endpoint_id TEXT PRIMARY KEY,
|
|
250
|
+
framework TEXT NOT NULL,
|
|
251
|
+
http_method TEXT NOT NULL,
|
|
252
|
+
route_path TEXT NOT NULL,
|
|
253
|
+
normalized_route TEXT NOT NULL,
|
|
254
|
+
handler_name TEXT NOT NULL,
|
|
255
|
+
handler_canonical_id TEXT NOT NULL,
|
|
256
|
+
file_path TEXT NOT NULL,
|
|
257
|
+
line INTEGER NOT NULL,
|
|
258
|
+
evidence TEXT NOT NULL,
|
|
259
|
+
confidence TEXT NOT NULL DEFAULT 'HIGH',
|
|
260
|
+
FOREIGN KEY(file_path) REFERENCES files(path) ON DELETE CASCADE
|
|
261
|
+
);
|
|
262
|
+
|
|
263
|
+
CREATE INDEX IF NOT EXISTS idx_routes_file ON framework_routes (file_path);
|
|
264
|
+
CREATE INDEX IF NOT EXISTS idx_routes_handler ON framework_routes (handler_canonical_id);
|
|
265
|
+
CREATE INDEX IF NOT EXISTS idx_routes_path ON framework_routes (route_path);
|
|
266
|
+
|
|
267
|
+
CREATE TABLE IF NOT EXISTS "references" (
|
|
268
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
269
|
+
source_symbol_id TEXT NOT NULL,
|
|
270
|
+
target_symbol_id TEXT,
|
|
271
|
+
relationship TEXT NOT NULL,
|
|
272
|
+
confidence TEXT NOT NULL,
|
|
273
|
+
path TEXT NOT NULL,
|
|
274
|
+
start_line INTEGER NOT NULL,
|
|
275
|
+
end_line INTEGER NOT NULL,
|
|
276
|
+
evidence TEXT NOT NULL DEFAULT '',
|
|
277
|
+
source_hash TEXT NOT NULL DEFAULT '',
|
|
278
|
+
indexed_commit TEXT,
|
|
279
|
+
evidence_status TEXT NOT NULL DEFAULT 'current',
|
|
280
|
+
FOREIGN KEY(path) REFERENCES files(path) ON DELETE CASCADE
|
|
281
|
+
);
|
|
282
|
+
|
|
283
|
+
CREATE INDEX IF NOT EXISTS idx_refs_source ON "references" (source_symbol_id);
|
|
284
|
+
CREATE INDEX IF NOT EXISTS idx_refs_target ON "references" (target_symbol_id);
|
|
285
|
+
CREATE INDEX IF NOT EXISTS idx_refs_path ON "references" (path);
|
|
286
|
+
CREATE INDEX IF NOT EXISTS idx_refs_rel ON "references" (relationship);
|
|
287
|
+
|
|
288
|
+
CREATE TABLE IF NOT EXISTS graph_edges (
|
|
289
|
+
source TEXT NOT NULL,
|
|
290
|
+
target TEXT NOT NULL,
|
|
291
|
+
relationship TEXT NOT NULL,
|
|
292
|
+
confidence TEXT NOT NULL,
|
|
293
|
+
file TEXT NOT NULL,
|
|
294
|
+
start_line INTEGER NOT NULL,
|
|
295
|
+
end_line INTEGER NOT NULL,
|
|
296
|
+
evidence TEXT NOT NULL DEFAULT '',
|
|
297
|
+
evidence_id TEXT NOT NULL DEFAULT '',
|
|
298
|
+
source_hash TEXT NOT NULL DEFAULT '',
|
|
299
|
+
indexed_commit TEXT,
|
|
300
|
+
evidence_class TEXT NOT NULL DEFAULT 'AST_VERIFIED',
|
|
301
|
+
reason TEXT,
|
|
302
|
+
FOREIGN KEY(file) REFERENCES files(path) ON DELETE CASCADE
|
|
303
|
+
);
|
|
304
|
+
|
|
305
|
+
CREATE INDEX IF NOT EXISTS idx_edges_source ON graph_edges (source);
|
|
306
|
+
CREATE INDEX IF NOT EXISTS idx_edges_target ON graph_edges (target);
|
|
307
|
+
CREATE INDEX IF NOT EXISTS idx_edges_rel ON graph_edges (relationship);
|
|
308
|
+
CREATE INDEX IF NOT EXISTS idx_edges_file ON graph_edges (file);
|
|
309
|
+
CREATE INDEX IF NOT EXISTS idx_edges_evidence ON graph_edges (evidence_id);
|
|
310
|
+
CREATE INDEX IF NOT EXISTS idx_edges_target_rel ON graph_edges (target, relationship);
|
|
311
|
+
CREATE INDEX IF NOT EXISTS idx_edges_source_rel ON graph_edges (source, relationship);
|
|
312
|
+
|
|
313
|
+
CREATE TABLE IF NOT EXISTS context_cache (
|
|
314
|
+
cache_key TEXT PRIMARY KEY,
|
|
315
|
+
data TEXT NOT NULL,
|
|
316
|
+
created_at INTEGER NOT NULL
|
|
317
|
+
);
|
|
318
|
+
|
|
319
|
+
CREATE TABLE IF NOT EXISTS db_entities (
|
|
320
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
321
|
+
canonical_id TEXT NOT NULL,
|
|
322
|
+
kind TEXT NOT NULL,
|
|
323
|
+
name TEXT NOT NULL,
|
|
324
|
+
dialect TEXT NOT NULL DEFAULT 'UNKNOWN',
|
|
325
|
+
schema_name TEXT NOT NULL DEFAULT 'UNKNOWN',
|
|
326
|
+
table_name TEXT NOT NULL DEFAULT '',
|
|
327
|
+
column_name TEXT NOT NULL DEFAULT '',
|
|
328
|
+
data_type TEXT NOT NULL DEFAULT '',
|
|
329
|
+
nullable INTEGER NOT NULL DEFAULT 1,
|
|
330
|
+
default_value TEXT,
|
|
331
|
+
is_primary_key INTEGER NOT NULL DEFAULT 0,
|
|
332
|
+
is_foreign_key INTEGER NOT NULL DEFAULT 0,
|
|
333
|
+
is_unique INTEGER NOT NULL DEFAULT 0,
|
|
334
|
+
is_indexed INTEGER NOT NULL DEFAULT 0,
|
|
335
|
+
target_table TEXT,
|
|
336
|
+
target_column TEXT,
|
|
337
|
+
orm_model_id TEXT,
|
|
338
|
+
framework TEXT NOT NULL DEFAULT '',
|
|
339
|
+
file_path TEXT NOT NULL,
|
|
340
|
+
start_line INTEGER NOT NULL DEFAULT 1,
|
|
341
|
+
end_line INTEGER NOT NULL DEFAULT 1,
|
|
342
|
+
evidence TEXT NOT NULL DEFAULT '',
|
|
343
|
+
confidence TEXT NOT NULL DEFAULT 'HIGH',
|
|
344
|
+
evidence_class TEXT NOT NULL DEFAULT 'AST_VERIFIED',
|
|
345
|
+
status TEXT NOT NULL DEFAULT 'FACT',
|
|
346
|
+
metadata_json TEXT NOT NULL DEFAULT '{}',
|
|
347
|
+
FOREIGN KEY(file_path) REFERENCES files(path) ON DELETE CASCADE
|
|
348
|
+
);
|
|
349
|
+
|
|
350
|
+
CREATE INDEX IF NOT EXISTS idx_db_entities_canonical ON db_entities (canonical_id);
|
|
351
|
+
CREATE INDEX IF NOT EXISTS idx_db_entities_kind ON db_entities (kind);
|
|
352
|
+
CREATE INDEX IF NOT EXISTS idx_db_entities_table ON db_entities (table_name);
|
|
353
|
+
CREATE INDEX IF NOT EXISTS idx_db_entities_file ON db_entities (file_path);
|
|
354
|
+
|
|
355
|
+
CREATE TABLE IF NOT EXISTS db_queries (
|
|
356
|
+
query_id TEXT PRIMARY KEY,
|
|
357
|
+
caller_symbol_id TEXT NOT NULL,
|
|
358
|
+
operation TEXT NOT NULL,
|
|
359
|
+
relationship TEXT NOT NULL,
|
|
360
|
+
table_name TEXT NOT NULL,
|
|
361
|
+
table_canonical_id TEXT NOT NULL,
|
|
362
|
+
columns_json TEXT NOT NULL DEFAULT '[]',
|
|
363
|
+
framework TEXT NOT NULL DEFAULT 'raw_sql',
|
|
364
|
+
normalized_sql TEXT NOT NULL DEFAULT '',
|
|
365
|
+
file_path TEXT NOT NULL,
|
|
366
|
+
start_line INTEGER NOT NULL DEFAULT 1,
|
|
367
|
+
end_line INTEGER NOT NULL DEFAULT 1,
|
|
368
|
+
evidence TEXT NOT NULL DEFAULT '',
|
|
369
|
+
confidence TEXT NOT NULL DEFAULT 'HIGH',
|
|
370
|
+
evidence_class TEXT NOT NULL DEFAULT 'AST_VERIFIED',
|
|
371
|
+
status TEXT NOT NULL DEFAULT 'FACT',
|
|
372
|
+
FOREIGN KEY(file_path) REFERENCES files(path) ON DELETE CASCADE
|
|
373
|
+
);
|
|
374
|
+
|
|
375
|
+
CREATE INDEX IF NOT EXISTS idx_db_queries_caller ON db_queries (caller_symbol_id);
|
|
376
|
+
CREATE INDEX IF NOT EXISTS idx_db_queries_table ON db_queries (table_name);
|
|
377
|
+
CREATE INDEX IF NOT EXISTS idx_db_queries_op ON db_queries (operation);
|
|
378
|
+
CREATE INDEX IF NOT EXISTS idx_db_queries_file ON db_queries (file_path);
|
|
379
|
+
|
|
380
|
+
CREATE TABLE IF NOT EXISTS db_migrations (
|
|
381
|
+
migration_id TEXT PRIMARY KEY,
|
|
382
|
+
canonical_id TEXT NOT NULL,
|
|
383
|
+
system TEXT NOT NULL,
|
|
384
|
+
revision TEXT NOT NULL,
|
|
385
|
+
down_revision TEXT,
|
|
386
|
+
file_path TEXT NOT NULL,
|
|
387
|
+
start_line INTEGER NOT NULL DEFAULT 1,
|
|
388
|
+
end_line INTEGER NOT NULL DEFAULT 1,
|
|
389
|
+
created_tables_json TEXT NOT NULL DEFAULT '[]',
|
|
390
|
+
dropped_tables_json TEXT NOT NULL DEFAULT '[]',
|
|
391
|
+
added_columns_json TEXT NOT NULL DEFAULT '[]',
|
|
392
|
+
removed_columns_json TEXT NOT NULL DEFAULT '[]',
|
|
393
|
+
renamed_columns_json TEXT NOT NULL DEFAULT '[]',
|
|
394
|
+
indexes_json TEXT NOT NULL DEFAULT '[]',
|
|
395
|
+
foreign_keys_json TEXT NOT NULL DEFAULT '[]',
|
|
396
|
+
constraints_json TEXT NOT NULL DEFAULT '[]',
|
|
397
|
+
affected_tables_json TEXT NOT NULL DEFAULT '[]',
|
|
398
|
+
evidence TEXT NOT NULL DEFAULT '',
|
|
399
|
+
confidence TEXT NOT NULL DEFAULT 'HIGH',
|
|
400
|
+
evidence_class TEXT NOT NULL DEFAULT 'AST_VERIFIED',
|
|
401
|
+
FOREIGN KEY(file_path) REFERENCES files(path) ON DELETE CASCADE
|
|
402
|
+
);
|
|
403
|
+
|
|
404
|
+
CREATE INDEX IF NOT EXISTS idx_db_migrations_file ON db_migrations (file_path);
|
|
405
|
+
CREATE INDEX IF NOT EXISTS idx_db_migrations_rev ON db_migrations (revision);
|
|
406
|
+
|
|
407
|
+
CREATE TABLE IF NOT EXISTS runtime_observations (
|
|
408
|
+
event_id TEXT PRIMARY KEY,
|
|
409
|
+
trace_id TEXT NOT NULL,
|
|
410
|
+
span_id TEXT NOT NULL,
|
|
411
|
+
parent_span_id TEXT NOT NULL DEFAULT '',
|
|
412
|
+
timestamp TEXT NOT NULL DEFAULT '',
|
|
413
|
+
source_format TEXT NOT NULL DEFAULT 'json',
|
|
414
|
+
http_method TEXT NOT NULL DEFAULT '',
|
|
415
|
+
route_path TEXT NOT NULL DEFAULT '',
|
|
416
|
+
handler_symbol TEXT NOT NULL DEFAULT '',
|
|
417
|
+
caller_symbol TEXT NOT NULL DEFAULT '',
|
|
418
|
+
callee_symbol TEXT NOT NULL DEFAULT '',
|
|
419
|
+
service_name TEXT NOT NULL DEFAULT '',
|
|
420
|
+
db_operation TEXT NOT NULL DEFAULT '',
|
|
421
|
+
db_table TEXT NOT NULL DEFAULT '',
|
|
422
|
+
db_columns_json TEXT NOT NULL DEFAULT '[]',
|
|
423
|
+
normalized_sql TEXT NOT NULL DEFAULT '',
|
|
424
|
+
status_code INTEGER,
|
|
425
|
+
exception_type TEXT NOT NULL DEFAULT '',
|
|
426
|
+
duration_ms REAL NOT NULL DEFAULT 0.0,
|
|
427
|
+
evidence_class TEXT NOT NULL DEFAULT 'RUNTIME_OBSERVED',
|
|
428
|
+
runtime_generation INTEGER NOT NULL DEFAULT 1
|
|
429
|
+
);
|
|
430
|
+
|
|
431
|
+
CREATE INDEX IF NOT EXISTS idx_rt_obs_trace ON runtime_observations (trace_id);
|
|
432
|
+
CREATE INDEX IF NOT EXISTS idx_rt_obs_route ON runtime_observations (route_path);
|
|
433
|
+
CREATE INDEX IF NOT EXISTS idx_rt_obs_caller ON runtime_observations (caller_symbol);
|
|
434
|
+
CREATE INDEX IF NOT EXISTS idx_rt_obs_table ON runtime_observations (db_table);
|
|
435
|
+
|
|
436
|
+
CREATE TABLE IF NOT EXISTS runtime_edges (
|
|
437
|
+
edge_id TEXT PRIMARY KEY,
|
|
438
|
+
source TEXT NOT NULL,
|
|
439
|
+
target TEXT NOT NULL,
|
|
440
|
+
relationship TEXT NOT NULL,
|
|
441
|
+
operation TEXT NOT NULL DEFAULT '',
|
|
442
|
+
evidence_class TEXT NOT NULL DEFAULT 'RUNTIME_OBSERVED',
|
|
443
|
+
observation_count INTEGER NOT NULL DEFAULT 1,
|
|
444
|
+
first_seen TEXT NOT NULL DEFAULT '',
|
|
445
|
+
last_seen TEXT NOT NULL DEFAULT '',
|
|
446
|
+
avg_duration_ms REAL NOT NULL DEFAULT 0.0,
|
|
447
|
+
max_duration_ms REAL NOT NULL DEFAULT 0.0,
|
|
448
|
+
sample_trace_id TEXT NOT NULL DEFAULT '',
|
|
449
|
+
sample_span_id TEXT NOT NULL DEFAULT '',
|
|
450
|
+
normalized_sql TEXT NOT NULL DEFAULT '',
|
|
451
|
+
status_code INTEGER,
|
|
452
|
+
exception_type TEXT NOT NULL DEFAULT '',
|
|
453
|
+
runtime_generation INTEGER NOT NULL DEFAULT 1
|
|
454
|
+
);
|
|
455
|
+
|
|
456
|
+
CREATE INDEX IF NOT EXISTS idx_rt_edges_source ON runtime_edges (source);
|
|
457
|
+
CREATE INDEX IF NOT EXISTS idx_rt_edges_target ON runtime_edges (target);
|
|
458
|
+
CREATE INDEX IF NOT EXISTS idx_rt_edges_rel ON runtime_edges (relationship);
|
|
459
|
+
"""
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
_DB_EDGE_RELATIONSHIPS = (
|
|
463
|
+
"MAPS_TO_TABLE",
|
|
464
|
+
"FOREIGN_KEY_TO",
|
|
465
|
+
"READS_TABLE",
|
|
466
|
+
"WRITES_TABLE",
|
|
467
|
+
"UPDATES_TABLE",
|
|
468
|
+
"DELETES_FROM_TABLE",
|
|
469
|
+
"REFERENCES_COLUMN",
|
|
470
|
+
"MIGRATES_TABLE",
|
|
471
|
+
)
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
class Indexer:
|
|
475
|
+
def __init__(
|
|
476
|
+
self,
|
|
477
|
+
repository: Path,
|
|
478
|
+
settings: Settings | None = None,
|
|
479
|
+
governor: ResourceGovernor | None = None,
|
|
480
|
+
*,
|
|
481
|
+
commit_batch_size: int = 500,
|
|
482
|
+
max_reexport_depth: int = DEFAULT_MAX_REEXPORT_DEPTH,
|
|
483
|
+
max_wildcard_expansions: int = DEFAULT_MAX_WILDCARD_EXPANSIONS,
|
|
484
|
+
) -> None:
|
|
485
|
+
self.repository = repository.resolve(strict=True)
|
|
486
|
+
self.settings = settings or Settings()
|
|
487
|
+
self.db_path = self.settings.db_path or self.repository / ".codegraph.sqlite3"
|
|
488
|
+
self.governor = governor or get_global_governor()
|
|
489
|
+
self.commit_batch_size = max(1, commit_batch_size)
|
|
490
|
+
self.max_reexport_depth = max_reexport_depth
|
|
491
|
+
self.max_wildcard_expansions = max_wildcard_expansions
|
|
492
|
+
self.last_epistemic_findings: list[dict[str, object]] = []
|
|
493
|
+
self.last_telemetry: IndexingTelemetry | None = None
|
|
494
|
+
self._pool: SQLiteConnectionPool | None = None
|
|
495
|
+
self._schema_lock = threading.Lock()
|
|
496
|
+
self._schema_verified = False
|
|
497
|
+
|
|
498
|
+
@property
|
|
499
|
+
def pool(self) -> SQLiteConnectionPool:
|
|
500
|
+
if self._pool is None:
|
|
501
|
+
self._pool = SQLiteConnectionPool(
|
|
502
|
+
self.db_path,
|
|
503
|
+
max_size=16,
|
|
504
|
+
timeout=30.0,
|
|
505
|
+
initializer=self._ensure_schema,
|
|
506
|
+
)
|
|
507
|
+
return self._pool
|
|
508
|
+
|
|
509
|
+
def connect(self) -> sqlite3.Connection:
|
|
510
|
+
"""Create a dedicated unpooled connection (for indexing or long-lived writes)."""
|
|
511
|
+
con = sqlite3.connect(
|
|
512
|
+
self.db_path,
|
|
513
|
+
check_same_thread=False,
|
|
514
|
+
timeout=30.0,
|
|
515
|
+
)
|
|
516
|
+
con.row_factory = make_compat_row
|
|
517
|
+
con.execute("PRAGMA journal_mode = WAL")
|
|
518
|
+
con.execute("PRAGMA synchronous = NORMAL")
|
|
519
|
+
con.execute("PRAGMA foreign_keys = ON")
|
|
520
|
+
con.execute("PRAGMA busy_timeout = 30000")
|
|
521
|
+
con.execute("PRAGMA cache_size = -64000")
|
|
522
|
+
con.execute("PRAGMA temp_store = MEMORY")
|
|
523
|
+
self._ensure_schema(con)
|
|
524
|
+
return con
|
|
525
|
+
|
|
526
|
+
@contextmanager
|
|
527
|
+
def session(self) -> Iterator[sqlite3.Connection]:
|
|
528
|
+
"""Borrow a pooled connection; commit on success, rollback on failure, return to pool."""
|
|
529
|
+
con, is_owner = self.pool.acquire()
|
|
530
|
+
try:
|
|
531
|
+
yield con
|
|
532
|
+
self.pool.release(con, is_owner=is_owner, rollback=False)
|
|
533
|
+
except BaseException:
|
|
534
|
+
self.pool.release(con, is_owner=is_owner, rollback=True)
|
|
535
|
+
raise
|
|
536
|
+
|
|
537
|
+
def reset_pool(self) -> None:
|
|
538
|
+
"""Close pooled connections and clear schema verification flag."""
|
|
539
|
+
if self._pool is not None:
|
|
540
|
+
self._pool.close_all()
|
|
541
|
+
self._pool = None
|
|
542
|
+
self._schema_verified = False
|
|
543
|
+
|
|
544
|
+
def close(self) -> None:
|
|
545
|
+
"""Dispose of resources and close pooled connections."""
|
|
546
|
+
self.reset_pool()
|
|
547
|
+
|
|
548
|
+
def __del__(self) -> None:
|
|
549
|
+
try:
|
|
550
|
+
self.close()
|
|
551
|
+
except Exception:
|
|
552
|
+
pass
|
|
553
|
+
|
|
554
|
+
def _ensure_schema(self, con: sqlite3.Connection) -> None:
|
|
555
|
+
"""Migrate schema via PRAGMA user_version non-destructively."""
|
|
556
|
+
if self._schema_verified:
|
|
557
|
+
return
|
|
558
|
+
with self._schema_lock:
|
|
559
|
+
if self._schema_verified:
|
|
560
|
+
return
|
|
561
|
+
current_version = con.execute("PRAGMA user_version").fetchone()[0]
|
|
562
|
+
|
|
563
|
+
if current_version == 0:
|
|
564
|
+
try:
|
|
565
|
+
row = con.execute("SELECT version FROM schema_version").fetchone()
|
|
566
|
+
if row:
|
|
567
|
+
current_version = int(row[0])
|
|
568
|
+
except sqlite3.OperationalError:
|
|
569
|
+
pass
|
|
570
|
+
|
|
571
|
+
target_version = SCHEMA_VERSION
|
|
572
|
+
|
|
573
|
+
if current_version < target_version:
|
|
574
|
+
if current_version > 0:
|
|
575
|
+
self._migrate_schema(con, current_version, target_version)
|
|
576
|
+
else:
|
|
577
|
+
con.executescript(SCHEMA)
|
|
578
|
+
|
|
579
|
+
con.execute(f"PRAGMA user_version = {target_version}")
|
|
580
|
+
try:
|
|
581
|
+
con.execute("DROP TABLE IF EXISTS schema_version")
|
|
582
|
+
except sqlite3.OperationalError:
|
|
583
|
+
pass
|
|
584
|
+
con.commit()
|
|
585
|
+
else:
|
|
586
|
+
existing_core = con.execute(
|
|
587
|
+
"SELECT count(*) FROM sqlite_master WHERE type='table' "
|
|
588
|
+
"AND name IN ('files', 'symbols', 'graph_edges', 'db_tables', 'runtime_traces')"
|
|
589
|
+
).fetchone()[0]
|
|
590
|
+
if existing_core < 5:
|
|
591
|
+
con.executescript(SCHEMA)
|
|
592
|
+
con.commit()
|
|
593
|
+
|
|
594
|
+
from codegraph.async_queue import ensure_async_tables
|
|
595
|
+
|
|
596
|
+
ensure_async_tables(con)
|
|
597
|
+
self._schema_verified = True
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def _migrate_schema(
|
|
601
|
+
self, con: sqlite3.Connection, from_version: int, to_version: int
|
|
602
|
+
) -> None:
|
|
603
|
+
"""Non-destructive schema migration using ALTER TABLE."""
|
|
604
|
+
del from_version, to_version
|
|
605
|
+
# Ensure new tables and indexes are created
|
|
606
|
+
con.executescript(SCHEMA)
|
|
607
|
+
|
|
608
|
+
# Add missing columns to existing tables
|
|
609
|
+
def existing_columns(tbl: str) -> set[str]:
|
|
610
|
+
try:
|
|
611
|
+
return {
|
|
612
|
+
str(r["name"])
|
|
613
|
+
for r in con.execute(f"PRAGMA table_info({tbl})").fetchall()
|
|
614
|
+
}
|
|
615
|
+
except sqlite3.OperationalError:
|
|
616
|
+
return set()
|
|
617
|
+
|
|
618
|
+
files_cols = existing_columns("files")
|
|
619
|
+
if "status" not in files_cols:
|
|
620
|
+
con.execute("ALTER TABLE files ADD COLUMN status TEXT NOT NULL DEFAULT 'ok'")
|
|
621
|
+
if "parse_error" not in files_cols:
|
|
622
|
+
con.execute("ALTER TABLE files ADD COLUMN parse_error TEXT")
|
|
623
|
+
if "last_valid_hash" not in files_cols:
|
|
624
|
+
con.execute("ALTER TABLE files ADD COLUMN last_valid_hash TEXT")
|
|
625
|
+
if "category" not in files_cols:
|
|
626
|
+
con.execute("ALTER TABLE files ADD COLUMN category TEXT NOT NULL DEFAULT 'SOURCE'")
|
|
627
|
+
|
|
628
|
+
symbols_cols = existing_columns("symbols")
|
|
629
|
+
for col_def in (
|
|
630
|
+
("id", "TEXT NOT NULL DEFAULT ''"),
|
|
631
|
+
("canonical_id", "TEXT NOT NULL DEFAULT ''"),
|
|
632
|
+
("language", "TEXT NOT NULL DEFAULT 'python'"),
|
|
633
|
+
("module", "TEXT NOT NULL DEFAULT ''"),
|
|
634
|
+
("scope", "TEXT NOT NULL DEFAULT ''"),
|
|
635
|
+
("signature", "TEXT NOT NULL DEFAULT ''"),
|
|
636
|
+
("content_hash", "TEXT NOT NULL DEFAULT ''"),
|
|
637
|
+
("parent_symbol_id", "TEXT"),
|
|
638
|
+
("visibility", "TEXT NOT NULL DEFAULT 'public'"),
|
|
639
|
+
("return_type", "TEXT"),
|
|
640
|
+
("parameter_count", "INTEGER"),
|
|
641
|
+
("documentation", "TEXT"),
|
|
642
|
+
):
|
|
643
|
+
if col_def[0] not in symbols_cols:
|
|
644
|
+
con.execute(f"ALTER TABLE symbols ADD COLUMN {col_def[0]} {col_def[1]}")
|
|
645
|
+
|
|
646
|
+
imports_cols = existing_columns("imports")
|
|
647
|
+
for col_def in (
|
|
648
|
+
("source_module", "TEXT NOT NULL DEFAULT ''"),
|
|
649
|
+
("imported_module", "TEXT NOT NULL DEFAULT ''"),
|
|
650
|
+
("imported_name", "TEXT"),
|
|
651
|
+
("local_name", "TEXT NOT NULL DEFAULT ''"),
|
|
652
|
+
("import_type", "TEXT NOT NULL DEFAULT 'module'"),
|
|
653
|
+
("resolved_path", "TEXT"),
|
|
654
|
+
("resolved_module", "TEXT"),
|
|
655
|
+
):
|
|
656
|
+
if col_def[0] not in imports_cols:
|
|
657
|
+
con.execute(f"ALTER TABLE imports ADD COLUMN {col_def[0]} {col_def[1]}")
|
|
658
|
+
|
|
659
|
+
files_cols = existing_columns("files")
|
|
660
|
+
if "category" not in files_cols:
|
|
661
|
+
con.execute("ALTER TABLE files ADD COLUMN category TEXT NOT NULL DEFAULT 'SOURCE'")
|
|
662
|
+
|
|
663
|
+
calls_cols = existing_columns("calls")
|
|
664
|
+
for col_def in (
|
|
665
|
+
("source_symbol_id", "TEXT"),
|
|
666
|
+
("resolved_symbol_id", "TEXT"),
|
|
667
|
+
):
|
|
668
|
+
if col_def[0] not in calls_cols:
|
|
669
|
+
con.execute(f"ALTER TABLE calls ADD COLUMN {col_def[0]} {col_def[1]}")
|
|
670
|
+
|
|
671
|
+
edges_cols = existing_columns("graph_edges")
|
|
672
|
+
if "evidence_class" not in edges_cols:
|
|
673
|
+
con.execute("ALTER TABLE graph_edges ADD COLUMN evidence_class TEXT NOT NULL DEFAULT 'AST_VERIFIED'")
|
|
674
|
+
if "reason" not in edges_cols:
|
|
675
|
+
con.execute("ALTER TABLE graph_edges ADD COLUMN reason TEXT")
|
|
676
|
+
|
|
677
|
+
try:
|
|
678
|
+
con.execute(
|
|
679
|
+
"INSERT OR IGNORE INTO raw_framework_routes("
|
|
680
|
+
"endpoint_id, framework, http_method, route_path, normalized_route, "
|
|
681
|
+
"handler_name, handler_canonical_id, file_path, line, evidence, confidence) "
|
|
682
|
+
"SELECT endpoint_id, framework, http_method, route_path, normalized_route, "
|
|
683
|
+
"handler_name, handler_canonical_id, file_path, line, evidence, confidence "
|
|
684
|
+
"FROM framework_routes"
|
|
685
|
+
)
|
|
686
|
+
except sqlite3.OperationalError:
|
|
687
|
+
pass
|
|
688
|
+
|
|
689
|
+
try:
|
|
690
|
+
con.execute(
|
|
691
|
+
"CREATE TABLE IF NOT EXISTS local_bindings ("
|
|
692
|
+
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
|
|
693
|
+
"target_name TEXT NOT NULL, "
|
|
694
|
+
"file_path TEXT NOT NULL, "
|
|
695
|
+
"line INTEGER NOT NULL, "
|
|
696
|
+
"column INTEGER, "
|
|
697
|
+
"scope TEXT NOT NULL DEFAULT '', "
|
|
698
|
+
"expr_kind TEXT NOT NULL DEFAULT 'IDENTIFIER', "
|
|
699
|
+
"source_expr TEXT NOT NULL DEFAULT '', "
|
|
700
|
+
"is_conditional INTEGER NOT NULL DEFAULT 0, "
|
|
701
|
+
"base_expr TEXT NOT NULL DEFAULT '', "
|
|
702
|
+
"attr_name TEXT NOT NULL DEFAULT '', "
|
|
703
|
+
"dict_entries_json TEXT NOT NULL DEFAULT '[]', "
|
|
704
|
+
"list_entries_json TEXT NOT NULL DEFAULT '[]', "
|
|
705
|
+
"subscript_target TEXT, "
|
|
706
|
+
"subscript_key TEXT, "
|
|
707
|
+
"subscript_index INTEGER, "
|
|
708
|
+
"FOREIGN KEY(file_path) REFERENCES files(path) ON DELETE CASCADE)"
|
|
709
|
+
)
|
|
710
|
+
con.execute("CREATE INDEX IF NOT EXISTS idx_bindings_file ON local_bindings (file_path)")
|
|
711
|
+
con.execute("CREATE INDEX IF NOT EXISTS idx_bindings_target ON local_bindings (target_name)")
|
|
712
|
+
con.execute("CREATE INDEX IF NOT EXISTS idx_bindings_file_target ON local_bindings (file_path, target_name)")
|
|
713
|
+
except sqlite3.OperationalError:
|
|
714
|
+
pass
|
|
715
|
+
|
|
716
|
+
def index(
|
|
717
|
+
self,
|
|
718
|
+
*,
|
|
719
|
+
progress_callback: Callable[[str, int, int, IndexingTelemetry], None] | None = None,
|
|
720
|
+
catch_interrupt: bool = False,
|
|
721
|
+
) -> dict[str, int]:
|
|
722
|
+
"""Perform an incremental indexing pass, followed by global reference resolution."""
|
|
723
|
+
indexed = unchanged = parse_failed_count = 0
|
|
724
|
+
stale: list[str] = []
|
|
725
|
+
dirty_files: set[str] = set()
|
|
726
|
+
files = []
|
|
727
|
+
telemetry = IndexingTelemetry()
|
|
728
|
+
self.last_telemetry = telemetry
|
|
729
|
+
from codegraph.freshness import current_commit, save_commit
|
|
730
|
+
|
|
731
|
+
with self.governor.task_scope(TaskPriority.BACKGROUND):
|
|
732
|
+
self.governor.set_indexing_active(True)
|
|
733
|
+
try:
|
|
734
|
+
with telemetry.phase("repository_scan") as p_scan:
|
|
735
|
+
files = scan(self.repository, self.settings.max_file_size, self.settings.exclude)
|
|
736
|
+
p_scan.files_processed = len(files)
|
|
737
|
+
p_scan.items_processed = len(files)
|
|
738
|
+
total_files = len(files)
|
|
739
|
+
if progress_callback is not None:
|
|
740
|
+
progress_callback("repository_scan", total_files, total_files, telemetry)
|
|
741
|
+
|
|
742
|
+
seen = {item.relative_path.as_posix() for item in files}
|
|
743
|
+
batch_limit = self.governor.policy.max_files_per_incremental_batch
|
|
744
|
+
uncommitted_in_batch = 0
|
|
745
|
+
|
|
746
|
+
con = self.connect()
|
|
747
|
+
try:
|
|
748
|
+
# Check parser version invalidation & previous interrupted state
|
|
749
|
+
last_parser_ver = None
|
|
750
|
+
resolution_dirty = False
|
|
751
|
+
try:
|
|
752
|
+
for mrow in con.execute(
|
|
753
|
+
"SELECT key, value FROM metadata WHERE key IN ('parser_version', 'resolution_dirty')"
|
|
754
|
+
).fetchall():
|
|
755
|
+
mkey = str(mrow["key"])
|
|
756
|
+
if mkey == "parser_version":
|
|
757
|
+
last_parser_ver = str(mrow["value"])
|
|
758
|
+
elif mkey == "resolution_dirty" and str(mrow["value"]) == "1":
|
|
759
|
+
resolution_dirty = True
|
|
760
|
+
except sqlite3.OperationalError:
|
|
761
|
+
pass
|
|
762
|
+
|
|
763
|
+
force_reparse = (last_parser_ver != PARSER_VERSION)
|
|
764
|
+
|
|
765
|
+
# Pre-load existing file hashes and statuses in a single query
|
|
766
|
+
existing_files: dict[str, tuple[str, str]] = {
|
|
767
|
+
str(r["path"]): (str(r["hash"]), str(r["status"]))
|
|
768
|
+
for r in con.execute("SELECT path, hash, status FROM files").fetchall()
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
for idx, item in enumerate(files):
|
|
772
|
+
if idx > 0 and idx % batch_limit == 0:
|
|
773
|
+
self.governor.yield_if_needed(TaskPriority.BACKGROUND)
|
|
774
|
+
|
|
775
|
+
relative = item.relative_path.as_posix()
|
|
776
|
+
t_read = time.perf_counter()
|
|
777
|
+
try:
|
|
778
|
+
content = item.path.read_text(encoding="utf-8", errors="replace")
|
|
779
|
+
except OSError:
|
|
780
|
+
telemetry.record("file_reading", time.perf_counter() - t_read, errors_skips=1)
|
|
781
|
+
continue
|
|
782
|
+
digest = hashlib.sha256(content.encode()).hexdigest()
|
|
783
|
+
telemetry.record(
|
|
784
|
+
"file_reading",
|
|
785
|
+
time.perf_counter() - t_read,
|
|
786
|
+
files_processed=1,
|
|
787
|
+
items_processed=len(content),
|
|
788
|
+
)
|
|
789
|
+
|
|
790
|
+
old_state = existing_files.get(relative)
|
|
791
|
+
if old_state is not None and old_state[0] == digest and not force_reparse:
|
|
792
|
+
if old_state[1] == "parse_failed":
|
|
793
|
+
parse_failed_count += 1
|
|
794
|
+
else:
|
|
795
|
+
unchanged += 1
|
|
796
|
+
telemetry.phases["file_reading"].errors_skips += 1
|
|
797
|
+
continue
|
|
798
|
+
|
|
799
|
+
t_cls = time.perf_counter()
|
|
800
|
+
cat_info = classify_file_detailed(relative, content)
|
|
801
|
+
cat_str = cat_info.category.value
|
|
802
|
+
telemetry.record(
|
|
803
|
+
"file_classification",
|
|
804
|
+
time.perf_counter() - t_cls,
|
|
805
|
+
files_processed=1,
|
|
806
|
+
items_processed=1,
|
|
807
|
+
)
|
|
808
|
+
|
|
809
|
+
success = self._replace_file(
|
|
810
|
+
con,
|
|
811
|
+
relative,
|
|
812
|
+
item.language,
|
|
813
|
+
content,
|
|
814
|
+
digest,
|
|
815
|
+
category=cat_str,
|
|
816
|
+
is_existing=(old_state is not None),
|
|
817
|
+
telemetry=telemetry,
|
|
818
|
+
)
|
|
819
|
+
if success:
|
|
820
|
+
indexed += 1
|
|
821
|
+
dirty_files.add(relative)
|
|
822
|
+
existing_files[relative] = (digest, "ok")
|
|
823
|
+
else:
|
|
824
|
+
parse_failed_count += 1
|
|
825
|
+
existing_files[relative] = (digest, "parse_failed")
|
|
826
|
+
|
|
827
|
+
uncommitted_in_batch += 1
|
|
828
|
+
if uncommitted_in_batch >= self.commit_batch_size:
|
|
829
|
+
t_commit = time.perf_counter()
|
|
830
|
+
con.execute(
|
|
831
|
+
"INSERT INTO metadata(key, value) VALUES('resolution_dirty', '1') "
|
|
832
|
+
"ON CONFLICT(key) DO UPDATE SET value='1'"
|
|
833
|
+
)
|
|
834
|
+
con.execute(
|
|
835
|
+
"INSERT INTO metadata(key, value) VALUES('index_status', 'in_progress') "
|
|
836
|
+
"ON CONFLICT(key) DO UPDATE SET value='in_progress'"
|
|
837
|
+
)
|
|
838
|
+
con.commit()
|
|
839
|
+
telemetry.batch_commits += 1
|
|
840
|
+
telemetry.observe_wal(self.db_path)
|
|
841
|
+
try:
|
|
842
|
+
con.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
|
843
|
+
telemetry.wal_checkpoints += 1
|
|
844
|
+
except sqlite3.OperationalError:
|
|
845
|
+
pass
|
|
846
|
+
telemetry.observe_wal(self.db_path)
|
|
847
|
+
telemetry.record(
|
|
848
|
+
"sqlite_writes",
|
|
849
|
+
time.perf_counter() - t_commit,
|
|
850
|
+
db_writes=1,
|
|
851
|
+
)
|
|
852
|
+
uncommitted_in_batch = 0
|
|
853
|
+
if progress_callback is not None:
|
|
854
|
+
progress_callback("ast_parsing", idx + 1, total_files, telemetry)
|
|
855
|
+
|
|
856
|
+
telemetry.post_parse_rss_mb = get_process_rss_mb()
|
|
857
|
+
telemetry.peak_rss_mb = max(telemetry.peak_rss_mb, telemetry.post_parse_rss_mb)
|
|
858
|
+
|
|
859
|
+
stale = [
|
|
860
|
+
p for p in existing_files
|
|
861
|
+
if p not in seen
|
|
862
|
+
]
|
|
863
|
+
if stale:
|
|
864
|
+
t_del = time.perf_counter()
|
|
865
|
+
for path in stale:
|
|
866
|
+
self._delete_file(con, path, telemetry=telemetry)
|
|
867
|
+
telemetry.record(
|
|
868
|
+
"sqlite_writes",
|
|
869
|
+
time.perf_counter() - t_del,
|
|
870
|
+
files_processed=len(stale),
|
|
871
|
+
db_writes=len(stale),
|
|
872
|
+
)
|
|
873
|
+
|
|
874
|
+
# Commit file indexing phase before global post-processing so completed file work is durable
|
|
875
|
+
head_commit = current_commit(self.repository)
|
|
876
|
+
now_ts = int(time.time())
|
|
877
|
+
need_resolution = (indexed > 0 or len(stale) > 0 or force_reparse or resolution_dirty)
|
|
878
|
+
|
|
879
|
+
if uncommitted_in_batch > 0 or stale:
|
|
880
|
+
t_commit = time.perf_counter()
|
|
881
|
+
if need_resolution:
|
|
882
|
+
con.execute(
|
|
883
|
+
"INSERT INTO metadata(key, value) VALUES('resolution_dirty', '1') "
|
|
884
|
+
"ON CONFLICT(key) DO UPDATE SET value='1'"
|
|
885
|
+
)
|
|
886
|
+
con.commit()
|
|
887
|
+
telemetry.batch_commits += 1
|
|
888
|
+
telemetry.observe_wal(self.db_path)
|
|
889
|
+
try:
|
|
890
|
+
con.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
|
891
|
+
telemetry.wal_checkpoints += 1
|
|
892
|
+
except sqlite3.OperationalError:
|
|
893
|
+
pass
|
|
894
|
+
telemetry.observe_wal(self.db_path)
|
|
895
|
+
telemetry.record("sqlite_writes", time.perf_counter() - t_commit, db_writes=1)
|
|
896
|
+
uncommitted_in_batch = 0
|
|
897
|
+
|
|
898
|
+
# Run global reference resolution pass if any files were changed, removed, or pending from interrupt
|
|
899
|
+
if need_resolution:
|
|
900
|
+
with telemetry.phase("post_processing") as p_post:
|
|
901
|
+
if progress_callback is not None:
|
|
902
|
+
progress_callback("post_processing", total_files, total_files, telemetry)
|
|
903
|
+
self._run_global_resolution(
|
|
904
|
+
con,
|
|
905
|
+
head_commit,
|
|
906
|
+
dirty_files=dirty_files if not (force_reparse or resolution_dirty) else None,
|
|
907
|
+
removed_files=set(stale) if not (force_reparse or resolution_dirty) else None,
|
|
908
|
+
telemetry=telemetry,
|
|
909
|
+
progress_callback=progress_callback,
|
|
910
|
+
)
|
|
911
|
+
from codegraph.async_queue import AsyncQueueScanner
|
|
912
|
+
|
|
913
|
+
AsyncQueueScanner(self.repository).scan_repository(con)
|
|
914
|
+
p_post.files_processed = indexed + len(stale)
|
|
915
|
+
p_post.items_processed = indexed + len(stale)
|
|
916
|
+
# Invalidate context cache on index update
|
|
917
|
+
t_rank = time.perf_counter()
|
|
918
|
+
try:
|
|
919
|
+
con.execute("DELETE FROM context_cache")
|
|
920
|
+
except sqlite3.OperationalError:
|
|
921
|
+
pass
|
|
922
|
+
telemetry.record("ranking_index_building", time.perf_counter() - t_rank, db_writes=1)
|
|
923
|
+
|
|
924
|
+
# Final commit & WAL checkpoint
|
|
925
|
+
with telemetry.phase("final_commit_checkpoint") as p_final:
|
|
926
|
+
save_commit(con, head_commit, now_ts)
|
|
927
|
+
current_gen = 0
|
|
928
|
+
try:
|
|
929
|
+
grow = con.execute("SELECT value FROM metadata WHERE key='index_generation'").fetchone()
|
|
930
|
+
if grow and grow[0]:
|
|
931
|
+
current_gen = int(grow[0])
|
|
932
|
+
except (sqlite3.OperationalError, ValueError):
|
|
933
|
+
pass
|
|
934
|
+
next_gen = current_gen + 1
|
|
935
|
+
|
|
936
|
+
con.executemany(
|
|
937
|
+
"INSERT INTO metadata(key, value) VALUES(?,?) "
|
|
938
|
+
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
|
939
|
+
[
|
|
940
|
+
("repository", str(self.repository)),
|
|
941
|
+
("parser_version", PARSER_VERSION),
|
|
942
|
+
("schema_version", str(SCHEMA_VERSION)),
|
|
943
|
+
("index_generation", str(next_gen)),
|
|
944
|
+
("index_timestamp", str(now_ts)),
|
|
945
|
+
("resolution_dirty", "0"),
|
|
946
|
+
("index_status", "ok"),
|
|
947
|
+
],
|
|
948
|
+
)
|
|
949
|
+
con.commit()
|
|
950
|
+
telemetry.batch_commits += 1
|
|
951
|
+
telemetry.observe_wal(self.db_path)
|
|
952
|
+
try:
|
|
953
|
+
con.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
|
954
|
+
telemetry.wal_checkpoints += 1
|
|
955
|
+
except sqlite3.OperationalError:
|
|
956
|
+
pass
|
|
957
|
+
telemetry.observe_wal(self.db_path)
|
|
958
|
+
p_final.files_processed = total_files
|
|
959
|
+
p_final.items_processed = indexed
|
|
960
|
+
p_final.db_writes = 8
|
|
961
|
+
if progress_callback is not None:
|
|
962
|
+
progress_callback("final_commit_checkpoint", total_files, total_files, telemetry)
|
|
963
|
+
|
|
964
|
+
except KeyboardInterrupt:
|
|
965
|
+
# Safe cancellation: roll back only the active uncommitted batch, preserve committed batches
|
|
966
|
+
telemetry.interrupted = True
|
|
967
|
+
telemetry.scanned_files = len(files)
|
|
968
|
+
telemetry.indexed_files = indexed
|
|
969
|
+
telemetry.unchanged_files = unchanged
|
|
970
|
+
telemetry.removed_files = len(stale)
|
|
971
|
+
telemetry.parse_failed_files = parse_failed_count
|
|
972
|
+
try:
|
|
973
|
+
con.rollback()
|
|
974
|
+
con.executemany(
|
|
975
|
+
"INSERT INTO metadata(key, value) VALUES(?,?) "
|
|
976
|
+
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
|
977
|
+
[
|
|
978
|
+
("resolution_dirty", "1"),
|
|
979
|
+
("index_status", "interrupted"),
|
|
980
|
+
("parser_version", PARSER_VERSION),
|
|
981
|
+
],
|
|
982
|
+
)
|
|
983
|
+
con.commit()
|
|
984
|
+
con.execute("PRAGMA wal_checkpoint(PASSIVE)")
|
|
985
|
+
telemetry.wal_checkpoints += 1
|
|
986
|
+
telemetry.observe_wal(self.db_path)
|
|
987
|
+
except sqlite3.OperationalError:
|
|
988
|
+
pass
|
|
989
|
+
telemetry.finish(self.db_path)
|
|
990
|
+
if catch_interrupt:
|
|
991
|
+
return {
|
|
992
|
+
"scanned": len(files),
|
|
993
|
+
"indexed": indexed,
|
|
994
|
+
"unchanged": unchanged,
|
|
995
|
+
"removed": len(stale),
|
|
996
|
+
"interrupted": 1,
|
|
997
|
+
}
|
|
998
|
+
raise
|
|
999
|
+
except BaseException:
|
|
1000
|
+
con.rollback()
|
|
1001
|
+
raise
|
|
1002
|
+
finally:
|
|
1003
|
+
con.close()
|
|
1004
|
+
finally:
|
|
1005
|
+
if not telemetry.interrupted:
|
|
1006
|
+
telemetry.scanned_files = len(files)
|
|
1007
|
+
telemetry.indexed_files = indexed
|
|
1008
|
+
telemetry.unchanged_files = unchanged
|
|
1009
|
+
telemetry.removed_files = len(stale)
|
|
1010
|
+
telemetry.parse_failed_files = parse_failed_count
|
|
1011
|
+
telemetry.finish(self.db_path)
|
|
1012
|
+
self.governor.set_indexing_active(False)
|
|
1013
|
+
|
|
1014
|
+
result: dict[str, int] = {
|
|
1015
|
+
"scanned": len(files),
|
|
1016
|
+
"indexed": indexed,
|
|
1017
|
+
"unchanged": unchanged,
|
|
1018
|
+
"removed": len(stale),
|
|
1019
|
+
}
|
|
1020
|
+
if parse_failed_count > 0:
|
|
1021
|
+
result["parse_failed"] = parse_failed_count
|
|
1022
|
+
self.reset_pool()
|
|
1023
|
+
return result
|
|
1024
|
+
|
|
1025
|
+
def reindex_paths(
|
|
1026
|
+
self,
|
|
1027
|
+
paths: Iterable[str | Path],
|
|
1028
|
+
) -> dict[str, Any]:
|
|
1029
|
+
"""Incrementally reindex a set of changed or deleted file paths in sub-15ms.
|
|
1030
|
+
|
|
1031
|
+
Bypasses full repository filesystem scanning and only touches the specified files,
|
|
1032
|
+
updating their AST facts, re-running affected reference resolution, busting in-memory
|
|
1033
|
+
caches, and advancing index_generation atomically.
|
|
1034
|
+
"""
|
|
1035
|
+
t0 = time.perf_counter()
|
|
1036
|
+
from codegraph.freshness import current_commit, save_commit
|
|
1037
|
+
from codegraph.watcher import _is_path_ignored
|
|
1038
|
+
|
|
1039
|
+
# 1. Normalize and deduplicate paths relative to self.repository
|
|
1040
|
+
normalized_paths: set[str] = set()
|
|
1041
|
+
for p in paths:
|
|
1042
|
+
try:
|
|
1043
|
+
p_path = Path(p)
|
|
1044
|
+
if p_path.is_absolute():
|
|
1045
|
+
rel = str(p_path.resolve().relative_to(self.repository)).replace("\\", "/")
|
|
1046
|
+
else:
|
|
1047
|
+
rel = str(p_path).replace("\\", "/").lstrip("./")
|
|
1048
|
+
except Exception:
|
|
1049
|
+
continue
|
|
1050
|
+
|
|
1051
|
+
if not rel or _is_path_ignored(rel):
|
|
1052
|
+
continue
|
|
1053
|
+
normalized_paths.add(rel)
|
|
1054
|
+
|
|
1055
|
+
if not normalized_paths:
|
|
1056
|
+
return {
|
|
1057
|
+
"status": "ok",
|
|
1058
|
+
"scanned": 0,
|
|
1059
|
+
"indexed": 0,
|
|
1060
|
+
"removed": 0,
|
|
1061
|
+
"reindexed": [],
|
|
1062
|
+
"deleted": [],
|
|
1063
|
+
"skipped": [],
|
|
1064
|
+
"parse_failed": [],
|
|
1065
|
+
"generation": 0,
|
|
1066
|
+
"elapsed_ms": round((time.perf_counter() - t0) * 1000.0, 2),
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
# 2. Inspect filesystem vs SQLite to determine what to delete, update, or skip
|
|
1070
|
+
con = self.connect()
|
|
1071
|
+
try:
|
|
1072
|
+
to_delete: list[str] = []
|
|
1073
|
+
to_update: list[tuple[str, str, str, str, bool]] = []
|
|
1074
|
+
skipped: list[str] = []
|
|
1075
|
+
|
|
1076
|
+
for rel_path in sorted(normalized_paths):
|
|
1077
|
+
full_path = self.repository / rel_path
|
|
1078
|
+
existing_row = con.execute("SELECT hash, status FROM files WHERE path=?", (rel_path,)).fetchone()
|
|
1079
|
+
|
|
1080
|
+
if not full_path.is_file():
|
|
1081
|
+
# File was deleted from disk
|
|
1082
|
+
if existing_row is not None:
|
|
1083
|
+
to_delete.append(rel_path)
|
|
1084
|
+
else:
|
|
1085
|
+
skipped.append(rel_path)
|
|
1086
|
+
continue
|
|
1087
|
+
|
|
1088
|
+
# File exists on disk
|
|
1089
|
+
suffix = full_path.suffix.lower()
|
|
1090
|
+
lang = LANGUAGES.get(suffix)
|
|
1091
|
+
if not lang:
|
|
1092
|
+
skipped.append(rel_path)
|
|
1093
|
+
continue
|
|
1094
|
+
|
|
1095
|
+
try:
|
|
1096
|
+
content = full_path.read_text(encoding="utf-8", errors="replace")
|
|
1097
|
+
except Exception:
|
|
1098
|
+
skipped.append(rel_path)
|
|
1099
|
+
continue
|
|
1100
|
+
|
|
1101
|
+
digest = hashlib.sha256(content.encode("utf-8")).hexdigest()
|
|
1102
|
+
|
|
1103
|
+
if existing_row is not None and existing_row[0] == digest and existing_row[1] == "ok":
|
|
1104
|
+
# Identical digest, no AST or file modification
|
|
1105
|
+
skipped.append(rel_path)
|
|
1106
|
+
else:
|
|
1107
|
+
to_update.append((rel_path, lang, content, digest, existing_row is not None))
|
|
1108
|
+
|
|
1109
|
+
# If nothing changed, return early
|
|
1110
|
+
grow = con.execute("SELECT value FROM metadata WHERE key='index_generation'").fetchone()
|
|
1111
|
+
current_gen = int(grow[0]) if (grow and grow[0]) else 0
|
|
1112
|
+
|
|
1113
|
+
if not to_delete and not to_update:
|
|
1114
|
+
return {
|
|
1115
|
+
"status": "ok",
|
|
1116
|
+
"scanned": len(normalized_paths),
|
|
1117
|
+
"indexed": 0,
|
|
1118
|
+
"removed": 0,
|
|
1119
|
+
"reindexed": [],
|
|
1120
|
+
"deleted": [],
|
|
1121
|
+
"skipped": skipped,
|
|
1122
|
+
"parse_failed": [],
|
|
1123
|
+
"generation": current_gen,
|
|
1124
|
+
"elapsed_ms": round((time.perf_counter() - t0) * 1000.0, 2),
|
|
1125
|
+
}
|
|
1126
|
+
|
|
1127
|
+
# 3. Apply atomic updates
|
|
1128
|
+
self.governor.set_indexing_active(True)
|
|
1129
|
+
reindexed: list[str] = []
|
|
1130
|
+
parse_failed: list[str] = []
|
|
1131
|
+
|
|
1132
|
+
for del_path in to_delete:
|
|
1133
|
+
self._delete_file(con, del_path)
|
|
1134
|
+
|
|
1135
|
+
for rel_path, lang, content, digest, is_existing in to_update:
|
|
1136
|
+
cat_info = classify_file_detailed(Path(rel_path), content)
|
|
1137
|
+
success = self._replace_file(
|
|
1138
|
+
con,
|
|
1139
|
+
rel_path,
|
|
1140
|
+
lang,
|
|
1141
|
+
content,
|
|
1142
|
+
digest,
|
|
1143
|
+
category=cat_info.category.value,
|
|
1144
|
+
is_existing=is_existing,
|
|
1145
|
+
)
|
|
1146
|
+
if success:
|
|
1147
|
+
reindexed.append(rel_path)
|
|
1148
|
+
else:
|
|
1149
|
+
parse_failed.append(rel_path)
|
|
1150
|
+
|
|
1151
|
+
# 4. Run reference resolution pass
|
|
1152
|
+
head_commit = current_commit(self.repository)
|
|
1153
|
+
now_ts = int(time.time())
|
|
1154
|
+
|
|
1155
|
+
dirty_set = set(reindexed)
|
|
1156
|
+
removed_set = set(to_delete)
|
|
1157
|
+
|
|
1158
|
+
self._run_global_resolution(
|
|
1159
|
+
con,
|
|
1160
|
+
head_commit,
|
|
1161
|
+
dirty_files=dirty_set,
|
|
1162
|
+
removed_files=removed_set,
|
|
1163
|
+
)
|
|
1164
|
+
|
|
1165
|
+
from codegraph.async_queue import AsyncQueueScanner
|
|
1166
|
+
|
|
1167
|
+
AsyncQueueScanner(self.repository).scan_repository(con)
|
|
1168
|
+
|
|
1169
|
+
# Invalidate context cache on index update
|
|
1170
|
+
try:
|
|
1171
|
+
con.execute("DELETE FROM context_cache")
|
|
1172
|
+
except sqlite3.OperationalError:
|
|
1173
|
+
pass
|
|
1174
|
+
|
|
1175
|
+
# 5. Advance generation and update metadata
|
|
1176
|
+
next_gen = current_gen + 1
|
|
1177
|
+
con.executemany(
|
|
1178
|
+
"INSERT INTO metadata(key, value) VALUES(?,?) "
|
|
1179
|
+
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
|
1180
|
+
[
|
|
1181
|
+
("repository", str(self.repository)),
|
|
1182
|
+
("parser_version", PARSER_VERSION),
|
|
1183
|
+
("schema_version", str(SCHEMA_VERSION)),
|
|
1184
|
+
("index_generation", str(next_gen)),
|
|
1185
|
+
("index_timestamp", str(now_ts)),
|
|
1186
|
+
("resolution_dirty", "0"),
|
|
1187
|
+
("index_status", "ok"),
|
|
1188
|
+
],
|
|
1189
|
+
)
|
|
1190
|
+
save_commit(con, head_commit, now_ts)
|
|
1191
|
+
con.commit()
|
|
1192
|
+
try:
|
|
1193
|
+
con.execute("PRAGMA wal_checkpoint(PASSIVE)")
|
|
1194
|
+
except sqlite3.OperationalError:
|
|
1195
|
+
pass
|
|
1196
|
+
except Exception:
|
|
1197
|
+
con.rollback()
|
|
1198
|
+
raise
|
|
1199
|
+
finally:
|
|
1200
|
+
self.governor.set_indexing_active(False)
|
|
1201
|
+
con.close()
|
|
1202
|
+
|
|
1203
|
+
# Invalidate in-memory caches and reset pool
|
|
1204
|
+
get_parse_cache().clear()
|
|
1205
|
+
get_graph_cache().clear()
|
|
1206
|
+
self.reset_pool()
|
|
1207
|
+
|
|
1208
|
+
elapsed_ms = round((time.perf_counter() - t0) * 1000.0, 2)
|
|
1209
|
+
return {
|
|
1210
|
+
"status": "ok",
|
|
1211
|
+
"scanned": len(normalized_paths),
|
|
1212
|
+
"indexed": len(reindexed),
|
|
1213
|
+
"removed": len(to_delete),
|
|
1214
|
+
"reindexed": reindexed,
|
|
1215
|
+
"deleted": to_delete,
|
|
1216
|
+
"skipped": skipped,
|
|
1217
|
+
"parse_failed": parse_failed,
|
|
1218
|
+
"generation": next_gen,
|
|
1219
|
+
"elapsed_ms": elapsed_ms,
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
def _delete_file(
|
|
1223
|
+
self,
|
|
1224
|
+
con: sqlite3.Connection,
|
|
1225
|
+
path: str,
|
|
1226
|
+
*,
|
|
1227
|
+
telemetry: IndexingTelemetry | None = None,
|
|
1228
|
+
) -> None:
|
|
1229
|
+
"""Atomically delete all indexed facts for a file, maintaining FTS synchronization."""
|
|
1230
|
+
ids = [r[0] for r in con.execute("SELECT id FROM chunks WHERE path=?", (path,))]
|
|
1231
|
+
if ids:
|
|
1232
|
+
t_fts = time.perf_counter()
|
|
1233
|
+
con.executemany("DELETE FROM chunks_fts WHERE rowid=?", [(chunk_id,) for chunk_id in ids])
|
|
1234
|
+
if telemetry is not None:
|
|
1235
|
+
telemetry.record(
|
|
1236
|
+
"fts_updates",
|
|
1237
|
+
time.perf_counter() - t_fts,
|
|
1238
|
+
items_processed=len(ids),
|
|
1239
|
+
db_writes=len(ids),
|
|
1240
|
+
)
|
|
1241
|
+
con.execute("DELETE FROM chunks WHERE path=?", (path,))
|
|
1242
|
+
con.execute("DELETE FROM symbols WHERE path=?", (path,))
|
|
1243
|
+
con.execute("DELETE FROM imports WHERE source_path=?", (path,))
|
|
1244
|
+
con.execute("DELETE FROM calls WHERE source_path=?", (path,))
|
|
1245
|
+
con.execute("DELETE FROM inheritance WHERE source_file=?", (path,))
|
|
1246
|
+
con.execute("DELETE FROM framework_routes WHERE file_path=?", (path,))
|
|
1247
|
+
con.execute("DELETE FROM raw_framework_routes WHERE file_path=?", (path,))
|
|
1248
|
+
con.execute("DELETE FROM router_mounts WHERE file_path=?", (path,))
|
|
1249
|
+
con.execute("DELETE FROM router_definitions WHERE file_path=?", (path,))
|
|
1250
|
+
con.execute("DELETE FROM local_bindings WHERE file_path=?", (path,))
|
|
1251
|
+
con.execute("DELETE FROM 'references' WHERE path=?", (path,))
|
|
1252
|
+
con.execute("DELETE FROM graph_edges WHERE file=?", (path,))
|
|
1253
|
+
con.execute("DELETE FROM db_entities WHERE file_path=?", (path,))
|
|
1254
|
+
con.execute("DELETE FROM db_queries WHERE file_path=?", (path,))
|
|
1255
|
+
con.execute("DELETE FROM db_migrations WHERE file_path=?", (path,))
|
|
1256
|
+
con.execute("DELETE FROM files WHERE path=?", (path,))
|
|
1257
|
+
|
|
1258
|
+
def _replace_file(
|
|
1259
|
+
self,
|
|
1260
|
+
con: sqlite3.Connection,
|
|
1261
|
+
path: str,
|
|
1262
|
+
language: str,
|
|
1263
|
+
content: str,
|
|
1264
|
+
digest: str,
|
|
1265
|
+
category: str = "SOURCE",
|
|
1266
|
+
*,
|
|
1267
|
+
is_existing: bool = True,
|
|
1268
|
+
telemetry: IndexingTelemetry | None = None,
|
|
1269
|
+
) -> bool:
|
|
1270
|
+
"""Parse and insert a file. If parse fails, preserves last-known-good state."""
|
|
1271
|
+
from codegraph.security.redaction import redact_secrets
|
|
1272
|
+
|
|
1273
|
+
t_parse = time.perf_counter()
|
|
1274
|
+
parse_cache = get_parse_cache()
|
|
1275
|
+
cached = parse_cache.get((digest, PARSER_VERSION, language))
|
|
1276
|
+
if cached is not None:
|
|
1277
|
+
result = cached
|
|
1278
|
+
else:
|
|
1279
|
+
result = parse(content, language, path)
|
|
1280
|
+
parse_cache.put((digest, PARSER_VERSION, language), result)
|
|
1281
|
+
parse_elapsed = time.perf_counter() - t_parse
|
|
1282
|
+
|
|
1283
|
+
if telemetry is not None:
|
|
1284
|
+
telemetry.record(
|
|
1285
|
+
"ast_parsing",
|
|
1286
|
+
parse_elapsed,
|
|
1287
|
+
files_processed=1,
|
|
1288
|
+
items_processed=len(result.symbols) + len(result.imports) + len(result.calls),
|
|
1289
|
+
errors_skips=1 if result.parse_failed else 0,
|
|
1290
|
+
)
|
|
1291
|
+
telemetry.record(
|
|
1292
|
+
"symbol_extraction",
|
|
1293
|
+
0.0,
|
|
1294
|
+
files_processed=1,
|
|
1295
|
+
items_processed=len(result.symbols),
|
|
1296
|
+
)
|
|
1297
|
+
telemetry.record(
|
|
1298
|
+
"import_extraction",
|
|
1299
|
+
0.0,
|
|
1300
|
+
files_processed=1,
|
|
1301
|
+
items_processed=len(result.imports),
|
|
1302
|
+
)
|
|
1303
|
+
if result.routes or result.mounts or result.router_definitions:
|
|
1304
|
+
telemetry.record(
|
|
1305
|
+
"framework_analysis",
|
|
1306
|
+
0.0,
|
|
1307
|
+
files_processed=1,
|
|
1308
|
+
items_processed=len(result.routes) + len(result.mounts) + len(result.router_definitions),
|
|
1309
|
+
)
|
|
1310
|
+
|
|
1311
|
+
now = int(time.time())
|
|
1312
|
+
|
|
1313
|
+
if result.parse_failed:
|
|
1314
|
+
t_sql = time.perf_counter()
|
|
1315
|
+
# Parse failure safety: check if file previously existed
|
|
1316
|
+
old = con.execute("SELECT hash, status FROM files WHERE path=?", (path,)).fetchone() if is_existing else None
|
|
1317
|
+
if old:
|
|
1318
|
+
# Retain last-known-good index records, mark file as parse_failed
|
|
1319
|
+
con.execute(
|
|
1320
|
+
"UPDATE files SET hash=?, status='parse_failed', parse_error=?, indexed_at=?, category=? WHERE path=?",
|
|
1321
|
+
(digest, redact_secrets(result.parse_error or ""), now, category, path),
|
|
1322
|
+
)
|
|
1323
|
+
else:
|
|
1324
|
+
con.execute(
|
|
1325
|
+
"INSERT INTO files(path, hash, language, indexed_at, status, parse_error, category) VALUES (?, ?, ?, ?, ?, ?, ?)",
|
|
1326
|
+
(path, digest, language, now, "parse_failed", redact_secrets(result.parse_error or ""), category),
|
|
1327
|
+
)
|
|
1328
|
+
if telemetry is not None:
|
|
1329
|
+
telemetry.record("sqlite_writes", time.perf_counter() - t_sql, files_processed=1, db_writes=1)
|
|
1330
|
+
return False
|
|
1331
|
+
|
|
1332
|
+
t_sql = time.perf_counter()
|
|
1333
|
+
db_write_count = 0
|
|
1334
|
+
fts_elapsed = 0.0
|
|
1335
|
+
fts_write_count = 0
|
|
1336
|
+
|
|
1337
|
+
# Successful parse: clean up old file state only if file previously existed in DB
|
|
1338
|
+
if is_existing:
|
|
1339
|
+
self._delete_file(con, path, telemetry=telemetry)
|
|
1340
|
+
db_write_count += 16
|
|
1341
|
+
|
|
1342
|
+
con.execute(
|
|
1343
|
+
"INSERT INTO files(path, hash, language, indexed_at, status, parse_error, last_valid_hash, category) "
|
|
1344
|
+
"VALUES (?, ?, ?, ?, 'ok', NULL, ?, ?)",
|
|
1345
|
+
(path, digest, language, now, digest, category),
|
|
1346
|
+
)
|
|
1347
|
+
db_write_count += 1
|
|
1348
|
+
|
|
1349
|
+
if category in ("MINIFIED", "BUNDLE", "BINARY"):
|
|
1350
|
+
if telemetry is not None:
|
|
1351
|
+
telemetry.record("sqlite_writes", time.perf_counter() - t_sql, files_processed=1, db_writes=db_write_count)
|
|
1352
|
+
return True
|
|
1353
|
+
|
|
1354
|
+
lines = content.splitlines()
|
|
1355
|
+
|
|
1356
|
+
if result.symbols:
|
|
1357
|
+
symbol_rows = []
|
|
1358
|
+
for s in result.symbols:
|
|
1359
|
+
dec_str = ",".join(s.decorators) if s.decorators else ""
|
|
1360
|
+
safe_doc = redact_secrets(s.documentation) if s.documentation else None
|
|
1361
|
+
symbol_rows.append(
|
|
1362
|
+
(
|
|
1363
|
+
s.name,
|
|
1364
|
+
s.qualified_name,
|
|
1365
|
+
s.kind,
|
|
1366
|
+
path,
|
|
1367
|
+
s.start_line,
|
|
1368
|
+
s.end_line,
|
|
1369
|
+
dec_str,
|
|
1370
|
+
s.id,
|
|
1371
|
+
s.canonical_id,
|
|
1372
|
+
s.language,
|
|
1373
|
+
s.module,
|
|
1374
|
+
s.scope,
|
|
1375
|
+
redact_secrets(s.signature),
|
|
1376
|
+
s.content_hash,
|
|
1377
|
+
s.parent_symbol_id,
|
|
1378
|
+
s.visibility,
|
|
1379
|
+
s.return_type,
|
|
1380
|
+
s.parameter_count,
|
|
1381
|
+
safe_doc,
|
|
1382
|
+
)
|
|
1383
|
+
)
|
|
1384
|
+
body = redact_secrets("\n".join(lines[s.start_line - 1 : s.end_line]))
|
|
1385
|
+
cur = con.execute(
|
|
1386
|
+
"INSERT INTO chunks(path, language, symbol, symbol_type, start_line, end_line, content, hash) "
|
|
1387
|
+
"VALUES (?,?,?,?,?,?,?,?)",
|
|
1388
|
+
(
|
|
1389
|
+
path,
|
|
1390
|
+
language,
|
|
1391
|
+
s.qualified_name,
|
|
1392
|
+
s.kind,
|
|
1393
|
+
s.start_line,
|
|
1394
|
+
s.end_line,
|
|
1395
|
+
body,
|
|
1396
|
+
digest,
|
|
1397
|
+
),
|
|
1398
|
+
)
|
|
1399
|
+
db_write_count += 1
|
|
1400
|
+
t_fts = time.perf_counter()
|
|
1401
|
+
con.execute(
|
|
1402
|
+
"INSERT INTO chunks_fts(rowid, content, path, symbol) VALUES (?,?,?,?)",
|
|
1403
|
+
(cur.lastrowid, body, path, s.qualified_name),
|
|
1404
|
+
)
|
|
1405
|
+
fts_elapsed += time.perf_counter() - t_fts
|
|
1406
|
+
fts_write_count += 1
|
|
1407
|
+
|
|
1408
|
+
con.executemany(
|
|
1409
|
+
"INSERT INTO symbols("
|
|
1410
|
+
"name, qualified_name, kind, path, start_line, end_line, decorators, "
|
|
1411
|
+
"id, canonical_id, language, module, scope, signature, content_hash, "
|
|
1412
|
+
"parent_symbol_id, visibility, return_type, parameter_count, documentation) "
|
|
1413
|
+
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
1414
|
+
symbol_rows,
|
|
1415
|
+
)
|
|
1416
|
+
db_write_count += len(symbol_rows)
|
|
1417
|
+
else:
|
|
1418
|
+
safe_content = redact_secrets(content)
|
|
1419
|
+
cur = con.execute(
|
|
1420
|
+
"INSERT INTO chunks(path, language, symbol, symbol_type, start_line, end_line, content, hash) "
|
|
1421
|
+
"VALUES (?,?,?,?,?,?,?,?)",
|
|
1422
|
+
(path, language, None, "module", 1, max(1, len(lines)), safe_content, digest),
|
|
1423
|
+
)
|
|
1424
|
+
db_write_count += 1
|
|
1425
|
+
t_fts = time.perf_counter()
|
|
1426
|
+
con.execute(
|
|
1427
|
+
"INSERT INTO chunks_fts(rowid, content, path, symbol) VALUES (?,?,?,?)",
|
|
1428
|
+
(cur.lastrowid, safe_content, path, None),
|
|
1429
|
+
)
|
|
1430
|
+
fts_elapsed += time.perf_counter() - t_fts
|
|
1431
|
+
fts_write_count += 1
|
|
1432
|
+
|
|
1433
|
+
if result.imports:
|
|
1434
|
+
con.executemany(
|
|
1435
|
+
"INSERT INTO imports(source_path, module, name, alias, full_name, line, "
|
|
1436
|
+
"source_module, imported_module, imported_name, local_name, import_type) "
|
|
1437
|
+
"VALUES (?,?,?,?,?,?,?,?,?,?,?)",
|
|
1438
|
+
[
|
|
1439
|
+
(
|
|
1440
|
+
path,
|
|
1441
|
+
imp.module,
|
|
1442
|
+
imp.name,
|
|
1443
|
+
imp.alias,
|
|
1444
|
+
imp.full,
|
|
1445
|
+
imp.line,
|
|
1446
|
+
imp.source_module,
|
|
1447
|
+
imp.imported_module,
|
|
1448
|
+
imp.imported_name,
|
|
1449
|
+
imp.local_name,
|
|
1450
|
+
imp.import_type,
|
|
1451
|
+
)
|
|
1452
|
+
for imp in result.imports
|
|
1453
|
+
],
|
|
1454
|
+
)
|
|
1455
|
+
db_write_count += len(result.imports)
|
|
1456
|
+
|
|
1457
|
+
if result.calls:
|
|
1458
|
+
con.executemany(
|
|
1459
|
+
"INSERT INTO calls(source_path, callee, qualified_callee, line, confidence, source_symbol_id) "
|
|
1460
|
+
"VALUES (?,?,?,?,?,?)",
|
|
1461
|
+
[
|
|
1462
|
+
(
|
|
1463
|
+
path,
|
|
1464
|
+
c.callee,
|
|
1465
|
+
c.qualified_callee,
|
|
1466
|
+
c.line,
|
|
1467
|
+
c.confidence,
|
|
1468
|
+
c.caller_canonical_id,
|
|
1469
|
+
)
|
|
1470
|
+
for c in result.calls
|
|
1471
|
+
],
|
|
1472
|
+
)
|
|
1473
|
+
db_write_count += len(result.calls)
|
|
1474
|
+
|
|
1475
|
+
if result.inheritance:
|
|
1476
|
+
con.executemany(
|
|
1477
|
+
"INSERT INTO inheritance(source_symbol, base_name, relationship, source_file, line, source_canonical_id) "
|
|
1478
|
+
"VALUES (?,?,?,?,?,?)",
|
|
1479
|
+
[
|
|
1480
|
+
(
|
|
1481
|
+
inh.source_symbol,
|
|
1482
|
+
inh.base_name,
|
|
1483
|
+
inh.relationship,
|
|
1484
|
+
path,
|
|
1485
|
+
inh.line,
|
|
1486
|
+
inh.source_canonical_id,
|
|
1487
|
+
)
|
|
1488
|
+
for inh in result.inheritance
|
|
1489
|
+
],
|
|
1490
|
+
)
|
|
1491
|
+
db_write_count += len(result.inheritance)
|
|
1492
|
+
|
|
1493
|
+
if result.routes:
|
|
1494
|
+
con.executemany(
|
|
1495
|
+
"INSERT INTO raw_framework_routes(endpoint_id, framework, http_method, route_path, "
|
|
1496
|
+
"normalized_route, handler_name, handler_canonical_id, file_path, line, evidence, confidence, router_name) "
|
|
1497
|
+
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?) "
|
|
1498
|
+
"ON CONFLICT(endpoint_id) DO UPDATE SET "
|
|
1499
|
+
"framework=excluded.framework, "
|
|
1500
|
+
"http_method=excluded.http_method, "
|
|
1501
|
+
"route_path=excluded.route_path, "
|
|
1502
|
+
"normalized_route=excluded.normalized_route, "
|
|
1503
|
+
"handler_name=excluded.handler_name, "
|
|
1504
|
+
"handler_canonical_id=excluded.handler_canonical_id, "
|
|
1505
|
+
"file_path=excluded.file_path, "
|
|
1506
|
+
"line=excluded.line, "
|
|
1507
|
+
"evidence=excluded.evidence, "
|
|
1508
|
+
"confidence=excluded.confidence, "
|
|
1509
|
+
"router_name=excluded.router_name",
|
|
1510
|
+
[
|
|
1511
|
+
(
|
|
1512
|
+
r.endpoint_id,
|
|
1513
|
+
r.framework,
|
|
1514
|
+
r.http_method,
|
|
1515
|
+
r.route_path,
|
|
1516
|
+
r.normalized_route,
|
|
1517
|
+
r.handler_name,
|
|
1518
|
+
r.handler_canonical_id,
|
|
1519
|
+
path,
|
|
1520
|
+
r.line,
|
|
1521
|
+
redact_secrets(r.evidence),
|
|
1522
|
+
r.confidence,
|
|
1523
|
+
r.router_name,
|
|
1524
|
+
)
|
|
1525
|
+
for r in result.routes
|
|
1526
|
+
],
|
|
1527
|
+
)
|
|
1528
|
+
con.executemany(
|
|
1529
|
+
"INSERT INTO framework_routes(endpoint_id, framework, http_method, route_path, "
|
|
1530
|
+
"normalized_route, handler_name, handler_canonical_id, file_path, line, evidence, confidence) "
|
|
1531
|
+
"VALUES (?,?,?,?,?,?,?,?,?,?,?) "
|
|
1532
|
+
"ON CONFLICT(endpoint_id) DO UPDATE SET "
|
|
1533
|
+
"framework=excluded.framework, "
|
|
1534
|
+
"http_method=excluded.http_method, "
|
|
1535
|
+
"route_path=excluded.route_path, "
|
|
1536
|
+
"normalized_route=excluded.normalized_route, "
|
|
1537
|
+
"handler_name=excluded.handler_name, "
|
|
1538
|
+
"handler_canonical_id=excluded.handler_canonical_id, "
|
|
1539
|
+
"file_path=excluded.file_path, "
|
|
1540
|
+
"line=excluded.line, "
|
|
1541
|
+
"evidence=excluded.evidence, "
|
|
1542
|
+
"confidence=excluded.confidence",
|
|
1543
|
+
[
|
|
1544
|
+
(
|
|
1545
|
+
r.endpoint_id,
|
|
1546
|
+
r.framework,
|
|
1547
|
+
r.http_method,
|
|
1548
|
+
r.route_path,
|
|
1549
|
+
r.normalized_route,
|
|
1550
|
+
r.handler_name,
|
|
1551
|
+
r.handler_canonical_id,
|
|
1552
|
+
path,
|
|
1553
|
+
r.line,
|
|
1554
|
+
redact_secrets(r.evidence),
|
|
1555
|
+
r.confidence,
|
|
1556
|
+
)
|
|
1557
|
+
for r in result.routes
|
|
1558
|
+
],
|
|
1559
|
+
)
|
|
1560
|
+
db_write_count += len(result.routes) * 2
|
|
1561
|
+
|
|
1562
|
+
if result.mounts:
|
|
1563
|
+
con.executemany(
|
|
1564
|
+
"INSERT INTO router_mounts(mount_id, framework, parent_router, child_router, prefix, file_path, line, evidence, confidence) "
|
|
1565
|
+
"VALUES (?,?,?,?,?,?,?,?,?) "
|
|
1566
|
+
"ON CONFLICT(mount_id) DO UPDATE SET "
|
|
1567
|
+
"prefix=excluded.prefix, evidence=excluded.evidence, confidence=excluded.confidence",
|
|
1568
|
+
[
|
|
1569
|
+
(
|
|
1570
|
+
f"MOUNT:{m.framework}:{path}:{m.line}:{m.parent_router}->{m.child_router}",
|
|
1571
|
+
m.framework,
|
|
1572
|
+
m.parent_router,
|
|
1573
|
+
m.child_router,
|
|
1574
|
+
m.prefix,
|
|
1575
|
+
path,
|
|
1576
|
+
m.line,
|
|
1577
|
+
redact_secrets(m.evidence),
|
|
1578
|
+
m.confidence,
|
|
1579
|
+
)
|
|
1580
|
+
for m in result.mounts
|
|
1581
|
+
],
|
|
1582
|
+
)
|
|
1583
|
+
db_write_count += len(result.mounts)
|
|
1584
|
+
|
|
1585
|
+
if result.router_definitions:
|
|
1586
|
+
con.executemany(
|
|
1587
|
+
"INSERT INTO router_definitions(framework, router_name, prefix, file_path, line, evidence) "
|
|
1588
|
+
"VALUES (?,?,?,?,?,?)",
|
|
1589
|
+
[
|
|
1590
|
+
(
|
|
1591
|
+
d.framework,
|
|
1592
|
+
d.router_name,
|
|
1593
|
+
d.prefix,
|
|
1594
|
+
path,
|
|
1595
|
+
d.line,
|
|
1596
|
+
redact_secrets(d.evidence),
|
|
1597
|
+
)
|
|
1598
|
+
for d in result.router_definitions
|
|
1599
|
+
],
|
|
1600
|
+
)
|
|
1601
|
+
db_write_count += len(result.router_definitions)
|
|
1602
|
+
|
|
1603
|
+
if result.bindings:
|
|
1604
|
+
con.executemany(
|
|
1605
|
+
"INSERT INTO local_bindings("
|
|
1606
|
+
"target_name, file_path, line, column, scope, expr_kind, source_expr, "
|
|
1607
|
+
"is_conditional, base_expr, attr_name, dict_entries_json, list_entries_json, "
|
|
1608
|
+
"subscript_target, subscript_key, subscript_index) "
|
|
1609
|
+
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
1610
|
+
[
|
|
1611
|
+
(
|
|
1612
|
+
b.target_name,
|
|
1613
|
+
path,
|
|
1614
|
+
b.line,
|
|
1615
|
+
b.column,
|
|
1616
|
+
b.scope,
|
|
1617
|
+
b.expr_kind,
|
|
1618
|
+
redact_secrets(b.source_expr),
|
|
1619
|
+
1 if b.is_conditional else 0,
|
|
1620
|
+
b.base_expr,
|
|
1621
|
+
b.attr_name,
|
|
1622
|
+
json.dumps(b.dict_entries),
|
|
1623
|
+
json.dumps(b.list_entries),
|
|
1624
|
+
b.subscript_target,
|
|
1625
|
+
b.subscript_key,
|
|
1626
|
+
b.subscript_index,
|
|
1627
|
+
)
|
|
1628
|
+
for b in result.bindings
|
|
1629
|
+
],
|
|
1630
|
+
)
|
|
1631
|
+
db_write_count += len(result.bindings)
|
|
1632
|
+
|
|
1633
|
+
total_sql_elapsed = max(0.0, (time.perf_counter() - t_sql) - fts_elapsed)
|
|
1634
|
+
if telemetry is not None:
|
|
1635
|
+
telemetry.record(
|
|
1636
|
+
"sqlite_writes",
|
|
1637
|
+
total_sql_elapsed,
|
|
1638
|
+
files_processed=1,
|
|
1639
|
+
items_processed=db_write_count,
|
|
1640
|
+
db_writes=db_write_count,
|
|
1641
|
+
)
|
|
1642
|
+
if fts_write_count > 0:
|
|
1643
|
+
telemetry.record(
|
|
1644
|
+
"fts_updates",
|
|
1645
|
+
fts_elapsed,
|
|
1646
|
+
files_processed=1,
|
|
1647
|
+
items_processed=fts_write_count,
|
|
1648
|
+
db_writes=fts_write_count,
|
|
1649
|
+
)
|
|
1650
|
+
|
|
1651
|
+
return True
|
|
1652
|
+
|
|
1653
|
+
def _run_global_resolution(
|
|
1654
|
+
self,
|
|
1655
|
+
con: sqlite3.Connection,
|
|
1656
|
+
indexed_commit: str | None,
|
|
1657
|
+
*,
|
|
1658
|
+
dirty_files: set[str] | None = None,
|
|
1659
|
+
removed_files: set[str] | None = None,
|
|
1660
|
+
telemetry: IndexingTelemetry | None = None,
|
|
1661
|
+
progress_callback: Callable[[str, int, int, IndexingTelemetry], None] | None = None,
|
|
1662
|
+
) -> None:
|
|
1663
|
+
"""Run repository-wide reference resolution, updating 'references' and 'graph_edges'."""
|
|
1664
|
+
from codegraph.security.redaction import redact_secrets
|
|
1665
|
+
|
|
1666
|
+
t_prep = time.perf_counter()
|
|
1667
|
+
# 1. Load all files
|
|
1668
|
+
file_rows = con.execute("SELECT path, hash, language FROM files WHERE status='ok'").fetchall()
|
|
1669
|
+
known_files = {str(r["path"]) for r in file_rows}
|
|
1670
|
+
file_hashes = {str(r["path"]): str(r["hash"]) for r in file_rows}
|
|
1671
|
+
file_langs = {str(r["path"]): str(r["language"]) for r in file_rows}
|
|
1672
|
+
|
|
1673
|
+
# 2. Load symbols
|
|
1674
|
+
sym_rows = con.execute(
|
|
1675
|
+
"SELECT name, qualified_name, kind, path, start_line, end_line, decorators, "
|
|
1676
|
+
"id, canonical_id, language, module, scope, signature, content_hash, "
|
|
1677
|
+
"parent_symbol_id, visibility, return_type, parameter_count, documentation "
|
|
1678
|
+
"FROM symbols"
|
|
1679
|
+
).fetchall()
|
|
1680
|
+
symbols = [
|
|
1681
|
+
Symbol(
|
|
1682
|
+
id=str(r["id"]),
|
|
1683
|
+
canonical_id=str(r["canonical_id"]),
|
|
1684
|
+
name=str(r["name"]),
|
|
1685
|
+
qualified_name=str(r["qualified_name"]),
|
|
1686
|
+
kind=str(r["kind"]),
|
|
1687
|
+
start_line=int(r["start_line"]),
|
|
1688
|
+
end_line=int(r["end_line"]),
|
|
1689
|
+
file_path=str(r["path"]),
|
|
1690
|
+
decorators=[d for d in str(r["decorators"]).split(",") if d],
|
|
1691
|
+
language=str(r["language"]),
|
|
1692
|
+
module=str(r["module"]),
|
|
1693
|
+
path=str(r["path"]),
|
|
1694
|
+
scope=str(r["scope"]),
|
|
1695
|
+
signature=str(r["signature"]),
|
|
1696
|
+
content_hash=str(r["content_hash"]),
|
|
1697
|
+
parent_symbol_id=r["parent_symbol_id"],
|
|
1698
|
+
visibility=str(r["visibility"]),
|
|
1699
|
+
return_type=r["return_type"],
|
|
1700
|
+
parameter_count=r["parameter_count"],
|
|
1701
|
+
documentation=r["documentation"],
|
|
1702
|
+
)
|
|
1703
|
+
for r in sym_rows
|
|
1704
|
+
]
|
|
1705
|
+
|
|
1706
|
+
# 3. Load imports
|
|
1707
|
+
imp_rows = con.execute(
|
|
1708
|
+
"SELECT source_path, module, name, alias, full_name, line, "
|
|
1709
|
+
"source_module, imported_module, imported_name, local_name, import_type "
|
|
1710
|
+
"FROM imports"
|
|
1711
|
+
).fetchall()
|
|
1712
|
+
imports = [
|
|
1713
|
+
ImportRef(
|
|
1714
|
+
module=str(r["module"]),
|
|
1715
|
+
source_file=str(r["source_path"]),
|
|
1716
|
+
name=r["name"],
|
|
1717
|
+
alias=r["alias"],
|
|
1718
|
+
full=r["full_name"],
|
|
1719
|
+
line=int(r["line"]),
|
|
1720
|
+
source_module=str(r["source_module"]),
|
|
1721
|
+
imported_module=str(r["imported_module"]),
|
|
1722
|
+
imported_name=r["imported_name"],
|
|
1723
|
+
local_name=str(r["local_name"]),
|
|
1724
|
+
import_type=str(r["import_type"]),
|
|
1725
|
+
is_reexport=(str(r["import_type"]) == "reexport"),
|
|
1726
|
+
)
|
|
1727
|
+
for r in imp_rows
|
|
1728
|
+
]
|
|
1729
|
+
|
|
1730
|
+
# 4. Load calls
|
|
1731
|
+
call_rows = con.execute(
|
|
1732
|
+
"SELECT source_path, callee, qualified_callee, line, confidence, source_symbol_id "
|
|
1733
|
+
"FROM calls"
|
|
1734
|
+
).fetchall()
|
|
1735
|
+
calls = [
|
|
1736
|
+
CallRef(
|
|
1737
|
+
callee=str(r["callee"]),
|
|
1738
|
+
source_file=str(r["source_path"]),
|
|
1739
|
+
confidence=str(r["confidence"]),
|
|
1740
|
+
qualified_callee=r["qualified_callee"],
|
|
1741
|
+
line=int(r["line"]),
|
|
1742
|
+
caller_canonical_id=r["source_symbol_id"],
|
|
1743
|
+
)
|
|
1744
|
+
for r in call_rows
|
|
1745
|
+
]
|
|
1746
|
+
|
|
1747
|
+
# 5. Load inheritance
|
|
1748
|
+
inh_rows = con.execute(
|
|
1749
|
+
"SELECT source_symbol, base_name, relationship, source_file, line, source_canonical_id "
|
|
1750
|
+
"FROM inheritance"
|
|
1751
|
+
).fetchall()
|
|
1752
|
+
inheritance = [
|
|
1753
|
+
InheritanceRef(
|
|
1754
|
+
source_symbol=str(r["source_symbol"]),
|
|
1755
|
+
base_name=str(r["base_name"]),
|
|
1756
|
+
relationship=str(r["relationship"]),
|
|
1757
|
+
source_file=str(r["source_file"]),
|
|
1758
|
+
line=int(r["line"]),
|
|
1759
|
+
source_canonical_id=str(r["source_canonical_id"]),
|
|
1760
|
+
)
|
|
1761
|
+
for r in inh_rows
|
|
1762
|
+
]
|
|
1763
|
+
if telemetry is not None:
|
|
1764
|
+
telemetry.record(
|
|
1765
|
+
"ranking_index_building",
|
|
1766
|
+
time.perf_counter() - t_prep,
|
|
1767
|
+
files_processed=len(known_files),
|
|
1768
|
+
items_processed=len(symbols) + len(imports) + len(calls),
|
|
1769
|
+
)
|
|
1770
|
+
|
|
1771
|
+
# 6. Load raw framework routes, router mounts, and router definitions for composition
|
|
1772
|
+
t_fw = time.perf_counter()
|
|
1773
|
+
from codegraph.frameworks import RouterDefinition, RouterMountDetection
|
|
1774
|
+
from codegraph.route_composer import compose_routes
|
|
1775
|
+
|
|
1776
|
+
raw_rt_rows = con.execute(
|
|
1777
|
+
"SELECT endpoint_id, framework, http_method, route_path, normalized_route, "
|
|
1778
|
+
"handler_name, handler_canonical_id, file_path, line, evidence, confidence, router_name "
|
|
1779
|
+
"FROM raw_framework_routes"
|
|
1780
|
+
).fetchall()
|
|
1781
|
+
raw_routes = [
|
|
1782
|
+
RouteDetection(
|
|
1783
|
+
framework=str(r["framework"]),
|
|
1784
|
+
http_method=str(r["http_method"]),
|
|
1785
|
+
route_path=str(r["route_path"]),
|
|
1786
|
+
normalized_route=str(r["normalized_route"]),
|
|
1787
|
+
handler_name=str(r["handler_name"]),
|
|
1788
|
+
handler_canonical_id=str(r["handler_canonical_id"]),
|
|
1789
|
+
file_path=str(r["file_path"]),
|
|
1790
|
+
line=int(r["line"]),
|
|
1791
|
+
evidence=str(r["evidence"]),
|
|
1792
|
+
confidence=str(r["confidence"]),
|
|
1793
|
+
endpoint_id_override=str(r["endpoint_id"]),
|
|
1794
|
+
router_name=str(r["router_name"]) if "router_name" in r.keys() else "",
|
|
1795
|
+
)
|
|
1796
|
+
for r in raw_rt_rows
|
|
1797
|
+
]
|
|
1798
|
+
|
|
1799
|
+
mount_rows = con.execute(
|
|
1800
|
+
"SELECT framework, parent_router, child_router, prefix, file_path, line, evidence, confidence "
|
|
1801
|
+
"FROM router_mounts"
|
|
1802
|
+
).fetchall()
|
|
1803
|
+
mount_detections = [
|
|
1804
|
+
RouterMountDetection(
|
|
1805
|
+
framework=str(r["framework"]),
|
|
1806
|
+
parent_router=str(r["parent_router"]),
|
|
1807
|
+
child_router=str(r["child_router"]),
|
|
1808
|
+
prefix=str(r["prefix"]),
|
|
1809
|
+
file_path=str(r["file_path"]),
|
|
1810
|
+
line=int(r["line"]),
|
|
1811
|
+
evidence=str(r["evidence"]),
|
|
1812
|
+
confidence=str(r["confidence"]),
|
|
1813
|
+
)
|
|
1814
|
+
for r in mount_rows
|
|
1815
|
+
]
|
|
1816
|
+
|
|
1817
|
+
rdef_rows = con.execute(
|
|
1818
|
+
"SELECT framework, router_name, prefix, file_path, line, evidence "
|
|
1819
|
+
"FROM router_definitions"
|
|
1820
|
+
).fetchall()
|
|
1821
|
+
router_defs = [
|
|
1822
|
+
RouterDefinition(
|
|
1823
|
+
framework=str(r["framework"]),
|
|
1824
|
+
router_name=str(r["router_name"]),
|
|
1825
|
+
prefix=str(r["prefix"]),
|
|
1826
|
+
file_path=str(r["file_path"]),
|
|
1827
|
+
line=int(r["line"]),
|
|
1828
|
+
evidence=str(r["evidence"]),
|
|
1829
|
+
)
|
|
1830
|
+
for r in rdef_rows
|
|
1831
|
+
]
|
|
1832
|
+
|
|
1833
|
+
# Compose routes across hierarchical router mounts
|
|
1834
|
+
composed_routes, verified_mount_edges = compose_routes(
|
|
1835
|
+
raw_routes=raw_routes,
|
|
1836
|
+
mounts=mount_detections,
|
|
1837
|
+
router_defs=router_defs,
|
|
1838
|
+
imports=imports,
|
|
1839
|
+
known_files=known_files,
|
|
1840
|
+
)
|
|
1841
|
+
if telemetry is not None:
|
|
1842
|
+
telemetry.record(
|
|
1843
|
+
"framework_analysis",
|
|
1844
|
+
time.perf_counter() - t_fw,
|
|
1845
|
+
files_processed=len(known_files),
|
|
1846
|
+
items_processed=len(composed_routes) + len(verified_mount_edges),
|
|
1847
|
+
)
|
|
1848
|
+
|
|
1849
|
+
# Repopulate framework_routes with the composed routes
|
|
1850
|
+
t_sql = time.perf_counter()
|
|
1851
|
+
con.execute("DELETE FROM framework_routes")
|
|
1852
|
+
if composed_routes:
|
|
1853
|
+
con.executemany(
|
|
1854
|
+
"INSERT INTO framework_routes(endpoint_id, framework, http_method, route_path, "
|
|
1855
|
+
"normalized_route, handler_name, handler_canonical_id, file_path, line, evidence, confidence) "
|
|
1856
|
+
"VALUES (?,?,?,?,?,?,?,?,?,?,?)",
|
|
1857
|
+
[
|
|
1858
|
+
(
|
|
1859
|
+
r.endpoint_id,
|
|
1860
|
+
r.framework,
|
|
1861
|
+
r.http_method,
|
|
1862
|
+
r.route_path,
|
|
1863
|
+
r.normalized_route,
|
|
1864
|
+
r.handler_name,
|
|
1865
|
+
r.handler_canonical_id,
|
|
1866
|
+
r.file_path,
|
|
1867
|
+
r.line,
|
|
1868
|
+
redact_secrets(r.evidence),
|
|
1869
|
+
r.confidence,
|
|
1870
|
+
)
|
|
1871
|
+
for r in composed_routes
|
|
1872
|
+
],
|
|
1873
|
+
)
|
|
1874
|
+
if telemetry is not None:
|
|
1875
|
+
telemetry.record(
|
|
1876
|
+
"sqlite_writes",
|
|
1877
|
+
time.perf_counter() - t_sql,
|
|
1878
|
+
items_processed=len(composed_routes),
|
|
1879
|
+
db_writes=len(composed_routes) + 1,
|
|
1880
|
+
)
|
|
1881
|
+
|
|
1882
|
+
# 7. Load local bindings for data-flow and alias resolution
|
|
1883
|
+
t_sym_res = time.perf_counter()
|
|
1884
|
+
binding_rows = con.execute(
|
|
1885
|
+
"SELECT target_name, file_path, line, column, scope, expr_kind, source_expr, "
|
|
1886
|
+
"is_conditional, base_expr, attr_name, dict_entries_json, list_entries_json, "
|
|
1887
|
+
"subscript_target, subscript_key, subscript_index "
|
|
1888
|
+
"FROM local_bindings"
|
|
1889
|
+
).fetchall()
|
|
1890
|
+
bindings = [
|
|
1891
|
+
BindingRef(
|
|
1892
|
+
target_name=str(r["target_name"]),
|
|
1893
|
+
file_path=str(r["file_path"]),
|
|
1894
|
+
line=int(r["line"]),
|
|
1895
|
+
column=r["column"],
|
|
1896
|
+
scope=str(r["scope"]),
|
|
1897
|
+
expr_kind=str(r["expr_kind"]),
|
|
1898
|
+
source_expr=str(r["source_expr"]),
|
|
1899
|
+
is_conditional=bool(r["is_conditional"]),
|
|
1900
|
+
base_expr=str(r["base_expr"]),
|
|
1901
|
+
attr_name=str(r["attr_name"]),
|
|
1902
|
+
dict_entries=tuple(tuple(x) for x in json.loads(r["dict_entries_json"])),
|
|
1903
|
+
list_entries=tuple(json.loads(r["list_entries_json"])),
|
|
1904
|
+
subscript_target=r["subscript_target"],
|
|
1905
|
+
subscript_key=r["subscript_key"],
|
|
1906
|
+
subscript_index=r["subscript_index"],
|
|
1907
|
+
)
|
|
1908
|
+
for r in binding_rows
|
|
1909
|
+
]
|
|
1910
|
+
|
|
1911
|
+
from codegraph.monorepo import detect_workspace
|
|
1912
|
+
workspace = detect_workspace(self.repository, con=con)
|
|
1913
|
+
|
|
1914
|
+
from codegraph.tsconfig import TsConfigResolver
|
|
1915
|
+
tsconfig_resolver = TsConfigResolver.load_from_repo(self.repository)
|
|
1916
|
+
|
|
1917
|
+
# Run resolution engine with composed routes, mount edges, bindings, workspace, and tsconfig
|
|
1918
|
+
resolver = ReferenceResolver(
|
|
1919
|
+
symbols=symbols,
|
|
1920
|
+
imports=imports,
|
|
1921
|
+
calls=calls,
|
|
1922
|
+
inheritance=inheritance,
|
|
1923
|
+
routes=composed_routes,
|
|
1924
|
+
mount_edges=verified_mount_edges,
|
|
1925
|
+
known_files=known_files,
|
|
1926
|
+
file_hashes=file_hashes,
|
|
1927
|
+
indexed_commit=indexed_commit,
|
|
1928
|
+
bindings=bindings,
|
|
1929
|
+
workspace=workspace,
|
|
1930
|
+
tsconfig_resolver=tsconfig_resolver,
|
|
1931
|
+
repository=self.repository,
|
|
1932
|
+
max_reexport_depth=self.max_reexport_depth,
|
|
1933
|
+
max_wildcard_expansions=self.max_wildcard_expansions,
|
|
1934
|
+
)
|
|
1935
|
+
if telemetry is not None:
|
|
1936
|
+
telemetry.record(
|
|
1937
|
+
"symbol_resolution",
|
|
1938
|
+
time.perf_counter() - t_sym_res,
|
|
1939
|
+
files_processed=len(known_files),
|
|
1940
|
+
items_processed=len(symbols) + len(bindings),
|
|
1941
|
+
)
|
|
1942
|
+
if progress_callback is not None:
|
|
1943
|
+
progress_callback("symbol_resolution", len(known_files), len(known_files), telemetry)
|
|
1944
|
+
|
|
1945
|
+
t_rel_res = time.perf_counter()
|
|
1946
|
+
output = resolver.resolve_all()
|
|
1947
|
+
self.last_epistemic_findings = list(output.findings)
|
|
1948
|
+
|
|
1949
|
+
# Update framework_routes handler_canonical_id if resolved via local bindings / imports
|
|
1950
|
+
route_updates: list[tuple[str, str]] = []
|
|
1951
|
+
for rt in composed_routes:
|
|
1952
|
+
handler_sym, _conf, _reason = resolver.resolve_identifier_in_file(
|
|
1953
|
+
rt.handler_name, rt.file_path, use_line=rt.line
|
|
1954
|
+
)
|
|
1955
|
+
if handler_sym and handler_sym.canonical_id != rt.handler_canonical_id:
|
|
1956
|
+
route_updates.append((handler_sym.canonical_id, rt.endpoint_id))
|
|
1957
|
+
if route_updates:
|
|
1958
|
+
con.executemany(
|
|
1959
|
+
"UPDATE framework_routes SET handler_canonical_id=? WHERE endpoint_id=?",
|
|
1960
|
+
route_updates,
|
|
1961
|
+
)
|
|
1962
|
+
|
|
1963
|
+
num_symbols = len(symbols)
|
|
1964
|
+
# Release in-memory IR lists and resolver caches before DB pass & SQL serialization
|
|
1965
|
+
symbols.clear()
|
|
1966
|
+
imports.clear()
|
|
1967
|
+
calls.clear()
|
|
1968
|
+
inheritance.clear()
|
|
1969
|
+
bindings.clear()
|
|
1970
|
+
del resolver
|
|
1971
|
+
|
|
1972
|
+
if telemetry is not None:
|
|
1973
|
+
budget_skips = sum(
|
|
1974
|
+
1 for f in output.findings if f.get("reason") == "resolution_budget_exceeded"
|
|
1975
|
+
)
|
|
1976
|
+
telemetry.record(
|
|
1977
|
+
"relationship_resolution",
|
|
1978
|
+
time.perf_counter() - t_rel_res,
|
|
1979
|
+
files_processed=len(known_files),
|
|
1980
|
+
items_processed=len(output.references) + len(output.resolved_imports) + len(output.resolved_calls),
|
|
1981
|
+
errors_skips=budget_skips,
|
|
1982
|
+
db_writes=len(route_updates),
|
|
1983
|
+
)
|
|
1984
|
+
if progress_callback is not None:
|
|
1985
|
+
progress_callback("relationship_resolution", len(known_files), len(known_files), telemetry)
|
|
1986
|
+
|
|
1987
|
+
# Run repository-wide Database Deep Intelligence extraction (Phases 1-8)
|
|
1988
|
+
with (telemetry.phase("database_analysis") if telemetry is not None else contextmanager(lambda: iter([None]))()) as p_db:
|
|
1989
|
+
db_edges = self._run_database_intelligence_pass(
|
|
1990
|
+
con,
|
|
1991
|
+
sorted(known_files),
|
|
1992
|
+
file_langs,
|
|
1993
|
+
dirty_files=dirty_files,
|
|
1994
|
+
removed_files=removed_files,
|
|
1995
|
+
)
|
|
1996
|
+
if p_db is not None:
|
|
1997
|
+
p_db.files_processed = len(dirty_files) if dirty_files is not None else len(known_files)
|
|
1998
|
+
p_db.items_processed = len(db_edges)
|
|
1999
|
+
if telemetry is not None and progress_callback is not None:
|
|
2000
|
+
progress_callback("database_analysis", len(known_files), len(known_files), telemetry)
|
|
2001
|
+
|
|
2002
|
+
t_edges = time.perf_counter()
|
|
2003
|
+
all_edges: list[Any] = [*output.edges, *db_edges]
|
|
2004
|
+
output.edges.clear()
|
|
2005
|
+
db_edges.clear()
|
|
2006
|
+
total_edges = len(all_edges)
|
|
2007
|
+
total_refs = len(output.references)
|
|
2008
|
+
if telemetry is not None:
|
|
2009
|
+
telemetry.symbols_count = num_symbols
|
|
2010
|
+
telemetry.edges_count = total_edges
|
|
2011
|
+
telemetry.record(
|
|
2012
|
+
"graph_edge_creation",
|
|
2013
|
+
time.perf_counter() - t_edges,
|
|
2014
|
+
files_processed=len(known_files),
|
|
2015
|
+
items_processed=total_edges,
|
|
2016
|
+
)
|
|
2017
|
+
|
|
2018
|
+
# Batch write references, graph_edges, and resolved imports/calls with bounded WAL checkpoints
|
|
2019
|
+
t_sql_post = time.perf_counter()
|
|
2020
|
+
post_chunk_size = 4000
|
|
2021
|
+
|
|
2022
|
+
def _checkpoint_post() -> None:
|
|
2023
|
+
con.commit()
|
|
2024
|
+
if telemetry is not None:
|
|
2025
|
+
telemetry.batch_commits += 1
|
|
2026
|
+
telemetry.observe_wal(self.db_path)
|
|
2027
|
+
try:
|
|
2028
|
+
con.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
|
2029
|
+
if telemetry is not None:
|
|
2030
|
+
telemetry.wal_checkpoints += 1
|
|
2031
|
+
except sqlite3.OperationalError:
|
|
2032
|
+
pass
|
|
2033
|
+
if telemetry is not None:
|
|
2034
|
+
telemetry.observe_wal(self.db_path)
|
|
2035
|
+
|
|
2036
|
+
con.execute("DELETE FROM 'references'")
|
|
2037
|
+
for i in range(0, total_refs, post_chunk_size):
|
|
2038
|
+
batch_refs = output.references[i : i + post_chunk_size]
|
|
2039
|
+
con.executemany(
|
|
2040
|
+
"INSERT INTO 'references'("
|
|
2041
|
+
"source_symbol_id, target_symbol_id, relationship, confidence, path, "
|
|
2042
|
+
"start_line, end_line, evidence, source_hash, indexed_commit, evidence_status) "
|
|
2043
|
+
"VALUES (?,?,?,?,?,?,?,?,?,?,?)",
|
|
2044
|
+
[
|
|
2045
|
+
(
|
|
2046
|
+
ref.source_symbol_id,
|
|
2047
|
+
ref.target_symbol_id,
|
|
2048
|
+
ref.relationship,
|
|
2049
|
+
ref.confidence,
|
|
2050
|
+
ref.path,
|
|
2051
|
+
ref.start_line,
|
|
2052
|
+
ref.end_line,
|
|
2053
|
+
redact_secrets(ref.evidence),
|
|
2054
|
+
ref.source_hash,
|
|
2055
|
+
ref.indexed_commit,
|
|
2056
|
+
ref.evidence_status,
|
|
2057
|
+
)
|
|
2058
|
+
for ref in batch_refs
|
|
2059
|
+
],
|
|
2060
|
+
)
|
|
2061
|
+
if total_refs > post_chunk_size:
|
|
2062
|
+
_checkpoint_post()
|
|
2063
|
+
output.references.clear()
|
|
2064
|
+
|
|
2065
|
+
con.execute("DELETE FROM graph_edges")
|
|
2066
|
+
for i in range(0, total_edges, post_chunk_size):
|
|
2067
|
+
batch_edges = all_edges[i : i + post_chunk_size]
|
|
2068
|
+
con.executemany(
|
|
2069
|
+
"INSERT INTO graph_edges("
|
|
2070
|
+
"source, target, relationship, confidence, file, start_line, end_line, "
|
|
2071
|
+
"evidence, evidence_id, source_hash, indexed_commit, evidence_class, reason) "
|
|
2072
|
+
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
2073
|
+
[
|
|
2074
|
+
(
|
|
2075
|
+
e.source,
|
|
2076
|
+
e.target,
|
|
2077
|
+
e.relationship,
|
|
2078
|
+
e.confidence,
|
|
2079
|
+
e.file,
|
|
2080
|
+
e.start_line,
|
|
2081
|
+
e.end_line,
|
|
2082
|
+
redact_secrets(e.evidence),
|
|
2083
|
+
f"edge_{i + offset}",
|
|
2084
|
+
file_hashes.get(e.file, ""),
|
|
2085
|
+
indexed_commit,
|
|
2086
|
+
getattr(e, "evidence_class", "AST_VERIFIED"),
|
|
2087
|
+
getattr(e, "reason", None),
|
|
2088
|
+
)
|
|
2089
|
+
for offset, e in enumerate(batch_edges)
|
|
2090
|
+
],
|
|
2091
|
+
)
|
|
2092
|
+
if total_edges > post_chunk_size:
|
|
2093
|
+
_checkpoint_post()
|
|
2094
|
+
all_edges.clear()
|
|
2095
|
+
|
|
2096
|
+
# Batch update resolved fields on imports and calls tables using composite indexes
|
|
2097
|
+
num_res_imports = len(output.resolved_imports)
|
|
2098
|
+
if output.resolved_imports:
|
|
2099
|
+
imp_updates = [
|
|
2100
|
+
(tgt_path, tgt_mod, src_path, ln, loc_name, loc_name, loc_name)
|
|
2101
|
+
for (src_path, ln, loc_name), (tgt_path, tgt_mod) in output.resolved_imports.items()
|
|
2102
|
+
]
|
|
2103
|
+
output.resolved_imports.clear()
|
|
2104
|
+
for i in range(0, len(imp_updates), post_chunk_size):
|
|
2105
|
+
con.executemany(
|
|
2106
|
+
"UPDATE imports SET resolved_path=?, resolved_module=? "
|
|
2107
|
+
"WHERE source_path=? AND line=? AND (local_name=? OR alias=? OR name=?)",
|
|
2108
|
+
imp_updates[i : i + post_chunk_size],
|
|
2109
|
+
)
|
|
2110
|
+
if len(imp_updates) > post_chunk_size:
|
|
2111
|
+
_checkpoint_post()
|
|
2112
|
+
|
|
2113
|
+
num_res_calls = len(output.resolved_calls)
|
|
2114
|
+
if output.resolved_calls:
|
|
2115
|
+
call_updates = [
|
|
2116
|
+
(resolved_id, conf, src_path, ln, callee_nm)
|
|
2117
|
+
for (src_path, ln, callee_nm), (resolved_id, conf) in output.resolved_calls.items()
|
|
2118
|
+
]
|
|
2119
|
+
output.resolved_calls.clear()
|
|
2120
|
+
for i in range(0, len(call_updates), post_chunk_size):
|
|
2121
|
+
con.executemany(
|
|
2122
|
+
"UPDATE calls SET resolved_symbol_id=?, confidence=? "
|
|
2123
|
+
"WHERE source_path=? AND line=? AND callee=?",
|
|
2124
|
+
call_updates[i : i + post_chunk_size],
|
|
2125
|
+
)
|
|
2126
|
+
if len(call_updates) > post_chunk_size:
|
|
2127
|
+
_checkpoint_post()
|
|
2128
|
+
|
|
2129
|
+
if telemetry is not None:
|
|
2130
|
+
post_writes = (
|
|
2131
|
+
total_refs
|
|
2132
|
+
+ total_edges
|
|
2133
|
+
+ num_res_imports
|
|
2134
|
+
+ num_res_calls
|
|
2135
|
+
)
|
|
2136
|
+
telemetry.record(
|
|
2137
|
+
"sqlite_writes",
|
|
2138
|
+
time.perf_counter() - t_sql_post,
|
|
2139
|
+
items_processed=post_writes,
|
|
2140
|
+
db_writes=post_writes,
|
|
2141
|
+
)
|
|
2142
|
+
|
|
2143
|
+
def _run_database_intelligence_pass(
|
|
2144
|
+
self,
|
|
2145
|
+
con: sqlite3.Connection,
|
|
2146
|
+
sorted_files: list[str],
|
|
2147
|
+
file_langs: dict[str, str],
|
|
2148
|
+
*,
|
|
2149
|
+
dirty_files: set[str] | None = None,
|
|
2150
|
+
removed_files: set[str] | None = None,
|
|
2151
|
+
) -> list[RelationshipRecord]:
|
|
2152
|
+
"""Extract ORM models, tables, columns, queries, migrations, and database edges across the repo.
|
|
2153
|
+
|
|
2154
|
+
Streams file contents instead of holding all repository files in memory at once,
|
|
2155
|
+
and supports incremental dirty-file extraction when the repository dialect and
|
|
2156
|
+
cross-file ORM model->table mappings are unchanged.
|
|
2157
|
+
"""
|
|
2158
|
+
from codegraph.database import (
|
|
2159
|
+
DatabaseEntity,
|
|
2160
|
+
DatabaseQueryFact,
|
|
2161
|
+
MigrationFact,
|
|
2162
|
+
detect_file_dialects,
|
|
2163
|
+
extract_database_from_file,
|
|
2164
|
+
has_potential_database_activity,
|
|
2165
|
+
has_potential_orm_models,
|
|
2166
|
+
)
|
|
2167
|
+
from codegraph.security.redaction import redact_secrets
|
|
2168
|
+
|
|
2169
|
+
def _compute_repo_dialect(dialects_map: dict[str, list[str]]) -> str:
|
|
2170
|
+
union_dialects: set[str] = set()
|
|
2171
|
+
for dlist in dialects_map.values():
|
|
2172
|
+
union_dialects.update(dlist)
|
|
2173
|
+
if len(union_dialects) == 1:
|
|
2174
|
+
return next(iter(union_dialects))
|
|
2175
|
+
if len(union_dialects) > 1:
|
|
2176
|
+
return "AMBIGUOUS"
|
|
2177
|
+
return "UNKNOWN"
|
|
2178
|
+
|
|
2179
|
+
def _flatten_model_tables(models_map: dict[str, dict[str, str]]) -> dict[str, str]:
|
|
2180
|
+
flat: dict[str, str] = {}
|
|
2181
|
+
for fpath in sorted(models_map):
|
|
2182
|
+
flat.update(models_map[fpath])
|
|
2183
|
+
return flat
|
|
2184
|
+
|
|
2185
|
+
def _insert_db_rows(
|
|
2186
|
+
entities: list[DatabaseEntity],
|
|
2187
|
+
queries: list[DatabaseQueryFact],
|
|
2188
|
+
migrations: list[MigrationFact],
|
|
2189
|
+
) -> None:
|
|
2190
|
+
if entities:
|
|
2191
|
+
con.executemany(
|
|
2192
|
+
"INSERT INTO db_entities("
|
|
2193
|
+
"canonical_id, kind, name, dialect, schema_name, table_name, column_name, "
|
|
2194
|
+
"data_type, nullable, default_value, is_primary_key, is_foreign_key, is_unique, "
|
|
2195
|
+
"is_indexed, target_table, target_column, orm_model_id, framework, file_path, "
|
|
2196
|
+
"start_line, end_line, evidence, confidence, evidence_class, status, metadata_json) "
|
|
2197
|
+
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
2198
|
+
[
|
|
2199
|
+
(
|
|
2200
|
+
e.canonical_id,
|
|
2201
|
+
e.kind,
|
|
2202
|
+
e.name,
|
|
2203
|
+
e.dialect,
|
|
2204
|
+
e.schema_name,
|
|
2205
|
+
e.table_name,
|
|
2206
|
+
e.column_name,
|
|
2207
|
+
e.data_type,
|
|
2208
|
+
1 if e.nullable else 0,
|
|
2209
|
+
redact_secrets(e.default_value) if e.default_value is not None else None,
|
|
2210
|
+
1 if e.is_primary_key else 0,
|
|
2211
|
+
1 if e.is_foreign_key else 0,
|
|
2212
|
+
1 if e.is_unique else 0,
|
|
2213
|
+
1 if e.is_indexed else 0,
|
|
2214
|
+
e.target_table,
|
|
2215
|
+
e.target_column,
|
|
2216
|
+
e.orm_model_id,
|
|
2217
|
+
e.framework,
|
|
2218
|
+
e.file_path,
|
|
2219
|
+
e.start_line,
|
|
2220
|
+
e.end_line,
|
|
2221
|
+
redact_secrets(e.evidence),
|
|
2222
|
+
e.confidence,
|
|
2223
|
+
e.evidence_class,
|
|
2224
|
+
e.status,
|
|
2225
|
+
json.dumps(e.metadata),
|
|
2226
|
+
)
|
|
2227
|
+
for e in entities
|
|
2228
|
+
],
|
|
2229
|
+
)
|
|
2230
|
+
if queries:
|
|
2231
|
+
con.executemany(
|
|
2232
|
+
"INSERT OR REPLACE INTO db_queries("
|
|
2233
|
+
"query_id, caller_symbol_id, operation, relationship, table_name, table_canonical_id, "
|
|
2234
|
+
"columns_json, framework, normalized_sql, file_path, start_line, end_line, "
|
|
2235
|
+
"evidence, confidence, evidence_class, status) "
|
|
2236
|
+
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
2237
|
+
[
|
|
2238
|
+
(
|
|
2239
|
+
q.query_id,
|
|
2240
|
+
q.caller_symbol_id,
|
|
2241
|
+
q.operation,
|
|
2242
|
+
q.relationship,
|
|
2243
|
+
q.table_name,
|
|
2244
|
+
q.table_canonical_id,
|
|
2245
|
+
json.dumps(list(q.columns)),
|
|
2246
|
+
q.framework,
|
|
2247
|
+
redact_secrets(q.normalized_sql),
|
|
2248
|
+
q.file_path,
|
|
2249
|
+
q.start_line,
|
|
2250
|
+
q.end_line,
|
|
2251
|
+
redact_secrets(q.evidence),
|
|
2252
|
+
q.confidence,
|
|
2253
|
+
q.evidence_class,
|
|
2254
|
+
q.status,
|
|
2255
|
+
)
|
|
2256
|
+
for q in queries
|
|
2257
|
+
],
|
|
2258
|
+
)
|
|
2259
|
+
if migrations:
|
|
2260
|
+
con.executemany(
|
|
2261
|
+
"INSERT OR REPLACE INTO db_migrations("
|
|
2262
|
+
"migration_id, canonical_id, system, revision, down_revision, file_path, "
|
|
2263
|
+
"start_line, end_line, created_tables_json, dropped_tables_json, added_columns_json, "
|
|
2264
|
+
"removed_columns_json, renamed_columns_json, indexes_json, foreign_keys_json, "
|
|
2265
|
+
"constraints_json, affected_tables_json, evidence, confidence, evidence_class) "
|
|
2266
|
+
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
2267
|
+
[
|
|
2268
|
+
(
|
|
2269
|
+
m.migration_id,
|
|
2270
|
+
m.canonical_id,
|
|
2271
|
+
m.system,
|
|
2272
|
+
m.revision,
|
|
2273
|
+
m.down_revision,
|
|
2274
|
+
m.file_path,
|
|
2275
|
+
m.start_line,
|
|
2276
|
+
m.end_line,
|
|
2277
|
+
json.dumps(list(m.created_tables)),
|
|
2278
|
+
json.dumps(list(m.dropped_tables)),
|
|
2279
|
+
json.dumps(list(m.added_columns)),
|
|
2280
|
+
json.dumps(list(m.removed_columns)),
|
|
2281
|
+
json.dumps(list(m.renamed_columns)),
|
|
2282
|
+
json.dumps(list(m.indexes)),
|
|
2283
|
+
json.dumps(list(m.foreign_keys)),
|
|
2284
|
+
json.dumps(list(m.constraints)),
|
|
2285
|
+
json.dumps(list(m.affected_tables)),
|
|
2286
|
+
redact_secrets(m.evidence),
|
|
2287
|
+
m.confidence,
|
|
2288
|
+
m.evidence_class,
|
|
2289
|
+
)
|
|
2290
|
+
for m in migrations
|
|
2291
|
+
],
|
|
2292
|
+
)
|
|
2293
|
+
|
|
2294
|
+
# Attempt fast incremental DB extraction when dirty_files is a small subset
|
|
2295
|
+
can_try_incremental = (
|
|
2296
|
+
dirty_files is not None
|
|
2297
|
+
and len(sorted_files) > 0
|
|
2298
|
+
and 0 < len(dirty_files) + len(removed_files or ()) <= max(64, len(sorted_files) // 5)
|
|
2299
|
+
)
|
|
2300
|
+
if can_try_incremental and dirty_files is not None:
|
|
2301
|
+
prev_dialect_row = con.execute(
|
|
2302
|
+
"SELECT value FROM metadata WHERE key='db_repo_dialect'"
|
|
2303
|
+
).fetchone()
|
|
2304
|
+
prev_dialects_row = con.execute(
|
|
2305
|
+
"SELECT value FROM metadata WHERE key='db_dialects_by_file_json'"
|
|
2306
|
+
).fetchone()
|
|
2307
|
+
prev_models_row = con.execute(
|
|
2308
|
+
"SELECT value FROM metadata WHERE key='db_models_by_file_json'"
|
|
2309
|
+
).fetchone()
|
|
2310
|
+
if prev_dialect_row and prev_dialects_row and prev_models_row:
|
|
2311
|
+
try:
|
|
2312
|
+
prev_repo_dialect = str(prev_dialect_row[0])
|
|
2313
|
+
dialects_by_file: dict[str, list[str]] = json.loads(str(prev_dialects_row[0]))
|
|
2314
|
+
models_by_file: dict[str, dict[str, str]] = json.loads(str(prev_models_row[0]))
|
|
2315
|
+
prev_known_model_tables = _flatten_model_tables(models_by_file)
|
|
2316
|
+
|
|
2317
|
+
changed_or_removed = set(dirty_files) | set(removed_files or ())
|
|
2318
|
+
for rem in changed_or_removed:
|
|
2319
|
+
dialects_by_file.pop(rem, None)
|
|
2320
|
+
models_by_file.pop(rem, None)
|
|
2321
|
+
|
|
2322
|
+
dirty_texts: dict[str, str] = {}
|
|
2323
|
+
for fpath in sorted(dirty_files):
|
|
2324
|
+
abs_p = self.repository / fpath
|
|
2325
|
+
try:
|
|
2326
|
+
text = abs_p.read_text(encoding="utf-8", errors="replace")
|
|
2327
|
+
except OSError:
|
|
2328
|
+
continue
|
|
2329
|
+
dirty_texts[fpath] = text
|
|
2330
|
+
f_dialects = sorted(detect_file_dialects(text, fpath))
|
|
2331
|
+
if f_dialects:
|
|
2332
|
+
dialects_by_file[fpath] = f_dialects
|
|
2333
|
+
|
|
2334
|
+
new_repo_dialect = _compute_repo_dialect(dialects_by_file)
|
|
2335
|
+
for fpath, text in dirty_texts.items():
|
|
2336
|
+
pass1 = extract_database_from_file(
|
|
2337
|
+
text,
|
|
2338
|
+
fpath,
|
|
2339
|
+
language=file_langs.get(fpath, "python"),
|
|
2340
|
+
repo_dialect=new_repo_dialect,
|
|
2341
|
+
)
|
|
2342
|
+
f_models: dict[str, str] = {}
|
|
2343
|
+
for ent in pass1.entities:
|
|
2344
|
+
if ent.kind == "ORMModel" and ent.table_name:
|
|
2345
|
+
f_models[ent.name] = ent.table_name
|
|
2346
|
+
f_models[ent.canonical_id] = ent.table_name
|
|
2347
|
+
if f_models:
|
|
2348
|
+
models_by_file[fpath] = f_models
|
|
2349
|
+
|
|
2350
|
+
new_known_model_tables = _flatten_model_tables(models_by_file)
|
|
2351
|
+
if (
|
|
2352
|
+
new_repo_dialect == prev_repo_dialect
|
|
2353
|
+
and new_known_model_tables == prev_known_model_tables
|
|
2354
|
+
):
|
|
2355
|
+
# Dialect and ORM model->table mappings are unchanged!
|
|
2356
|
+
# Only update DB records for dirty_files and preserve unchanged files' DB edges.
|
|
2357
|
+
for fpath in changed_or_removed:
|
|
2358
|
+
con.execute("DELETE FROM db_entities WHERE file_path=?", (fpath,))
|
|
2359
|
+
con.execute("DELETE FROM db_queries WHERE file_path=?", (fpath,))
|
|
2360
|
+
con.execute("DELETE FROM db_migrations WHERE file_path=?", (fpath,))
|
|
2361
|
+
|
|
2362
|
+
inc_entities: list[DatabaseEntity] = []
|
|
2363
|
+
inc_queries: list[DatabaseQueryFact] = []
|
|
2364
|
+
inc_migrations: list[MigrationFact] = []
|
|
2365
|
+
inc_edges: list[RelationshipRecord] = []
|
|
2366
|
+
|
|
2367
|
+
for fpath in sorted(dirty_texts):
|
|
2368
|
+
pass2 = extract_database_from_file(
|
|
2369
|
+
dirty_texts[fpath],
|
|
2370
|
+
fpath,
|
|
2371
|
+
language=file_langs.get(fpath, "python"),
|
|
2372
|
+
repo_dialect=new_repo_dialect,
|
|
2373
|
+
known_model_tables=new_known_model_tables,
|
|
2374
|
+
)
|
|
2375
|
+
inc_entities.extend(pass2.entities)
|
|
2376
|
+
inc_queries.extend(pass2.queries)
|
|
2377
|
+
inc_migrations.extend(pass2.migrations)
|
|
2378
|
+
inc_edges.extend(pass2.edges)
|
|
2379
|
+
|
|
2380
|
+
_insert_db_rows(inc_entities, inc_queries, inc_migrations)
|
|
2381
|
+
|
|
2382
|
+
placeholders = ",".join("?" for _ in _DB_EDGE_RELATIONSHIPS)
|
|
2383
|
+
existing_db_edge_rows = con.execute(
|
|
2384
|
+
f"SELECT source, target, relationship, confidence, file, "
|
|
2385
|
+
f"start_line, end_line, evidence, evidence_class, reason "
|
|
2386
|
+
f"FROM graph_edges WHERE relationship IN ({placeholders})",
|
|
2387
|
+
_DB_EDGE_RELATIONSHIPS,
|
|
2388
|
+
).fetchall()
|
|
2389
|
+
preserved_edges = [
|
|
2390
|
+
RelationshipRecord(
|
|
2391
|
+
source=str(r["source"]),
|
|
2392
|
+
target=str(r["target"]),
|
|
2393
|
+
relationship=str(r["relationship"]),
|
|
2394
|
+
file=str(r["file"]),
|
|
2395
|
+
start_line=int(r["start_line"]),
|
|
2396
|
+
end_line=int(r["end_line"]),
|
|
2397
|
+
evidence=str(r["evidence"]),
|
|
2398
|
+
evidence_class=str(r["evidence_class"]),
|
|
2399
|
+
confidence=str(r["confidence"]),
|
|
2400
|
+
reason=r["reason"],
|
|
2401
|
+
)
|
|
2402
|
+
for r in existing_db_edge_rows
|
|
2403
|
+
if str(r["file"]) not in changed_or_removed
|
|
2404
|
+
]
|
|
2405
|
+
con.executemany(
|
|
2406
|
+
"INSERT INTO metadata(key, value) VALUES(?,?) "
|
|
2407
|
+
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
|
2408
|
+
[
|
|
2409
|
+
("db_dialects_by_file_json", json.dumps(dialects_by_file)),
|
|
2410
|
+
("db_models_by_file_json", json.dumps(models_by_file)),
|
|
2411
|
+
],
|
|
2412
|
+
)
|
|
2413
|
+
return [*preserved_edges, *inc_edges]
|
|
2414
|
+
except (sqlite3.OperationalError, ValueError, KeyError, TypeError):
|
|
2415
|
+
pass
|
|
2416
|
+
|
|
2417
|
+
# Full streaming pass: never hold all file_contents in RAM at once
|
|
2418
|
+
dialects_by_file_full: dict[str, list[str]] = {}
|
|
2419
|
+
models_by_file_full: dict[str, dict[str, str]] = {}
|
|
2420
|
+
db_candidate_files: list[str] = []
|
|
2421
|
+
|
|
2422
|
+
# Pass 1: stream files to detect dialect and ORM model -> table mappings
|
|
2423
|
+
for fpath in sorted_files:
|
|
2424
|
+
abs_p = self.repository / fpath
|
|
2425
|
+
try:
|
|
2426
|
+
text = abs_p.read_text(encoding="utf-8", errors="replace")
|
|
2427
|
+
except OSError:
|
|
2428
|
+
continue
|
|
2429
|
+
if not text:
|
|
2430
|
+
continue
|
|
2431
|
+
lang = file_langs.get(fpath, "python")
|
|
2432
|
+
f_dialects = sorted(detect_file_dialects(text, fpath))
|
|
2433
|
+
if f_dialects:
|
|
2434
|
+
dialects_by_file_full[fpath] = f_dialects
|
|
2435
|
+
|
|
2436
|
+
if has_potential_database_activity(text, fpath, lang):
|
|
2437
|
+
db_candidate_files.append(fpath)
|
|
2438
|
+
|
|
2439
|
+
if has_potential_orm_models(text, fpath, lang):
|
|
2440
|
+
pass1 = extract_database_from_file(
|
|
2441
|
+
text,
|
|
2442
|
+
fpath,
|
|
2443
|
+
language=lang,
|
|
2444
|
+
repo_dialect="UNKNOWN",
|
|
2445
|
+
)
|
|
2446
|
+
f_models_full: dict[str, str] = {}
|
|
2447
|
+
for ent in pass1.entities:
|
|
2448
|
+
if ent.kind == "ORMModel" and ent.table_name:
|
|
2449
|
+
f_models_full[ent.name] = ent.table_name
|
|
2450
|
+
f_models_full[ent.canonical_id] = ent.table_name
|
|
2451
|
+
if f_models_full:
|
|
2452
|
+
models_by_file_full[fpath] = f_models_full
|
|
2453
|
+
|
|
2454
|
+
repo_dialect = _compute_repo_dialect(dialects_by_file_full)
|
|
2455
|
+
known_model_tables = _flatten_model_tables(models_by_file_full)
|
|
2456
|
+
|
|
2457
|
+
# Pass 2: stream only db_candidate_files (files that actually contain DB constructs)
|
|
2458
|
+
all_entities: list[DatabaseEntity] = []
|
|
2459
|
+
all_queries: list[DatabaseQueryFact] = []
|
|
2460
|
+
all_migrations: list[MigrationFact] = []
|
|
2461
|
+
all_edges: list[RelationshipRecord] = []
|
|
2462
|
+
|
|
2463
|
+
for fpath in db_candidate_files:
|
|
2464
|
+
abs_p = self.repository / fpath
|
|
2465
|
+
try:
|
|
2466
|
+
content = abs_p.read_text(encoding="utf-8", errors="replace")
|
|
2467
|
+
except OSError:
|
|
2468
|
+
continue
|
|
2469
|
+
if not content:
|
|
2470
|
+
continue
|
|
2471
|
+
pass2 = extract_database_from_file(
|
|
2472
|
+
content,
|
|
2473
|
+
fpath,
|
|
2474
|
+
language=file_langs.get(fpath, "python"),
|
|
2475
|
+
repo_dialect=repo_dialect,
|
|
2476
|
+
known_model_tables=known_model_tables,
|
|
2477
|
+
)
|
|
2478
|
+
all_entities.extend(pass2.entities)
|
|
2479
|
+
all_queries.extend(pass2.queries)
|
|
2480
|
+
all_migrations.extend(pass2.migrations)
|
|
2481
|
+
all_edges.extend(pass2.edges)
|
|
2482
|
+
|
|
2483
|
+
con.execute("DELETE FROM db_entities")
|
|
2484
|
+
con.execute("DELETE FROM db_queries")
|
|
2485
|
+
con.execute("DELETE FROM db_migrations")
|
|
2486
|
+
_insert_db_rows(all_entities, all_queries, all_migrations)
|
|
2487
|
+
|
|
2488
|
+
con.executemany(
|
|
2489
|
+
"INSERT INTO metadata(key, value) VALUES(?,?) "
|
|
2490
|
+
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
|
2491
|
+
[
|
|
2492
|
+
("db_repo_dialect", repo_dialect),
|
|
2493
|
+
("db_dialects_by_file_json", json.dumps(dialects_by_file_full)),
|
|
2494
|
+
("db_models_by_file_json", json.dumps(models_by_file_full)),
|
|
2495
|
+
],
|
|
2496
|
+
)
|
|
2497
|
+
|
|
2498
|
+
return list(all_edges)
|
|
2499
|
+
|
|
2500
|
+
|
|
2501
|
+
def check_database_health(con: sqlite3.Connection, repository: Path | None = None) -> dict[str, object]:
|
|
2502
|
+
"""Inspect database integrity, FTS synchronization, orphan records, and consistency."""
|
|
2503
|
+
issues: list[str] = []
|
|
2504
|
+
|
|
2505
|
+
# 1. Foreign keys
|
|
2506
|
+
fk_enabled = con.execute("PRAGMA foreign_keys").fetchone()[0]
|
|
2507
|
+
if not fk_enabled:
|
|
2508
|
+
issues.append("Foreign keys are disabled in SQLite connection.")
|
|
2509
|
+
|
|
2510
|
+
# 2. Schema and user version
|
|
2511
|
+
user_ver = con.execute("PRAGMA user_version").fetchone()[0]
|
|
2512
|
+
if user_ver < SCHEMA_VERSION:
|
|
2513
|
+
issues.append(f"Schema version mismatch: database is v{user_ver}, expected v{SCHEMA_VERSION}.")
|
|
2514
|
+
|
|
2515
|
+
# 3. FTS synchronization check
|
|
2516
|
+
chunks_count = con.execute("SELECT count(*) FROM chunks").fetchone()[0]
|
|
2517
|
+
fts_count = con.execute("SELECT count(*) FROM chunks_fts").fetchone()[0]
|
|
2518
|
+
if chunks_count != fts_count:
|
|
2519
|
+
issues.append(f"FTS desynchronization: chunks table has {chunks_count} rows, chunks_fts has {fts_count} rows.")
|
|
2520
|
+
|
|
2521
|
+
# 4. Orphan symbols
|
|
2522
|
+
orphan_syms = con.execute(
|
|
2523
|
+
"SELECT count(*) FROM symbols s LEFT JOIN files f ON s.path = f.path WHERE f.path IS NULL"
|
|
2524
|
+
).fetchone()[0]
|
|
2525
|
+
if orphan_syms > 0:
|
|
2526
|
+
issues.append(f"Found {orphan_syms} orphan symbol(s) referencing non-existent files.")
|
|
2527
|
+
|
|
2528
|
+
# 5. Orphan references
|
|
2529
|
+
orphan_refs = con.execute(
|
|
2530
|
+
"SELECT count(*) FROM 'references' r LEFT JOIN files f ON r.path = f.path WHERE f.path IS NULL"
|
|
2531
|
+
).fetchone()[0]
|
|
2532
|
+
if orphan_refs > 0:
|
|
2533
|
+
issues.append(f"Found {orphan_refs} orphan reference(s) referencing non-existent files.")
|
|
2534
|
+
|
|
2535
|
+
# 6. Duplicate canonical IDs within same file
|
|
2536
|
+
dups = con.execute(
|
|
2537
|
+
"SELECT canonical_id, path, count(*) FROM symbols "
|
|
2538
|
+
"GROUP BY canonical_id, path HAVING count(*) > 1"
|
|
2539
|
+
).fetchall()
|
|
2540
|
+
if dups:
|
|
2541
|
+
issues.append(f"Found {len(dups)} duplicate canonical symbol ID(s) in same file.")
|
|
2542
|
+
|
|
2543
|
+
# 7. Check parser version
|
|
2544
|
+
parser_ver = None
|
|
2545
|
+
try:
|
|
2546
|
+
prow = con.execute("SELECT value FROM metadata WHERE key='parser_version'").fetchone()
|
|
2547
|
+
if prow:
|
|
2548
|
+
parser_ver = str(prow[0])
|
|
2549
|
+
except sqlite3.OperationalError:
|
|
2550
|
+
pass
|
|
2551
|
+
if parser_ver and parser_ver != PARSER_VERSION:
|
|
2552
|
+
issues.append(f"Parser version mismatch: indexed with {parser_ver}, active parser is {PARSER_VERSION}.")
|
|
2553
|
+
|
|
2554
|
+
# 8. Orphan framework routes
|
|
2555
|
+
orphan_routes = con.execute(
|
|
2556
|
+
"SELECT count(*) FROM framework_routes r LEFT JOIN files f ON r.file_path = f.path WHERE f.path IS NULL"
|
|
2557
|
+
).fetchone()[0]
|
|
2558
|
+
if orphan_routes > 0:
|
|
2559
|
+
issues.append(f"Found {orphan_routes} orphan route(s) referencing non-existent files.")
|
|
2560
|
+
|
|
2561
|
+
# 9. Duplicate endpoint IDs
|
|
2562
|
+
dup_routes = con.execute(
|
|
2563
|
+
"SELECT endpoint_id, count(*) FROM framework_routes GROUP BY endpoint_id HAVING count(*) > 1"
|
|
2564
|
+
).fetchall()
|
|
2565
|
+
if dup_routes:
|
|
2566
|
+
issues.append(f"Found {len(dup_routes)} duplicate route endpoint ID(s).")
|
|
2567
|
+
|
|
2568
|
+
status = "OK" if not issues else "ISSUES_FOUND"
|
|
2569
|
+
return {
|
|
2570
|
+
"status": status,
|
|
2571
|
+
"database_version": user_ver,
|
|
2572
|
+
"expected_version": SCHEMA_VERSION,
|
|
2573
|
+
"parser_version": parser_ver or PARSER_VERSION,
|
|
2574
|
+
"chunks_count": chunks_count,
|
|
2575
|
+
"fts_count": fts_count,
|
|
2576
|
+
"foreign_keys": bool(fk_enabled),
|
|
2577
|
+
"issues": issues,
|
|
2578
|
+
}
|