codegraph-engine 3.0.0__1-py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codegraph/__init__.py +37 -0
- codegraph/agent.py +26 -0
- codegraph/agent_brain.py +2212 -0
- codegraph/agent_capabilities.py +4058 -0
- codegraph/agent_rules.py +531 -0
- codegraph/api_drift.py +455 -0
- codegraph/architecture.py +371 -0
- codegraph/async_queue/__init__.py +50 -0
- codegraph/async_queue/detectors/__init__.py +48 -0
- codegraph/async_queue/detectors/celery.py +160 -0
- codegraph/async_queue/detectors/kafka.py +168 -0
- codegraph/async_queue/detectors/rabbitmq.py +116 -0
- codegraph/async_queue/detectors/redis.py +130 -0
- codegraph/async_queue/detectors/sqs.py +135 -0
- codegraph/async_queue/models.py +161 -0
- codegraph/async_queue/otel_models.py +74 -0
- codegraph/async_queue/otel_parser.py +337 -0
- codegraph/async_queue/reconciler.py +427 -0
- codegraph/async_queue/scanner.py +291 -0
- codegraph/async_queue/traversal.py +447 -0
- codegraph/audit.py +106 -0
- codegraph/binding_resolver.py +616 -0
- codegraph/branch_comparison.py +197 -0
- codegraph/cache.py +95 -0
- codegraph/change_impact.py +547 -0
- codegraph/cli.py +2436 -0
- codegraph/cli_output.py +205 -0
- codegraph/config.py +45 -0
- codegraph/constraints.py +238 -0
- codegraph/context.py +2182 -0
- codegraph/context_freshness.py +351 -0
- codegraph/database/__init__.py +72 -0
- codegraph/database/extractor.py +3651 -0
- codegraph/database/interrogation.py +1494 -0
- codegraph/database/models.py +310 -0
- codegraph/database/schema_drift.py +862 -0
- codegraph/epistemic.py +200 -0
- codegraph/errors.py +282 -0
- codegraph/evidence/__init__.py +15 -0
- codegraph/evidence/citations.py +418 -0
- codegraph/evidence_contract.py +486 -0
- codegraph/frameworks.py +960 -0
- codegraph/frameworks_nestjs.py +502 -0
- codegraph/frameworks_nextjs.py +337 -0
- codegraph/frameworks_react.py +457 -0
- codegraph/freshness.py +335 -0
- codegraph/frontend_html.py +109 -0
- codegraph/git.py +362 -0
- codegraph/git_state.py +545 -0
- codegraph/graph/__init__.py +46 -0
- codegraph/graph/models.py +56 -0
- codegraph/graph/traversal.py +1993 -0
- codegraph/indexing/__init__.py +24 -0
- codegraph/indexing/classifier.py +564 -0
- codegraph/indexing/indexer.py +2578 -0
- codegraph/indexing/models.py +427 -0
- codegraph/indexing/parser.py +3404 -0
- codegraph/indexing/pool.py +136 -0
- codegraph/indexing/scanner.py +211 -0
- codegraph/indexing/telemetry.py +326 -0
- codegraph/indexing/templates.py +418 -0
- codegraph/indexing/test_framework.py +116 -0
- codegraph/installer.py +1702 -0
- codegraph/interrogation.py +2355 -0
- codegraph/llm/__init__.py +3 -0
- codegraph/llm/base.py +15 -0
- codegraph/llm/context.py +20 -0
- codegraph/mcp/__init__.py +3 -0
- codegraph/mcp/server.py +1476 -0
- codegraph/mcp_diagnostics.py +882 -0
- codegraph/memory/__init__.py +3 -0
- codegraph/memory/store.py +46 -0
- codegraph/models.py +310 -0
- codegraph/monorepo.py +925 -0
- codegraph/observability.py +262 -0
- codegraph/optimizer.py +1014 -0
- codegraph/planner.py +417 -0
- codegraph/process_lifecycle.py +1153 -0
- codegraph/py.typed +1 -0
- codegraph/query_expansion.py +199 -0
- codegraph/ranking.py +408 -0
- codegraph/refactor/__init__.py +36 -0
- codegraph/refactor/models.py +114 -0
- codegraph/refactor/renamer.py +844 -0
- codegraph/refactor/transactions.py +414 -0
- codegraph/resolver.py +3968 -0
- codegraph/resources/__init__.py +46 -0
- codegraph/resources/cache.py +117 -0
- codegraph/resources/coalescer.py +83 -0
- codegraph/resources/debouncer.py +98 -0
- codegraph/resources/governor.py +219 -0
- codegraph/resources/monitor.py +102 -0
- codegraph/resources/policy.py +123 -0
- codegraph/retrieval_policy.py +320 -0
- codegraph/roblox.py +2174 -0
- codegraph/route_composer.py +363 -0
- codegraph/route_topology.py +172 -0
- codegraph/runtime/__init__.py +51 -0
- codegraph/runtime/collector.py +155 -0
- codegraph/runtime/db_watcher.py +191 -0
- codegraph/runtime/ingestor.py +692 -0
- codegraph/runtime/models.py +130 -0
- codegraph/runtime/port_inspector.py +304 -0
- codegraph/runtime/reconciliation.py +492 -0
- codegraph/runtime/runner.py +353 -0
- codegraph/runtime/trace_linker.py +175 -0
- codegraph/search/__init__.py +25 -0
- codegraph/search/hybrid.py +846 -0
- codegraph/search/semantic.py +28 -0
- codegraph/security/__init__.py +51 -0
- codegraph/security/paths.py +189 -0
- codegraph/security/redaction.py +633 -0
- codegraph/semantic_conflicts.py +270 -0
- codegraph/semantic_decorators.py +372 -0
- codegraph/structural_diff.py +542 -0
- codegraph/styles.py +144 -0
- codegraph/symbol_history.py +277 -0
- codegraph/target_resolver.py +348 -0
- codegraph/task.py +637 -0
- codegraph/tool_selection_eval.py +1158 -0
- codegraph/tsconfig.py +236 -0
- codegraph/ui/__init__.py +6 -0
- codegraph/ui/server.py +755 -0
- codegraph/watcher.py +323 -0
- codegraph_engine-3.0.0.dist-info/METADATA +859 -0
- codegraph_engine-3.0.0.dist-info/RECORD +130 -0
- codegraph_engine-3.0.0.dist-info/WHEEL +6 -0
- codegraph_engine-3.0.0.dist-info/entry_points.txt +2 -0
- codegraph_engine-3.0.0.dist-info/licenses/LICENSE +21 -0
- codegraph_engine-3.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1494 @@
|
|
|
1
|
+
"""Database Intelligence Interrogation & Bidirectional Code-Database Impact Engine (Phases 8, 9, 17).
|
|
2
|
+
|
|
3
|
+
Provides deterministic, evidence-backed queries across:
|
|
4
|
+
- Tables, Columns, Primary Keys, Foreign Keys, Indexes, Constraints, Views, Sequences
|
|
5
|
+
- ORM Models & Field Mappings
|
|
6
|
+
- SQL & ORM Queries (Readers, Writers, Callers)
|
|
7
|
+
- Schema Migrations & Column/Table History
|
|
8
|
+
- Bidirectional Code <-> Database Impact Analysis (`DIRECT`, `FRAMEWORK`, `SEMANTIC`, `DI`, `TEST`, `DATABASE`, `RUNTIME`)
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import sqlite3
|
|
14
|
+
from collections import deque
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from codegraph.security.redaction import REDACTED_DB_CREDENTIALS, redact_payload, redact_secrets
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _table_matches(candidate_table: str, candidate_cid: str, query_str: str) -> bool:
|
|
21
|
+
if not query_str:
|
|
22
|
+
return True
|
|
23
|
+
q = query_str.strip().lower()
|
|
24
|
+
t = (candidate_table or "").strip().lower()
|
|
25
|
+
cid = (candidate_cid or "").strip().lower()
|
|
26
|
+
if q in (t, cid):
|
|
27
|
+
return True
|
|
28
|
+
if cid.endswith(f".{q}"):
|
|
29
|
+
return True
|
|
30
|
+
return False
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def find_db_tables(
|
|
34
|
+
con: sqlite3.Connection,
|
|
35
|
+
repo_or_name: Any = None,
|
|
36
|
+
name: str | None = None,
|
|
37
|
+
dialect: str | None = None,
|
|
38
|
+
schema: str | None = None,
|
|
39
|
+
*,
|
|
40
|
+
table: str | None = None,
|
|
41
|
+
) -> dict[str, Any]:
|
|
42
|
+
"""Discover database tables and views in the repository."""
|
|
43
|
+
if isinstance(repo_or_name, str) and not name and not table:
|
|
44
|
+
name = repo_or_name
|
|
45
|
+
elif table and not name:
|
|
46
|
+
name = table
|
|
47
|
+
rows = con.execute(
|
|
48
|
+
"SELECT canonical_id, kind, name, dialect, schema_name, table_name, orm_model_id, "
|
|
49
|
+
"framework, file_path, start_line, end_line, evidence, confidence, evidence_class, status, metadata_json "
|
|
50
|
+
"FROM db_entities WHERE kind IN ('Table', 'View') "
|
|
51
|
+
"ORDER BY table_name, canonical_id, file_path, start_line"
|
|
52
|
+
).fetchall()
|
|
53
|
+
|
|
54
|
+
tables_by_cid: dict[str, dict[str, Any]] = {}
|
|
55
|
+
for r in rows:
|
|
56
|
+
t_name = str(r["table_name"] or r["name"])
|
|
57
|
+
cid = str(r["canonical_id"])
|
|
58
|
+
if name and not _table_matches(t_name, cid, name):
|
|
59
|
+
continue
|
|
60
|
+
if dialect and str(r["dialect"]).lower() != dialect.strip().lower():
|
|
61
|
+
continue
|
|
62
|
+
if schema and str(r["schema_name"]).lower() != schema.strip().lower():
|
|
63
|
+
continue
|
|
64
|
+
|
|
65
|
+
if cid not in tables_by_cid:
|
|
66
|
+
tables_by_cid[cid] = {
|
|
67
|
+
"canonical_id": cid,
|
|
68
|
+
"kind": str(r["kind"]),
|
|
69
|
+
"name": t_name,
|
|
70
|
+
"table_name": t_name,
|
|
71
|
+
"dialect": str(r["dialect"]),
|
|
72
|
+
"schema_name": str(r["schema_name"]),
|
|
73
|
+
"orm_models": [str(r["orm_model_id"])] if r["orm_model_id"] else [],
|
|
74
|
+
"framework": str(r["framework"]),
|
|
75
|
+
"file": str(r["file_path"]),
|
|
76
|
+
"start_line": int(r["start_line"]),
|
|
77
|
+
"end_line": int(r["end_line"]),
|
|
78
|
+
"evidence": redact_secrets(str(r["evidence"])),
|
|
79
|
+
"confidence": str(r["confidence"]),
|
|
80
|
+
"evidence_class": str(r["evidence_class"]),
|
|
81
|
+
"status": str(r["status"]),
|
|
82
|
+
}
|
|
83
|
+
else:
|
|
84
|
+
if r["orm_model_id"] and str(r["orm_model_id"]) not in tables_by_cid[cid]["orm_models"]:
|
|
85
|
+
tables_by_cid[cid]["orm_models"].append(str(r["orm_model_id"]))
|
|
86
|
+
|
|
87
|
+
results = sorted(tables_by_cid.values(), key=lambda x: (str(x["table_name"]), str(x["canonical_id"])))
|
|
88
|
+
return {
|
|
89
|
+
"status": "ok",
|
|
90
|
+
"count": len(results),
|
|
91
|
+
"tables": results,
|
|
92
|
+
"results": results,
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def find_db_columns(
|
|
97
|
+
con: sqlite3.Connection,
|
|
98
|
+
repo_or_table: Any = None,
|
|
99
|
+
column: str | None = None,
|
|
100
|
+
*,
|
|
101
|
+
table: str | None = None,
|
|
102
|
+
) -> dict[str, Any]:
|
|
103
|
+
"""Discover database columns, types, nullability, PK/FK flags, and ORM field mappings."""
|
|
104
|
+
if isinstance(repo_or_table, str) and not table:
|
|
105
|
+
table = repo_or_table
|
|
106
|
+
rows = con.execute(
|
|
107
|
+
"SELECT canonical_id, name, dialect, schema_name, table_name, column_name, data_type, "
|
|
108
|
+
"nullable, default_value, is_primary_key, is_foreign_key, is_unique, is_indexed, "
|
|
109
|
+
"target_table, target_column, orm_model_id, framework, file_path, start_line, end_line, "
|
|
110
|
+
"evidence, confidence, evidence_class, status "
|
|
111
|
+
"FROM db_entities WHERE kind = 'Column' "
|
|
112
|
+
"ORDER BY table_name, column_name, file_path, start_line"
|
|
113
|
+
).fetchall()
|
|
114
|
+
|
|
115
|
+
results: list[dict[str, Any]] = []
|
|
116
|
+
seen: set[tuple[str, str]] = set()
|
|
117
|
+
for r in rows:
|
|
118
|
+
t_name = str(r["table_name"])
|
|
119
|
+
c_name = str(r["column_name"] or r["name"])
|
|
120
|
+
cid = str(r["canonical_id"])
|
|
121
|
+
if table and not _table_matches(t_name, cid, table):
|
|
122
|
+
continue
|
|
123
|
+
if column and column.strip().lower() not in (c_name.lower(), cid.lower(), f"{t_name.lower()}.{c_name.lower()}"):
|
|
124
|
+
continue
|
|
125
|
+
dedup_key = (cid, str(r["file_path"]))
|
|
126
|
+
if dedup_key in seen:
|
|
127
|
+
continue
|
|
128
|
+
seen.add(dedup_key)
|
|
129
|
+
fk_target: str | None = None
|
|
130
|
+
if r["target_table"] and r["target_column"]:
|
|
131
|
+
fk_target = f"{r['target_table']}.{r['target_column']}"
|
|
132
|
+
elif r["target_table"]:
|
|
133
|
+
fk_target = str(r["target_table"])
|
|
134
|
+
results.append({
|
|
135
|
+
"canonical_id": cid,
|
|
136
|
+
"table_name": t_name,
|
|
137
|
+
"column_name": c_name,
|
|
138
|
+
"name": c_name,
|
|
139
|
+
"dialect": str(r["dialect"]),
|
|
140
|
+
"schema_name": str(r["schema_name"]),
|
|
141
|
+
"data_type": str(r["data_type"]),
|
|
142
|
+
"nullable": bool(r["nullable"]),
|
|
143
|
+
"default_value": redact_secrets(str(r["default_value"])) if r["default_value"] is not None else None,
|
|
144
|
+
"is_primary_key": bool(r["is_primary_key"]),
|
|
145
|
+
"is_foreign_key": bool(r["is_foreign_key"]),
|
|
146
|
+
"is_unique": bool(r["is_unique"]),
|
|
147
|
+
"is_indexed": bool(r["is_indexed"]),
|
|
148
|
+
"target_table": r["target_table"],
|
|
149
|
+
"target_column": r["target_column"],
|
|
150
|
+
"foreign_key_target": fk_target,
|
|
151
|
+
"orm_model_id": r["orm_model_id"],
|
|
152
|
+
"framework": str(r["framework"]),
|
|
153
|
+
"file": str(r["file_path"]),
|
|
154
|
+
"start_line": int(r["start_line"]),
|
|
155
|
+
"end_line": int(r["end_line"]),
|
|
156
|
+
"evidence": redact_secrets(str(r["evidence"])),
|
|
157
|
+
"confidence": str(r["confidence"]),
|
|
158
|
+
"evidence_class": str(r["evidence_class"]),
|
|
159
|
+
"status": str(r["status"]),
|
|
160
|
+
})
|
|
161
|
+
|
|
162
|
+
return {
|
|
163
|
+
"status": "ok",
|
|
164
|
+
"count": len(results),
|
|
165
|
+
"columns": results,
|
|
166
|
+
"results": results,
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def find_db_models(
|
|
171
|
+
con: sqlite3.Connection,
|
|
172
|
+
repo_or_name: Any = None,
|
|
173
|
+
name: str | None = None,
|
|
174
|
+
table: str | None = None,
|
|
175
|
+
framework: str | None = None,
|
|
176
|
+
*,
|
|
177
|
+
model: str | None = None,
|
|
178
|
+
) -> dict[str, Any]:
|
|
179
|
+
"""Discover ORM models and their mapped database tables."""
|
|
180
|
+
if isinstance(repo_or_name, str) and not name and not model:
|
|
181
|
+
name = repo_or_name
|
|
182
|
+
elif model and not name:
|
|
183
|
+
name = model
|
|
184
|
+
rows = con.execute(
|
|
185
|
+
"SELECT canonical_id, name, dialect, schema_name, table_name, framework, "
|
|
186
|
+
"file_path, start_line, end_line, evidence, confidence, evidence_class, status "
|
|
187
|
+
"FROM db_entities WHERE kind = 'ORMModel' "
|
|
188
|
+
"ORDER BY name, canonical_id, file_path, start_line"
|
|
189
|
+
).fetchall()
|
|
190
|
+
|
|
191
|
+
results: list[dict[str, Any]] = []
|
|
192
|
+
for r in rows:
|
|
193
|
+
m_name = str(r["name"])
|
|
194
|
+
cid = str(r["canonical_id"])
|
|
195
|
+
t_name = str(r["table_name"])
|
|
196
|
+
fw = str(r["framework"])
|
|
197
|
+
if name and name.strip().lower() not in (m_name.lower(), cid.lower()):
|
|
198
|
+
continue
|
|
199
|
+
if table and table.strip().lower() != t_name.lower():
|
|
200
|
+
continue
|
|
201
|
+
if framework and framework.strip().lower() != fw.lower():
|
|
202
|
+
continue
|
|
203
|
+
|
|
204
|
+
# Attach mapped fields
|
|
205
|
+
col_rows = con.execute(
|
|
206
|
+
"SELECT column_name, data_type, is_primary_key, is_foreign_key, target_table, target_column "
|
|
207
|
+
"FROM db_entities WHERE kind = 'Column' AND orm_model_id = ? ORDER BY start_line, column_name",
|
|
208
|
+
(cid,),
|
|
209
|
+
).fetchall()
|
|
210
|
+
fields = [
|
|
211
|
+
{
|
|
212
|
+
"column": str(cr["column_name"]),
|
|
213
|
+
"data_type": str(cr["data_type"]),
|
|
214
|
+
"is_primary_key": bool(cr["is_primary_key"]),
|
|
215
|
+
"is_foreign_key": bool(cr["is_foreign_key"]),
|
|
216
|
+
"target_table": cr["target_table"],
|
|
217
|
+
"target_column": cr["target_column"],
|
|
218
|
+
}
|
|
219
|
+
for cr in col_rows
|
|
220
|
+
]
|
|
221
|
+
results.append({
|
|
222
|
+
"canonical_id": cid,
|
|
223
|
+
"name": m_name,
|
|
224
|
+
"model_name": m_name,
|
|
225
|
+
"table_name": t_name,
|
|
226
|
+
"dialect": str(r["dialect"]),
|
|
227
|
+
"schema_name": str(r["schema_name"]),
|
|
228
|
+
"framework": fw,
|
|
229
|
+
"fields": fields,
|
|
230
|
+
"file": str(r["file_path"]),
|
|
231
|
+
"start_line": int(r["start_line"]),
|
|
232
|
+
"end_line": int(r["end_line"]),
|
|
233
|
+
"evidence": redact_secrets(str(r["evidence"])),
|
|
234
|
+
"confidence": str(r["confidence"]),
|
|
235
|
+
"evidence_class": str(r["evidence_class"]),
|
|
236
|
+
"status": str(r["status"]),
|
|
237
|
+
})
|
|
238
|
+
|
|
239
|
+
return {
|
|
240
|
+
"status": "ok",
|
|
241
|
+
"count": len(results),
|
|
242
|
+
"models": results,
|
|
243
|
+
"results": results,
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def find_db_queries(
|
|
248
|
+
con: sqlite3.Connection,
|
|
249
|
+
repo_or_table: Any = None,
|
|
250
|
+
table: str | None = None,
|
|
251
|
+
operation: str | None = None,
|
|
252
|
+
symbol: str | None = None,
|
|
253
|
+
) -> dict[str, Any]:
|
|
254
|
+
"""Discover static database queries (ORM and raw SQL) across the repository."""
|
|
255
|
+
if isinstance(repo_or_table, str) and not table:
|
|
256
|
+
table = repo_or_table
|
|
257
|
+
rows = con.execute(
|
|
258
|
+
"SELECT query_id, caller_symbol_id, operation, relationship, table_name, table_canonical_id, "
|
|
259
|
+
"columns_json, framework, normalized_sql, file_path, start_line, end_line, "
|
|
260
|
+
"evidence, confidence, evidence_class, status "
|
|
261
|
+
"FROM db_queries ORDER BY file_path, start_line, query_id"
|
|
262
|
+
).fetchall()
|
|
263
|
+
|
|
264
|
+
results: list[dict[str, Any]] = []
|
|
265
|
+
for r in rows:
|
|
266
|
+
t_name = str(r["table_name"])
|
|
267
|
+
t_cid = str(r["table_canonical_id"])
|
|
268
|
+
op = str(r["operation"])
|
|
269
|
+
caller = str(r["caller_symbol_id"])
|
|
270
|
+
if table and not _table_matches(t_name, t_cid, table):
|
|
271
|
+
continue
|
|
272
|
+
if operation and operation.strip().upper() != op.upper():
|
|
273
|
+
continue
|
|
274
|
+
if symbol and symbol.strip() not in (caller, caller.split(".")[-1]):
|
|
275
|
+
continue
|
|
276
|
+
|
|
277
|
+
cols = json.loads(r["columns_json"]) if r["columns_json"] else []
|
|
278
|
+
results.append({
|
|
279
|
+
"query_id": str(r["query_id"]),
|
|
280
|
+
"symbol": caller,
|
|
281
|
+
"caller_symbol_id": caller,
|
|
282
|
+
"operation": op,
|
|
283
|
+
"relationship": str(r["relationship"]),
|
|
284
|
+
"table_name": t_name,
|
|
285
|
+
"table_canonical_id": t_cid,
|
|
286
|
+
"columns": cols,
|
|
287
|
+
"framework": str(r["framework"]),
|
|
288
|
+
"normalized_sql": redact_secrets(str(r["normalized_sql"])),
|
|
289
|
+
"file": str(r["file_path"]),
|
|
290
|
+
"start_line": int(r["start_line"]),
|
|
291
|
+
"end_line": int(r["end_line"]),
|
|
292
|
+
"evidence": redact_secrets(str(r["evidence"])),
|
|
293
|
+
"confidence": str(r["confidence"]),
|
|
294
|
+
"evidence_class": str(r["evidence_class"]),
|
|
295
|
+
"status": str(r["status"]),
|
|
296
|
+
})
|
|
297
|
+
|
|
298
|
+
return {
|
|
299
|
+
"status": "ok",
|
|
300
|
+
"count": len(results),
|
|
301
|
+
"queries": results,
|
|
302
|
+
"results": results,
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _find_table_accessors(
|
|
307
|
+
con: sqlite3.Connection,
|
|
308
|
+
target: str,
|
|
309
|
+
rel_filter: tuple[str, ...],
|
|
310
|
+
include_upstream_routes: bool = True,
|
|
311
|
+
) -> dict[str, Any]:
|
|
312
|
+
"""Shared implementation for find_db_readers, find_db_writers, and find_db_callers."""
|
|
313
|
+
q = target.strip().lower()
|
|
314
|
+
# Match graph_edges where relationship is in rel_filter and target matches table or column
|
|
315
|
+
placeholders = ",".join("?" for _ in rel_filter)
|
|
316
|
+
edge_rows = con.execute(
|
|
317
|
+
f"SELECT source, target, relationship, confidence, file, start_line, end_line, "
|
|
318
|
+
f"evidence, evidence_class, reason "
|
|
319
|
+
f"FROM graph_edges WHERE relationship IN ({placeholders}) "
|
|
320
|
+
f"ORDER BY file, start_line, source, target",
|
|
321
|
+
rel_filter,
|
|
322
|
+
).fetchall()
|
|
323
|
+
|
|
324
|
+
direct_accessors: list[dict[str, Any]] = []
|
|
325
|
+
seed_symbols: set[str] = set()
|
|
326
|
+
|
|
327
|
+
for r in edge_rows:
|
|
328
|
+
tgt = str(r["target"])
|
|
329
|
+
tgt_lower = tgt.lower()
|
|
330
|
+
if not (
|
|
331
|
+
tgt_lower == q
|
|
332
|
+
or tgt_lower.endswith(f".{q}")
|
|
333
|
+
or f".{q}." in tgt_lower
|
|
334
|
+
):
|
|
335
|
+
continue
|
|
336
|
+
src = str(r["source"])
|
|
337
|
+
seed_symbols.add(src)
|
|
338
|
+
ev_cls = str(r["evidence_class"] or "AST_VERIFIED")
|
|
339
|
+
st = "UNKNOWN" if ev_cls == "UNKNOWN" else ("INFERENCE" if ev_cls in ("POSSIBLE", "AMBIGUOUS") else "FACT")
|
|
340
|
+
direct_accessors.append({
|
|
341
|
+
"source": src,
|
|
342
|
+
"symbol": src,
|
|
343
|
+
"target": tgt,
|
|
344
|
+
"relationship": str(r["relationship"]),
|
|
345
|
+
"confidence": str(r["confidence"]),
|
|
346
|
+
"evidence_class": ev_cls,
|
|
347
|
+
"status": st,
|
|
348
|
+
"file": str(r["file"]),
|
|
349
|
+
"start_line": int(r["start_line"]),
|
|
350
|
+
"end_line": int(r["end_line"]),
|
|
351
|
+
"evidence": redact_secrets(str(r["evidence"])),
|
|
352
|
+
"reason": r["reason"],
|
|
353
|
+
"distance": 1,
|
|
354
|
+
})
|
|
355
|
+
|
|
356
|
+
# Traverse upstream callers (service -> controller -> route) deterministically
|
|
357
|
+
upstream_callers: list[dict[str, Any]] = []
|
|
358
|
+
reaching_routes: list[dict[str, Any]] = []
|
|
359
|
+
|
|
360
|
+
if include_upstream_routes and seed_symbols:
|
|
361
|
+
visited: set[str] = set(seed_symbols)
|
|
362
|
+
queue: deque[tuple[str, int, list[str]]] = deque((s, 1, [s]) for s in sorted(seed_symbols))
|
|
363
|
+
|
|
364
|
+
while queue:
|
|
365
|
+
curr_sym, dist, path_chain = queue.popleft()
|
|
366
|
+
if dist >= 5:
|
|
367
|
+
continue
|
|
368
|
+
|
|
369
|
+
# Find callers of curr_sym in graph_edges
|
|
370
|
+
caller_rows = con.execute(
|
|
371
|
+
"SELECT source, target, relationship, confidence, file, start_line, end_line, evidence, evidence_class "
|
|
372
|
+
"FROM graph_edges WHERE target = ? AND relationship IN ('CALLS', 'HANDLED_BY', 'ROUTE_HANDLER', 'ROUTES_TO', 'INJECTS', 'PROVIDES') "
|
|
373
|
+
"ORDER BY file, start_line, source",
|
|
374
|
+
(curr_sym,),
|
|
375
|
+
).fetchall()
|
|
376
|
+
|
|
377
|
+
for cr in caller_rows:
|
|
378
|
+
caller_src = str(cr["source"])
|
|
379
|
+
rel_type = str(cr["relationship"])
|
|
380
|
+
new_chain = [caller_src, *path_chain]
|
|
381
|
+
if caller_src.startswith("ENDPOINT:") or rel_type in ("HANDLED_BY", "ROUTE_HANDLER", "ROUTES_TO"):
|
|
382
|
+
ep_parts = caller_src.split(":", 2)
|
|
383
|
+
inferred_method = ep_parts[1] if len(ep_parts) >= 3 else "ANY"
|
|
384
|
+
inferred_route = ep_parts[2] if len(ep_parts) >= 3 else caller_src
|
|
385
|
+
if " /" in caller_src:
|
|
386
|
+
before_path, after_slash = caller_src.split(" /", 1)
|
|
387
|
+
inferred_method = before_path.split(":")[-1].strip() or inferred_method
|
|
388
|
+
inferred_route = "/" + after_slash.split("#", 1)[0].strip()
|
|
389
|
+
reaching_routes.append({
|
|
390
|
+
"endpoint": caller_src,
|
|
391
|
+
"http_method": inferred_method,
|
|
392
|
+
"route_path": inferred_route,
|
|
393
|
+
"handler": curr_sym,
|
|
394
|
+
"relationship": rel_type,
|
|
395
|
+
"confidence": str(cr["confidence"]),
|
|
396
|
+
"evidence_class": str(cr["evidence_class"]),
|
|
397
|
+
"file": str(cr["file"]),
|
|
398
|
+
"start_line": int(cr["start_line"]),
|
|
399
|
+
"evidence": redact_secrets(str(cr["evidence"])),
|
|
400
|
+
"chain": new_chain,
|
|
401
|
+
})
|
|
402
|
+
if caller_src not in visited:
|
|
403
|
+
visited.add(caller_src)
|
|
404
|
+
upstream_callers.append({
|
|
405
|
+
"source": caller_src,
|
|
406
|
+
"symbol": caller_src,
|
|
407
|
+
"calls": curr_sym,
|
|
408
|
+
"relationship": rel_type,
|
|
409
|
+
"confidence": str(cr["confidence"]),
|
|
410
|
+
"evidence_class": str(cr["evidence_class"]),
|
|
411
|
+
"file": str(cr["file"]),
|
|
412
|
+
"start_line": int(cr["start_line"]),
|
|
413
|
+
"end_line": int(cr["end_line"]),
|
|
414
|
+
"evidence": redact_secrets(str(cr["evidence"])),
|
|
415
|
+
"distance": dist + 1,
|
|
416
|
+
"chain": new_chain,
|
|
417
|
+
})
|
|
418
|
+
queue.append((caller_src, dist + 1, new_chain))
|
|
419
|
+
|
|
420
|
+
# Also check framework_routes table directly for handlers matching curr_sym
|
|
421
|
+
rt_rows = con.execute(
|
|
422
|
+
"SELECT endpoint_id, framework, http_method, route_path, handler_canonical_id, file_path, line, evidence, confidence "
|
|
423
|
+
"FROM framework_routes WHERE handler_canonical_id = ? OR handler_name = ? ORDER BY endpoint_id",
|
|
424
|
+
(curr_sym, curr_sym.split(".")[-1]),
|
|
425
|
+
).fetchall()
|
|
426
|
+
for rr in rt_rows:
|
|
427
|
+
ep_id = str(rr["endpoint_id"])
|
|
428
|
+
if not any(rt["endpoint"] == ep_id for rt in reaching_routes):
|
|
429
|
+
reaching_routes.append({
|
|
430
|
+
"endpoint": ep_id,
|
|
431
|
+
"http_method": str(rr["http_method"]),
|
|
432
|
+
"route_path": str(rr["route_path"]),
|
|
433
|
+
"framework": str(rr["framework"]),
|
|
434
|
+
"handler": curr_sym,
|
|
435
|
+
"relationship": "HANDLED_BY",
|
|
436
|
+
"confidence": str(rr["confidence"]),
|
|
437
|
+
"evidence_class": "FRAMEWORK_VERIFIED",
|
|
438
|
+
"file": str(rr["file_path"]),
|
|
439
|
+
"start_line": int(rr["line"]),
|
|
440
|
+
"evidence": redact_secrets(str(rr["evidence"])),
|
|
441
|
+
"chain": [ep_id, *path_chain],
|
|
442
|
+
})
|
|
443
|
+
|
|
444
|
+
return {
|
|
445
|
+
"status": "ok",
|
|
446
|
+
"target": target,
|
|
447
|
+
"count": len(direct_accessors),
|
|
448
|
+
"direct_accessors": direct_accessors,
|
|
449
|
+
"upstream_callers": upstream_callers,
|
|
450
|
+
"routes": reaching_routes,
|
|
451
|
+
"upstream_routes": reaching_routes,
|
|
452
|
+
"results": direct_accessors,
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
def _resolve_target_arg(
|
|
457
|
+
repo_or_target: Any = None,
|
|
458
|
+
table: str | None = None,
|
|
459
|
+
column: str | None = None,
|
|
460
|
+
table_or_column: str | None = None,
|
|
461
|
+
) -> str:
|
|
462
|
+
if isinstance(repo_or_target, str) and repo_or_target.strip():
|
|
463
|
+
return repo_or_target.strip()
|
|
464
|
+
if table_or_column and table_or_column.strip():
|
|
465
|
+
return table_or_column.strip()
|
|
466
|
+
if table and column:
|
|
467
|
+
return f"{table.strip()}.{column.strip()}"
|
|
468
|
+
if table:
|
|
469
|
+
return table.strip()
|
|
470
|
+
if column:
|
|
471
|
+
return column.strip()
|
|
472
|
+
return ""
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def find_db_writers(
|
|
476
|
+
con: sqlite3.Connection,
|
|
477
|
+
repo_or_target: Any = None,
|
|
478
|
+
*,
|
|
479
|
+
table: str | None = None,
|
|
480
|
+
column: str | None = None,
|
|
481
|
+
table_or_column: str | None = None,
|
|
482
|
+
) -> dict[str, Any]:
|
|
483
|
+
"""Find all functions, services, and routes that write (INSERT/UPDATE/DELETE) to a table or column."""
|
|
484
|
+
target = _resolve_target_arg(repo_or_target, table=table, column=column, table_or_column=table_or_column)
|
|
485
|
+
res = _find_table_accessors(
|
|
486
|
+
con,
|
|
487
|
+
target,
|
|
488
|
+
("WRITES_TABLE", "WRITES_COLUMN", "WRITES_COLLECTION"),
|
|
489
|
+
include_upstream_routes=True,
|
|
490
|
+
)
|
|
491
|
+
res["writers"] = res["direct_accessors"]
|
|
492
|
+
return res
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def find_db_readers(
|
|
496
|
+
con: sqlite3.Connection,
|
|
497
|
+
repo_or_target: Any = None,
|
|
498
|
+
*,
|
|
499
|
+
table: str | None = None,
|
|
500
|
+
column: str | None = None,
|
|
501
|
+
table_or_column: str | None = None,
|
|
502
|
+
) -> dict[str, Any]:
|
|
503
|
+
"""Find all functions, services, and routes that read (SELECT/JOIN) from a table or column."""
|
|
504
|
+
target = _resolve_target_arg(repo_or_target, table=table, column=column, table_or_column=table_or_column)
|
|
505
|
+
res = _find_table_accessors(
|
|
506
|
+
con,
|
|
507
|
+
target,
|
|
508
|
+
("READS_TABLE", "READS_COLUMN", "READS_COLLECTION"),
|
|
509
|
+
include_upstream_routes=True,
|
|
510
|
+
)
|
|
511
|
+
res["readers"] = res["direct_accessors"]
|
|
512
|
+
return res
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
def find_db_callers(
|
|
516
|
+
con: sqlite3.Connection,
|
|
517
|
+
repo_or_target: Any = None,
|
|
518
|
+
*,
|
|
519
|
+
table: str | None = None,
|
|
520
|
+
column: str | None = None,
|
|
521
|
+
table_or_column: str | None = None,
|
|
522
|
+
) -> dict[str, Any]:
|
|
523
|
+
"""Find all code symbols and routes that read, write, map to, or query a database table or column."""
|
|
524
|
+
target = _resolve_target_arg(repo_or_target, table=table, column=column, table_or_column=table_or_column)
|
|
525
|
+
res = _find_table_accessors(
|
|
526
|
+
con,
|
|
527
|
+
target,
|
|
528
|
+
(
|
|
529
|
+
"READS_TABLE",
|
|
530
|
+
"WRITES_TABLE",
|
|
531
|
+
"READS_COLUMN",
|
|
532
|
+
"WRITES_COLUMN",
|
|
533
|
+
"MAPS_TO_TABLE",
|
|
534
|
+
"MAPS_TO_COLUMN",
|
|
535
|
+
"POSSIBLE_TABLE",
|
|
536
|
+
"UNKNOWN_TABLE",
|
|
537
|
+
),
|
|
538
|
+
include_upstream_routes=True,
|
|
539
|
+
)
|
|
540
|
+
res["callers"] = res["direct_accessors"]
|
|
541
|
+
return res
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
def find_db_relationships(
|
|
545
|
+
con: sqlite3.Connection,
|
|
546
|
+
repo_or_target: Any = None,
|
|
547
|
+
relationship_type: str | None = None,
|
|
548
|
+
*,
|
|
549
|
+
target: str | None = None,
|
|
550
|
+
table: str | None = None,
|
|
551
|
+
) -> dict[str, Any]:
|
|
552
|
+
"""Discover explicit database relationships (FOREIGN_KEY_TO, MAPS_TO_TABLE, MAPS_TO_COLUMN,
|
|
553
|
+
ORM_RELATION, HAS_PRIMARY_KEY, HAS_INDEX, HAS_UNIQUE_CONSTRAINT, HAS_CHECK_CONSTRAINT,
|
|
554
|
+
MIGRATES_TABLE, READS_TABLE, WRITES_TABLE).
|
|
555
|
+
"""
|
|
556
|
+
if isinstance(repo_or_target, str) and not target and not table:
|
|
557
|
+
target = repo_or_target
|
|
558
|
+
elif table and not target:
|
|
559
|
+
target = table
|
|
560
|
+
db_rels = (
|
|
561
|
+
"MAPS_TO_TABLE",
|
|
562
|
+
"MAPS_TO_COLUMN",
|
|
563
|
+
"READS_TABLE",
|
|
564
|
+
"WRITES_TABLE",
|
|
565
|
+
"READS_COLUMN",
|
|
566
|
+
"WRITES_COLUMN",
|
|
567
|
+
"REFERENCES_TABLE",
|
|
568
|
+
"REFERENCES_COLUMN",
|
|
569
|
+
"FOREIGN_KEY_TO",
|
|
570
|
+
"HAS_PRIMARY_KEY",
|
|
571
|
+
"HAS_INDEX",
|
|
572
|
+
"HAS_UNIQUE_CONSTRAINT",
|
|
573
|
+
"HAS_CHECK_CONSTRAINT",
|
|
574
|
+
"MIGRATES_TABLE",
|
|
575
|
+
"QUERIES_DATABASE",
|
|
576
|
+
"ORM_RELATION",
|
|
577
|
+
"POSSIBLE_TABLE",
|
|
578
|
+
"UNKNOWN_TABLE",
|
|
579
|
+
)
|
|
580
|
+
placeholders = ",".join("?" for _ in db_rels)
|
|
581
|
+
rows = con.execute(
|
|
582
|
+
f"SELECT source, target, relationship, confidence, file, start_line, end_line, "
|
|
583
|
+
f"evidence, evidence_class, reason "
|
|
584
|
+
f"FROM graph_edges WHERE relationship IN ({placeholders}) "
|
|
585
|
+
f"ORDER BY relationship, source, target, file, start_line",
|
|
586
|
+
db_rels,
|
|
587
|
+
).fetchall()
|
|
588
|
+
|
|
589
|
+
results: list[dict[str, Any]] = []
|
|
590
|
+
q = (target or "").strip().lower()
|
|
591
|
+
rel_f = (relationship_type or "").strip().upper()
|
|
592
|
+
|
|
593
|
+
for r in rows:
|
|
594
|
+
src = str(r["source"])
|
|
595
|
+
tgt = str(r["target"])
|
|
596
|
+
rel = str(r["relationship"])
|
|
597
|
+
if rel_f and rel != rel_f:
|
|
598
|
+
continue
|
|
599
|
+
if q and not (
|
|
600
|
+
q in src.lower()
|
|
601
|
+
or q in tgt.lower()
|
|
602
|
+
):
|
|
603
|
+
continue
|
|
604
|
+
ev_cls = str(r["evidence_class"] or "AST_VERIFIED")
|
|
605
|
+
st = "UNKNOWN" if ev_cls == "UNKNOWN" else ("INFERENCE" if ev_cls in ("POSSIBLE", "AMBIGUOUS") else "FACT")
|
|
606
|
+
results.append({
|
|
607
|
+
"source": src,
|
|
608
|
+
"target": tgt,
|
|
609
|
+
"relationship": rel,
|
|
610
|
+
"confidence": str(r["confidence"]),
|
|
611
|
+
"evidence_class": ev_cls,
|
|
612
|
+
"status": st,
|
|
613
|
+
"file": str(r["file"]),
|
|
614
|
+
"start_line": int(r["start_line"]),
|
|
615
|
+
"end_line": int(r["end_line"]),
|
|
616
|
+
"evidence": redact_secrets(str(r["evidence"])),
|
|
617
|
+
"reason": r["reason"],
|
|
618
|
+
})
|
|
619
|
+
|
|
620
|
+
return {
|
|
621
|
+
"status": "ok",
|
|
622
|
+
"count": len(results),
|
|
623
|
+
"relationships": results,
|
|
624
|
+
"results": results,
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def get_db_table(
|
|
629
|
+
con: sqlite3.Connection,
|
|
630
|
+
repo_or_table: Any = None,
|
|
631
|
+
*,
|
|
632
|
+
table: str | None = None,
|
|
633
|
+
) -> dict[str, Any]:
|
|
634
|
+
"""Get comprehensive details for a single database table: columns, PKs, FKs, indexes,
|
|
635
|
+
constraints, ORM models, readers, writers, migrations, and runtime observations.
|
|
636
|
+
"""
|
|
637
|
+
if isinstance(repo_or_table, str) and not table:
|
|
638
|
+
table = repo_or_table
|
|
639
|
+
tbl_name = (table or "").strip()
|
|
640
|
+
tbl_res = find_db_tables(con, name=tbl_name)
|
|
641
|
+
cols_res = find_db_columns(con, table=tbl_name)
|
|
642
|
+
models_res = find_db_models(con, table=tbl_name)
|
|
643
|
+
readers_res = find_db_readers(con, tbl_name)
|
|
644
|
+
writers_res = find_db_writers(con, tbl_name)
|
|
645
|
+
rels_res = find_db_relationships(con, target=tbl_name)
|
|
646
|
+
|
|
647
|
+
# Migrations affecting this table
|
|
648
|
+
mig_rows = con.execute(
|
|
649
|
+
"SELECT migration_id, canonical_id, system, revision, down_revision, file_path, "
|
|
650
|
+
"start_line, end_line, created_tables_json, dropped_tables_json, added_columns_json, "
|
|
651
|
+
"removed_columns_json, renamed_columns_json, affected_tables_json, evidence, confidence "
|
|
652
|
+
"FROM db_migrations ORDER BY file_path, revision"
|
|
653
|
+
).fetchall()
|
|
654
|
+
q_lower = tbl_name.lower().split(".")[-1]
|
|
655
|
+
migrations: list[dict[str, Any]] = []
|
|
656
|
+
for mr in mig_rows:
|
|
657
|
+
aff = [str(x).lower() for x in json.loads(mr["affected_tables_json"] or "[]")]
|
|
658
|
+
if q_lower in aff:
|
|
659
|
+
migrations.append({
|
|
660
|
+
"migration_id": str(mr["migration_id"]),
|
|
661
|
+
"canonical_id": str(mr["canonical_id"]),
|
|
662
|
+
"system": str(mr["system"]),
|
|
663
|
+
"revision": str(mr["revision"]),
|
|
664
|
+
"down_revision": mr["down_revision"],
|
|
665
|
+
"file": str(mr["file_path"]),
|
|
666
|
+
"created_tables": json.loads(mr["created_tables_json"] or "[]"),
|
|
667
|
+
"dropped_tables": json.loads(mr["dropped_tables_json"] or "[]"),
|
|
668
|
+
"added_columns": json.loads(mr["added_columns_json"] or "[]"),
|
|
669
|
+
"removed_columns": json.loads(mr["removed_columns_json"] or "[]"),
|
|
670
|
+
"renamed_columns": json.loads(mr["renamed_columns_json"] or "[]"),
|
|
671
|
+
"evidence": redact_secrets(str(mr["evidence"])),
|
|
672
|
+
"confidence": str(mr["confidence"]),
|
|
673
|
+
})
|
|
674
|
+
|
|
675
|
+
# Runtime observations for this table
|
|
676
|
+
runtime_obs: list[dict[str, Any]] = []
|
|
677
|
+
try:
|
|
678
|
+
rt_rows = con.execute(
|
|
679
|
+
"SELECT source, target, relationship, operation, observation_count, first_seen, last_seen, "
|
|
680
|
+
"avg_duration_ms, normalized_sql "
|
|
681
|
+
"FROM runtime_edges WHERE LOWER(target) LIKE ? ORDER BY source, operation",
|
|
682
|
+
(f"%{q_lower}%",),
|
|
683
|
+
).fetchall()
|
|
684
|
+
for rr in rt_rows:
|
|
685
|
+
runtime_obs.append({
|
|
686
|
+
"source": str(rr["source"]),
|
|
687
|
+
"target": str(rr["target"]),
|
|
688
|
+
"relationship": str(rr["relationship"]),
|
|
689
|
+
"operation": str(rr["operation"]),
|
|
690
|
+
"evidence_class": "RUNTIME_OBSERVED",
|
|
691
|
+
"observation_count": int(rr["observation_count"]),
|
|
692
|
+
"first_seen": rr["first_seen"],
|
|
693
|
+
"last_seen": rr["last_seen"],
|
|
694
|
+
"avg_duration_ms": float(rr["avg_duration_ms"] or 0.0),
|
|
695
|
+
"normalized_sql": redact_secrets(str(rr["normalized_sql"] or "")),
|
|
696
|
+
})
|
|
697
|
+
except sqlite3.OperationalError:
|
|
698
|
+
pass
|
|
699
|
+
|
|
700
|
+
primary_table = tbl_res["tables"][0] if tbl_res["tables"] else {
|
|
701
|
+
"canonical_id": f"db.UNKNOWN.UNKNOWN.{q_lower}",
|
|
702
|
+
"name": q_lower,
|
|
703
|
+
"table_name": q_lower,
|
|
704
|
+
"dialect": "UNKNOWN",
|
|
705
|
+
"schema_name": "UNKNOWN",
|
|
706
|
+
"status": "UNKNOWN",
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
pks = [c["column_name"] for c in cols_res["columns"] if c.get("is_primary_key")]
|
|
710
|
+
fks = [
|
|
711
|
+
{
|
|
712
|
+
"column": c["column_name"],
|
|
713
|
+
"target_table": c["target_table"],
|
|
714
|
+
"target_column": c["target_column"],
|
|
715
|
+
}
|
|
716
|
+
for c in cols_res["columns"]
|
|
717
|
+
if c.get("is_foreign_key")
|
|
718
|
+
]
|
|
719
|
+
dedup_routes = sorted(
|
|
720
|
+
{r["endpoint"]: r for r in [*readers_res["routes"], *writers_res["routes"]]}.values(),
|
|
721
|
+
key=lambda x: str(x["endpoint"]),
|
|
722
|
+
)
|
|
723
|
+
|
|
724
|
+
return redact_payload({
|
|
725
|
+
"status": "ok",
|
|
726
|
+
"canonical_id": primary_table["canonical_id"],
|
|
727
|
+
"table_name": primary_table["table_name"],
|
|
728
|
+
"table": primary_table,
|
|
729
|
+
"columns": cols_res["columns"],
|
|
730
|
+
"primary_keys": pks,
|
|
731
|
+
"foreign_keys": fks,
|
|
732
|
+
"orm_models": models_res["models"],
|
|
733
|
+
"models": models_res["models"],
|
|
734
|
+
"readers": readers_res["direct_accessors"],
|
|
735
|
+
"writers": writers_res["direct_accessors"],
|
|
736
|
+
"routes": dedup_routes,
|
|
737
|
+
"upstream_routes": dedup_routes,
|
|
738
|
+
"relationships": rels_res["relationships"],
|
|
739
|
+
"migrations": migrations,
|
|
740
|
+
"runtime_observations": runtime_obs,
|
|
741
|
+
"credentials_policy": REDACTED_DB_CREDENTIALS,
|
|
742
|
+
})
|
|
743
|
+
|
|
744
|
+
|
|
745
|
+
def get_db_schema(
|
|
746
|
+
con: sqlite3.Connection,
|
|
747
|
+
repo_or_dialect: Any = None,
|
|
748
|
+
schema: str | None = None,
|
|
749
|
+
*,
|
|
750
|
+
dialect: str | None = None,
|
|
751
|
+
limit: int | None = None,
|
|
752
|
+
offset: int = 0,
|
|
753
|
+
) -> dict[str, Any]:
|
|
754
|
+
"""Return the complete repository database schema overview with safe credential redaction."""
|
|
755
|
+
if isinstance(repo_or_dialect, str) and not dialect:
|
|
756
|
+
dialect = repo_or_dialect
|
|
757
|
+
tables = find_db_tables(con, dialect=dialect, schema=schema)["tables"]
|
|
758
|
+
columns = find_db_columns(con)["columns"]
|
|
759
|
+
models = find_db_models(con)["models"]
|
|
760
|
+
rels = find_db_relationships(con)["relationships"]
|
|
761
|
+
|
|
762
|
+
mig_rows = con.execute(
|
|
763
|
+
"SELECT migration_id, canonical_id, system, revision, down_revision, file_path, "
|
|
764
|
+
"created_tables_json, dropped_tables_json, added_columns_json, removed_columns_json, "
|
|
765
|
+
"renamed_columns_json, affected_tables_json, evidence, confidence "
|
|
766
|
+
"FROM db_migrations ORDER BY file_path, revision"
|
|
767
|
+
).fetchall()
|
|
768
|
+
migrations = [
|
|
769
|
+
{
|
|
770
|
+
"migration_id": str(r["migration_id"]),
|
|
771
|
+
"canonical_id": str(r["canonical_id"]),
|
|
772
|
+
"system": str(r["system"]),
|
|
773
|
+
"revision": str(r["revision"]),
|
|
774
|
+
"down_revision": r["down_revision"],
|
|
775
|
+
"file": str(r["file_path"]),
|
|
776
|
+
"created_tables": json.loads(r["created_tables_json"] or "[]"),
|
|
777
|
+
"dropped_tables": json.loads(r["dropped_tables_json"] or "[]"),
|
|
778
|
+
"added_columns": json.loads(r["added_columns_json"] or "[]"),
|
|
779
|
+
"removed_columns": json.loads(r["removed_columns_json"] or "[]"),
|
|
780
|
+
"renamed_columns": json.loads(r["renamed_columns_json"] or "[]"),
|
|
781
|
+
"affected_tables": json.loads(r["affected_tables_json"] or "[]"),
|
|
782
|
+
"evidence": redact_secrets(str(r["evidence"])),
|
|
783
|
+
"confidence": str(r["confidence"]),
|
|
784
|
+
}
|
|
785
|
+
for r in mig_rows
|
|
786
|
+
]
|
|
787
|
+
|
|
788
|
+
env_rows = con.execute(
|
|
789
|
+
"SELECT source, target, file, start_line, evidence FROM graph_edges "
|
|
790
|
+
"WHERE relationship = 'READS_ENV' ORDER BY file, start_line"
|
|
791
|
+
).fetchall()
|
|
792
|
+
env_vars = [
|
|
793
|
+
{
|
|
794
|
+
"source": str(er["source"]),
|
|
795
|
+
"env_var": str(er["target"]),
|
|
796
|
+
"file": str(er["file"]),
|
|
797
|
+
"start_line": int(er["start_line"]),
|
|
798
|
+
"evidence": redact_secrets(str(er["evidence"])),
|
|
799
|
+
}
|
|
800
|
+
for er in env_rows
|
|
801
|
+
]
|
|
802
|
+
|
|
803
|
+
dialects = sorted({t["dialect"] for t in tables if t.get("dialect")}) or ["UNKNOWN"]
|
|
804
|
+
|
|
805
|
+
safe_tables = tables
|
|
806
|
+
safe_models = models
|
|
807
|
+
safe_migrations = migrations
|
|
808
|
+
safe_cols = columns
|
|
809
|
+
safe_rels = rels
|
|
810
|
+
|
|
811
|
+
if limit is not None:
|
|
812
|
+
safe_offset = max(0, offset)
|
|
813
|
+
safe_limit = max(0, limit)
|
|
814
|
+
safe_tables = tables[safe_offset : safe_offset + safe_limit]
|
|
815
|
+
safe_models = models[safe_offset : safe_offset + safe_limit]
|
|
816
|
+
safe_migrations = migrations[safe_offset : safe_offset + safe_limit]
|
|
817
|
+
safe_cols = columns[: max(safe_limit * 5, 20)]
|
|
818
|
+
safe_rels = rels[: max(safe_limit * 5, 20)]
|
|
819
|
+
|
|
820
|
+
result_dict: dict[str, Any] = {
|
|
821
|
+
"status": "ok",
|
|
822
|
+
"dialects": dialects,
|
|
823
|
+
"table_count": len(tables),
|
|
824
|
+
"column_count": len(columns),
|
|
825
|
+
"model_count": len(models),
|
|
826
|
+
"migration_count": len(migrations),
|
|
827
|
+
"tables": safe_tables,
|
|
828
|
+
"columns": safe_cols,
|
|
829
|
+
"models": safe_models,
|
|
830
|
+
"relationships": safe_rels,
|
|
831
|
+
"migrations": safe_migrations,
|
|
832
|
+
"env_variables": env_vars,
|
|
833
|
+
"database_credentials": "REDACTED",
|
|
834
|
+
}
|
|
835
|
+
if limit is not None:
|
|
836
|
+
result_dict["limit"] = safe_limit
|
|
837
|
+
result_dict["offset"] = safe_offset
|
|
838
|
+
result_dict["has_more"] = (safe_offset + len(safe_tables)) < len(tables)
|
|
839
|
+
|
|
840
|
+
return redact_payload(result_dict)
|
|
841
|
+
|
|
842
|
+
|
|
843
|
+
def get_db_impact(
|
|
844
|
+
con: sqlite3.Connection,
|
|
845
|
+
repo_or_target: Any = None,
|
|
846
|
+
max_depth: int = 5,
|
|
847
|
+
*,
|
|
848
|
+
target: str | None = None,
|
|
849
|
+
table: str | None = None,
|
|
850
|
+
column: str | None = None,
|
|
851
|
+
) -> dict[str, Any]:
|
|
852
|
+
"""Bidirectional Database <-> Code impact analysis (Phases 8 & 17).
|
|
853
|
+
|
|
854
|
+
Supports:
|
|
855
|
+
1. DATABASE -> CODE:
|
|
856
|
+
"If I change `products.price` or `users`, what ORM fields, models, queries, services, routes, and tests are affected?"
|
|
857
|
+
2. CODE / ROUTE -> DATABASE:
|
|
858
|
+
"If I change this route or service, which database tables and columns can it touch?"
|
|
859
|
+
3. MIGRATION -> CODE:
|
|
860
|
+
"If I change this schema migration, what tables and code are affected?"
|
|
861
|
+
|
|
862
|
+
Preserves impact categories:
|
|
863
|
+
DIRECT, FRAMEWORK, SEMANTIC, DI, TEST, DATABASE, RUNTIME
|
|
864
|
+
"""
|
|
865
|
+
q = _resolve_target_arg(repo_or_target, table=table, column=column, table_or_column=target)
|
|
866
|
+
q_lower = q.lower()
|
|
867
|
+
|
|
868
|
+
affected_columns: list[dict[str, Any]] = []
|
|
869
|
+
affected_tables: list[dict[str, Any]] = []
|
|
870
|
+
affected_models: list[dict[str, Any]] = []
|
|
871
|
+
affected_queries: list[dict[str, Any]] = []
|
|
872
|
+
affected_services: list[dict[str, Any]] = []
|
|
873
|
+
affected_writers: list[dict[str, Any]] = []
|
|
874
|
+
affected_readers: list[dict[str, Any]] = []
|
|
875
|
+
affected_routes: list[dict[str, Any]] = []
|
|
876
|
+
affected_tests: list[dict[str, Any]] = []
|
|
877
|
+
affected_migrations: list[dict[str, Any]] = []
|
|
878
|
+
impact_items: list[dict[str, Any]] = []
|
|
879
|
+
|
|
880
|
+
# Check if target is a table, table.column, ORM model, route, or code symbol
|
|
881
|
+
col_matches = find_db_columns(con, column=q)["columns"]
|
|
882
|
+
tbl_matches = find_db_tables(con, name=q.split(".")[0] if "." in q else q)["tables"]
|
|
883
|
+
|
|
884
|
+
# Case A: Target matches a database column or table (DATABASE -> CODE impact)
|
|
885
|
+
if col_matches or tbl_matches or q_lower.startswith("db."):
|
|
886
|
+
target_table_names = {c["table_name"] for c in col_matches} | {t["table_name"] for t in tbl_matches}
|
|
887
|
+
if not target_table_names and "." in q_lower:
|
|
888
|
+
parts = q_lower.split(".")
|
|
889
|
+
target_table_names.add(parts[-2] if len(parts) >= 2 else parts[0])
|
|
890
|
+
|
|
891
|
+
for c in col_matches:
|
|
892
|
+
affected_columns.append(c)
|
|
893
|
+
impact_items.append({
|
|
894
|
+
"symbol": c["canonical_id"],
|
|
895
|
+
"category": "DATABASE",
|
|
896
|
+
"relationship": "MAPS_TO_COLUMN",
|
|
897
|
+
"confidence": c["confidence"],
|
|
898
|
+
"file": c["file"],
|
|
899
|
+
"start_line": c["start_line"],
|
|
900
|
+
"evidence": c["evidence"],
|
|
901
|
+
})
|
|
902
|
+
|
|
903
|
+
for t_name in sorted(target_table_names):
|
|
904
|
+
t_info = get_db_table(con, table=t_name)
|
|
905
|
+
if t_info.get("table"):
|
|
906
|
+
affected_tables.append(t_info["table"])
|
|
907
|
+
for w_item in t_info.get("writers", []):
|
|
908
|
+
affected_writers.append(w_item)
|
|
909
|
+
for r_item in t_info.get("readers", []):
|
|
910
|
+
affected_readers.append(r_item)
|
|
911
|
+
for m in find_db_models(con, table=t_name)["models"]:
|
|
912
|
+
affected_models.append(m)
|
|
913
|
+
impact_items.append({
|
|
914
|
+
"symbol": m["canonical_id"],
|
|
915
|
+
"category": "DATABASE",
|
|
916
|
+
"relationship": "MAPS_TO_TABLE",
|
|
917
|
+
"confidence": m["confidence"],
|
|
918
|
+
"file": m["file"],
|
|
919
|
+
"start_line": m["start_line"],
|
|
920
|
+
"evidence": m["evidence"],
|
|
921
|
+
})
|
|
922
|
+
for q_item in find_db_queries(con, table=t_name)["queries"]:
|
|
923
|
+
affected_queries.append(q_item)
|
|
924
|
+
impact_items.append({
|
|
925
|
+
"symbol": q_item["symbol"],
|
|
926
|
+
"category": "DIRECT",
|
|
927
|
+
"relationship": q_item["relationship"],
|
|
928
|
+
"confidence": q_item["confidence"],
|
|
929
|
+
"file": q_item["file"],
|
|
930
|
+
"start_line": q_item["start_line"],
|
|
931
|
+
"evidence": q_item["evidence"],
|
|
932
|
+
})
|
|
933
|
+
callers_info = find_db_callers(con, table=t_name)
|
|
934
|
+
for acc in [*callers_info["direct_accessors"], *callers_info["upstream_callers"]]:
|
|
935
|
+
sym_id = str(acc["symbol"])
|
|
936
|
+
cat = "DIRECT" if acc.get("distance", 1) == 1 else "SEMANTIC"
|
|
937
|
+
if "test" in str(acc.get("file", "")).lower() or sym_id.split(".")[-1].startswith("test_"):
|
|
938
|
+
cat = "TEST"
|
|
939
|
+
affected_tests.append(acc)
|
|
940
|
+
else:
|
|
941
|
+
affected_services.append(acc)
|
|
942
|
+
impact_items.append({
|
|
943
|
+
"symbol": sym_id,
|
|
944
|
+
"category": cat,
|
|
945
|
+
"relationship": acc["relationship"],
|
|
946
|
+
"confidence": acc["confidence"],
|
|
947
|
+
"file": acc["file"],
|
|
948
|
+
"start_line": acc["start_line"],
|
|
949
|
+
"evidence": acc["evidence"],
|
|
950
|
+
})
|
|
951
|
+
for rt in callers_info["routes"]:
|
|
952
|
+
affected_routes.append(rt)
|
|
953
|
+
impact_items.append({
|
|
954
|
+
"symbol": rt["endpoint"],
|
|
955
|
+
"category": "FRAMEWORK",
|
|
956
|
+
"relationship": rt["relationship"],
|
|
957
|
+
"confidence": rt["confidence"],
|
|
958
|
+
"file": rt["file"],
|
|
959
|
+
"start_line": rt["start_line"],
|
|
960
|
+
"evidence": rt["evidence"],
|
|
961
|
+
})
|
|
962
|
+
for mig in t_info.get("migrations", []):
|
|
963
|
+
affected_migrations.append(mig)
|
|
964
|
+
|
|
965
|
+
# Also check tests linked to any affected service or model
|
|
966
|
+
candidate_syms = {m["canonical_id"] for m in affected_models} | {s["symbol"] for s in affected_services}
|
|
967
|
+
for csym in sorted(candidate_syms):
|
|
968
|
+
test_rows = con.execute(
|
|
969
|
+
"SELECT source, target, relationship, confidence, file, start_line, evidence "
|
|
970
|
+
"FROM graph_edges WHERE (target = ? OR target = ?) AND relationship LIKE 'TESTS%' ORDER BY file, start_line",
|
|
971
|
+
(csym, csym.split(".")[-1]),
|
|
972
|
+
).fetchall()
|
|
973
|
+
for tr in test_rows:
|
|
974
|
+
t_entry = {
|
|
975
|
+
"symbol": str(tr["source"]),
|
|
976
|
+
"target": str(tr["target"]),
|
|
977
|
+
"relationship": str(tr["relationship"]),
|
|
978
|
+
"confidence": str(tr["confidence"]),
|
|
979
|
+
"file": str(tr["file"]),
|
|
980
|
+
"start_line": int(tr["start_line"]),
|
|
981
|
+
"evidence": redact_secrets(str(tr["evidence"])),
|
|
982
|
+
}
|
|
983
|
+
affected_tests.append(t_entry)
|
|
984
|
+
impact_items.append({
|
|
985
|
+
"symbol": str(tr["source"]),
|
|
986
|
+
"category": "TEST",
|
|
987
|
+
"relationship": str(tr["relationship"]),
|
|
988
|
+
"confidence": str(tr["confidence"]),
|
|
989
|
+
"file": str(tr["file"]),
|
|
990
|
+
"start_line": int(tr["start_line"]),
|
|
991
|
+
"evidence": redact_secrets(str(tr["evidence"])),
|
|
992
|
+
})
|
|
993
|
+
# Also check CALLS from test_* symbols to affected services
|
|
994
|
+
caller_test_rows = con.execute(
|
|
995
|
+
"SELECT source, target, relationship, confidence, file, start_line, evidence "
|
|
996
|
+
"FROM graph_edges WHERE (target = ? OR target = ?) AND relationship = 'CALLS' ORDER BY file, start_line",
|
|
997
|
+
(csym, csym.split(".")[-1]),
|
|
998
|
+
).fetchall()
|
|
999
|
+
for ctr in caller_test_rows:
|
|
1000
|
+
c_src = str(ctr["source"])
|
|
1001
|
+
c_file = str(ctr["file"])
|
|
1002
|
+
if "test" in c_file.lower() or c_src.split(".")[-1].startswith("test_"):
|
|
1003
|
+
t_entry = {
|
|
1004
|
+
"symbol": c_src,
|
|
1005
|
+
"target": str(ctr["target"]),
|
|
1006
|
+
"relationship": "TESTS",
|
|
1007
|
+
"confidence": str(ctr["confidence"]),
|
|
1008
|
+
"file": c_file,
|
|
1009
|
+
"start_line": int(ctr["start_line"]),
|
|
1010
|
+
"evidence": redact_secrets(str(ctr["evidence"])),
|
|
1011
|
+
}
|
|
1012
|
+
if not any(x["symbol"] == c_src for x in affected_tests):
|
|
1013
|
+
affected_tests.append(t_entry)
|
|
1014
|
+
impact_items.append({
|
|
1015
|
+
"symbol": c_src,
|
|
1016
|
+
"category": "TEST",
|
|
1017
|
+
"relationship": "TESTS",
|
|
1018
|
+
"confidence": str(ctr["confidence"]),
|
|
1019
|
+
"file": c_file,
|
|
1020
|
+
"start_line": int(ctr["start_line"]),
|
|
1021
|
+
"evidence": redact_secrets(str(ctr["evidence"])),
|
|
1022
|
+
})
|
|
1023
|
+
|
|
1024
|
+
# Case B: Target is a route, function, or service (CODE -> DATABASE forward traversal)
|
|
1025
|
+
visited_fwd: set[str] = {q}
|
|
1026
|
+
fwd_queue: deque[tuple[str, int]] = deque([(q, 0)])
|
|
1027
|
+
# Also resolve short symbol names or route paths
|
|
1028
|
+
sym_rows = con.execute(
|
|
1029
|
+
"SELECT canonical_id FROM symbols WHERE canonical_id = ? OR qualified_name = ? OR name = ?",
|
|
1030
|
+
(q, q, q),
|
|
1031
|
+
).fetchall()
|
|
1032
|
+
for sr in sym_rows:
|
|
1033
|
+
cid = str(sr["canonical_id"])
|
|
1034
|
+
if cid not in visited_fwd:
|
|
1035
|
+
visited_fwd.add(cid)
|
|
1036
|
+
fwd_queue.append((cid, 0))
|
|
1037
|
+
|
|
1038
|
+
rt_seed_rows = con.execute(
|
|
1039
|
+
"SELECT endpoint_id, handler_canonical_id FROM framework_routes WHERE route_path = ? OR endpoint_id = ?",
|
|
1040
|
+
(q, q),
|
|
1041
|
+
).fetchall()
|
|
1042
|
+
for rsr in rt_seed_rows:
|
|
1043
|
+
h_cid = str(rsr["handler_canonical_id"])
|
|
1044
|
+
if h_cid not in visited_fwd:
|
|
1045
|
+
visited_fwd.add(h_cid)
|
|
1046
|
+
fwd_queue.append((h_cid, 1))
|
|
1047
|
+
|
|
1048
|
+
while fwd_queue:
|
|
1049
|
+
curr, depth = fwd_queue.popleft()
|
|
1050
|
+
if depth >= max_depth:
|
|
1051
|
+
continue
|
|
1052
|
+
out_rows = con.execute(
|
|
1053
|
+
"SELECT source, target, relationship, confidence, file, start_line, evidence, evidence_class "
|
|
1054
|
+
"FROM graph_edges WHERE source = ? ORDER BY file, start_line, target",
|
|
1055
|
+
(curr,),
|
|
1056
|
+
).fetchall()
|
|
1057
|
+
for orow in out_rows:
|
|
1058
|
+
tgt = str(orow["target"])
|
|
1059
|
+
rel = str(orow["relationship"])
|
|
1060
|
+
if rel in ("READS_TABLE", "WRITES_TABLE", "READS_COLUMN", "WRITES_COLUMN", "MAPS_TO_TABLE", "POSSIBLE_TABLE", "UNKNOWN_TABLE"):
|
|
1061
|
+
affected_tables.append({
|
|
1062
|
+
"canonical_id": tgt,
|
|
1063
|
+
"relationship": rel,
|
|
1064
|
+
"source": curr,
|
|
1065
|
+
"confidence": str(orow["confidence"]),
|
|
1066
|
+
"evidence_class": str(orow["evidence_class"]),
|
|
1067
|
+
"file": str(orow["file"]),
|
|
1068
|
+
"start_line": int(orow["start_line"]),
|
|
1069
|
+
"evidence": redact_secrets(str(orow["evidence"])),
|
|
1070
|
+
})
|
|
1071
|
+
impact_items.append({
|
|
1072
|
+
"symbol": tgt,
|
|
1073
|
+
"category": "DATABASE",
|
|
1074
|
+
"relationship": rel,
|
|
1075
|
+
"confidence": str(orow["confidence"]),
|
|
1076
|
+
"file": str(orow["file"]),
|
|
1077
|
+
"start_line": int(orow["start_line"]),
|
|
1078
|
+
"evidence": redact_secrets(str(orow["evidence"])),
|
|
1079
|
+
})
|
|
1080
|
+
elif rel in ("CALLS", "HANDLED_BY", "ROUTE_HANDLER", "ROUTES_TO", "INJECTS", "PROVIDES", "RESOLVES_TO") and tgt not in visited_fwd:
|
|
1081
|
+
visited_fwd.add(tgt)
|
|
1082
|
+
fwd_queue.append((tgt, depth + 1))
|
|
1083
|
+
|
|
1084
|
+
return redact_payload({
|
|
1085
|
+
"status": "ok",
|
|
1086
|
+
"target": q,
|
|
1087
|
+
"impact_count": len(impact_items),
|
|
1088
|
+
"columns": affected_columns,
|
|
1089
|
+
"tables": affected_tables,
|
|
1090
|
+
"models": affected_models,
|
|
1091
|
+
"affected_models": affected_models,
|
|
1092
|
+
"queries": affected_queries,
|
|
1093
|
+
"affected_queries": affected_queries,
|
|
1094
|
+
"services": affected_services,
|
|
1095
|
+
"affected_services": affected_services,
|
|
1096
|
+
"writers": affected_writers,
|
|
1097
|
+
"affected_writers": affected_writers,
|
|
1098
|
+
"readers": affected_readers,
|
|
1099
|
+
"affected_readers": affected_readers,
|
|
1100
|
+
"routes": affected_routes,
|
|
1101
|
+
"affected_routes": affected_routes,
|
|
1102
|
+
"tests": affected_tests,
|
|
1103
|
+
"affected_tests": affected_tests,
|
|
1104
|
+
"migrations": affected_migrations,
|
|
1105
|
+
"affected_migrations": affected_migrations,
|
|
1106
|
+
"impacts": impact_items,
|
|
1107
|
+
"results": impact_items,
|
|
1108
|
+
})
|
|
1109
|
+
|
|
1110
|
+
|
|
1111
|
+
def _normalize_table_name(raw: str) -> str:
|
|
1112
|
+
if not raw:
|
|
1113
|
+
return ""
|
|
1114
|
+
clean = raw.strip().strip('"`')
|
|
1115
|
+
if clean.startswith("db."):
|
|
1116
|
+
parts = clean.split(".")
|
|
1117
|
+
cand = parts[-1].strip('"`')
|
|
1118
|
+
return "UNKNOWN" if cand.upper() == "UNKNOWN" else cand.lower()
|
|
1119
|
+
cand = clean.split(".")[-1].strip('"`')
|
|
1120
|
+
return "UNKNOWN" if cand.upper() == "UNKNOWN" else cand.lower()
|
|
1121
|
+
|
|
1122
|
+
|
|
1123
|
+
def _normalize_col_name(raw: str) -> str:
|
|
1124
|
+
clean = raw.strip().strip('"`')
|
|
1125
|
+
return clean.split(".")[-1].strip('"`').lower()
|
|
1126
|
+
|
|
1127
|
+
|
|
1128
|
+
def _infer_table_from_column_target(tgt: str) -> str:
|
|
1129
|
+
clean = tgt.strip().strip('"`')
|
|
1130
|
+
parts = clean.split(".")
|
|
1131
|
+
if len(parts) >= 2:
|
|
1132
|
+
cand = parts[-2].strip('"`')
|
|
1133
|
+
return "UNKNOWN" if cand.upper() == "UNKNOWN" else cand.lower()
|
|
1134
|
+
return ""
|
|
1135
|
+
|
|
1136
|
+
|
|
1137
|
+
def _aggregate_table_access(raw_items: list[dict[str, Any]], max_cols: int) -> list[dict[str, Any]]:
|
|
1138
|
+
by_table: dict[str, dict[str, Any]] = {}
|
|
1139
|
+
for item in raw_items:
|
|
1140
|
+
t = item["table"]
|
|
1141
|
+
if not t:
|
|
1142
|
+
continue
|
|
1143
|
+
if t not in by_table:
|
|
1144
|
+
by_table[t] = {
|
|
1145
|
+
"table": t,
|
|
1146
|
+
"columns": set(),
|
|
1147
|
+
"operations": set(),
|
|
1148
|
+
"evidence_classes": set(),
|
|
1149
|
+
"confidences": set(),
|
|
1150
|
+
"call_chains": [],
|
|
1151
|
+
"evidence": [],
|
|
1152
|
+
}
|
|
1153
|
+
entry = by_table[t]
|
|
1154
|
+
for c in item.get("columns", []):
|
|
1155
|
+
if c:
|
|
1156
|
+
entry["columns"].add(c)
|
|
1157
|
+
if item.get("operation"):
|
|
1158
|
+
entry["operations"].add(item["operation"])
|
|
1159
|
+
entry["evidence_classes"].add(item.get("evidence_class", "AST_VERIFIED"))
|
|
1160
|
+
entry["confidences"].add(item.get("confidence", "HIGH"))
|
|
1161
|
+
if item.get("call_chain"):
|
|
1162
|
+
entry["call_chains"].append(item["call_chain"])
|
|
1163
|
+
if item.get("evidence"):
|
|
1164
|
+
entry["evidence"].append({
|
|
1165
|
+
"file": item.get("file", ""),
|
|
1166
|
+
"line": item.get("line", 0),
|
|
1167
|
+
"snippet": item.get("evidence", ""),
|
|
1168
|
+
})
|
|
1169
|
+
|
|
1170
|
+
results: list[dict[str, Any]] = []
|
|
1171
|
+
for t_name in sorted(by_table.keys()):
|
|
1172
|
+
data = by_table[t_name]
|
|
1173
|
+
sorted_cols = sorted(list(data["columns"]))[:max_cols]
|
|
1174
|
+
ops = set(data["operations"])
|
|
1175
|
+
# If specific mutation operations exist, drop generic "WRITE"
|
|
1176
|
+
if len(ops) > 1 and "WRITE" in ops and any(x in ops for x in ("INSERT", "UPDATE", "DELETE", "CREATE", "DROP", "ALTER")):
|
|
1177
|
+
ops.remove("WRITE")
|
|
1178
|
+
# If specific read operations exist, drop generic "READ"
|
|
1179
|
+
if len(ops) > 1 and "READ" in ops and "SELECT" in ops:
|
|
1180
|
+
ops.remove("READ")
|
|
1181
|
+
|
|
1182
|
+
sorted_ops = sorted(list(ops))
|
|
1183
|
+
ev_classes = data["evidence_classes"]
|
|
1184
|
+
|
|
1185
|
+
if "UNKNOWN" in ev_classes:
|
|
1186
|
+
synth_ev = "UNKNOWN"
|
|
1187
|
+
synth_conf = "UNKNOWN"
|
|
1188
|
+
elif "POSSIBLE" in ev_classes or "AMBIGUOUS" in ev_classes:
|
|
1189
|
+
synth_ev = "POSSIBLE"
|
|
1190
|
+
synth_conf = "LOW"
|
|
1191
|
+
elif "DATAFLOW_VERIFIED" in ev_classes:
|
|
1192
|
+
synth_ev = "DATAFLOW_VERIFIED"
|
|
1193
|
+
synth_conf = "HIGH"
|
|
1194
|
+
elif "FRAMEWORK_VERIFIED" in ev_classes:
|
|
1195
|
+
synth_ev = "FRAMEWORK_VERIFIED"
|
|
1196
|
+
synth_conf = "HIGH"
|
|
1197
|
+
else:
|
|
1198
|
+
synth_ev = "AST_VERIFIED"
|
|
1199
|
+
synth_conf = "HIGH"
|
|
1200
|
+
|
|
1201
|
+
shortest_chain = min(data["call_chains"], key=len) if data["call_chains"] else []
|
|
1202
|
+
|
|
1203
|
+
results.append({
|
|
1204
|
+
"table": t_name,
|
|
1205
|
+
"columns": sorted_cols,
|
|
1206
|
+
"operations": sorted_ops,
|
|
1207
|
+
"evidence_class": synth_ev,
|
|
1208
|
+
"confidence": synth_conf,
|
|
1209
|
+
"call_chain": shortest_chain,
|
|
1210
|
+
"evidence": data["evidence"][:10],
|
|
1211
|
+
})
|
|
1212
|
+
|
|
1213
|
+
return results
|
|
1214
|
+
|
|
1215
|
+
|
|
1216
|
+
def get_route_db_lineage(
|
|
1217
|
+
con: sqlite3.Connection,
|
|
1218
|
+
route_or_handler: str,
|
|
1219
|
+
method: str | None = None,
|
|
1220
|
+
*,
|
|
1221
|
+
max_depth: int = 4,
|
|
1222
|
+
max_tables: int = 25,
|
|
1223
|
+
max_columns_per_table: int = 50,
|
|
1224
|
+
) -> dict[str, Any]:
|
|
1225
|
+
"""Deterministically trace route or handler through services, repositories, and ORM/SQL down to database mutations and reads.
|
|
1226
|
+
|
|
1227
|
+
Enforces bounded BFS (max_depth=4), cycle protection via visited set, deterministic ordering (table ASC, column ASC),
|
|
1228
|
+
strict output budgets, and preserves epistemic states (AST_VERIFIED, STATIC_VERIFIED, DATAFLOW_VERIFIED, FRAMEWORK_VERIFIED, POSSIBLE, UNKNOWN).
|
|
1229
|
+
"""
|
|
1230
|
+
raw = (route_or_handler or "").strip()
|
|
1231
|
+
if not raw:
|
|
1232
|
+
return {
|
|
1233
|
+
"status": "ok",
|
|
1234
|
+
"route": "",
|
|
1235
|
+
"method": (method or "").upper() or "ANY",
|
|
1236
|
+
"handler": "",
|
|
1237
|
+
"db_reads": [],
|
|
1238
|
+
"db_writes": [],
|
|
1239
|
+
"tables": [],
|
|
1240
|
+
"evidence_class": "STATIC_VERIFIED",
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
clean_target = raw
|
|
1244
|
+
req_method = (method or "").strip().upper() or None
|
|
1245
|
+
|
|
1246
|
+
if " " in clean_target:
|
|
1247
|
+
parts = clean_target.split(" ", 1)
|
|
1248
|
+
if parts[0].upper() in ("GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"):
|
|
1249
|
+
req_method = req_method or parts[0].upper()
|
|
1250
|
+
clean_target = parts[1].strip()
|
|
1251
|
+
|
|
1252
|
+
route_query = (
|
|
1253
|
+
"SELECT route_path, http_method, handler_name, handler_canonical_id, file_path, line "
|
|
1254
|
+
"FROM framework_routes WHERE route_path = ? OR endpoint_id = ?"
|
|
1255
|
+
)
|
|
1256
|
+
params: list[Any] = [clean_target, clean_target]
|
|
1257
|
+
if req_method:
|
|
1258
|
+
route_query += " AND UPPER(http_method) = ?"
|
|
1259
|
+
params.append(req_method)
|
|
1260
|
+
|
|
1261
|
+
try:
|
|
1262
|
+
route_rows = con.execute(route_query, params).fetchall()
|
|
1263
|
+
except sqlite3.OperationalError:
|
|
1264
|
+
route_rows = []
|
|
1265
|
+
|
|
1266
|
+
if not route_rows and req_method:
|
|
1267
|
+
try:
|
|
1268
|
+
route_rows = con.execute(
|
|
1269
|
+
"SELECT route_path, http_method, handler_name, handler_canonical_id, file_path, line "
|
|
1270
|
+
"FROM framework_routes WHERE route_path = ? OR endpoint_id = ?",
|
|
1271
|
+
(clean_target, clean_target),
|
|
1272
|
+
).fetchall()
|
|
1273
|
+
except sqlite3.OperationalError:
|
|
1274
|
+
route_rows = []
|
|
1275
|
+
|
|
1276
|
+
seeds: list[tuple[str, str]] = []
|
|
1277
|
+
route_info: dict[str, Any] = {}
|
|
1278
|
+
|
|
1279
|
+
if route_rows:
|
|
1280
|
+
primary_r = route_rows[0]
|
|
1281
|
+
route_info = {
|
|
1282
|
+
"route": str(primary_r["route_path"]),
|
|
1283
|
+
"method": str(primary_r["http_method"] or req_method or "").upper(),
|
|
1284
|
+
"file": str(primary_r["file_path"]),
|
|
1285
|
+
"line": int(primary_r["line"]),
|
|
1286
|
+
}
|
|
1287
|
+
for rr in route_rows:
|
|
1288
|
+
h_cid = str(rr["handler_canonical_id"] or "")
|
|
1289
|
+
h_name = str(rr["handler_name"] or "")
|
|
1290
|
+
if h_cid:
|
|
1291
|
+
seeds.append((h_cid, h_name or h_cid.split(".")[-1]))
|
|
1292
|
+
elif h_name:
|
|
1293
|
+
seeds.append((h_name, h_name))
|
|
1294
|
+
else:
|
|
1295
|
+
try:
|
|
1296
|
+
sym_rows = con.execute(
|
|
1297
|
+
"SELECT canonical_id, qualified_name, name, file_path, start_line FROM symbols "
|
|
1298
|
+
"WHERE canonical_id = ? OR qualified_name = ? OR name = ?",
|
|
1299
|
+
(clean_target, clean_target, clean_target),
|
|
1300
|
+
).fetchall()
|
|
1301
|
+
except sqlite3.OperationalError:
|
|
1302
|
+
sym_rows = []
|
|
1303
|
+
if sym_rows:
|
|
1304
|
+
for sr in sym_rows:
|
|
1305
|
+
cid = str(sr["canonical_id"] or sr["qualified_name"] or sr["name"])
|
|
1306
|
+
seeds.append((cid, str(sr["name"])))
|
|
1307
|
+
else:
|
|
1308
|
+
seeds.append((clean_target, clean_target))
|
|
1309
|
+
|
|
1310
|
+
visited: set[str] = set()
|
|
1311
|
+
queue: deque[tuple[str, int, list[str]]] = deque()
|
|
1312
|
+
for s_id, s_disp in seeds:
|
|
1313
|
+
if s_id not in visited:
|
|
1314
|
+
visited.add(s_id)
|
|
1315
|
+
queue.append((s_id, 0, [s_disp]))
|
|
1316
|
+
|
|
1317
|
+
raw_reads: list[dict[str, Any]] = []
|
|
1318
|
+
raw_writes: list[dict[str, Any]] = []
|
|
1319
|
+
|
|
1320
|
+
while queue:
|
|
1321
|
+
curr_sym, depth, chain = queue.popleft()
|
|
1322
|
+
if depth > max_depth:
|
|
1323
|
+
continue
|
|
1324
|
+
|
|
1325
|
+
curr_bare = curr_sym.split(".")[-1]
|
|
1326
|
+
|
|
1327
|
+
# 1. Query db_queries table
|
|
1328
|
+
try:
|
|
1329
|
+
q_rows = con.execute(
|
|
1330
|
+
"SELECT query_id, caller_symbol_id, operation, relationship, table_name, table_canonical_id, "
|
|
1331
|
+
"columns_json, file_path, start_line, end_line, evidence, confidence, evidence_class, status "
|
|
1332
|
+
"FROM db_queries "
|
|
1333
|
+
"WHERE caller_symbol_id = ? OR caller_symbol_id LIKE ? OR caller_symbol_id LIKE ? OR caller_symbol_id = ?",
|
|
1334
|
+
(curr_sym, f"%.{curr_sym}", f"{curr_sym}.%", curr_bare),
|
|
1335
|
+
).fetchall()
|
|
1336
|
+
except sqlite3.OperationalError:
|
|
1337
|
+
q_rows = []
|
|
1338
|
+
|
|
1339
|
+
for qr in q_rows:
|
|
1340
|
+
op = str(qr["operation"] or "").upper()
|
|
1341
|
+
rel = str(qr["relationship"] or "")
|
|
1342
|
+
tbl_raw = str(qr["table_name"] or qr["table_canonical_id"] or "")
|
|
1343
|
+
tbl_name = _normalize_table_name(tbl_raw)
|
|
1344
|
+
if not tbl_name:
|
|
1345
|
+
continue
|
|
1346
|
+
|
|
1347
|
+
cols_raw: list[Any] = []
|
|
1348
|
+
try:
|
|
1349
|
+
cols_raw = json.loads(qr["columns_json"] or "[]")
|
|
1350
|
+
except Exception:
|
|
1351
|
+
cols_raw = []
|
|
1352
|
+
cols = [_normalize_col_name(str(c)) for c in cols_raw if str(c).strip()]
|
|
1353
|
+
|
|
1354
|
+
ev_cls = str(qr["evidence_class"] or "AST_VERIFIED")
|
|
1355
|
+
conf = str(qr["confidence"] or "HIGH")
|
|
1356
|
+
item = {
|
|
1357
|
+
"table": tbl_name,
|
|
1358
|
+
"columns": cols,
|
|
1359
|
+
"operation": op,
|
|
1360
|
+
"evidence_class": ev_cls,
|
|
1361
|
+
"confidence": conf,
|
|
1362
|
+
"call_chain": list(chain),
|
|
1363
|
+
"file": str(qr["file_path"]),
|
|
1364
|
+
"line": int(qr["start_line"]),
|
|
1365
|
+
"evidence": redact_secrets(str(qr["evidence"])),
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1368
|
+
if op in ("SELECT", "FETCH", "READ", "FIND") or rel in ("READS_TABLE", "READS_COLUMN"):
|
|
1369
|
+
raw_reads.append(item)
|
|
1370
|
+
elif op in ("INSERT", "UPDATE", "DELETE", "WRITE", "SAVE", "CREATE", "DROP", "ALTER") or rel in ("WRITES_TABLE", "WRITES_COLUMN"):
|
|
1371
|
+
raw_writes.append(item)
|
|
1372
|
+
elif op == "UNKNOWN" or rel == "UNKNOWN_TABLE" or ev_cls == "UNKNOWN":
|
|
1373
|
+
raw_writes.append(item)
|
|
1374
|
+
raw_reads.append(item)
|
|
1375
|
+
|
|
1376
|
+
# 2. Query graph_edges table for outgoing edges from curr_sym
|
|
1377
|
+
try:
|
|
1378
|
+
edge_rows = con.execute(
|
|
1379
|
+
"SELECT source, target, relationship, confidence, file, start_line, end_line, evidence, evidence_class, reason "
|
|
1380
|
+
"FROM graph_edges WHERE source = ? OR source = ?",
|
|
1381
|
+
(curr_sym, curr_bare),
|
|
1382
|
+
).fetchall()
|
|
1383
|
+
except sqlite3.OperationalError:
|
|
1384
|
+
edge_rows = []
|
|
1385
|
+
|
|
1386
|
+
for er in edge_rows:
|
|
1387
|
+
rel = str(er["relationship"])
|
|
1388
|
+
tgt = str(er["target"])
|
|
1389
|
+
ev_cls = str(er["evidence_class"] or "AST_VERIFIED")
|
|
1390
|
+
conf = str(er["confidence"] or "HIGH")
|
|
1391
|
+
ev_str = redact_secrets(str(er["evidence"] or ""))
|
|
1392
|
+
|
|
1393
|
+
if rel in ("READS_TABLE", "READS_COLUMN"):
|
|
1394
|
+
tbl_name = _normalize_table_name(tgt) if rel == "READS_TABLE" else _infer_table_from_column_target(tgt)
|
|
1395
|
+
col_name = _normalize_col_name(tgt) if rel == "READS_COLUMN" else None
|
|
1396
|
+
if tbl_name:
|
|
1397
|
+
raw_reads.append({
|
|
1398
|
+
"table": tbl_name,
|
|
1399
|
+
"columns": [col_name] if col_name else [],
|
|
1400
|
+
"operation": "SELECT",
|
|
1401
|
+
"evidence_class": ev_cls,
|
|
1402
|
+
"confidence": conf,
|
|
1403
|
+
"call_chain": list(chain),
|
|
1404
|
+
"file": str(er["file"]),
|
|
1405
|
+
"line": int(er["start_line"]),
|
|
1406
|
+
"evidence": ev_str,
|
|
1407
|
+
})
|
|
1408
|
+
elif rel in ("WRITES_TABLE", "WRITES_COLUMN"):
|
|
1409
|
+
tbl_name = _normalize_table_name(tgt) if rel == "WRITES_TABLE" else _infer_table_from_column_target(tgt)
|
|
1410
|
+
col_name = _normalize_col_name(tgt) if rel == "WRITES_COLUMN" else None
|
|
1411
|
+
if tbl_name:
|
|
1412
|
+
raw_writes.append({
|
|
1413
|
+
"table": tbl_name,
|
|
1414
|
+
"columns": [col_name] if col_name else [],
|
|
1415
|
+
"operation": "WRITE",
|
|
1416
|
+
"evidence_class": ev_cls,
|
|
1417
|
+
"confidence": conf,
|
|
1418
|
+
"call_chain": list(chain),
|
|
1419
|
+
"file": str(er["file"]),
|
|
1420
|
+
"line": int(er["start_line"]),
|
|
1421
|
+
"evidence": ev_str,
|
|
1422
|
+
})
|
|
1423
|
+
elif rel in ("UNKNOWN_TABLE", "POSSIBLE_TABLE"):
|
|
1424
|
+
tbl_name = _normalize_table_name(tgt)
|
|
1425
|
+
if tbl_name:
|
|
1426
|
+
u_item = {
|
|
1427
|
+
"table": tbl_name,
|
|
1428
|
+
"columns": [],
|
|
1429
|
+
"operation": "UNKNOWN",
|
|
1430
|
+
"evidence_class": "UNKNOWN" if rel == "UNKNOWN_TABLE" else "POSSIBLE",
|
|
1431
|
+
"confidence": "UNKNOWN" if rel == "UNKNOWN_TABLE" else "LOW",
|
|
1432
|
+
"call_chain": list(chain),
|
|
1433
|
+
"file": str(er["file"]),
|
|
1434
|
+
"line": int(er["start_line"]),
|
|
1435
|
+
"evidence": ev_str,
|
|
1436
|
+
}
|
|
1437
|
+
raw_writes.append(u_item)
|
|
1438
|
+
raw_reads.append(u_item)
|
|
1439
|
+
elif rel in (
|
|
1440
|
+
"CALLS",
|
|
1441
|
+
"INJECTS",
|
|
1442
|
+
"PROVIDES",
|
|
1443
|
+
"RESOLVES_DEPENDENCY",
|
|
1444
|
+
"DISPATCHES_TO",
|
|
1445
|
+
"ROUTES_TO",
|
|
1446
|
+
"HANDLED_BY",
|
|
1447
|
+
"CALLS_CELERY_TASK",
|
|
1448
|
+
"DISPATCHES_SIGNAL",
|
|
1449
|
+
):
|
|
1450
|
+
if depth + 1 <= max_depth and tgt not in visited:
|
|
1451
|
+
visited.add(tgt)
|
|
1452
|
+
next_disp = tgt.split(".")[-1]
|
|
1453
|
+
queue.append((tgt, depth + 1, chain + [next_disp]))
|
|
1454
|
+
|
|
1455
|
+
# 3. Class method expansion
|
|
1456
|
+
try:
|
|
1457
|
+
def_rows = con.execute(
|
|
1458
|
+
"SELECT target FROM graph_edges WHERE (source = ? OR source = ?) AND relationship = 'DEFINES'",
|
|
1459
|
+
(curr_sym, curr_bare),
|
|
1460
|
+
).fetchall()
|
|
1461
|
+
for dr in def_rows:
|
|
1462
|
+
dtgt = str(dr["target"])
|
|
1463
|
+
if depth + 1 <= max_depth and dtgt not in visited:
|
|
1464
|
+
visited.add(dtgt)
|
|
1465
|
+
queue.append((dtgt, depth + 1, chain + [dtgt.split(".")[-1]]))
|
|
1466
|
+
except sqlite3.OperationalError:
|
|
1467
|
+
pass
|
|
1468
|
+
|
|
1469
|
+
aggregated_reads = _aggregate_table_access(raw_reads, max_columns_per_table)[:max_tables]
|
|
1470
|
+
aggregated_writes = _aggregate_table_access(raw_writes, max_columns_per_table)[:max_tables]
|
|
1471
|
+
|
|
1472
|
+
all_tables = sorted(list(set(r["table"] for r in aggregated_reads) | set(w["table"] for w in aggregated_writes)))
|
|
1473
|
+
|
|
1474
|
+
all_ev_classes = set(r["evidence_class"] for r in aggregated_reads) | set(w["evidence_class"] for w in aggregated_writes)
|
|
1475
|
+
if "UNKNOWN" in all_ev_classes:
|
|
1476
|
+
overall_ev = "UNKNOWN"
|
|
1477
|
+
elif "POSSIBLE" in all_ev_classes:
|
|
1478
|
+
overall_ev = "POSSIBLE"
|
|
1479
|
+
elif all_ev_classes:
|
|
1480
|
+
overall_ev = "AST_VERIFIED"
|
|
1481
|
+
else:
|
|
1482
|
+
overall_ev = "STATIC_VERIFIED"
|
|
1483
|
+
|
|
1484
|
+
return {
|
|
1485
|
+
"status": "ok",
|
|
1486
|
+
"route": route_info.get("route", clean_target),
|
|
1487
|
+
"method": route_info.get("method", req_method or "ANY"),
|
|
1488
|
+
"handler": seeds[0][0] if seeds else clean_target,
|
|
1489
|
+
"db_reads": aggregated_reads,
|
|
1490
|
+
"db_writes": aggregated_writes,
|
|
1491
|
+
"tables": all_tables,
|
|
1492
|
+
"evidence_class": overall_ev,
|
|
1493
|
+
}
|
|
1494
|
+
|