sql-code-graph 1.33.1__py3-none-any.whl → 1.34.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sql_code_graph-1.33.1.dist-info → sql_code_graph-1.34.1.dist-info}/METADATA +1 -1
- {sql_code_graph-1.33.1.dist-info → sql_code_graph-1.34.1.dist-info}/RECORD +15 -15
- sqlcg/__init__.py +1 -1
- sqlcg/cli/commands/git.py +46 -17
- sqlcg/cli/commands/reindex.py +65 -0
- sqlcg/core/duckdb_backend.py +46 -0
- sqlcg/core/graph_db.py +10 -0
- sqlcg/core/queries.py +2 -0
- sqlcg/core/queries.sql +47 -0
- sqlcg/core/schema.py +2 -1
- sqlcg/indexer/indexer.py +67 -0
- sqlcg/parsers/ansi_parser.py +2 -0
- sqlcg/parsers/base.py +88 -1
- {sql_code_graph-1.33.1.dist-info → sql_code_graph-1.34.1.dist-info}/WHEEL +0 -0
- {sql_code_graph-1.33.1.dist-info → sql_code_graph-1.34.1.dist-info}/entry_points.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sql-code-graph
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.34.1
|
|
4
4
|
Summary: SQL code graph analyzer and lineage tracer
|
|
5
5
|
Project-URL: Homepage, https://github.com/Warhorze/sql-code-graph
|
|
6
6
|
Project-URL: Repository, https://github.com/Warhorze/sql-code-graph
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
sqlcg/__init__.py,sha256=
|
|
1
|
+
sqlcg/__init__.py,sha256=41PZf0zcZBhUQbwRo78_ftWrsT-lF8uXFf2sQm3NIKc,116
|
|
2
2
|
sqlcg/__main__.py,sha256=1YoFLcqEgTwYq1J3TbUwpkdG0zeeLIf2fJvwWI-CLFU,109
|
|
3
3
|
sqlcg/cli/__init__.py,sha256=W8fD0LpMq2xm_5WKGNMvJh2WBL1ho5E8hUeAqXQYT1g,28
|
|
4
4
|
sqlcg/cli/coverage.py,sha256=Xm9ITzZDHv2mJ70Q5jCacVuhDStVrE3gq12_-Ypvtd8,43823
|
|
@@ -9,31 +9,31 @@ sqlcg/cli/commands/catalog.py,sha256=xCIWp_9xcDPHIMUB0xrZR3-G0WMvKbfq4yMwe2sDT2g
|
|
|
9
9
|
sqlcg/cli/commands/db.py,sha256=QEDPs0boeFe8ETE8BWe7I2JLSXROUI571SHXMTUqxh8,7982
|
|
10
10
|
sqlcg/cli/commands/find.py,sha256=uMdG08bU-AvB9RqsTKPfdVnJet7b3Ebj0xbsjqTclYA,2678
|
|
11
11
|
sqlcg/cli/commands/gain.py,sha256=PQCWjlisjUY4Gz_MkppXPI1_w8xsB4DPlN7uAxx0GvY,10149
|
|
12
|
-
sqlcg/cli/commands/git.py,sha256=
|
|
12
|
+
sqlcg/cli/commands/git.py,sha256=IB6Pypo_0RYd8rsKQLdP73OnlZUidZJ8_czQe1SNdOc,8857
|
|
13
13
|
sqlcg/cli/commands/index.py,sha256=KGy0vPlvFcMetBjT5wIEvEcy9MEcLt7lhNaKfRWUkgU,19502
|
|
14
14
|
sqlcg/cli/commands/install.py,sha256=4aZzvDNdl2Ea5v-ufR8Oj1qq30oGS_9jX17etYpoan8,10148
|
|
15
15
|
sqlcg/cli/commands/mcp.py,sha256=drh5udcroZFddmJAG3q9JWxekDqTO9H4xC-oIqiis-Y,14871
|
|
16
|
-
sqlcg/cli/commands/reindex.py,sha256=
|
|
16
|
+
sqlcg/cli/commands/reindex.py,sha256=6j1QL3TjrsnWaFSIKvpGJ3LF8CoCjAbue0UFtmk9FS0,22704
|
|
17
17
|
sqlcg/cli/commands/report.py,sha256=JU0qjyMxwOukE7bN3XvvIzOI7zMg_Gsnvk_8F6pKNpA,4915
|
|
18
18
|
sqlcg/cli/commands/uninstall.py,sha256=WrA1FnINxnd6mmE4-_QBK0aHBnibstJeAT8swnQKG4M,8962
|
|
19
19
|
sqlcg/cli/commands/watch.py,sha256=_OvUQ4m_Pp2lAzI_f9UX2Gwb56zbsaSC4Tj8h51uciA,2806
|
|
20
20
|
sqlcg/core/__init__.py,sha256=dXvLWpbQ72f5CM6sKSvBDnEGqGuIZaN5MmHyD8Vf1aA,154
|
|
21
21
|
sqlcg/core/config.py,sha256=QxhIiF_let_ablMa-WttGbcAlCCDIpT4lR4tUVymIEI,19091
|
|
22
|
-
sqlcg/core/duckdb_backend.py,sha256=
|
|
22
|
+
sqlcg/core/duckdb_backend.py,sha256=zPsxiO2_qcE6axVZu8P3gUNHnJ1cgCCFv1XXy_t_LEw,42298
|
|
23
23
|
sqlcg/core/freshness.py,sha256=Cm-w7jACPrt8TIstZpCzDJpYsHCiBo63ni5js8wCGbg,6248
|
|
24
|
-
sqlcg/core/graph_db.py,sha256=
|
|
24
|
+
sqlcg/core/graph_db.py,sha256=3iZ1J32cEn7_cVhNA1tLiNuJHCyatN4F40HXLLfmwHY,10815
|
|
25
25
|
sqlcg/core/jobs.py,sha256=Je-fCdSKRgiSsv1W8SgNAlp36a7t7-pJZ-qKPbka9OE,3298
|
|
26
26
|
sqlcg/core/noise_match.py,sha256=8jxMBum0HB7wnVyS8hHaklO279QOSNKDmPtv-0RujD8,4781
|
|
27
27
|
sqlcg/core/queries.cypher,sha256=cvPOVe5GUOzJN4bxUvDxNI--xIIP8gm42TR-gUnea4U,4685
|
|
28
|
-
sqlcg/core/queries.py,sha256=
|
|
29
|
-
sqlcg/core/queries.sql,sha256=
|
|
28
|
+
sqlcg/core/queries.py,sha256=mxpG_zBENG7kcoz3YolMvHv7s_HagWluakTUw3nYSts,3631
|
|
29
|
+
sqlcg/core/queries.sql,sha256=gT8vl7CgOUZXjElpMZrHvuE_bs2ZJ4FLFA83EKaeRWM,15601
|
|
30
30
|
sqlcg/core/schema.cypher,sha256=StaJqIzOLwF2X9HYvfHSdGiYGFb_YkiHubPrTu-Qpmg,2898
|
|
31
|
-
sqlcg/core/schema.py,sha256=
|
|
31
|
+
sqlcg/core/schema.py,sha256=pVGzf__czAddmz-fsXgXqq2DTxN9EKom5RvIHLctIPU,1519
|
|
32
32
|
sqlcg/indexer/__init__.py,sha256=Wh20Unz2OHs1oIyWLrpurPAasF0BET2g4iXtNk7mh2U,56
|
|
33
33
|
sqlcg/indexer/dbt_adapter.py,sha256=EB5x1WU5Z9d-I97ADDj88S_hG1C4z4nbrv8JUCzXfy8,686
|
|
34
34
|
sqlcg/indexer/error_classify.py,sha256=-sp8cRmuOBHu_CxnCtaXf34YxHFYwIFNjIrn4LaEv6M,7142
|
|
35
35
|
sqlcg/indexer/git_delta.py,sha256=zYdH5q-jV7w_ne8Oxdywsy0N3rwUjpd5RjEDurlrMSA,5026
|
|
36
|
-
sqlcg/indexer/indexer.py,sha256=
|
|
36
|
+
sqlcg/indexer/indexer.py,sha256=de-3aYaAP_EdG-bwtFwp4e2NGKw6kpDpxb_wsOsQhdU,101712
|
|
37
37
|
sqlcg/indexer/pool.py,sha256=iMmCQtpDRKBTQBep2_EUq9THcsE18Zgk0hdaFB_CwiA,19006
|
|
38
38
|
sqlcg/indexer/walker.py,sha256=Cft6JiJtdBFy0HR6L9pJdr5Fg0eRR3XBW1OMtM2apto,1947
|
|
39
39
|
sqlcg/indexer/watcher.py,sha256=mJQq1LASRLKKwhz0WhCUWPLLqyPR2_-FD_8efYU6gE8,8442
|
|
@@ -43,8 +43,8 @@ sqlcg/lineage/schema_resolver.py,sha256=iXt6LYF6UVWsGUpcfbmjmGn9wCgXl721lTGf_8Aa
|
|
|
43
43
|
sqlcg/metrics/__init__.py,sha256=hLJ6wm4St8qqYwKh3o9QG7lcEt1BEYM31ccqO9tGpIg,133
|
|
44
44
|
sqlcg/metrics/store.py,sha256=KuDtxvyAgug9_KtiSCpvgKM2VZM7VSaI3D11uMLjJJk,10604
|
|
45
45
|
sqlcg/parsers/__init__.py,sha256=AamA8wBbDZV9_zEtZCI4Hyen5UAVKHmBwjTghTt2PZE,785
|
|
46
|
-
sqlcg/parsers/ansi_parser.py,sha256=
|
|
47
|
-
sqlcg/parsers/base.py,sha256=
|
|
46
|
+
sqlcg/parsers/ansi_parser.py,sha256=Iayv93KDOs3qKrQlrPcpQRd4_K8dEsGHOaTY076e-GY,38032
|
|
47
|
+
sqlcg/parsers/base.py,sha256=QDBDqF55vf4tmlfq_oyB9ZivHvhAuhMXU6sH3mQ5k5Q,97743
|
|
48
48
|
sqlcg/parsers/bigquery_parser.py,sha256=mOnWTfXB_Dp4JwFE1PVYOB6CDPf5nYE0Dea8kJCl9uQ,2827
|
|
49
49
|
sqlcg/parsers/postgres_parser.py,sha256=lYfUpQY6j4Qm7ndXBtXbgPoGzYqYddWt5YeFnWKdA6I,946
|
|
50
50
|
sqlcg/parsers/registry.py,sha256=LXy1F6rqQI6VdxpRvZg_tNpoEucW3mXZHYBMlMONbX4,1496
|
|
@@ -65,7 +65,7 @@ sqlcg/utils/__init__.py,sha256=--iqt5ThTXmT8Wz7da8hs3n0zDfYPl8P-z5OgRJ_77E,154
|
|
|
65
65
|
sqlcg/utils/hashing.py,sha256=H25-sYfxHKb3_IERFnHyAIYNiXN470Oqo5sJT_D3YOA,438
|
|
66
66
|
sqlcg/utils/ignore.py,sha256=6__DfiYgynI2b8nH52O2BzI1EZcZ_LRazhqEOJVGGcE,1955
|
|
67
67
|
sqlcg/utils/logging.py,sha256=u0fCmYsLj9o81vawm3xZTHaw68GQYVm7JxG-gP81u8A,840
|
|
68
|
-
sql_code_graph-1.
|
|
69
|
-
sql_code_graph-1.
|
|
70
|
-
sql_code_graph-1.
|
|
71
|
-
sql_code_graph-1.
|
|
68
|
+
sql_code_graph-1.34.1.dist-info/METADATA,sha256=UtXmf-S4GYLSS7MuPpYJxhaJ7fOmbZwSR54RnSyVGPw,17791
|
|
69
|
+
sql_code_graph-1.34.1.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
70
|
+
sql_code_graph-1.34.1.dist-info/entry_points.txt,sha256=Wfe49sVzV9p4eVFGo5RxcV-frr3HOP0yzzst8JBxQLQ,46
|
|
71
|
+
sql_code_graph-1.34.1.dist-info/RECORD,,
|
sqlcg/__init__.py
CHANGED
sqlcg/cli/commands/git.py
CHANGED
|
@@ -23,6 +23,29 @@ class _HookSpec(NamedTuple):
|
|
|
23
23
|
# Hook script templates — use {sqlcg_bin} as the placeholder for the resolved binary.
|
|
24
24
|
# The sentinel comments (e.g. "# sqlcg post-checkout hook") must stay byte-for-byte
|
|
25
25
|
# unchanged so R9 idempotency is preserved: _install_single_hook matches them verbatim.
|
|
26
|
+
#
|
|
27
|
+
# Run-time binary resolution (#126 + #1): the hook no longer trusts the single embedded
|
|
28
|
+
# absolute literal to stay valid forever (a venv rebuild or tool upgrade makes the
|
|
29
|
+
# install-time path stale and the resync would then run a wrong/missing binary). Instead
|
|
30
|
+
# the shared _RESOLVE_PREAMBLE resolves the binary at *run time* with a hybrid order:
|
|
31
|
+
# 1. the preferred path rendered in at install time ({sqlcg_bin}, normally the stable
|
|
32
|
+
# user-base binary from _resolve_sqlcg_bin) — the fast path, used if still executable;
|
|
33
|
+
# 2. `command -v sqlcg` on $PATH — covers uv-tool, user-base-on-PATH, and active-venv;
|
|
34
|
+
# 3. neither resolves → print a loud stderr error and exit non-zero (never silent).
|
|
35
|
+
# The shell never recomputes getuserbase() — the preferred path is rendered in by Python.
|
|
36
|
+
# It exports SQLCG_BIN for the hook body to invoke.
|
|
37
|
+
_RESOLVE_PREAMBLE = (
|
|
38
|
+
'if [ -x "{sqlcg_bin}" ]; then\n'
|
|
39
|
+
' SQLCG_BIN="{sqlcg_bin}"\n'
|
|
40
|
+
"elif command -v sqlcg >/dev/null 2>&1; then\n"
|
|
41
|
+
' SQLCG_BIN="$(command -v sqlcg)"\n'
|
|
42
|
+
"else\n"
|
|
43
|
+
" echo \"sqlcg: cannot find the sqlcg binary (tried '{sqlcg_bin}' and \\$PATH)"
|
|
44
|
+
" -- run 'sqlcg git install-hooks' to repoint this hook\" >&2\n"
|
|
45
|
+
" exit 1\n"
|
|
46
|
+
"fi\n"
|
|
47
|
+
)
|
|
48
|
+
|
|
26
49
|
_HOOKS: list[_HookSpec] = [
|
|
27
50
|
_HookSpec(
|
|
28
51
|
filename="post-checkout",
|
|
@@ -32,7 +55,8 @@ _HOOKS: list[_HookSpec] = [
|
|
|
32
55
|
"# sqlcg post-checkout hook — incremental resync after branch switch\n"
|
|
33
56
|
"# $3 == 1 means branch checkout (not file checkout); skip file checkouts\n"
|
|
34
57
|
'[ "$3" = "1" ] || exit 0\n'
|
|
35
|
-
|
|
58
|
+
+ _RESOLVE_PREAMBLE
|
|
59
|
+
+ '"$SQLCG_BIN" reindex --from "$1" --to "$2"'
|
|
36
60
|
' "$(git rev-parse --show-toplevel)" --dialect auto --quiet --notify'
|
|
37
61
|
' || echo "sqlcg: graph not updated (reindex failed)'
|
|
38
62
|
" -- run 'sqlcg mcp status'\" >&2\n"
|
|
@@ -41,22 +65,27 @@ _HOOKS: list[_HookSpec] = [
|
|
|
41
65
|
_HookSpec(
|
|
42
66
|
filename="post-merge",
|
|
43
67
|
sentinel="# sqlcg post-merge hook",
|
|
44
|
-
script_template=
|
|
45
|
-
#!/bin/sh
|
|
46
|
-
# sqlcg post-merge hook — incremental resync after pull/merge
|
|
47
|
-
# git sets ORIG_HEAD to the pre-merge HEAD; pass it as --from so --notify
|
|
48
|
-
# through a running server (same path as post-checkout). If
|
|
49
|
-
# first-ever merge / gc'd), fall back to the
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|| echo "sqlcg: graph not updated (reindex failed)
|
|
58
|
-
|
|
59
|
-
""
|
|
68
|
+
script_template=(
|
|
69
|
+
"#!/bin/sh\n"
|
|
70
|
+
"# sqlcg post-merge hook — incremental resync after pull/merge\n"
|
|
71
|
+
"# git sets ORIG_HEAD to the pre-merge HEAD; pass it as --from so --notify\n"
|
|
72
|
+
"# can route through a running server (same path as post-checkout). If\n"
|
|
73
|
+
"# ORIG_HEAD is unset (e.g. first-ever merge / gc'd), fall back to the\n"
|
|
74
|
+
"# standalone stored-SHA delta (direct write).\n"
|
|
75
|
+
+ _RESOLVE_PREAMBLE
|
|
76
|
+
+ "PREV=$(git rev-parse --verify --quiet ORIG_HEAD)\n"
|
|
77
|
+
"TOP=$(git rev-parse --show-toplevel)\n"
|
|
78
|
+
'if [ -n "$PREV" ]; then\n'
|
|
79
|
+
' "$SQLCG_BIN" reindex --from "$PREV" --to HEAD "$TOP" --dialect auto'
|
|
80
|
+
" --quiet --notify \\\n"
|
|
81
|
+
' || echo "sqlcg: graph not updated (reindex failed)'
|
|
82
|
+
" -- run 'sqlcg mcp status'\" >&2\n"
|
|
83
|
+
"else\n"
|
|
84
|
+
' "$SQLCG_BIN" reindex "$TOP" --dialect auto --quiet --notify \\\n'
|
|
85
|
+
' || echo "sqlcg: graph not updated (reindex failed)'
|
|
86
|
+
" -- run 'sqlcg mcp status'\" >&2\n"
|
|
87
|
+
"fi\n"
|
|
88
|
+
),
|
|
60
89
|
),
|
|
61
90
|
]
|
|
62
91
|
|
sqlcg/cli/commands/reindex.py
CHANGED
|
@@ -129,6 +129,11 @@ def reindex_cmd( # noqa: B008
|
|
|
129
129
|
# below) and return to git immediately. The respawn guard env var prevents
|
|
130
130
|
# the detached child from detaching again.
|
|
131
131
|
if _is_hook_path and os.environ.get(_HOOK_DETACHED_ENV) != "1":
|
|
132
|
+
# #1 — loud foreground preflight BEFORE detaching. The detached child's
|
|
133
|
+
# schema gate (below) writes its Exit(1) to the hook-resync log, invisible
|
|
134
|
+
# to the user, so a stale-binary schema skew dies silently (exit 0 to git).
|
|
135
|
+
# Surface a real mismatch in the foreground, where git's terminal can see it.
|
|
136
|
+
_hook_preflight_schema_check()
|
|
132
137
|
_spawn_detached_hook_resync(path)
|
|
133
138
|
return
|
|
134
139
|
|
|
@@ -291,6 +296,66 @@ def _spawn_detached_hook_resync(path: Path) -> None:
|
|
|
291
296
|
console.print(f"sqlcg: resyncing graph in background (log: {log_path})")
|
|
292
297
|
|
|
293
298
|
|
|
299
|
+
def _hook_preflight_schema_check() -> None:
|
|
300
|
+
"""Surface a stale-binary schema/version skew in the FOREGROUND hook process (#1).
|
|
301
|
+
|
|
302
|
+
On the hook path the resync is detached (#77), so the existing schema gate
|
|
303
|
+
(``reindex_cmd`` at ~:150) runs in the detached child whose stdout/stderr go to
|
|
304
|
+
``hook-resync.log`` — its ``Exit(1)`` never reaches the user's terminal and the
|
|
305
|
+
graph silently never refreshes. This preflight runs *before* the detach and, on a
|
|
306
|
+
REAL mismatch, prints a visible, actionable stderr message and exits non-zero
|
|
307
|
+
without detaching.
|
|
308
|
+
|
|
309
|
+
Git runs post-checkout/post-merge AFTER the operation and ignores the hook exit
|
|
310
|
+
status, so the **stderr message** is the load-bearing part (not the exit code).
|
|
311
|
+
|
|
312
|
+
BOUNDED, FAIL-SOFT open (plan-review B1 — MANDATED). DuckDB takes an exclusive
|
|
313
|
+
file lock: per ``config.py`` ``get_backend`` docstring, a second process cannot
|
|
314
|
+
open the DB at all (even read-only) while another holds it — it raises
|
|
315
|
+
``duckdb.IOException`` (#63). On rapid successive checkouts a prior detached
|
|
316
|
+
resync still holds the lock, so this open WILL contend. We therefore do a SINGLE
|
|
317
|
+
open attempt (NOT ``_open_backend_with_lock_retry``'s ~31 s retry, which would
|
|
318
|
+
stall ``git checkout`` — a #77-class regression) and, on ``IOException`` (lock
|
|
319
|
+
held ⇒ a resync is already in flight), treat it as "cannot determine — fall
|
|
320
|
+
through to the existing detach path unchanged": never block, never emit a
|
|
321
|
+
schema-mismatch error. Only a SUCCESSFUL read showing a real mismatch triggers
|
|
322
|
+
the loud exit. The backend is opened read-only and released before returning so
|
|
323
|
+
the detached child can still acquire the lock.
|
|
324
|
+
"""
|
|
325
|
+
import duckdb
|
|
326
|
+
|
|
327
|
+
from sqlcg.core.config import get_backend
|
|
328
|
+
from sqlcg.core.schema import SCHEMA_VERSION
|
|
329
|
+
|
|
330
|
+
try:
|
|
331
|
+
backend = get_backend(read_only=True)
|
|
332
|
+
except duckdb.IOException:
|
|
333
|
+
# Lock held by an in-flight resync — cannot determine; fall through to detach.
|
|
334
|
+
return
|
|
335
|
+
|
|
336
|
+
try:
|
|
337
|
+
with backend:
|
|
338
|
+
stored_version = backend.get_schema_version()
|
|
339
|
+
if stored_version is None or stored_version == SCHEMA_VERSION:
|
|
340
|
+
# Matches (or no stored version yet on a fresh graph) — detach as today.
|
|
341
|
+
return
|
|
342
|
+
indexed_tool_version = backend.get_indexed_version()
|
|
343
|
+
except duckdb.IOException:
|
|
344
|
+
# Lock raced after the open (e.g. closed-then-reopened by the writer);
|
|
345
|
+
# fail soft exactly as the open-time IOException path.
|
|
346
|
+
return
|
|
347
|
+
|
|
348
|
+
skew = f" (graph last indexed by sqlcg {indexed_tool_version})" if indexed_tool_version else ""
|
|
349
|
+
print(
|
|
350
|
+
f"sqlcg: graph schema is v{stored_version} but this tool needs "
|
|
351
|
+
f"v{SCHEMA_VERSION}{skew} — the hook binary is stale. "
|
|
352
|
+
"Run 'sqlcg git install-hooks' to repoint the hook, then "
|
|
353
|
+
"'sqlcg db reset && sqlcg index <root>'.",
|
|
354
|
+
file=sys.stderr,
|
|
355
|
+
)
|
|
356
|
+
raise typer.Exit(1)
|
|
357
|
+
|
|
358
|
+
|
|
294
359
|
def _open_backend_with_lock_retry() -> "GraphBackend":
|
|
295
360
|
"""Open the DuckDB backend, retrying with backoff if it is locked.
|
|
296
361
|
|
sqlcg/core/duckdb_backend.py
CHANGED
|
@@ -203,6 +203,14 @@ _EDGE_DDLS = [
|
|
|
203
203
|
PRIMARY KEY (src_key, dst_key)
|
|
204
204
|
)
|
|
205
205
|
""",
|
|
206
|
+
"""
|
|
207
|
+
CREATE TABLE IF NOT EXISTS "JOIN_COL_RESOLVE" (
|
|
208
|
+
src_key VARCHAR NOT NULL,
|
|
209
|
+
dst_key VARCHAR NOT NULL,
|
|
210
|
+
bare_col VARCHAR,
|
|
211
|
+
PRIMARY KEY (src_key, dst_key)
|
|
212
|
+
)
|
|
213
|
+
""",
|
|
206
214
|
]
|
|
207
215
|
|
|
208
216
|
_INDEX_DDLS = [
|
|
@@ -230,6 +238,8 @@ _INDEX_DDLS = [
|
|
|
230
238
|
'CREATE INDEX IF NOT EXISTS idx_STAR_SOURCE_dst ON "STAR_SOURCE" (dst_key)',
|
|
231
239
|
'CREATE INDEX IF NOT EXISTS idx_CONSUMED_BY_src ON "CONSUMED_BY" (src_key)',
|
|
232
240
|
'CREATE INDEX IF NOT EXISTS idx_CONSUMED_BY_dst ON "CONSUMED_BY" (dst_key)',
|
|
241
|
+
'CREATE INDEX IF NOT EXISTS idx_JOIN_COL_RESOLVE_src ON "JOIN_COL_RESOLVE" (src_key)',
|
|
242
|
+
'CREATE INDEX IF NOT EXISTS idx_JOIN_COL_RESOLVE_dst ON "JOIN_COL_RESOLVE" (dst_key)',
|
|
233
243
|
]
|
|
234
244
|
|
|
235
245
|
# Node label → column list (all columns of that table, in order)
|
|
@@ -304,6 +314,7 @@ _EDGE_EXTRA_COLUMNS: dict[str, list[str]] = {
|
|
|
304
314
|
RelType.DECLARES: [],
|
|
305
315
|
RelType.STAR_SOURCE: ["qualifier", "target_table", "confidence"],
|
|
306
316
|
RelType.CONSUMED_BY: [],
|
|
317
|
+
RelType.JOIN_COL_RESOLVE: ["bare_col"],
|
|
307
318
|
}
|
|
308
319
|
|
|
309
320
|
|
|
@@ -752,6 +763,7 @@ class DuckDBBackend(GraphBackend):
|
|
|
752
763
|
"UPDATES",
|
|
753
764
|
"DECLARES",
|
|
754
765
|
"STAR_SOURCE",
|
|
766
|
+
"JOIN_COL_RESOLVE",
|
|
755
767
|
):
|
|
756
768
|
self._conn.execute(
|
|
757
769
|
f'DELETE FROM "{edge_table}" WHERE src_key IN ({placeholders})',
|
|
@@ -891,6 +903,39 @@ class DuckDBBackend(GraphBackend):
|
|
|
891
903
|
row = result.fetchone()
|
|
892
904
|
return int(row[0]) if row else 0
|
|
893
905
|
|
|
906
|
+
def resolve_join_columns(self) -> int:
|
|
907
|
+
"""Resolve JOIN_COL_RESOLVE markers into concrete COLUMN_LINEAGE edges.
|
|
908
|
+
|
|
909
|
+
Runs AFTER ingestion AND after the catalog has been applied, so that
|
|
910
|
+
information_schema ``HAS_COLUMN`` rows are present. For each parse-time
|
|
911
|
+
marker (a bare unqualified column in a >=2-table join projection), it
|
|
912
|
+
finds which of the query's source tables owns a column with that bare
|
|
913
|
+
name (any HAS_COLUMN source), normalising casing on both sides:
|
|
914
|
+
|
|
915
|
+
- exactly one owner -> one high-confidence (0.9) edge (the dominant
|
|
916
|
+
live case: the second table is visible only via information_schema);
|
|
917
|
+
- more than one owner -> one low-confidence (0.5) edge per owner
|
|
918
|
+
(genuine ambiguity; over-attribute, never silently mis-pick);
|
|
919
|
+
- zero owners -> emit nothing (honest empty; XML-DDL gap).
|
|
920
|
+
|
|
921
|
+
Mirrors :meth:`expand_star_sources` (post-index SQL pass). The
|
|
922
|
+
suppressed sqlglot mis-bind edge is never emitted at parse time, so this
|
|
923
|
+
pass is the SOLE producer of these columns' edges — it MUST run in every
|
|
924
|
+
path that re-creates the markers (full index + both incremental paths).
|
|
925
|
+
|
|
926
|
+
Returns:
|
|
927
|
+
Number of COLUMN_LINEAGE JOIN_COL_RESOLVED edges present afterward.
|
|
928
|
+
"""
|
|
929
|
+
from sqlcg.core.queries import RESOLVE_JOIN_COLUMNS_QUERY
|
|
930
|
+
|
|
931
|
+
self._conn.execute(RESOLVE_JOIN_COLUMNS_QUERY)
|
|
932
|
+
result = self._conn.execute(
|
|
933
|
+
'SELECT count(*) AS n FROM "COLUMN_LINEAGE" WHERE transform = ?',
|
|
934
|
+
["JOIN_COL_RESOLVED"],
|
|
935
|
+
)
|
|
936
|
+
row = result.fetchone()
|
|
937
|
+
return int(row[0]) if row else 0
|
|
938
|
+
|
|
894
939
|
# ------------------------------------------------------------------
|
|
895
940
|
# Full-rebuild helpers
|
|
896
941
|
# ------------------------------------------------------------------
|
|
@@ -927,6 +972,7 @@ class DuckDBBackend(GraphBackend):
|
|
|
927
972
|
"DECLARES",
|
|
928
973
|
"STAR_SOURCE",
|
|
929
974
|
"CONSUMED_BY",
|
|
975
|
+
"JOIN_COL_RESOLVE",
|
|
930
976
|
]
|
|
931
977
|
for tbl in edge_tables + node_tables:
|
|
932
978
|
self._conn.execute(f'DELETE FROM "{tbl}"')
|
sqlcg/core/graph_db.py
CHANGED
|
@@ -273,6 +273,16 @@ class GraphBackend(ABC):
|
|
|
273
273
|
"""
|
|
274
274
|
raise NotImplementedError(f"{type(self).__name__} does not support expand_star_sources")
|
|
275
275
|
|
|
276
|
+
def resolve_join_columns(self) -> int:
|
|
277
|
+
"""Resolve JOIN_COL_RESOLVE markers into per-column COLUMN_LINEAGE edges.
|
|
278
|
+
|
|
279
|
+
Runs once per index after ingestion AND catalog apply, resolving bare
|
|
280
|
+
unqualified join columns against the full post-index HAS_COLUMN catalog
|
|
281
|
+
(including information_schema rows unavailable at parse time). Concrete
|
|
282
|
+
backends must override this; returns the total JOIN_COL_RESOLVED edge count.
|
|
283
|
+
"""
|
|
284
|
+
raise NotImplementedError(f"{type(self).__name__} does not support resolve_join_columns")
|
|
285
|
+
|
|
276
286
|
|
|
277
287
|
def indexed_repo_root(db: "GraphBackend") -> Path | None:
|
|
278
288
|
"""Return the indexed root path stored on the first Repo node, or None.
|
sqlcg/core/queries.py
CHANGED
|
@@ -51,6 +51,8 @@ LIST_DIALECTS_AND_REPOS_QUERY = _Q["LIST_DIALECTS_AND_REPOS"]
|
|
|
51
51
|
EXPAND_STAR_SOURCES_QUERY = _Q["EXPAND_STAR_SOURCES"]
|
|
52
52
|
EXPAND_STAR_SOURCES_HAS_COLUMN_QUERY = _Q["EXPAND_STAR_SOURCES_HAS_COLUMN"]
|
|
53
53
|
EXPAND_STAR_SOURCES_LINEAGE_QUERY = _Q["EXPAND_STAR_SOURCES_LINEAGE"]
|
|
54
|
+
RESOLVE_JOIN_COLUMNS_QUERY = _Q["RESOLVE_JOIN_COLUMNS"]
|
|
55
|
+
COUNT_JOIN_COL_RESOLVED_QUERY = _Q["COUNT_JOIN_COL_RESOLVED"]
|
|
54
56
|
COUNT_STAR_SOURCES_QUERY = _Q["COUNT_STAR_SOURCES"]
|
|
55
57
|
COUNT_STAR_EXPANSIONS_QUERY = _Q["COUNT_STAR_EXPANSIONS"]
|
|
56
58
|
FIND_DEFINITION_QUERY = _Q["FIND_DEFINITION"]
|
sqlcg/core/queries.sql
CHANGED
|
@@ -148,6 +148,53 @@ JOIN "SqlTable" tgt ON tgt.qualified = q.target_table
|
|
|
148
148
|
WHERE q.target_table <> ''
|
|
149
149
|
AND q.target_table <> t.qualified
|
|
150
150
|
|
|
151
|
+
-- RESOLVE_JOIN_COLUMNS
|
|
152
|
+
-- Resolve JOIN_COL_RESOLVE markers (bare unqualified join columns whose owning
|
|
153
|
+
-- table was only knowable post-index, e.g. catalogued via information_schema)
|
|
154
|
+
-- into concrete COLUMN_LINEAGE edges.
|
|
155
|
+
--
|
|
156
|
+
-- For each marker (src_key=query_id, dst_key=<dst_table>.<dst_col>, bare_col):
|
|
157
|
+
-- find which of the query's SELECTS_FROM source tables OWNS a column named
|
|
158
|
+
-- bare_col (any HAS_COLUMN source: ddl | information_schema | star_expansion |
|
|
159
|
+
-- usage). Casing is normalised on BOTH sides of the owner-match join because
|
|
160
|
+
-- information_schema stores col_name as exported (e.g. STATUS) while the parsed
|
|
161
|
+
-- bare name is lowercased (status).
|
|
162
|
+
--
|
|
163
|
+
-- Confidence contract:
|
|
164
|
+
-- - exactly one owning table -> HIGH confidence (0.9), the resolved edge
|
|
165
|
+
-- - more than one owning table -> one LOW confidence (0.5) edge per owner
|
|
166
|
+
-- (genuine ambiguity: over-attribute, never silently mis-pick)
|
|
167
|
+
-- - zero owning tables -> emit nothing (honest empty; XML-DDL gap)
|
|
168
|
+
-- A window COUNT over the owning tables per marker selects the confidence in one
|
|
169
|
+
-- pass.
|
|
170
|
+
-- params: []
|
|
171
|
+
INSERT OR REPLACE INTO "COLUMN_LINEAGE"
|
|
172
|
+
(src_key, dst_key, transform, confidence, query_id, inferred_from_source_name)
|
|
173
|
+
SELECT
|
|
174
|
+
src_key,
|
|
175
|
+
dst_key,
|
|
176
|
+
'JOIN_COL_RESOLVED' AS transform,
|
|
177
|
+
CASE WHEN n_owners = 1 THEN 0.9 ELSE 0.5 END AS confidence,
|
|
178
|
+
query_id,
|
|
179
|
+
FALSE AS inferred_from_source_name
|
|
180
|
+
FROM (
|
|
181
|
+
SELECT
|
|
182
|
+
c.id AS src_key,
|
|
183
|
+
m.dst_key AS dst_key,
|
|
184
|
+
m.src_key AS query_id,
|
|
185
|
+
count(*) OVER (PARTITION BY m.src_key, m.dst_key) AS n_owners
|
|
186
|
+
FROM "JOIN_COL_RESOLVE" m
|
|
187
|
+
JOIN "SELECTS_FROM" sf ON sf.src_key = m.src_key
|
|
188
|
+
JOIN "HAS_COLUMN" hc ON hc.src_key = sf.dst_key
|
|
189
|
+
JOIN "SqlColumn" c
|
|
190
|
+
ON c.id = hc.dst_key
|
|
191
|
+
AND lower(c.col_name) = lower(m.bare_col)
|
|
192
|
+
) owners
|
|
193
|
+
|
|
194
|
+
-- COUNT_JOIN_COL_RESOLVED
|
|
195
|
+
-- params: []
|
|
196
|
+
SELECT count(*) AS n FROM "COLUMN_LINEAGE" WHERE transform = 'JOIN_COL_RESOLVED'
|
|
197
|
+
|
|
151
198
|
-- COUNT_STAR_SOURCES
|
|
152
199
|
-- params: []
|
|
153
200
|
SELECT count(*) AS n FROM "STAR_SOURCE"
|
sqlcg/core/schema.py
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
from enum import StrEnum
|
|
4
4
|
from importlib.resources import files
|
|
5
5
|
|
|
6
|
-
SCHEMA_VERSION = "
|
|
6
|
+
SCHEMA_VERSION = "11"
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
class NodeLabel(StrEnum):
|
|
@@ -29,6 +29,7 @@ class RelType(StrEnum):
|
|
|
29
29
|
DECLARES = "DECLARES"
|
|
30
30
|
STAR_SOURCE = "STAR_SOURCE"
|
|
31
31
|
CONSUMED_BY = "CONSUMED_BY"
|
|
32
|
+
JOIN_COL_RESOLVE = "JOIN_COL_RESOLVE"
|
|
32
33
|
|
|
33
34
|
|
|
34
35
|
# Backward-compatible aliases
|
sqlcg/indexer/indexer.py
CHANGED
|
@@ -47,6 +47,7 @@ class FileRowSet:
|
|
|
47
47
|
selects_from_edges: list[dict] = field(default_factory=list)
|
|
48
48
|
column_lineage_edges: list[dict] = field(default_factory=list)
|
|
49
49
|
star_source_edges: list[dict] = field(default_factory=list)
|
|
50
|
+
join_col_resolve_edges: list[dict] = field(default_factory=list)
|
|
50
51
|
counts: dict = field(
|
|
51
52
|
default_factory=lambda: {"tables": 0, "edges": 0, "columns_defined": 0, "star_sources": 0}
|
|
52
53
|
)
|
|
@@ -71,6 +72,7 @@ class BatchRowBuffer:
|
|
|
71
72
|
selects_from_edges: list[dict] = field(default_factory=list)
|
|
72
73
|
column_lineage_edges: list[dict] = field(default_factory=list)
|
|
73
74
|
star_source_edges: list[dict] = field(default_factory=list)
|
|
75
|
+
join_col_resolve_edges: list[dict] = field(default_factory=list)
|
|
74
76
|
|
|
75
77
|
def extend(self, rows: FileRowSet) -> None:
|
|
76
78
|
"""Accumulate one file's row sets into this buffer."""
|
|
@@ -84,6 +86,7 @@ class BatchRowBuffer:
|
|
|
84
86
|
self.selects_from_edges.extend(rows.selects_from_edges)
|
|
85
87
|
self.column_lineage_edges.extend(rows.column_lineage_edges)
|
|
86
88
|
self.star_source_edges.extend(rows.star_source_edges)
|
|
89
|
+
self.join_col_resolve_edges.extend(rows.join_col_resolve_edges)
|
|
87
90
|
|
|
88
91
|
@classmethod
|
|
89
92
|
def from_single(cls, rows: FileRowSet) -> "BatchRowBuffer":
|
|
@@ -292,6 +295,9 @@ def _flush_row_batch(
|
|
|
292
295
|
star_source_edges = list(
|
|
293
296
|
{(r["src_key"], r["dst_key"]): r for r in buf.star_source_edges}.values()
|
|
294
297
|
)
|
|
298
|
+
join_col_resolve_edges = list(
|
|
299
|
+
{(r["src_key"], r["dst_key"]): r for r in buf.join_col_resolve_edges}.values()
|
|
300
|
+
)
|
|
295
301
|
|
|
296
302
|
# --- Phase B2: usage-derived catalog harvest ---
|
|
297
303
|
# Post-dedup: for each COLUMN_LINEAGE src endpoint whose table is a physical
|
|
@@ -345,6 +351,11 @@ def _flush_row_batch(
|
|
|
345
351
|
star_source_edges = [
|
|
346
352
|
r for r in star_source_edges if r["dst_key"] not in excluded_tables
|
|
347
353
|
]
|
|
354
|
+
join_col_resolve_edges = [
|
|
355
|
+
r
|
|
356
|
+
for r in join_col_resolve_edges
|
|
357
|
+
if _table_of_column_id(r["dst_key"]) not in excluded_tables
|
|
358
|
+
]
|
|
348
359
|
column_lineage_edges = [
|
|
349
360
|
r
|
|
350
361
|
for r in column_lineage_edges
|
|
@@ -372,6 +383,9 @@ def _flush_row_batch(
|
|
|
372
383
|
NodeLabel.COLUMN, NodeLabel.COLUMN, RelType.COLUMN_LINEAGE, column_lineage_edges
|
|
373
384
|
)
|
|
374
385
|
db.upsert_edges_bulk(NodeLabel.QUERY, NodeLabel.TABLE, RelType.STAR_SOURCE, star_source_edges)
|
|
386
|
+
db.upsert_edges_bulk(
|
|
387
|
+
NodeLabel.QUERY, NodeLabel.COLUMN, RelType.JOIN_COL_RESOLVE, join_col_resolve_edges
|
|
388
|
+
)
|
|
375
389
|
|
|
376
390
|
|
|
377
391
|
def _subprocess_parse_worker(parser_cls, dialect, path, sql, q):
|
|
@@ -882,6 +896,12 @@ class Indexer:
|
|
|
882
896
|
if profile:
|
|
883
897
|
_t_catalog_end = time.perf_counter()
|
|
884
898
|
|
|
899
|
+
# Post-catalog: resolve JOIN_COL_RESOLVE markers against the full HAS_COLUMN
|
|
900
|
+
# catalog (Bug #5, PR-5 option b). MUST run after the catalog is applied so
|
|
901
|
+
# information_schema columns are present; the suppressed sqlglot mis-bind
|
|
902
|
+
# edge is never emitted, so this is the sole producer of the resolved edges.
|
|
903
|
+
join_cols_resolved = self._resolve_join_columns(db)
|
|
904
|
+
|
|
885
905
|
# Classify all errors into buckets for measurement and reporting
|
|
886
906
|
error_summary: dict[str, int] = {
|
|
887
907
|
"E1": 0,
|
|
@@ -931,6 +951,7 @@ class Indexer:
|
|
|
931
951
|
"columns_defined": nonlocal_counts["columns_defined"],
|
|
932
952
|
"star_sources": nonlocal_counts["star_sources"],
|
|
933
953
|
"star_edges_expanded": star_edges_expanded,
|
|
954
|
+
"join_cols_resolved": join_cols_resolved,
|
|
934
955
|
"quality": nonlocal_counts["quality"],
|
|
935
956
|
"error_summary": error_summary,
|
|
936
957
|
"degraded_files": degraded_files,
|
|
@@ -1329,6 +1350,14 @@ class Indexer:
|
|
|
1329
1350
|
# ---- Step 8: Single star expansion (same as index_repo) ------------------
|
|
1330
1351
|
self._expand_star_sources(db)
|
|
1331
1352
|
|
|
1353
|
+
# ---- Step 8b: Resolve JOIN_COL_RESOLVE markers (Bug #5, PR-5 option b) ----
|
|
1354
|
+
# The incremental path does NOT reapply the catalog — it resolves against the
|
|
1355
|
+
# information_schema HAS_COLUMN rows persisted by the prior full index. Without
|
|
1356
|
+
# this, a delta touching a join file would re-create the markers but never
|
|
1357
|
+
# resolve them, silently dropping the join-column edges until the next full
|
|
1358
|
+
# index (plan-review BLOCKER).
|
|
1359
|
+
self._resolve_join_columns(db)
|
|
1360
|
+
|
|
1332
1361
|
# ---- Step 9: Persist new SHA on success ----------------------------------
|
|
1333
1362
|
db.set_indexed_sha(new_sha)
|
|
1334
1363
|
|
|
@@ -1382,6 +1411,13 @@ class Indexer:
|
|
|
1382
1411
|
# Re-run star expansion after re-indexing
|
|
1383
1412
|
self._expand_star_sources(db)
|
|
1384
1413
|
|
|
1414
|
+
# Resolve JOIN_COL_RESOLVE markers (Bug #5, PR-5 option b) against the
|
|
1415
|
+
# information_schema HAS_COLUMN rows persisted by the prior full index. The
|
|
1416
|
+
# single-file path does NOT reapply the catalog; without re-resolving here a
|
|
1417
|
+
# reindex of a join file would re-create the markers but drop the resolved
|
|
1418
|
+
# join-column edges until the next full index (plan-review BLOCKER).
|
|
1419
|
+
self._resolve_join_columns(db)
|
|
1420
|
+
|
|
1385
1421
|
def _index_single_file(self, parser, path: Path, sql: str, timeout: int) -> ParsedFile:
|
|
1386
1422
|
"""Parse one file, with optional timeout via subprocess isolation.
|
|
1387
1423
|
|
|
@@ -1706,6 +1742,21 @@ class Indexer:
|
|
|
1706
1742
|
)
|
|
1707
1743
|
rows.counts["star_sources"] += 1
|
|
1708
1744
|
|
|
1745
|
+
# JOIN_COL_RESOLVE markers (Bug #5, PR-5 option b): bare unqualified
|
|
1746
|
+
# join-projection columns whose owning source table is only knowable
|
|
1747
|
+
# post-index (e.g. catalogued via information_schema). The sqlglot
|
|
1748
|
+
# mis-bind edge was suppressed at parse time; the post-index resolver
|
|
1749
|
+
# (DuckDBBackend.resolve_join_columns) is the SOLE producer of these
|
|
1750
|
+
# columns' COLUMN_LINEAGE edges. src_key=query_id, dst_key=<dst>.<col>.
|
|
1751
|
+
for jcr in stmt.join_col_resolves:
|
|
1752
|
+
rows.join_col_resolve_edges.append(
|
|
1753
|
+
{
|
|
1754
|
+
"src_key": query_id,
|
|
1755
|
+
"dst_key": jcr.dst.full_id,
|
|
1756
|
+
"bare_col": jcr.bare_col,
|
|
1757
|
+
}
|
|
1758
|
+
)
|
|
1759
|
+
|
|
1709
1760
|
# Upsert target table node (if not already a defined_table)
|
|
1710
1761
|
# so that star expansion can create destination columns.
|
|
1711
1762
|
# #44: when a canonical_by_bare index is available, attempt to resolve
|
|
@@ -1930,6 +1981,22 @@ class Indexer:
|
|
|
1930
1981
|
"""
|
|
1931
1982
|
return db.expand_star_sources()
|
|
1932
1983
|
|
|
1984
|
+
def _resolve_join_columns(self, db: GraphBackend) -> int:
|
|
1985
|
+
"""Run the post-index join-column resolution (Bug #5, PR-5 option b).
|
|
1986
|
+
|
|
1987
|
+
Calls DuckDBBackend.resolve_join_columns(), which resolves JOIN_COL_RESOLVE
|
|
1988
|
+
markers (bare unqualified join columns) against the full post-index
|
|
1989
|
+
HAS_COLUMN catalog — including information_schema rows unavailable at parse
|
|
1990
|
+
time. MUST run AFTER the catalog is applied (full-index path) and AFTER star
|
|
1991
|
+
expansion in every path that re-creates the markers; the suppressed sqlglot
|
|
1992
|
+
mis-bind edge is never emitted, so this pass is the SOLE producer of these
|
|
1993
|
+
columns' edges.
|
|
1994
|
+
|
|
1995
|
+
Returns:
|
|
1996
|
+
Number of COLUMN_LINEAGE JOIN_COL_RESOLVED edges after resolution.
|
|
1997
|
+
"""
|
|
1998
|
+
return db.resolve_join_columns()
|
|
1999
|
+
|
|
1933
2000
|
def _ingest_external_consumers(self, db: GraphBackend, path: Path) -> dict:
|
|
1934
2001
|
"""Ingest declared external downstream consumers from .sqlcg.toml.
|
|
1935
2002
|
|
sqlcg/parsers/ansi_parser.py
CHANGED
|
@@ -622,6 +622,7 @@ class AnsiParser(SqlParser):
|
|
|
622
622
|
)
|
|
623
623
|
column_lineage = extraction.edges
|
|
624
624
|
star_sources = extraction.star_sources
|
|
625
|
+
join_col_resolves = extraction.join_col_resolves
|
|
625
626
|
stmt_qualify_failed = extraction.qualify_failed
|
|
626
627
|
|
|
627
628
|
# Extract defined columns for CREATE TABLE statements
|
|
@@ -666,6 +667,7 @@ class AnsiParser(SqlParser):
|
|
|
666
667
|
parsing_mode="sqlglot",
|
|
667
668
|
defined_columns=defined_columns,
|
|
668
669
|
star_sources=star_sources,
|
|
670
|
+
join_col_resolves=join_col_resolves,
|
|
669
671
|
defined_body=defined_body,
|
|
670
672
|
clone_source=clone_source,
|
|
671
673
|
)
|
sqlcg/parsers/base.py
CHANGED
|
@@ -204,6 +204,32 @@ class StarSource:
|
|
|
204
204
|
qualifier: str | None = None
|
|
205
205
|
|
|
206
206
|
|
|
207
|
+
@dataclass(frozen=True)
|
|
208
|
+
class JoinColResolve:
|
|
209
|
+
"""A deferred-resolution marker for a bare unqualified join-projection column.
|
|
210
|
+
|
|
211
|
+
Emitted (instead of the sqlglot mis-bind edge) when a top-level INSERT/CREATE
|
|
212
|
+
SELECT projection contains a bare column (``exp.Column`` with no ``.table``)
|
|
213
|
+
AND the statement has >=2 source tables. The owning source table is only
|
|
214
|
+
knowable AFTER indexing (e.g. catalogued via information_schema), so the
|
|
215
|
+
binding is deferred to ``DuckDBBackend.resolve_join_columns`` — the post-index
|
|
216
|
+
pass is the SOLE producer of the resolved COLUMN_LINEAGE edge for this column.
|
|
217
|
+
|
|
218
|
+
Mirrors the over-attribution-instead-of-mis-bind discipline of the
|
|
219
|
+
``CTE_PROJECTION_AMBIGUOUS`` precedent (see PR-5 in
|
|
220
|
+
plan/sprints/bugfix_lineage_correctness_validation.md).
|
|
221
|
+
|
|
222
|
+
Attributes:
|
|
223
|
+
dst: The destination ColumnRef the projection would have produced
|
|
224
|
+
(``<dst_table>.<dst_col_name>``); its full_id is the marker's dst_key.
|
|
225
|
+
bare_col: The bare (unqualified) column name to resolve against source
|
|
226
|
+
tables' HAS_COLUMN catalog.
|
|
227
|
+
"""
|
|
228
|
+
|
|
229
|
+
dst: ColumnRef
|
|
230
|
+
bare_col: str = ""
|
|
231
|
+
|
|
232
|
+
|
|
207
233
|
@dataclass(frozen=True)
|
|
208
234
|
class LineageExtraction:
|
|
209
235
|
"""Result of column lineage extraction, including both edges and star markers.
|
|
@@ -211,6 +237,8 @@ class LineageExtraction:
|
|
|
211
237
|
Attributes:
|
|
212
238
|
edges: List of LineageEdge objects extracted from the statement
|
|
213
239
|
star_sources: List of StarSource markers for SELECT * projections
|
|
240
|
+
join_col_resolves: List of JoinColResolve markers for bare unqualified
|
|
241
|
+
>=2-table join projections, resolved post-index against HAS_COLUMN.
|
|
214
242
|
qualify_failed: True when the qualify() step raised an exception for
|
|
215
243
|
this statement (schema-free retry may still yield a partial scope).
|
|
216
244
|
Propagated to QueryNode.qualify_failed for per-statement storage.
|
|
@@ -218,6 +246,7 @@ class LineageExtraction:
|
|
|
218
246
|
|
|
219
247
|
edges: list[LineageEdge]
|
|
220
248
|
star_sources: list[StarSource]
|
|
249
|
+
join_col_resolves: list[JoinColResolve] = field(default_factory=list)
|
|
221
250
|
qualify_failed: bool = False
|
|
222
251
|
|
|
223
252
|
|
|
@@ -268,6 +297,7 @@ class QueryNode:
|
|
|
268
297
|
confidence: float = 1.0
|
|
269
298
|
parsing_mode: str = "sqlglot"
|
|
270
299
|
star_sources: list[StarSource] = field(default_factory=list)
|
|
300
|
+
join_col_resolves: list[JoinColResolve] = field(default_factory=list)
|
|
271
301
|
defined_columns: list[str] = field(default_factory=list)
|
|
272
302
|
defined_body: Any | None = None
|
|
273
303
|
start_line: int = 0 # 1-based start line of statement in file; 0 = unknown
|
|
@@ -413,6 +443,22 @@ def normalize_keys(parsed: "ParsedFile", aliases: dict[str, str]) -> None:
|
|
|
413
443
|
if stmt.clone_source is not None:
|
|
414
444
|
stmt.clone_source = _norm_ref(stmt.clone_source)
|
|
415
445
|
|
|
446
|
+
# join_col_resolves (JoinColResolve.dst is a ColumnRef over a TableRef).
|
|
447
|
+
# The dst table is the INSERT/CREATE target, which is aliased above; keep
|
|
448
|
+
# the marker's dst_key in lockstep so the post-index resolver writes a
|
|
449
|
+
# COLUMN_LINEAGE edge with the same dst key as the rest of the graph.
|
|
450
|
+
new_join_col_resolves: list[JoinColResolve] = []
|
|
451
|
+
for jcr in stmt.join_col_resolves:
|
|
452
|
+
new_dst_table = _norm_ref(jcr.dst.table)
|
|
453
|
+
if new_dst_table is None:
|
|
454
|
+
# Empty-identity target — drop the marker (skip already logged).
|
|
455
|
+
continue
|
|
456
|
+
if new_dst_table is not jcr.dst.table:
|
|
457
|
+
new_dst = dataclasses.replace(jcr.dst, table=new_dst_table)
|
|
458
|
+
jcr = dataclasses.replace(jcr, dst=new_dst)
|
|
459
|
+
new_join_col_resolves.append(jcr)
|
|
460
|
+
stmt.join_col_resolves = new_join_col_resolves
|
|
461
|
+
|
|
416
462
|
# column_lineage — LineageEdge and ColumnRef are frozen; rewrite via replace
|
|
417
463
|
new_edges: list[LineageEdge] = []
|
|
418
464
|
for edge in stmt.column_lineage:
|
|
@@ -1067,6 +1113,7 @@ class SqlParser(ABC):
|
|
|
1067
1113
|
|
|
1068
1114
|
edges: list[LineageEdge] = []
|
|
1069
1115
|
star_sources: list[StarSource] = []
|
|
1116
|
+
join_col_resolves: list[JoinColResolve] = []
|
|
1070
1117
|
_qualify_failed: bool = False
|
|
1071
1118
|
|
|
1072
1119
|
# NEW (T-07-06): Record MERGE statements explicitly as deferred.
|
|
@@ -1417,6 +1464,23 @@ class SqlParser(ABC):
|
|
|
1417
1464
|
except Exception:
|
|
1418
1465
|
pass
|
|
1419
1466
|
|
|
1467
|
+
# Bug #5 (PR-5, option b): count distinct PHYSICAL source tables ONCE per
|
|
1468
|
+
# statement (never per column — CLAUDE.md hot-loop invariant). When a
|
|
1469
|
+
# top-level INSERT/CREATE-SELECT join has >=2 physical sources, bare
|
|
1470
|
+
# unqualified projection columns cannot be reliably bound at parse time
|
|
1471
|
+
# (the owning table may be catalogued only via information_schema, which
|
|
1472
|
+
# is unavailable to qualify()). For those columns we SUPPRESS the sqlglot
|
|
1473
|
+
# mis-bind edge and record a JoinColResolve marker instead — resolved
|
|
1474
|
+
# post-index against the full HAS_COLUMN catalog by
|
|
1475
|
+
# DuckDBBackend.resolve_join_columns(). This mirrors the
|
|
1476
|
+
# CTE_PROJECTION_AMBIGUOUS over-attribution discipline above, but defers
|
|
1477
|
+
# resolution rather than over-attributing immediately. No new
|
|
1478
|
+
# qualify/build_scope/exp.expand/sg_lineage call is added.
|
|
1479
|
+
_physical_source_count = sum(
|
|
1480
|
+
1 for _s in (query_sources or []) if getattr(_s, "role", "table") == "table"
|
|
1481
|
+
)
|
|
1482
|
+
_join_resolve_active = dst_table is not None and _physical_source_count >= 2
|
|
1483
|
+
|
|
1420
1484
|
# Extract output columns — skip positions handled by the positional INSERT block
|
|
1421
1485
|
for loop_idx, col_expr in enumerate(col_expressions):
|
|
1422
1486
|
if loop_idx in positional_col_names:
|
|
@@ -1490,6 +1554,26 @@ class SqlParser(ABC):
|
|
|
1490
1554
|
expr_type = type(col_expr).__name__
|
|
1491
1555
|
out.errors.append(f"col_lineage_skip:func_fallback:{expr_type}")
|
|
1492
1556
|
continue
|
|
1557
|
+
|
|
1558
|
+
# Bug #5 (PR-5, option b): bare unqualified column in a >=2-physical-
|
|
1559
|
+
# table join. SUPPRESS the sqlglot mis-bind edge (continue past
|
|
1560
|
+
# sg_lineage) and record a JoinColResolve marker; the post-index
|
|
1561
|
+
# resolver is the SOLE producer of this column's edge(s). A qualified
|
|
1562
|
+
# reference (o.amount) has c.table set and fails the bare test, so it
|
|
1563
|
+
# takes the existing sg_lineage path unchanged. This is the same
|
|
1564
|
+
# cardinality as the existing edge append — no new hot-loop op.
|
|
1565
|
+
if _join_resolve_active and dst_table is not None:
|
|
1566
|
+
_bare_cols = [c for c in col_expr.find_all(exp.Column) if not c.table]
|
|
1567
|
+
if _bare_cols:
|
|
1568
|
+
_bare_name = _bare_cols[0].name or col_name
|
|
1569
|
+
join_col_resolves.append(
|
|
1570
|
+
JoinColResolve(
|
|
1571
|
+
dst=ColumnRef(dst_table, col_name),
|
|
1572
|
+
bare_col=_bare_name,
|
|
1573
|
+
)
|
|
1574
|
+
)
|
|
1575
|
+
continue # skip sg_lineage for this projection
|
|
1576
|
+
|
|
1493
1577
|
# Schema validation: if schema is loaded and column isn't in it,
|
|
1494
1578
|
# emit a reduced-confidence edge rather than a full-confidence one.
|
|
1495
1579
|
# Skip when the expression has an explicit alias — the alias is the
|
|
@@ -1789,7 +1873,10 @@ class SqlParser(ABC):
|
|
|
1789
1873
|
out.errors.append(f"col_lineage:statement:{exc}")
|
|
1790
1874
|
|
|
1791
1875
|
return LineageExtraction(
|
|
1792
|
-
edges=edges,
|
|
1876
|
+
edges=edges,
|
|
1877
|
+
star_sources=star_sources,
|
|
1878
|
+
join_col_resolves=join_col_resolves,
|
|
1879
|
+
qualify_failed=_qualify_failed,
|
|
1793
1880
|
)
|
|
1794
1881
|
|
|
1795
1882
|
def _resolve_star_source(
|
|
File without changes
|
|
File without changes
|