pretensor 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pretensor/__init__.py +50 -0
- pretensor/benchmark/__init__.py +54 -0
- pretensor/benchmark/cli.py +294 -0
- pretensor/benchmark/fixtures.py +84 -0
- pretensor/benchmark/l1/__init__.py +23 -0
- pretensor/benchmark/l1/metrics.py +141 -0
- pretensor/benchmark/l1/pipeline.py +188 -0
- pretensor/benchmark/l1/runner.py +245 -0
- pretensor/benchmark/l2/__init__.py +27 -0
- pretensor/benchmark/l2/gold.py +236 -0
- pretensor/benchmark/l2/metrics.py +146 -0
- pretensor/benchmark/l2/pipeline.py +124 -0
- pretensor/benchmark/l2/runner.py +530 -0
- pretensor/benchmark/l3/__init__.py +73 -0
- pretensor/benchmark/l3/agent.py +316 -0
- pretensor/benchmark/l3/db.py +188 -0
- pretensor/benchmark/l3/gold.py +85 -0
- pretensor/benchmark/l3/llm_client.py +395 -0
- pretensor/benchmark/l3/mcp_client.py +357 -0
- pretensor/benchmark/l3/pretensor_runner.py +456 -0
- pretensor/benchmark/l3/prompt.py +132 -0
- pretensor/benchmark/l3/runner.py +358 -0
- pretensor/benchmark/l3/sql_equivalence.py +176 -0
- pretensor/benchmark/release_gate.py +448 -0
- pretensor/benchmark/results.py +298 -0
- pretensor/benchmark/runner.py +109 -0
- pretensor/cli/__init__.py +1 -0
- pretensor/cli/commands/_source_runner.py +147 -0
- pretensor/cli/commands/analyze.py +201 -0
- pretensor/cli/commands/connections/__init__.py +7 -0
- pretensor/cli/commands/connections/add_remove.py +126 -0
- pretensor/cli/commands/connections/register.py +12 -0
- pretensor/cli/commands/export.py +131 -0
- pretensor/cli/commands/index.py +559 -0
- pretensor/cli/commands/list.py +76 -0
- pretensor/cli/commands/quickstart.py +207 -0
- pretensor/cli/commands/reindex.py +646 -0
- pretensor/cli/commands/semantic.py +190 -0
- pretensor/cli/commands/serve.py +144 -0
- pretensor/cli/commands/sync_grants.py +149 -0
- pretensor/cli/commands/validate.py +176 -0
- pretensor/cli/config_file.py +442 -0
- pretensor/cli/constants.py +10 -0
- pretensor/cli/dbt_enrichment.py +96 -0
- pretensor/cli/main.py +109 -0
- pretensor/cli/paths.py +43 -0
- pretensor/cli/plugin.py +52 -0
- pretensor/config.py +226 -0
- pretensor/connectors/__init__.py +29 -0
- pretensor/connectors/base.py +165 -0
- pretensor/connectors/bigquery.py +468 -0
- pretensor/connectors/inspect.py +321 -0
- pretensor/connectors/lineage_sqlglot.py +97 -0
- pretensor/connectors/models.py +130 -0
- pretensor/connectors/mysql.py +402 -0
- pretensor/connectors/pg_array_parse.py +53 -0
- pretensor/connectors/postgres.py +938 -0
- pretensor/connectors/registry.py +93 -0
- pretensor/connectors/snapshot.py +244 -0
- pretensor/connectors/snowflake.py +908 -0
- pretensor/core/__init__.py +1 -0
- pretensor/core/builder.py +307 -0
- pretensor/core/dsn_crypto.py +51 -0
- pretensor/core/graph_schema_manager.py +246 -0
- pretensor/core/graph_store.py +1226 -0
- pretensor/core/ids.py +101 -0
- pretensor/core/portable_export.py +276 -0
- pretensor/core/query_runner.py +67 -0
- pretensor/core/registry.py +209 -0
- pretensor/core/schema.py +473 -0
- pretensor/core/secure_io.py +93 -0
- pretensor/core/store.py +469 -0
- pretensor/enrichment/__init__.py +1 -0
- pretensor/enrichment/analyze/__init__.py +0 -0
- pretensor/enrichment/analyze/classify.py +49 -0
- pretensor/enrichment/analyze/extract_python.py +196 -0
- pretensor/enrichment/analyze/parse.py +141 -0
- pretensor/enrichment/analyze/pipeline.py +195 -0
- pretensor/enrichment/analyze/summary.py +38 -0
- pretensor/enrichment/analyze/walker.py +98 -0
- pretensor/enrichment/analyze/writers.py +214 -0
- pretensor/enrichment/dbt/__init__.py +30 -0
- pretensor/enrichment/dbt/lineage.py +100 -0
- pretensor/enrichment/dbt/manifest.py +300 -0
- pretensor/enrichment/dbt/metadata.py +263 -0
- pretensor/enrichment/dbt/pipeline.py +77 -0
- pretensor/enrichment/dbt/resolution.py +101 -0
- pretensor/enrichment/dbt/signals.py +305 -0
- pretensor/entities/__init__.py +27 -0
- pretensor/entities/builder.py +63 -0
- pretensor/entities/classifier.py +383 -0
- pretensor/entities/llm_extract.py +66 -0
- pretensor/errors.py +35 -0
- pretensor/graph_models/__init__.py +17 -0
- pretensor/graph_models/base.py +11 -0
- pretensor/graph_models/consumer.py +71 -0
- pretensor/graph_models/edge.py +35 -0
- pretensor/graph_models/entity.py +21 -0
- pretensor/graph_models/node.py +79 -0
- pretensor/graph_models/relationship.py +33 -0
- pretensor/integrations/__init__.py +42 -0
- pretensor/integrations/_base.py +138 -0
- pretensor/integrations/google_adk.py +49 -0
- pretensor/integrations/langchain.py +55 -0
- pretensor/integrations/llamaindex.py +53 -0
- pretensor/intelligence/__init__.py +33 -0
- pretensor/intelligence/cluster_labeler.py +425 -0
- pretensor/intelligence/clustering.py +168 -0
- pretensor/intelligence/combining.py +32 -0
- pretensor/intelligence/discovery.py +114 -0
- pretensor/intelligence/embeddings.py +317 -0
- pretensor/intelligence/graph_export.py +200 -0
- pretensor/intelligence/heuristic.py +544 -0
- pretensor/intelligence/join_paths/__init__.py +130 -0
- pretensor/intelligence/join_paths/on_demand.py +516 -0
- pretensor/intelligence/join_paths/storage.py +70 -0
- pretensor/intelligence/llm_infer.py +78 -0
- pretensor/intelligence/llm_runtime.py +62 -0
- pretensor/intelligence/metric_templates.py +193 -0
- pretensor/intelligence/pipeline.py +364 -0
- pretensor/intelligence/role_exemplars.py +263 -0
- pretensor/intelligence/schema_classification.py +360 -0
- pretensor/intelligence/scoring.py +76 -0
- pretensor/intelligence/semantic.py +240 -0
- pretensor/intelligence/shadow_alias.py +101 -0
- pretensor/intelligence/statistical.py +50 -0
- pretensor/intelligence/steps.py +191 -0
- pretensor/intelligence/steps_embedding.py +168 -0
- pretensor/introspection/__init__.py +6 -0
- pretensor/introspection/inspector.py +5 -0
- pretensor/introspection/models/__init__.py +0 -0
- pretensor/introspection/models/base.py +5 -0
- pretensor/introspection/models/config.py +237 -0
- pretensor/introspection/models/dsn.py +550 -0
- pretensor/introspection/models/plan.py +116 -0
- pretensor/introspection/models/schema.py +10 -0
- pretensor/introspection/models/semantic.py +121 -0
- pretensor/introspection/models/validation.py +116 -0
- pretensor/introspection/snapshot.py +46 -0
- pretensor/mcp/__init__.py +16 -0
- pretensor/mcp/config_json.py +24 -0
- pretensor/mcp/payload_types.py +274 -0
- pretensor/mcp/resources/__init__.py +17 -0
- pretensor/mcp/resources/markdown.py +314 -0
- pretensor/mcp/server.py +285 -0
- pretensor/mcp/service.py +49 -0
- pretensor/mcp/service_context.py +142 -0
- pretensor/mcp/service_registry.py +294 -0
- pretensor/mcp/store_cache.py +43 -0
- pretensor/mcp/tool_registry.py +136 -0
- pretensor/mcp/tools/__init__.py +1 -0
- pretensor/mcp/tools/_rank.py +244 -0
- pretensor/mcp/tools/_timed.py +26 -0
- pretensor/mcp/tools/compile_metric.py +144 -0
- pretensor/mcp/tools/consumers.py +161 -0
- pretensor/mcp/tools/context.py +1121 -0
- pretensor/mcp/tools/cypher.py +509 -0
- pretensor/mcp/tools/detect_changes.py +254 -0
- pretensor/mcp/tools/impact.py +271 -0
- pretensor/mcp/tools/list.py +131 -0
- pretensor/mcp/tools/schema.py +170 -0
- pretensor/mcp/tools/search.py +316 -0
- pretensor/mcp/tools/semantic_search.py +282 -0
- pretensor/mcp/tools/traverse.py +1027 -0
- pretensor/mcp/tools/validate_sql.py +150 -0
- pretensor/observability.py +203 -0
- pretensor/py.typed +0 -0
- pretensor/quickstart/README.md +29 -0
- pretensor/quickstart/__init__.py +6 -0
- pretensor/quickstart/docker-compose.yml +18 -0
- pretensor/quickstart/pagila_data.sql +63 -0
- pretensor/quickstart/pagila_ddl.sql +92 -0
- pretensor/search/__init__.py +6 -0
- pretensor/search/base.py +80 -0
- pretensor/search/index.py +435 -0
- pretensor/semantic/__init__.py +24 -0
- pretensor/semantic/base.py +123 -0
- pretensor/semantic/compiler.py +487 -0
- pretensor/semantic/yaml_layer.py +180 -0
- pretensor/skills/__init__.py +5 -0
- pretensor/skills/generator.py +235 -0
- pretensor/staleness/__init__.py +15 -0
- pretensor/staleness/graph_patcher.py +355 -0
- pretensor/staleness/impact_analyzer.py +162 -0
- pretensor/staleness/snapshot_store.py +38 -0
- pretensor/validation/__init__.py +9 -0
- pretensor/validation/query_validator.py +436 -0
- pretensor/visibility/__init__.py +23 -0
- pretensor/visibility/config.py +126 -0
- pretensor/visibility/filter.py +143 -0
- pretensor/visibility/kuzu_helpers.py +32 -0
- pretensor/visibility/runtime.py +36 -0
- pretensor/visibility/sync_grants.py +188 -0
- pretensor-0.1.0.dist-info/METADATA +251 -0
- pretensor-0.1.0.dist-info/RECORD +198 -0
- pretensor-0.1.0.dist-info/WHEEL +4 -0
- pretensor-0.1.0.dist-info/entry_points.txt +2 -0
- pretensor-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,1121 @@
|
|
|
1
|
+
"""MCP ``context`` tool and table resolution helpers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Literal
|
|
9
|
+
|
|
10
|
+
from pretensor.config import GraphConfig
|
|
11
|
+
from pretensor.core.store import KuzuStore
|
|
12
|
+
from pretensor.intelligence.cluster_labeler import HEURISTIC_CLUSTER_DESCRIPTION
|
|
13
|
+
from pretensor.intelligence.embeddings import embeddings_disabled_via_env
|
|
14
|
+
from pretensor.intelligence.shadow_alias import get_shadow_alias_node_ids
|
|
15
|
+
from pretensor.mcp.tool_registry import McpTool
|
|
16
|
+
from pretensor.visibility.filter import VisibilityFilter
|
|
17
|
+
|
|
18
|
+
from ..payload_types import (
|
|
19
|
+
ClusterInfo,
|
|
20
|
+
ColumnInfo,
|
|
21
|
+
ContextPayload,
|
|
22
|
+
LineageRef,
|
|
23
|
+
RelationshipInfo,
|
|
24
|
+
SimilarTable,
|
|
25
|
+
snippet,
|
|
26
|
+
stale_threshold_days,
|
|
27
|
+
staleness_days,
|
|
28
|
+
)
|
|
29
|
+
from ..service_context import (
|
|
30
|
+
get_effective_graph_config,
|
|
31
|
+
get_effective_visibility_filter,
|
|
32
|
+
)
|
|
33
|
+
from ..service_registry import (
|
|
34
|
+
graph_path_for_entry,
|
|
35
|
+
load_registry,
|
|
36
|
+
open_store_for_entry,
|
|
37
|
+
release_store,
|
|
38
|
+
resolve_registry_entry,
|
|
39
|
+
)
|
|
40
|
+
from ._rank import CosineHit, score_embedding_rows
|
|
41
|
+
|
|
42
|
+
logger = logging.getLogger(__name__)
|
|
43
|
+
|
|
44
|
+
_SIMILAR_K_MIN = 1
|
|
45
|
+
_SIMILAR_K_MAX = 50
|
|
46
|
+
|
|
47
|
+
_SCHEMA_PATTERN_LABELS: dict[str, str] = {
|
|
48
|
+
"star": "Star schema",
|
|
49
|
+
"snowflake": "Snowflake schema",
|
|
50
|
+
"constellation": "Constellation schema",
|
|
51
|
+
"erd": "ER-style schema",
|
|
52
|
+
"unknown": "Unknown schema pattern",
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _schema_pattern_display(raw: str) -> str:
|
|
57
|
+
key = raw.strip().lower()
|
|
58
|
+
if key in _SCHEMA_PATTERN_LABELS:
|
|
59
|
+
return _SCHEMA_PATTERN_LABELS[key]
|
|
60
|
+
return raw.replace("_", " ").title()
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _tags_from_kuzu_array(raw: object) -> list[str]:
|
|
64
|
+
if raw is None:
|
|
65
|
+
return []
|
|
66
|
+
if isinstance(raw, (list, tuple)):
|
|
67
|
+
seen: set[str] = set()
|
|
68
|
+
out: list[str] = []
|
|
69
|
+
for x in raw:
|
|
70
|
+
s = str(x).strip()
|
|
71
|
+
if s and s not in seen:
|
|
72
|
+
seen.add(s)
|
|
73
|
+
out.append(s)
|
|
74
|
+
return out
|
|
75
|
+
return []
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def find_table_rows(
|
|
79
|
+
store: KuzuStore,
|
|
80
|
+
table: str,
|
|
81
|
+
*,
|
|
82
|
+
visibility_filter: VisibilityFilter | None = None,
|
|
83
|
+
) -> list[tuple[Any, ...]]:
|
|
84
|
+
"""Return SchemaTable rows matching bare name or ``schema.table``."""
|
|
85
|
+
table = table.strip()
|
|
86
|
+
if not table:
|
|
87
|
+
return []
|
|
88
|
+
_table_return = """
|
|
89
|
+
RETURN t.node_id, t.schema_name, t.table_name, t.comment,
|
|
90
|
+
t.description, COALESCE(t.tags, CAST([] AS STRING[])) AS tags,
|
|
91
|
+
t.row_count,
|
|
92
|
+
t.entity_type, t.connection_name, t.database, t.table_type,
|
|
93
|
+
t.seq_scan_count, t.idx_scan_count, t.insert_count, t.update_count,
|
|
94
|
+
t.delete_count, t.is_partitioned, t.partition_key, t.grants_json,
|
|
95
|
+
t.access_read_count, t.access_write_count, t.days_since_last_access,
|
|
96
|
+
t.potentially_unused, t.table_bytes, t.clustering_key,
|
|
97
|
+
t.role, t.role_confidence, t.classification_signals,
|
|
98
|
+
t.has_external_consumers, t.test_count,
|
|
99
|
+
t.staleness_status, t.staleness_as_of
|
|
100
|
+
"""
|
|
101
|
+
if "." in table:
|
|
102
|
+
schema_name, _, table_name = table.partition(".")
|
|
103
|
+
params = {"schema": schema_name, "tbl": table_name}
|
|
104
|
+
rows_dot = store.query_all_rows(
|
|
105
|
+
f"""
|
|
106
|
+
MATCH (t:SchemaTable)
|
|
107
|
+
WHERE t.schema_name = $schema AND t.table_name = $tbl
|
|
108
|
+
{_table_return}
|
|
109
|
+
""",
|
|
110
|
+
params,
|
|
111
|
+
)
|
|
112
|
+
if rows_dot:
|
|
113
|
+
return _filter_visible_table_rows(rows_dot, visibility_filter)
|
|
114
|
+
# Case-insensitive fallback (e.g. Snowflake stores identifiers uppercase)
|
|
115
|
+
rows_dot = store.query_all_rows(
|
|
116
|
+
f"""
|
|
117
|
+
MATCH (t:SchemaTable)
|
|
118
|
+
WHERE lower(t.schema_name) = lower($schema)
|
|
119
|
+
AND lower(t.table_name) = lower($tbl)
|
|
120
|
+
{_table_return}
|
|
121
|
+
""",
|
|
122
|
+
params,
|
|
123
|
+
)
|
|
124
|
+
return _filter_visible_table_rows(rows_dot, visibility_filter)
|
|
125
|
+
rows = store.query_all_rows(
|
|
126
|
+
f"""
|
|
127
|
+
MATCH (t:SchemaTable)
|
|
128
|
+
WHERE t.table_name = $tbl
|
|
129
|
+
{_table_return}
|
|
130
|
+
""",
|
|
131
|
+
{"tbl": table},
|
|
132
|
+
)
|
|
133
|
+
if rows:
|
|
134
|
+
return _filter_visible_table_rows(rows, visibility_filter)
|
|
135
|
+
# Case-insensitive fallback
|
|
136
|
+
rows = store.query_all_rows(
|
|
137
|
+
f"""
|
|
138
|
+
MATCH (t:SchemaTable)
|
|
139
|
+
WHERE lower(t.table_name) = lower($tbl)
|
|
140
|
+
{_table_return}
|
|
141
|
+
""",
|
|
142
|
+
{"tbl": table},
|
|
143
|
+
)
|
|
144
|
+
return _filter_visible_table_rows(rows, visibility_filter)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _filter_visible_table_rows(
|
|
148
|
+
rows: list[tuple[Any, ...]],
|
|
149
|
+
visibility_filter: VisibilityFilter | None,
|
|
150
|
+
) -> list[tuple[Any, ...]]:
|
|
151
|
+
if visibility_filter is None:
|
|
152
|
+
return rows
|
|
153
|
+
out: list[tuple[Any, ...]] = []
|
|
154
|
+
for m in rows:
|
|
155
|
+
connection_name = str(m[8])
|
|
156
|
+
schema_name = str(m[1])
|
|
157
|
+
table_name = str(m[2])
|
|
158
|
+
if visibility_filter.is_table_visible(connection_name, schema_name, table_name):
|
|
159
|
+
out.append(m)
|
|
160
|
+
return out
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def suggest_table_candidates(
|
|
164
|
+
store: KuzuStore,
|
|
165
|
+
table: str,
|
|
166
|
+
) -> list[str]:
|
|
167
|
+
"""Return up to 5 qualified table names whose name matches *table* case-insensitively."""
|
|
168
|
+
bare = table.split(".")[-1].strip()
|
|
169
|
+
if not bare:
|
|
170
|
+
return []
|
|
171
|
+
rows = store.query_all_rows(
|
|
172
|
+
"""
|
|
173
|
+
MATCH (t:SchemaTable)
|
|
174
|
+
WHERE lower(t.table_name) = lower($term)
|
|
175
|
+
RETURN t.schema_name, t.table_name
|
|
176
|
+
LIMIT 5
|
|
177
|
+
""",
|
|
178
|
+
{"term": bare},
|
|
179
|
+
)
|
|
180
|
+
return [f"{r[0]}.{r[1]}" for r in rows]
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def columns_for_table_node_id(
|
|
184
|
+
store: KuzuStore,
|
|
185
|
+
table_node_id: str,
|
|
186
|
+
*,
|
|
187
|
+
visibility_filter: VisibilityFilter | None = None,
|
|
188
|
+
) -> list[ColumnInfo]:
|
|
189
|
+
"""Return column metadata linked from a ``SchemaTable`` node."""
|
|
190
|
+
rows = store.query_all_rows(
|
|
191
|
+
"""
|
|
192
|
+
MATCH (t:SchemaTable {node_id: $tid})-[:HAS_COLUMN]->(c:SchemaColumn)
|
|
193
|
+
RETURN c.column_name, c.data_type, c.nullable, c.is_primary_key, c.is_foreign_key,
|
|
194
|
+
c.comment, c.description, c.is_indexed, c.check_constraints_json,
|
|
195
|
+
c.most_common_values_json, c.histogram_bounds_json, c.stats_correlation
|
|
196
|
+
ORDER BY c.ordinal_position, c.column_name
|
|
197
|
+
""",
|
|
198
|
+
{"tid": table_node_id},
|
|
199
|
+
)
|
|
200
|
+
tbl_meta: tuple[str, str, str] | None = None
|
|
201
|
+
if visibility_filter is not None:
|
|
202
|
+
trows = store.query_all_rows(
|
|
203
|
+
"""
|
|
204
|
+
MATCH (t:SchemaTable {node_id: $tid})
|
|
205
|
+
RETURN t.connection_name, t.schema_name, t.table_name
|
|
206
|
+
""",
|
|
207
|
+
{"tid": table_node_id},
|
|
208
|
+
)
|
|
209
|
+
if trows:
|
|
210
|
+
cn, sn, tn = trows[0]
|
|
211
|
+
tbl_meta = (str(cn), str(sn), str(tn))
|
|
212
|
+
out: list[ColumnInfo] = []
|
|
213
|
+
for row in rows:
|
|
214
|
+
(
|
|
215
|
+
col_name,
|
|
216
|
+
dtype,
|
|
217
|
+
nullable,
|
|
218
|
+
is_pk,
|
|
219
|
+
is_fk,
|
|
220
|
+
cmt,
|
|
221
|
+
col_desc_raw,
|
|
222
|
+
indexed,
|
|
223
|
+
checks_json,
|
|
224
|
+
mcv_json,
|
|
225
|
+
hb_json,
|
|
226
|
+
scorr,
|
|
227
|
+
) = row
|
|
228
|
+
cname = str(col_name)
|
|
229
|
+
if visibility_filter is not None and tbl_meta is not None:
|
|
230
|
+
cn, sn, tn = tbl_meta
|
|
231
|
+
if not visibility_filter.visible_columns(cn, sn, tn, [cname]):
|
|
232
|
+
continue
|
|
233
|
+
checks: list[str] = []
|
|
234
|
+
if checks_json:
|
|
235
|
+
try:
|
|
236
|
+
parsed = json.loads(str(checks_json))
|
|
237
|
+
if isinstance(parsed, list):
|
|
238
|
+
checks = [str(x) for x in parsed]
|
|
239
|
+
except json.JSONDecodeError:
|
|
240
|
+
checks = []
|
|
241
|
+
col_desc = str(col_desc_raw).strip() if col_desc_raw else ""
|
|
242
|
+
cmt_s = str(cmt).strip() if cmt else ""
|
|
243
|
+
item: ColumnInfo = {
|
|
244
|
+
"column_name": cname,
|
|
245
|
+
"data_type": str(dtype or ""),
|
|
246
|
+
"nullable": bool(nullable) if nullable is not None else True,
|
|
247
|
+
"is_primary_key": bool(is_pk) if is_pk is not None else False,
|
|
248
|
+
"is_foreign_key": bool(is_fk) if is_fk is not None else False,
|
|
249
|
+
"description": col_desc if col_desc else cmt_s,
|
|
250
|
+
}
|
|
251
|
+
if indexed is not None:
|
|
252
|
+
item["is_indexed"] = bool(indexed)
|
|
253
|
+
if checks:
|
|
254
|
+
item["check_constraints"] = checks
|
|
255
|
+
if mcv_json:
|
|
256
|
+
try:
|
|
257
|
+
mcv_parsed = json.loads(str(mcv_json))
|
|
258
|
+
if isinstance(mcv_parsed, list):
|
|
259
|
+
item["most_common_values"] = [str(x) for x in mcv_parsed]
|
|
260
|
+
except json.JSONDecodeError:
|
|
261
|
+
pass
|
|
262
|
+
if hb_json:
|
|
263
|
+
try:
|
|
264
|
+
hb_parsed = json.loads(str(hb_json))
|
|
265
|
+
if isinstance(hb_parsed, list):
|
|
266
|
+
item["histogram_bounds"] = [str(x) for x in hb_parsed]
|
|
267
|
+
except json.JSONDecodeError:
|
|
268
|
+
pass
|
|
269
|
+
if scorr is not None:
|
|
270
|
+
item["stats_correlation"] = float(scorr)
|
|
271
|
+
out.append(item)
|
|
272
|
+
return out
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def relationships_for_table(
|
|
276
|
+
store: KuzuStore,
|
|
277
|
+
node_id: str,
|
|
278
|
+
*,
|
|
279
|
+
visibility_filter: VisibilityFilter | None = None,
|
|
280
|
+
shadow_alias_ids: frozenset[str] | None = None,
|
|
281
|
+
) -> list[RelationshipInfo]:
|
|
282
|
+
src_cn = ""
|
|
283
|
+
if visibility_filter is not None:
|
|
284
|
+
meta = store.query_all_rows(
|
|
285
|
+
"""
|
|
286
|
+
MATCH (t:SchemaTable {node_id: $id})
|
|
287
|
+
RETURN t.connection_name
|
|
288
|
+
""",
|
|
289
|
+
{"id": node_id},
|
|
290
|
+
)
|
|
291
|
+
if meta:
|
|
292
|
+
src_cn = str(meta[0][0] or "")
|
|
293
|
+
_shadows = shadow_alias_ids or frozenset()
|
|
294
|
+
fk_rows = store.query_all_rows(
|
|
295
|
+
"""
|
|
296
|
+
MATCH (t:SchemaTable {node_id: $id})-[r:FK_REFERENCES]->(t2:SchemaTable)
|
|
297
|
+
RETURN t2.node_id, t2.schema_name, t2.table_name,
|
|
298
|
+
r.source_column, r.target_column, r.constraint_name
|
|
299
|
+
""",
|
|
300
|
+
{"id": node_id},
|
|
301
|
+
)
|
|
302
|
+
inf_rows = store.query_all_rows(
|
|
303
|
+
"""
|
|
304
|
+
MATCH (t:SchemaTable {node_id: $id})-[r:INFERRED_JOIN]->(t2:SchemaTable)
|
|
305
|
+
RETURN t2.node_id, t2.schema_name, t2.table_name, r.source_column, r.target_column,
|
|
306
|
+
r.source, r.confidence, r.reasoning
|
|
307
|
+
""",
|
|
308
|
+
{"id": node_id},
|
|
309
|
+
)
|
|
310
|
+
out: list[RelationshipInfo] = []
|
|
311
|
+
# Group FK edges by constraint_name to collapse composite FKs into one row.
|
|
312
|
+
fk_groups: dict[tuple[str, str | None], list[tuple[str, str]]] = {}
|
|
313
|
+
fk_group_target: dict[tuple[str, str | None], str] = {}
|
|
314
|
+
for t2_nid, sn, tn, scol, tcol, cname in fk_rows:
|
|
315
|
+
if str(t2_nid) in _shadows:
|
|
316
|
+
continue
|
|
317
|
+
if visibility_filter is not None and not visibility_filter.is_table_visible(
|
|
318
|
+
src_cn, str(sn), str(tn)
|
|
319
|
+
):
|
|
320
|
+
continue
|
|
321
|
+
target = f"{sn}.{tn}"
|
|
322
|
+
cname_str = str(cname) if cname is not None else None
|
|
323
|
+
key = (target, cname_str)
|
|
324
|
+
fk_groups.setdefault(key, []).append((str(scol), str(tcol)))
|
|
325
|
+
fk_group_target[key] = target
|
|
326
|
+
for (target, cname_str), col_pairs in fk_groups.items():
|
|
327
|
+
# Sort pairs for deterministic output across Kuzu return orders; pairing is preserved.
|
|
328
|
+
ordered_pairs = sorted(col_pairs)
|
|
329
|
+
src_cols = [p[0] for p in ordered_pairs]
|
|
330
|
+
tgt_cols = [p[1] for p in ordered_pairs]
|
|
331
|
+
rel: RelationshipInfo = {
|
|
332
|
+
"target_table": target,
|
|
333
|
+
"rel_type": "FK_REFERENCES",
|
|
334
|
+
"source_column": src_cols[0],
|
|
335
|
+
"target_column": tgt_cols[0],
|
|
336
|
+
"source_columns": src_cols,
|
|
337
|
+
"target_columns": tgt_cols,
|
|
338
|
+
"constraint_name": cname_str,
|
|
339
|
+
"source": None,
|
|
340
|
+
"confidence": None,
|
|
341
|
+
"reasoning": None,
|
|
342
|
+
}
|
|
343
|
+
out.append(rel)
|
|
344
|
+
for row in inf_rows:
|
|
345
|
+
t2_nid, sn, tn, scol, tcol, src, conf, reason = row
|
|
346
|
+
if str(t2_nid) in _shadows:
|
|
347
|
+
continue
|
|
348
|
+
if visibility_filter is not None and not visibility_filter.is_table_visible(
|
|
349
|
+
src_cn, str(sn), str(tn)
|
|
350
|
+
):
|
|
351
|
+
continue
|
|
352
|
+
s_str = str(scol) if scol is not None else ""
|
|
353
|
+
t_str = str(tcol) if tcol is not None else ""
|
|
354
|
+
inferred_rel: RelationshipInfo = {
|
|
355
|
+
"target_table": f"{sn}.{tn}",
|
|
356
|
+
"rel_type": "INFERRED_JOIN",
|
|
357
|
+
"source_column": s_str,
|
|
358
|
+
"target_column": t_str,
|
|
359
|
+
"source_columns": [s_str] if s_str else [],
|
|
360
|
+
"target_columns": [t_str] if t_str else [],
|
|
361
|
+
"constraint_name": None,
|
|
362
|
+
"source": str(src) if src is not None else None,
|
|
363
|
+
"confidence": float(conf) if conf is not None else None,
|
|
364
|
+
"reasoning": str(reason) if reason is not None else None,
|
|
365
|
+
}
|
|
366
|
+
out.append(inferred_rel)
|
|
367
|
+
return out
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def lineage_for_table(
|
|
371
|
+
store: KuzuStore,
|
|
372
|
+
node_id: str,
|
|
373
|
+
*,
|
|
374
|
+
visibility_filter: VisibilityFilter | None = None,
|
|
375
|
+
) -> tuple[list[LineageRef], list[LineageRef]]:
|
|
376
|
+
"""Inbound and outbound LINEAGE edges (structural table-level flow)."""
|
|
377
|
+
base_cn = ""
|
|
378
|
+
if visibility_filter is not None:
|
|
379
|
+
meta = store.query_all_rows(
|
|
380
|
+
"""
|
|
381
|
+
MATCH (t:SchemaTable {node_id: $id})
|
|
382
|
+
RETURN t.connection_name
|
|
383
|
+
""",
|
|
384
|
+
{"id": node_id},
|
|
385
|
+
)
|
|
386
|
+
if meta:
|
|
387
|
+
base_cn = str(meta[0][0] or "")
|
|
388
|
+
in_rows = store.query_all_rows(
|
|
389
|
+
"""
|
|
390
|
+
MATCH (src:SchemaTable)-[r:LINEAGE]->(t:SchemaTable {node_id: $id})
|
|
391
|
+
RETURN src.schema_name, src.table_name, r.lineage_type, r.confidence, r.source
|
|
392
|
+
ORDER BY src.schema_name, src.table_name
|
|
393
|
+
""",
|
|
394
|
+
{"id": node_id},
|
|
395
|
+
)
|
|
396
|
+
out_rows = store.query_all_rows(
|
|
397
|
+
"""
|
|
398
|
+
MATCH (t:SchemaTable {node_id: $id})-[r:LINEAGE]->(dst:SchemaTable)
|
|
399
|
+
RETURN dst.schema_name, dst.table_name, r.lineage_type, r.confidence, r.source
|
|
400
|
+
ORDER BY dst.schema_name, dst.table_name
|
|
401
|
+
""",
|
|
402
|
+
{"id": node_id},
|
|
403
|
+
)
|
|
404
|
+
lineage_in: list[LineageRef] = []
|
|
405
|
+
for sn, tn, ltype, conf, src in in_rows:
|
|
406
|
+
if visibility_filter is not None and not visibility_filter.is_table_visible(
|
|
407
|
+
base_cn, str(sn), str(tn)
|
|
408
|
+
):
|
|
409
|
+
continue
|
|
410
|
+
item: LineageRef = {
|
|
411
|
+
"table": f"{sn}.{tn}",
|
|
412
|
+
"lineage_type": str(ltype or ""),
|
|
413
|
+
"confidence": float(conf) if conf is not None else 1.0,
|
|
414
|
+
"source": str(src or ""),
|
|
415
|
+
}
|
|
416
|
+
lineage_in.append(item)
|
|
417
|
+
lineage_out: list[LineageRef] = []
|
|
418
|
+
for sn, tn, ltype, conf, src in out_rows:
|
|
419
|
+
if visibility_filter is not None and not visibility_filter.is_table_visible(
|
|
420
|
+
base_cn, str(sn), str(tn)
|
|
421
|
+
):
|
|
422
|
+
continue
|
|
423
|
+
item2: LineageRef = {
|
|
424
|
+
"table": f"{sn}.{tn}",
|
|
425
|
+
"lineage_type": str(ltype or ""),
|
|
426
|
+
"confidence": float(conf) if conf is not None else 1.0,
|
|
427
|
+
"source": str(src or ""),
|
|
428
|
+
}
|
|
429
|
+
lineage_out.append(item2)
|
|
430
|
+
return lineage_in, lineage_out
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def _lineage_markdown_section(
|
|
434
|
+
lineage_in: list[LineageRef], lineage_out: list[LineageRef]
|
|
435
|
+
) -> str:
|
|
436
|
+
"""Return a markdown section for lineage edges, or empty string when none exist."""
|
|
437
|
+
if not lineage_in and not lineage_out:
|
|
438
|
+
return ""
|
|
439
|
+
lines: list[str] = ["## Lineage", ""]
|
|
440
|
+
if lineage_in:
|
|
441
|
+
lines.append("**Feeds this table**")
|
|
442
|
+
for ref in lineage_in:
|
|
443
|
+
lt = ref.get("lineage_type", "")
|
|
444
|
+
cf = ref.get("confidence")
|
|
445
|
+
tbl = ref.get("table", "")
|
|
446
|
+
conf_s = f" (confidence {cf:.2f})" if isinstance(cf, float) else ""
|
|
447
|
+
lines.append(f"- `{tbl}` — {lt}{conf_s}")
|
|
448
|
+
lines.append("")
|
|
449
|
+
if lineage_out:
|
|
450
|
+
lines.append("**This table feeds**")
|
|
451
|
+
for ref in lineage_out:
|
|
452
|
+
lt = ref.get("lineage_type", "")
|
|
453
|
+
cf = ref.get("confidence")
|
|
454
|
+
tbl = ref.get("table", "")
|
|
455
|
+
conf_s = f" (confidence {cf:.2f})" if isinstance(cf, float) else ""
|
|
456
|
+
lines.append(f"- `{tbl}` — {lt}{conf_s}")
|
|
457
|
+
return "\n".join(lines).strip()
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
_DEPRECATION_ACCESS_DAYS = 90
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def cluster_for_table_node_id(
|
|
464
|
+
store: KuzuStore, table_node_id: str, database_key: str
|
|
465
|
+
) -> ClusterInfo | None:
|
|
466
|
+
rows = store.query_all_rows(
|
|
467
|
+
"""
|
|
468
|
+
MATCH (t:SchemaTable {node_id: $tid})-[:IN_CLUSTER]->(c:Cluster)
|
|
469
|
+
WHERE c.database_key = $db
|
|
470
|
+
RETURN c.node_id, c.label, c.description, c.cohesion_score, c.schema_pattern, c.stale
|
|
471
|
+
ORDER BY CASE WHEN c.description = $heuristic THEN 1 ELSE 0 END,
|
|
472
|
+
c.cohesion_score DESC
|
|
473
|
+
LIMIT 1
|
|
474
|
+
""",
|
|
475
|
+
{
|
|
476
|
+
"tid": table_node_id,
|
|
477
|
+
"db": database_key,
|
|
478
|
+
"heuristic": HEURISTIC_CLUSTER_DESCRIPTION,
|
|
479
|
+
},
|
|
480
|
+
)
|
|
481
|
+
if not rows:
|
|
482
|
+
return None
|
|
483
|
+
cid, lab, desc, coh, spattern, stale = rows[0]
|
|
484
|
+
desc_str = str(desc or "")
|
|
485
|
+
if desc_str == HEURISTIC_CLUSTER_DESCRIPTION:
|
|
486
|
+
desc_str = ""
|
|
487
|
+
info: ClusterInfo = {
|
|
488
|
+
"cluster_id": str(cid),
|
|
489
|
+
"label": str(lab or ""),
|
|
490
|
+
"description": desc_str,
|
|
491
|
+
"cohesion_score": float(coh) if coh is not None else 0.0,
|
|
492
|
+
}
|
|
493
|
+
if spattern is not None and str(spattern).strip():
|
|
494
|
+
info["schema_pattern"] = str(spattern)
|
|
495
|
+
if bool(stale):
|
|
496
|
+
info["stale"] = True
|
|
497
|
+
info["stale_warning"] = (
|
|
498
|
+
"This cluster was last computed before recent schema changes. "
|
|
499
|
+
"Re-run `pretensor reindex <dsn> --recompute-intelligence` "
|
|
500
|
+
"for accurate cluster membership."
|
|
501
|
+
)
|
|
502
|
+
return info
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def resolve_table_node_id(
|
|
506
|
+
store: KuzuStore,
|
|
507
|
+
table: str,
|
|
508
|
+
database_key: str,
|
|
509
|
+
*,
|
|
510
|
+
visibility_filter: VisibilityFilter | None = None,
|
|
511
|
+
) -> tuple[str | None, str | None]:
|
|
512
|
+
"""Return ``(node_id, error_message)`` for a bare or qualified table name."""
|
|
513
|
+
matches = find_table_rows(store, table, visibility_filter=visibility_filter)
|
|
514
|
+
if not matches:
|
|
515
|
+
msg = f"No table matched {table!r}"
|
|
516
|
+
candidates = suggest_table_candidates(store, table)
|
|
517
|
+
if candidates:
|
|
518
|
+
msg += f". Did you mean: {', '.join(candidates)}"
|
|
519
|
+
return None, msg
|
|
520
|
+
scoped = [m for m in matches if str(m[7]) == database_key]
|
|
521
|
+
if not scoped:
|
|
522
|
+
scoped = list(matches)
|
|
523
|
+
if len(scoped) > 1:
|
|
524
|
+
opts = [f"{m[1]}.{m[2]}" for m in scoped]
|
|
525
|
+
return None, json.dumps(
|
|
526
|
+
{"error": "Ambiguous table name; use schema.table", "candidates": opts}
|
|
527
|
+
)
|
|
528
|
+
return str(scoped[0][0]), None
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def qualified_from_node_id(store: KuzuStore, node_id: str) -> str:
|
|
532
|
+
rows = store.query_all_rows(
|
|
533
|
+
"""
|
|
534
|
+
MATCH (t:SchemaTable {node_id: $id})
|
|
535
|
+
RETURN t.schema_name, t.table_name
|
|
536
|
+
""",
|
|
537
|
+
{"id": node_id},
|
|
538
|
+
)
|
|
539
|
+
if not rows:
|
|
540
|
+
return node_id
|
|
541
|
+
sn, tn = rows[0]
|
|
542
|
+
return f"{sn}.{tn}"
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
def shadow_aliases_for_base(
|
|
546
|
+
store: KuzuStore, node_id: str, shadow_ids: frozenset[str]
|
|
547
|
+
) -> list[str]:
|
|
548
|
+
"""Return qualified names of shadow alias views that project from this base table."""
|
|
549
|
+
if not shadow_ids:
|
|
550
|
+
return []
|
|
551
|
+
# Shadow aliases have a LINEAGE edge pointing *to* the view *from* the base.
|
|
552
|
+
out_rows = store.query_all_rows(
|
|
553
|
+
"""
|
|
554
|
+
MATCH (base:SchemaTable {node_id: $id})-[:LINEAGE]->(v:SchemaTable)
|
|
555
|
+
RETURN v.node_id, v.schema_name, v.table_name
|
|
556
|
+
""",
|
|
557
|
+
{"id": node_id},
|
|
558
|
+
)
|
|
559
|
+
aliases: list[str] = []
|
|
560
|
+
for vid, sn, tn in out_rows:
|
|
561
|
+
if str(vid) in shadow_ids:
|
|
562
|
+
aliases.append(f"{sn}.{tn}")
|
|
563
|
+
return sorted(aliases)
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def shadow_of_for_view(
|
|
567
|
+
store: KuzuStore, node_id: str, shadow_ids: frozenset[str]
|
|
568
|
+
) -> str | None:
|
|
569
|
+
"""If *node_id* is a shadow alias, return the qualified name of its base table."""
|
|
570
|
+
if node_id not in shadow_ids:
|
|
571
|
+
return None
|
|
572
|
+
rows = store.query_all_rows(
|
|
573
|
+
"""
|
|
574
|
+
MATCH (src:SchemaTable)-[:LINEAGE]->(v:SchemaTable {node_id: $id})
|
|
575
|
+
RETURN src.schema_name, src.table_name
|
|
576
|
+
LIMIT 1
|
|
577
|
+
""",
|
|
578
|
+
{"id": node_id},
|
|
579
|
+
)
|
|
580
|
+
if not rows:
|
|
581
|
+
return None
|
|
582
|
+
sn, tn = rows[0]
|
|
583
|
+
return f"{sn}.{tn}"
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
def entity_for_table(store: KuzuStore, node_id: str) -> tuple[str | None, str | None]:
|
|
587
|
+
rows = store.query_all_rows(
|
|
588
|
+
"""
|
|
589
|
+
MATCH (e:Entity)-[:REPRESENTS]->(t:SchemaTable {node_id: $id})
|
|
590
|
+
RETURN e.name, e.description
|
|
591
|
+
LIMIT 1
|
|
592
|
+
""",
|
|
593
|
+
{"id": node_id},
|
|
594
|
+
)
|
|
595
|
+
if not rows:
|
|
596
|
+
return None, None
|
|
597
|
+
name, desc = rows[0]
|
|
598
|
+
return (str(name) if name is not None else None, str(desc) if desc else None)
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
def similar_tables_for_table(
|
|
602
|
+
store: KuzuStore,
|
|
603
|
+
*,
|
|
604
|
+
table_node_id: str,
|
|
605
|
+
database_key: str,
|
|
606
|
+
k: int,
|
|
607
|
+
visibility_filter: VisibilityFilter | None = None,
|
|
608
|
+
) -> tuple[list[SimilarTable], str | None]:
|
|
609
|
+
"""Return ``(hits, reason)`` — top-``k`` cross-cluster nearest neighbors.
|
|
610
|
+
|
|
611
|
+
"Cross-cluster" means the candidate shares no cluster membership with
|
|
612
|
+
the target: the target's full cluster set is subtracted from every
|
|
613
|
+
candidate's cluster set, and any candidate whose sets intersect is
|
|
614
|
+
dropped. This handles tables that belong to multiple clusters (e.g.
|
|
615
|
+
both a heuristic graph-connectivity cluster and a labeled cluster).
|
|
616
|
+
|
|
617
|
+
The scan is scoped to ``database_key`` (clusters are per-database).
|
|
618
|
+
When the target has no ``embedding``, returns ``([], "no_embedding")``
|
|
619
|
+
— the hint is surfaced on the payload; ``query`` is where text
|
|
620
|
+
fallback lives.
|
|
621
|
+
|
|
622
|
+
**Unclustered target.** If the target table itself has no
|
|
623
|
+
``IN_CLUSTER`` membership, the cross-cluster filter degenerates: there
|
|
624
|
+
are no clusters to subtract. In that case the function returns the
|
|
625
|
+
raw top-K cosine neighbors (all visible embedded tables in the
|
|
626
|
+
database, sorted by score). This is an honest "best effort" — the
|
|
627
|
+
user asked for similar tables and the cluster signal isn't available
|
|
628
|
+
to refine — rather than empty. The hint is the same envelope shape
|
|
629
|
+
as the happy path; callers can detect this case by checking that
|
|
630
|
+
the target's cluster set on the response (not exposed today) was
|
|
631
|
+
empty.
|
|
632
|
+
"""
|
|
633
|
+
rows = store.query_all_rows(
|
|
634
|
+
"MATCH (t:SchemaTable {node_id: $tid}) RETURN t.embedding",
|
|
635
|
+
{"tid": table_node_id},
|
|
636
|
+
)
|
|
637
|
+
if not rows:
|
|
638
|
+
return [], "no_embedding"
|
|
639
|
+
raw = rows[0][0]
|
|
640
|
+
if raw is None:
|
|
641
|
+
return [], "no_embedding"
|
|
642
|
+
# Kill switch: suppress even stored-vector reads. Checked after the
|
|
643
|
+
# target-vector fetch so the no-vectors lane keeps the exact
|
|
644
|
+
# "no_embedding" reason regardless of the env var (parity), while a
|
|
645
|
+
# store that does carry vectors reports the honest reason.
|
|
646
|
+
if embeddings_disabled_via_env():
|
|
647
|
+
return [], "disabled"
|
|
648
|
+
qvec = [float(x) for x in raw]
|
|
649
|
+
|
|
650
|
+
target_cluster_rows = store.query_all_rows(
|
|
651
|
+
"""
|
|
652
|
+
MATCH (t:SchemaTable {node_id: $tid})-[:IN_CLUSTER]->(c:Cluster)
|
|
653
|
+
RETURN c.node_id
|
|
654
|
+
""",
|
|
655
|
+
{"tid": table_node_id},
|
|
656
|
+
)
|
|
657
|
+
target_clusters: set[str] = {
|
|
658
|
+
str(r[0]) for r in target_cluster_rows if r[0] is not None
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
# Pre-fetch every (table → {cluster, ...}) membership in one Cypher
|
|
662
|
+
# query so the cross-cluster filter sees the full membership set for
|
|
663
|
+
# multi-cluster tables. ``score_embedding_rows`` dedupes its output
|
|
664
|
+
# to one row per node_id (preventing RRF double-counting upstream),
|
|
665
|
+
# so we can't piggyback the membership info on the scored rows —
|
|
666
|
+
# only the first-seen cluster would survive there.
|
|
667
|
+
membership_rows = store.query_all_rows(
|
|
668
|
+
"""
|
|
669
|
+
MATCH (t:SchemaTable {database: $db})-[:IN_CLUSTER]->(c:Cluster)
|
|
670
|
+
RETURN t.node_id, c.node_id
|
|
671
|
+
""",
|
|
672
|
+
{"db": database_key},
|
|
673
|
+
)
|
|
674
|
+
clusters_by_node: dict[str, set[str]] = {}
|
|
675
|
+
for row in membership_rows:
|
|
676
|
+
nid = str(row[0])
|
|
677
|
+
cid = row[1]
|
|
678
|
+
if cid is None:
|
|
679
|
+
continue
|
|
680
|
+
clusters_by_node.setdefault(nid, set()).add(str(cid))
|
|
681
|
+
|
|
682
|
+
scored, _had_vector = score_embedding_rows(
|
|
683
|
+
store.iter_table_embeddings(database=database_key),
|
|
684
|
+
qvec,
|
|
685
|
+
visibility_filter=visibility_filter,
|
|
686
|
+
)
|
|
687
|
+
|
|
688
|
+
kept: list[CosineHit] = []
|
|
689
|
+
for hit in scored:
|
|
690
|
+
if hit.node_id == table_node_id:
|
|
691
|
+
continue
|
|
692
|
+
candidate_clusters = clusters_by_node.get(hit.node_id, set())
|
|
693
|
+
if target_clusters and candidate_clusters & target_clusters:
|
|
694
|
+
continue
|
|
695
|
+
kept.append(hit)
|
|
696
|
+
kept.sort(key=lambda h: h.score, reverse=True)
|
|
697
|
+
return [
|
|
698
|
+
SimilarTable(
|
|
699
|
+
table_id=hit.node_id,
|
|
700
|
+
qualified_name=f"{hit.schema_name}.{hit.table_name}",
|
|
701
|
+
score=hit.score,
|
|
702
|
+
cluster_id=hit.cluster_id,
|
|
703
|
+
)
|
|
704
|
+
for hit in kept[:k]
|
|
705
|
+
], None
|
|
706
|
+
|
|
707
|
+
|
|
708
|
+
def context_payload(
|
|
709
|
+
graph_dir: Path,
|
|
710
|
+
*,
|
|
711
|
+
table: str,
|
|
712
|
+
db: str | None = None,
|
|
713
|
+
detail: Literal["summary", "standard", "full"] = "standard",
|
|
714
|
+
config: GraphConfig | None = None,
|
|
715
|
+
visibility_filter: VisibilityFilter | None = None,
|
|
716
|
+
include_similar: bool = False,
|
|
717
|
+
similar_k: int = 5,
|
|
718
|
+
) -> dict[str, Any]:
|
|
719
|
+
"""360° view for one physical table."""
|
|
720
|
+
reg = load_registry(graph_dir)
|
|
721
|
+
entry = resolve_registry_entry(reg, db)
|
|
722
|
+
if entry is None and db is not None:
|
|
723
|
+
return {"error": f"Unknown database connection or name: {db!r}"}
|
|
724
|
+
if entry is None:
|
|
725
|
+
return {
|
|
726
|
+
"error": "Multiple indexed databases — pass `db` (connection name or logical database name)."
|
|
727
|
+
}
|
|
728
|
+
gp = graph_path_for_entry(entry)
|
|
729
|
+
if not gp.exists():
|
|
730
|
+
return {"error": f"Graph file missing for {entry.connection_name!r}: {gp}"}
|
|
731
|
+
|
|
732
|
+
vf = visibility_filter or get_effective_visibility_filter()
|
|
733
|
+
store = open_store_for_entry(entry)
|
|
734
|
+
try:
|
|
735
|
+
matches = find_table_rows(store, table, visibility_filter=vf)
|
|
736
|
+
if not matches:
|
|
737
|
+
msg = f"No table matched {table!r} in {entry.connection_name!r}"
|
|
738
|
+
candidates = suggest_table_candidates(store, table)
|
|
739
|
+
if candidates:
|
|
740
|
+
msg += f". Did you mean: {', '.join(candidates)}"
|
|
741
|
+
return {"error": msg}
|
|
742
|
+
if len(matches) > 1:
|
|
743
|
+
opts = [f"{m[1]}.{m[2]}" for m in matches]
|
|
744
|
+
return {
|
|
745
|
+
"error": "Ambiguous table name; use schema.table",
|
|
746
|
+
"candidates": opts,
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
(
|
|
750
|
+
node_id,
|
|
751
|
+
schema_name,
|
|
752
|
+
table_name,
|
|
753
|
+
comment,
|
|
754
|
+
table_desc_raw,
|
|
755
|
+
tags_array_raw,
|
|
756
|
+
row_count,
|
|
757
|
+
entity_type,
|
|
758
|
+
connection_name,
|
|
759
|
+
database,
|
|
760
|
+
table_type,
|
|
761
|
+
seq_scan,
|
|
762
|
+
idx_scan,
|
|
763
|
+
ins_ct,
|
|
764
|
+
upd_ct,
|
|
765
|
+
del_ct,
|
|
766
|
+
is_part,
|
|
767
|
+
part_key,
|
|
768
|
+
grants_json,
|
|
769
|
+
acc_read,
|
|
770
|
+
acc_write,
|
|
771
|
+
days_since_acc,
|
|
772
|
+
pot_unused,
|
|
773
|
+
table_bytes,
|
|
774
|
+
cluster_key,
|
|
775
|
+
role_raw,
|
|
776
|
+
role_conf_raw,
|
|
777
|
+
class_sig_json,
|
|
778
|
+
has_ext_raw,
|
|
779
|
+
test_count_raw,
|
|
780
|
+
staleness_status_raw,
|
|
781
|
+
staleness_as_of_raw,
|
|
782
|
+
) = matches[0]
|
|
783
|
+
|
|
784
|
+
qualified = f"{schema_name}.{table_name}"
|
|
785
|
+
table_description = str(table_desc_raw).strip() if table_desc_raw else ""
|
|
786
|
+
tags_list = _tags_from_kuzu_array(tags_array_raw)
|
|
787
|
+
has_ext_bool: bool | None = None
|
|
788
|
+
if has_ext_raw is not None:
|
|
789
|
+
has_ext_bool = bool(has_ext_raw)
|
|
790
|
+
test_count_val: int | None = None
|
|
791
|
+
if test_count_raw is not None:
|
|
792
|
+
try:
|
|
793
|
+
test_count_val = int(test_count_raw)
|
|
794
|
+
except (TypeError, ValueError):
|
|
795
|
+
test_count_val = None
|
|
796
|
+
staleness_status = (
|
|
797
|
+
str(staleness_status_raw).strip() if staleness_status_raw else ""
|
|
798
|
+
)
|
|
799
|
+
staleness_as_of = (
|
|
800
|
+
str(staleness_as_of_raw).strip() if staleness_as_of_raw else ""
|
|
801
|
+
)
|
|
802
|
+
description = table_description if table_description else str(comment or "")
|
|
803
|
+
shadow_ids = get_shadow_alias_node_ids(store, str(database))
|
|
804
|
+
rels = relationships_for_table(
|
|
805
|
+
store,
|
|
806
|
+
str(node_id),
|
|
807
|
+
visibility_filter=vf,
|
|
808
|
+
shadow_alias_ids=shadow_ids,
|
|
809
|
+
)
|
|
810
|
+
lineage_in, lineage_out = lineage_for_table(
|
|
811
|
+
store, str(node_id), visibility_filter=vf
|
|
812
|
+
)
|
|
813
|
+
lineage_md = _lineage_markdown_section(lineage_in, lineage_out)
|
|
814
|
+
ent_name, ent_desc = entity_for_table(store, str(node_id))
|
|
815
|
+
cluster_info = cluster_for_table_node_id(store, str(node_id), str(database))
|
|
816
|
+
alias_names = shadow_aliases_for_base(store, str(node_id), shadow_ids)
|
|
817
|
+
shadow_of_name = shadow_of_for_view(store, str(node_id), shadow_ids)
|
|
818
|
+
columns = columns_for_table_node_id(store, str(node_id), visibility_filter=vf)
|
|
819
|
+
|
|
820
|
+
role_str = str(role_raw).strip() if role_raw else ""
|
|
821
|
+
role_conf_f: float | None = None
|
|
822
|
+
if role_conf_raw is not None:
|
|
823
|
+
try:
|
|
824
|
+
role_conf_f = float(role_conf_raw)
|
|
825
|
+
except (TypeError, ValueError):
|
|
826
|
+
role_conf_f = None
|
|
827
|
+
class_signals: list[str] = []
|
|
828
|
+
if class_sig_json:
|
|
829
|
+
try:
|
|
830
|
+
parsed = json.loads(str(class_sig_json))
|
|
831
|
+
if isinstance(parsed, list):
|
|
832
|
+
class_signals = [str(x) for x in parsed if isinstance(x, str)]
|
|
833
|
+
except json.JSONDecodeError:
|
|
834
|
+
class_signals = []
|
|
835
|
+
pattern_label = ""
|
|
836
|
+
if cluster_info:
|
|
837
|
+
raw_pattern = cluster_info.get("schema_pattern")
|
|
838
|
+
if raw_pattern:
|
|
839
|
+
pattern_label = _schema_pattern_display(str(raw_pattern))
|
|
840
|
+
role_pretty = role_str.replace("_", " ").title() if role_str else ""
|
|
841
|
+
class_summary = ""
|
|
842
|
+
if role_pretty:
|
|
843
|
+
if pattern_label:
|
|
844
|
+
class_summary = (
|
|
845
|
+
f"This is a {role_pretty} table (confidence: {role_conf_f:.2f}) "
|
|
846
|
+
f"in a {pattern_label} cluster."
|
|
847
|
+
if role_conf_f is not None
|
|
848
|
+
else f"This is a {role_pretty} table in a {pattern_label} cluster."
|
|
849
|
+
)
|
|
850
|
+
else:
|
|
851
|
+
class_summary = (
|
|
852
|
+
f"This is a {role_pretty} table (confidence: {role_conf_f:.2f})."
|
|
853
|
+
if role_conf_f is not None
|
|
854
|
+
else f"This is a {role_pretty} table."
|
|
855
|
+
)
|
|
856
|
+
|
|
857
|
+
cfg = get_effective_graph_config(config)
|
|
858
|
+
days = staleness_days(entry.last_indexed_at)
|
|
859
|
+
stale = days > stale_threshold_days(cfg)
|
|
860
|
+
warn = (
|
|
861
|
+
f"Graph index is {days} days old (threshold {stale_threshold_days(cfg)} days); "
|
|
862
|
+
"run `pretensor reindex <dsn>` or `pretensor index`."
|
|
863
|
+
if stale
|
|
864
|
+
else None
|
|
865
|
+
)
|
|
866
|
+
|
|
867
|
+
dep_sig: str | None = None
|
|
868
|
+
days_acc_int: int | None = (
|
|
869
|
+
int(days_since_acc) if days_since_acc is not None else None
|
|
870
|
+
)
|
|
871
|
+
pot_u = bool(pot_unused) if pot_unused is not None else False
|
|
872
|
+
if (
|
|
873
|
+
pot_u
|
|
874
|
+
and days_acc_int is not None
|
|
875
|
+
and days_acc_int > _DEPRECATION_ACCESS_DAYS
|
|
876
|
+
and not lineage_in
|
|
877
|
+
):
|
|
878
|
+
dep_sig = (
|
|
879
|
+
f"No inbound lineage and not accessed in {days_acc_int} days "
|
|
880
|
+
"— likely deprecated"
|
|
881
|
+
)
|
|
882
|
+
|
|
883
|
+
base: ContextPayload = {
|
|
884
|
+
"connection_name": str(connection_name),
|
|
885
|
+
"database": str(database),
|
|
886
|
+
"schema_name": str(schema_name),
|
|
887
|
+
"table_name": str(table_name),
|
|
888
|
+
"table_type": str(table_type) if table_type else None,
|
|
889
|
+
"qualified_name": qualified,
|
|
890
|
+
"description": description,
|
|
891
|
+
"row_count": int(row_count) if row_count is not None else None,
|
|
892
|
+
"entity_type": str(entity_type) if entity_type else None,
|
|
893
|
+
"entity_name": ent_name,
|
|
894
|
+
"entity_description": ent_desc,
|
|
895
|
+
"role": role_str or None,
|
|
896
|
+
"role_confidence": role_conf_f,
|
|
897
|
+
"cluster": cluster_info,
|
|
898
|
+
"columns": columns,
|
|
899
|
+
"relationships": rels,
|
|
900
|
+
"deprecation_signal": dep_sig,
|
|
901
|
+
"staleness_warning": warn,
|
|
902
|
+
"detail": detail,
|
|
903
|
+
}
|
|
904
|
+
# Only surface lineage when there's something to show. Empty arrays read as
|
|
905
|
+
# "broken feature" on graphs without dbt manifests or view definitions.
|
|
906
|
+
if lineage_in or lineage_out:
|
|
907
|
+
base["lineage_in"] = lineage_in
|
|
908
|
+
base["lineage_out"] = lineage_out
|
|
909
|
+
if lineage_md:
|
|
910
|
+
base["lineage_markdown"] = lineage_md
|
|
911
|
+
if alias_names:
|
|
912
|
+
base["aliases"] = alias_names
|
|
913
|
+
if shadow_of_name is not None:
|
|
914
|
+
base["shadow_of"] = shadow_of_name
|
|
915
|
+
if tags_list:
|
|
916
|
+
base["tags"] = tags_list
|
|
917
|
+
if has_ext_bool is True:
|
|
918
|
+
base["has_external_consumers"] = True
|
|
919
|
+
if test_count_val is not None:
|
|
920
|
+
base["test_count"] = test_count_val
|
|
921
|
+
if staleness_status:
|
|
922
|
+
base["staleness_status"] = staleness_status
|
|
923
|
+
if staleness_as_of:
|
|
924
|
+
base["staleness_as_of"] = staleness_as_of
|
|
925
|
+
if detail == "full":
|
|
926
|
+
base["classification_signals"] = class_signals
|
|
927
|
+
base["classification_summary"] = class_summary
|
|
928
|
+
elif detail == "standard":
|
|
929
|
+
if class_summary:
|
|
930
|
+
base["classification_summary"] = class_summary
|
|
931
|
+
|
|
932
|
+
usage_stats: dict[str, int] = {}
|
|
933
|
+
for label, val in (
|
|
934
|
+
("seq_scan_count", seq_scan),
|
|
935
|
+
("idx_scan_count", idx_scan),
|
|
936
|
+
("insert_count", ins_ct),
|
|
937
|
+
("update_count", upd_ct),
|
|
938
|
+
("delete_count", del_ct),
|
|
939
|
+
):
|
|
940
|
+
if val is not None:
|
|
941
|
+
usage_stats[label] = int(val)
|
|
942
|
+
if usage_stats and detail != "summary":
|
|
943
|
+
base["usage_stats"] = usage_stats
|
|
944
|
+
|
|
945
|
+
if detail != "summary" and (
|
|
946
|
+
is_part is True or (part_key is not None and str(part_key).strip() != "")
|
|
947
|
+
):
|
|
948
|
+
base["partition"] = {
|
|
949
|
+
"is_partitioned": bool(is_part) if is_part is not None else True,
|
|
950
|
+
"partition_key": str(part_key) if part_key else None,
|
|
951
|
+
}
|
|
952
|
+
|
|
953
|
+
if grants_json and detail != "summary":
|
|
954
|
+
try:
|
|
955
|
+
gparsed = json.loads(str(grants_json))
|
|
956
|
+
if isinstance(gparsed, list):
|
|
957
|
+
base["grants"] = [dict(x) for x in gparsed if isinstance(x, dict)]
|
|
958
|
+
except json.JSONDecodeError:
|
|
959
|
+
pass
|
|
960
|
+
|
|
961
|
+
access_blob: dict[str, object] = {}
|
|
962
|
+
if acc_read is not None:
|
|
963
|
+
access_blob["read_operations_1y"] = int(acc_read)
|
|
964
|
+
if acc_write is not None:
|
|
965
|
+
access_blob["write_operations_1y"] = int(acc_write)
|
|
966
|
+
if days_since_acc is not None:
|
|
967
|
+
access_blob["days_since_last_access"] = int(days_since_acc)
|
|
968
|
+
if pot_unused is not None:
|
|
969
|
+
access_blob["potentially_unused_90d"] = bool(pot_unused)
|
|
970
|
+
if table_bytes is not None:
|
|
971
|
+
access_blob["table_bytes"] = int(table_bytes)
|
|
972
|
+
if cluster_key:
|
|
973
|
+
access_blob["clustering_key"] = str(cluster_key)
|
|
974
|
+
if access_blob and detail != "summary":
|
|
975
|
+
base["access_patterns"] = access_blob
|
|
976
|
+
|
|
977
|
+
if include_similar and detail != "summary":
|
|
978
|
+
k_clamped = max(_SIMILAR_K_MIN, min(_SIMILAR_K_MAX, int(similar_k)))
|
|
979
|
+
try:
|
|
980
|
+
similar_hits, similar_reason = similar_tables_for_table(
|
|
981
|
+
store,
|
|
982
|
+
table_node_id=str(node_id),
|
|
983
|
+
database_key=str(database),
|
|
984
|
+
k=k_clamped,
|
|
985
|
+
visibility_filter=vf,
|
|
986
|
+
)
|
|
987
|
+
except Exception as exc: # noqa: BLE001 — context must never raise
|
|
988
|
+
logger.warning("context: similar_tables computation failed (%s)", exc)
|
|
989
|
+
similar_hits, similar_reason = [], "error"
|
|
990
|
+
# Always emit the block when the caller asked for it, so the
|
|
991
|
+
# response shape is stable: callers branch on the reason, not
|
|
992
|
+
# on key presence.
|
|
993
|
+
base["similar_tables"] = similar_hits
|
|
994
|
+
if similar_reason is not None:
|
|
995
|
+
base["similar_reason"] = similar_reason
|
|
996
|
+
|
|
997
|
+
if detail == "summary":
|
|
998
|
+
sum_blob: dict[str, object] = {
|
|
999
|
+
"qualified_name": base["qualified_name"],
|
|
1000
|
+
"description": snippet(description, 120),
|
|
1001
|
+
"column_count": len(columns),
|
|
1002
|
+
"relationship_count": len(rels),
|
|
1003
|
+
"lineage_in_count": len(lineage_in),
|
|
1004
|
+
"lineage_out_count": len(lineage_out),
|
|
1005
|
+
"entity_name": ent_name,
|
|
1006
|
+
}
|
|
1007
|
+
if role_str:
|
|
1008
|
+
sum_blob["role"] = role_str
|
|
1009
|
+
if dep_sig:
|
|
1010
|
+
sum_blob["deprecation_signal"] = dep_sig
|
|
1011
|
+
return {
|
|
1012
|
+
"summary": sum_blob,
|
|
1013
|
+
"staleness_warning": warn,
|
|
1014
|
+
}
|
|
1015
|
+
return dict(base)
|
|
1016
|
+
finally:
|
|
1017
|
+
release_store(store)
|
|
1018
|
+
|
|
1019
|
+
|
|
1020
|
+
__all__ = [
|
|
1021
|
+
"cluster_for_table_node_id",
|
|
1022
|
+
"columns_for_table_node_id",
|
|
1023
|
+
"context_payload",
|
|
1024
|
+
"create_tool",
|
|
1025
|
+
"entity_for_table",
|
|
1026
|
+
"find_table_rows",
|
|
1027
|
+
"lineage_for_table",
|
|
1028
|
+
"qualified_from_node_id",
|
|
1029
|
+
"relationships_for_table",
|
|
1030
|
+
"resolve_table_node_id",
|
|
1031
|
+
"shadow_aliases_for_base",
|
|
1032
|
+
"shadow_of_for_view",
|
|
1033
|
+
"similar_tables_for_table",
|
|
1034
|
+
"suggest_table_candidates",
|
|
1035
|
+
]
|
|
1036
|
+
|
|
1037
|
+
|
|
1038
|
+
def create_tool(graph_dir: Path) -> McpTool:
|
|
1039
|
+
from ._timed import timed_tool
|
|
1040
|
+
|
|
1041
|
+
async def _handle(args: dict) -> dict:
|
|
1042
|
+
table = str(args.get("table", "")).strip()
|
|
1043
|
+
if not table:
|
|
1044
|
+
return {"error": "Missing `table`"}
|
|
1045
|
+
db = args.get("db")
|
|
1046
|
+
db_s = str(db) if db is not None else None
|
|
1047
|
+
detail = str(args.get("detail", "standard"))
|
|
1048
|
+
if detail not in ("summary", "standard", "full"):
|
|
1049
|
+
detail = "standard"
|
|
1050
|
+
include_similar = bool(args.get("include_similar", False))
|
|
1051
|
+
similar_k_raw = args.get("similar_k", 5)
|
|
1052
|
+
try:
|
|
1053
|
+
similar_k = int(similar_k_raw)
|
|
1054
|
+
except (TypeError, ValueError):
|
|
1055
|
+
similar_k = 5
|
|
1056
|
+
with timed_tool(
|
|
1057
|
+
"context",
|
|
1058
|
+
graph_dir,
|
|
1059
|
+
table=table,
|
|
1060
|
+
db=db_s,
|
|
1061
|
+
detail=detail,
|
|
1062
|
+
include_similar=include_similar,
|
|
1063
|
+
):
|
|
1064
|
+
return context_payload(
|
|
1065
|
+
graph_dir,
|
|
1066
|
+
table=table,
|
|
1067
|
+
db=db_s,
|
|
1068
|
+
detail=detail, # type: ignore[arg-type]
|
|
1069
|
+
include_similar=include_similar,
|
|
1070
|
+
similar_k=similar_k,
|
|
1071
|
+
)
|
|
1072
|
+
|
|
1073
|
+
return McpTool(
|
|
1074
|
+
name="context",
|
|
1075
|
+
description=(
|
|
1076
|
+
"Full context for one physical table: description (dbt-aware), columns (SchemaColumn), "
|
|
1077
|
+
"relationships (FK + inferred), LINEAGE in/out, optional deprecation signal, "
|
|
1078
|
+
"linked entity (if any), cluster when present."
|
|
1079
|
+
),
|
|
1080
|
+
input_schema={
|
|
1081
|
+
"type": "object",
|
|
1082
|
+
"properties": {
|
|
1083
|
+
"table": {
|
|
1084
|
+
"type": "string",
|
|
1085
|
+
"description": "Table name or schema.table",
|
|
1086
|
+
},
|
|
1087
|
+
"db": {
|
|
1088
|
+
"type": ["string", "null"],
|
|
1089
|
+
"description": "Connection name or logical database when multiple graphs exist",
|
|
1090
|
+
},
|
|
1091
|
+
"detail": {
|
|
1092
|
+
"type": "string",
|
|
1093
|
+
"enum": ["summary", "standard", "full"],
|
|
1094
|
+
"default": "standard",
|
|
1095
|
+
"description": "summary | standard (default) | full",
|
|
1096
|
+
},
|
|
1097
|
+
"include_similar": {
|
|
1098
|
+
"type": "boolean",
|
|
1099
|
+
"default": False,
|
|
1100
|
+
"description": (
|
|
1101
|
+
"When true, include a `similar_tables` block of "
|
|
1102
|
+
"cross-cluster nearest neighbors by cosine over "
|
|
1103
|
+
"`SchemaTable.embedding`. Default off."
|
|
1104
|
+
),
|
|
1105
|
+
},
|
|
1106
|
+
"similar_k": {
|
|
1107
|
+
"type": "integer",
|
|
1108
|
+
"default": 5,
|
|
1109
|
+
"minimum": 1,
|
|
1110
|
+
"maximum": 50,
|
|
1111
|
+
"description": (
|
|
1112
|
+
"Max number of similar tables to return when "
|
|
1113
|
+
"`include_similar` is true."
|
|
1114
|
+
),
|
|
1115
|
+
},
|
|
1116
|
+
},
|
|
1117
|
+
"required": ["table"],
|
|
1118
|
+
"additionalProperties": False,
|
|
1119
|
+
},
|
|
1120
|
+
handler=_handle,
|
|
1121
|
+
)
|