nl2sql-engine 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (192) hide show
  1. nl2sql/__init__.py +38 -0
  2. nl2sql/adapters/__init__.py +0 -0
  3. nl2sql/adapters/duckdb/__init__.py +0 -0
  4. nl2sql/adapters/duckdb/adapter.py +71 -0
  5. nl2sql/adapters/mssql/__init__.py +0 -0
  6. nl2sql/adapters/mssql/adapter.py +122 -0
  7. nl2sql/adapters/mysql/__init__.py +0 -0
  8. nl2sql/adapters/mysql/adapter.py +123 -0
  9. nl2sql/adapters/postgres/__init__.py +0 -0
  10. nl2sql/adapters/postgres/adapter.py +115 -0
  11. nl2sql/adapters/sqlalchemy_base/__init__.py +17 -0
  12. nl2sql/adapters/sqlalchemy_base/adapter.py +476 -0
  13. nl2sql/adapters/sqlalchemy_base/models.py +36 -0
  14. nl2sql/adapters/sqlite/__init__.py +0 -0
  15. nl2sql/adapters/sqlite/adapter.py +88 -0
  16. nl2sql/aggregation/__init__.py +3 -0
  17. nl2sql/aggregation/aggregator.py +98 -0
  18. nl2sql/aggregation/engines/__init__.py +3 -0
  19. nl2sql/aggregation/engines/polars_duckdb.py +125 -0
  20. nl2sql/api/__init__.py +0 -0
  21. nl2sql/api/auth_api.py +60 -0
  22. nl2sql/api/benchmark_api.py +114 -0
  23. nl2sql/api/datasource_api.py +132 -0
  24. nl2sql/api/indexing_api.py +59 -0
  25. nl2sql/api/llm_api.py +82 -0
  26. nl2sql/api/policy_api.py +135 -0
  27. nl2sql/api/query_api.py +138 -0
  28. nl2sql/api/result_api.py +24 -0
  29. nl2sql/api/settings_api.py +65 -0
  30. nl2sql/auth/__init__.py +8 -0
  31. nl2sql/auth/models.py +36 -0
  32. nl2sql/auth/rbac.py +25 -0
  33. nl2sql/cli/__init__.py +0 -0
  34. nl2sql/cli/checks.py +53 -0
  35. nl2sql/cli/commands/__init__.py +0 -0
  36. nl2sql/cli/commands/benchmark.py +34 -0
  37. nl2sql/cli/commands/doctor.py +49 -0
  38. nl2sql/cli/commands/indexing.py +126 -0
  39. nl2sql/cli/commands/info.py +25 -0
  40. nl2sql/cli/commands/install.py +27 -0
  41. nl2sql/cli/commands/policy.py +57 -0
  42. nl2sql/cli/commands/run.py +166 -0
  43. nl2sql/cli/commands/setup.py +415 -0
  44. nl2sql/cli/commands/visualize.py +34 -0
  45. nl2sql/cli/common/decorators.py +34 -0
  46. nl2sql/cli/config.py +24 -0
  47. nl2sql/cli/console.py +52 -0
  48. nl2sql/cli/demo/__init__.py +1 -0
  49. nl2sql/cli/demo/data.py +87 -0
  50. nl2sql/cli/demo/defaults.py +122 -0
  51. nl2sql/cli/demo/factory.py +289 -0
  52. nl2sql/cli/demo/manager.py +230 -0
  53. nl2sql/cli/demo/schemas.py +336 -0
  54. nl2sql/cli/demo/writers/__init__.py +0 -0
  55. nl2sql/cli/demo/writers/docker.py +182 -0
  56. nl2sql/cli/demo/writers/sqlite.py +88 -0
  57. nl2sql/cli/generators/datasources/__init__.py +3 -0
  58. nl2sql/cli/generators/datasources/generator.py +24 -0
  59. nl2sql/cli/generators/datasources/templates.py +7 -0
  60. nl2sql/cli/generators/env/__init__.py +3 -0
  61. nl2sql/cli/generators/env/generator.py +46 -0
  62. nl2sql/cli/generators/env/templates.py +25 -0
  63. nl2sql/cli/generators/llm/__init__.py +3 -0
  64. nl2sql/cli/generators/llm/generator.py +24 -0
  65. nl2sql/cli/generators/llm/templates.py +4 -0
  66. nl2sql/cli/generators/policies/__init__.py +3 -0
  67. nl2sql/cli/generators/policies/generator.py +20 -0
  68. nl2sql/cli/generators/policies/templates.py +2 -0
  69. nl2sql/cli/main.py +195 -0
  70. nl2sql/cli/reporting.py +878 -0
  71. nl2sql/cli/types.py +13 -0
  72. nl2sql/common/__init__.py +1 -0
  73. nl2sql/common/cancellation.py +25 -0
  74. nl2sql/common/context.py +5 -0
  75. nl2sql/common/errors.py +109 -0
  76. nl2sql/common/event_logger.py +88 -0
  77. nl2sql/common/exceptions.py +3 -0
  78. nl2sql/common/logger.py +119 -0
  79. nl2sql/common/metrics.py +50 -0
  80. nl2sql/common/resilience.py +59 -0
  81. nl2sql/common/settings.py +195 -0
  82. nl2sql/configs/__init__.py +6 -0
  83. nl2sql/configs/datasources.py +10 -0
  84. nl2sql/configs/llm.py +36 -0
  85. nl2sql/configs/manager.py +176 -0
  86. nl2sql/configs/policies.py +14 -0
  87. nl2sql/configs/sample_questions.py +11 -0
  88. nl2sql/configs/secrets.py +11 -0
  89. nl2sql/context.py +106 -0
  90. nl2sql/datasources/__init__.py +21 -0
  91. nl2sql/datasources/discovery.py +28 -0
  92. nl2sql/datasources/models.py +21 -0
  93. nl2sql/datasources/protocols.py +3 -0
  94. nl2sql/datasources/registry.py +172 -0
  95. nl2sql/evaluation/__init__.py +6 -0
  96. nl2sql/evaluation/benchmark_runner.py +320 -0
  97. nl2sql/evaluation/evaluator.py +134 -0
  98. nl2sql/evaluation/types.py +22 -0
  99. nl2sql/execution/__init__.py +4 -0
  100. nl2sql/execution/artifacts/__init__.py +3 -0
  101. nl2sql/execution/artifacts/parquet.py +41 -0
  102. nl2sql/execution/artifacts/store.py +165 -0
  103. nl2sql/execution/contracts.py +57 -0
  104. nl2sql/execution/execution_store.py +25 -0
  105. nl2sql/execution/executor/__init__.py +3 -0
  106. nl2sql/execution/executor/sql_executor.py +116 -0
  107. nl2sql/indexing/__init__.py +7 -0
  108. nl2sql/indexing/chunk_builder.py +227 -0
  109. nl2sql/indexing/embeddings.py +180 -0
  110. nl2sql/indexing/enrichment_service.py +316 -0
  111. nl2sql/indexing/models.py +209 -0
  112. nl2sql/indexing/orchestrator.py +90 -0
  113. nl2sql/indexing/vector_store.py +422 -0
  114. nl2sql/llm/__init__.py +8 -0
  115. nl2sql/llm/models.py +10 -0
  116. nl2sql/llm/registry.py +214 -0
  117. nl2sql/pipeline/__init__.py +1 -0
  118. nl2sql/pipeline/graph.py +73 -0
  119. nl2sql/pipeline/graph_utils.py +141 -0
  120. nl2sql/pipeline/nodes/__init__.py +25 -0
  121. nl2sql/pipeline/nodes/aggregator/__init__.py +4 -0
  122. nl2sql/pipeline/nodes/aggregator/node.py +55 -0
  123. nl2sql/pipeline/nodes/aggregator/prompts.py +20 -0
  124. nl2sql/pipeline/nodes/aggregator/schemas.py +28 -0
  125. nl2sql/pipeline/nodes/answer_synthesizer/__init__.py +4 -0
  126. nl2sql/pipeline/nodes/answer_synthesizer/node.py +98 -0
  127. nl2sql/pipeline/nodes/answer_synthesizer/prompts.py +19 -0
  128. nl2sql/pipeline/nodes/answer_synthesizer/schemas.py +24 -0
  129. nl2sql/pipeline/nodes/ast_planner/__init__.py +4 -0
  130. nl2sql/pipeline/nodes/ast_planner/node.py +104 -0
  131. nl2sql/pipeline/nodes/ast_planner/prompts.py +138 -0
  132. nl2sql/pipeline/nodes/ast_planner/schemas.py +236 -0
  133. nl2sql/pipeline/nodes/datasource_resolver/__init__.py +4 -0
  134. nl2sql/pipeline/nodes/datasource_resolver/node.py +253 -0
  135. nl2sql/pipeline/nodes/datasource_resolver/schemas.py +21 -0
  136. nl2sql/pipeline/nodes/decomposer/__init__.py +3 -0
  137. nl2sql/pipeline/nodes/decomposer/node.py +219 -0
  138. nl2sql/pipeline/nodes/decomposer/prompts.py +96 -0
  139. nl2sql/pipeline/nodes/decomposer/schemas.py +143 -0
  140. nl2sql/pipeline/nodes/executor/__init__.py +3 -0
  141. nl2sql/pipeline/nodes/executor/node.py +107 -0
  142. nl2sql/pipeline/nodes/generator/__init__.py +4 -0
  143. nl2sql/pipeline/nodes/generator/node.py +267 -0
  144. nl2sql/pipeline/nodes/generator/schemas.py +13 -0
  145. nl2sql/pipeline/nodes/global_planner/__init__.py +4 -0
  146. nl2sql/pipeline/nodes/global_planner/node.py +186 -0
  147. nl2sql/pipeline/nodes/global_planner/schemas.py +101 -0
  148. nl2sql/pipeline/nodes/refiner/__init__.py +4 -0
  149. nl2sql/pipeline/nodes/refiner/node.py +132 -0
  150. nl2sql/pipeline/nodes/refiner/prompts.py +28 -0
  151. nl2sql/pipeline/nodes/refiner/schemas.py +13 -0
  152. nl2sql/pipeline/nodes/schema_retriever/__init__.py +3 -0
  153. nl2sql/pipeline/nodes/schema_retriever/node.py +252 -0
  154. nl2sql/pipeline/nodes/schema_retriever/schema.py +27 -0
  155. nl2sql/pipeline/nodes/validator/__init__.py +7 -0
  156. nl2sql/pipeline/nodes/validator/node.py +839 -0
  157. nl2sql/pipeline/nodes/validator/schemas.py +12 -0
  158. nl2sql/pipeline/pipeline_runner.py +72 -0
  159. nl2sql/pipeline/routes.py +72 -0
  160. nl2sql/pipeline/runtime.py +153 -0
  161. nl2sql/pipeline/state.py +92 -0
  162. nl2sql/pipeline/subgraphs/__init__.py +5 -0
  163. nl2sql/pipeline/subgraphs/schemas.py +23 -0
  164. nl2sql/pipeline/subgraphs/sql_agent.py +167 -0
  165. nl2sql/public_api.py +199 -0
  166. nl2sql/schema/__init__.py +37 -0
  167. nl2sql/schema/in_memory_store.py +173 -0
  168. nl2sql/schema/protocol.py +88 -0
  169. nl2sql/schema/sqlite_store.py +233 -0
  170. nl2sql/schema/store.py +29 -0
  171. nl2sql/secrets/__init__.py +14 -0
  172. nl2sql/secrets/factory.py +85 -0
  173. nl2sql/secrets/interfaces.py +16 -0
  174. nl2sql/secrets/manager.py +139 -0
  175. nl2sql/secrets/models.py +56 -0
  176. nl2sql/secrets/providers/aws.py +30 -0
  177. nl2sql/secrets/providers/azure.py +49 -0
  178. nl2sql/secrets/providers/env.py +8 -0
  179. nl2sql/secrets/providers/hashi.py +46 -0
  180. nl2sql/services/__init__.py +0 -0
  181. nl2sql/services/callbacks/__init__.py +0 -0
  182. nl2sql/services/callbacks/monitor.py +84 -0
  183. nl2sql/services/callbacks/node_context.py +7 -0
  184. nl2sql/services/callbacks/node_handlers.py +187 -0
  185. nl2sql/services/callbacks/node_metrics.py +14 -0
  186. nl2sql/services/callbacks/presenter.py +12 -0
  187. nl2sql/services/callbacks/token_handler.py +56 -0
  188. nl2sql_engine-0.1.0.dist-info/METADATA +295 -0
  189. nl2sql_engine-0.1.0.dist-info/RECORD +192 -0
  190. nl2sql_engine-0.1.0.dist-info/WHEEL +5 -0
  191. nl2sql_engine-0.1.0.dist-info/entry_points.txt +9 -0
  192. nl2sql_engine-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,839 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import Set, Dict, Any, List, Optional, Tuple, TYPE_CHECKING
4
+ import traceback
5
+
6
+ from sqlglot import expressions as exp
7
+ from sqlglot.errors import SqlglotError
8
+ from sqlglot.optimizer.qualify import qualify
9
+
10
+ if TYPE_CHECKING:
11
+ from nl2sql.pipeline.state import SubgraphExecutionState
12
+ from nl2sql.common.errors import PipelineError, ErrorSeverity, ErrorCode
13
+ from nl2sql.pipeline.nodes.ast_planner.schemas import PlanModel, Expr
14
+ from nl2sql.pipeline.nodes.generator.node import SqlVisitor
15
+ from nl2sql.context import NL2SQLContext
16
+ from nl2sql.common.logger import get_logger
17
+ from nl2sql.common.settings import settings
18
+ from nl2sql.pipeline.nodes.validator.schemas import LogicalValidatorResponse
19
+
20
+
21
+ logger = get_logger("logical_validator")
22
+
23
+ _MAX_HINTED_COLUMNS = 15
24
+
25
+
26
+ class ValidationSqlVisitor(SqlVisitor):
27
+ """``SqlVisitor`` variant used to build a throw-away tree for validation.
28
+
29
+ Identical to the generator's visitor except that a ``*`` column name is
30
+ rendered as a real sqlglot star, so the optimizer expands it instead of
31
+ trying to resolve a column literally named ``*``.
32
+ """
33
+
34
+ def _visit_column(self, expr: Expr) -> exp.Expression:
35
+ """Converts a column expression, mapping wildcards to sqlglot stars."""
36
+ if (expr.column_name or "").strip() == "*":
37
+ if expr.alias:
38
+ return exp.Column(
39
+ this=exp.Star(),
40
+ table=exp.Identifier(this=expr.alias, quoted=False),
41
+ )
42
+ return exp.Star()
43
+ return super()._visit_column(expr)
44
+
45
+
46
+ class LogicalValidatorNode:
47
+ """Validates the generated AST (PlanModel).
48
+
49
+ Performs static validation on the AST structure and user policies.
50
+
51
+ Attributes:
52
+ registry (DatasourceRegistry): Registry to fetch schemas and profiles.
53
+ """
54
+
55
+ def __init__(self, ctx: NL2SQLContext):
56
+ """Initializes the LogicalValidatorNode.
57
+
58
+ Args:
59
+ registry (DatasourceRegistry): The registry of datasources.
60
+ """
61
+ self.registry = ctx.ds_registry
62
+ self.rbac = ctx.rbac
63
+ self.strict_columns = settings.logical_validator_strict_columns
64
+
65
+ def _normalize_table_key(
66
+ self,
67
+ table_name: str,
68
+ schema_name: Optional[str] = None,
69
+ database: Optional[str] = None,
70
+ ) -> str:
71
+ parts = []
72
+ if database:
73
+ parts.append(database)
74
+ if schema_name:
75
+ parts.append(schema_name)
76
+ parts.append(table_name)
77
+ return ".".join(parts).lower()
78
+
79
+ def _normalize_name(self, value: Optional[str]) -> str:
80
+ if not value:
81
+ return ""
82
+ return value.lower().split(".")[-1]
83
+
84
+ def _build_allowed_schema(
85
+ self, state: SubgraphExecutionState
86
+ ) -> Tuple[Dict[str, Set[str]], Dict[str, Dict[str, Dict[str, Any]]], List[Dict[str, Any]]]:
87
+ table_to_cols: Dict[str, Set[str]] = {}
88
+ table_to_stats: Dict[str, Dict[str, Dict[str, Any]]] = {}
89
+ relationships: List[Dict[str, Any]] = []
90
+
91
+ for rt in state.relevant_tables:
92
+ table_name = self._normalize_name(rt.name)
93
+ if not table_name:
94
+ continue
95
+ cols: Set[str] = set()
96
+ stats_map: Dict[str, Dict[str, Any]] = {}
97
+ for c in rt.columns or []:
98
+ col_name = self._normalize_name(c.name)
99
+ if not col_name:
100
+ continue
101
+ cols.add(col_name)
102
+ stats = getattr(c, "stats", None)
103
+ if isinstance(stats, dict) and stats:
104
+ stats_map[col_name] = stats
105
+ table_to_cols[table_name] = cols
106
+ table_to_stats[table_name] = stats_map
107
+
108
+ for rel in getattr(rt, "relationships", []) or []:
109
+ relationships.append(rel)
110
+
111
+ return table_to_cols, table_to_stats, relationships
112
+
113
+ def _extract_join_pairs(self, expr: Expr) -> List[Tuple[str, str, str, str]]:
114
+ pairs: List[Tuple[str, str, str, str]] = []
115
+
116
+ def walk(node: Optional[Expr]) -> None:
117
+ if not node:
118
+ return
119
+ if node.kind == "binary" and node.op == "=":
120
+ left = node.left
121
+ right = node.right
122
+ if left and right and left.kind == "column" and right.kind == "column":
123
+ if left.alias and right.alias and left.column_name and right.column_name:
124
+ pairs.append(
125
+ (
126
+ left.alias,
127
+ self._normalize_name(left.column_name),
128
+ right.alias,
129
+ self._normalize_name(right.column_name),
130
+ )
131
+ )
132
+ if node.kind == "binary":
133
+ walk(node.left)
134
+ walk(node.right)
135
+ elif node.kind == "func":
136
+ for arg in node.args:
137
+ walk(arg)
138
+ elif node.kind == "unary":
139
+ walk(node.expr)
140
+ elif node.kind == "case":
141
+ for when in node.whens:
142
+ walk(when.condition)
143
+ walk(when.result)
144
+ walk(node.else_expr)
145
+
146
+ walk(expr)
147
+ return pairs
148
+
149
+ def _extract_literal_checks(self, expr: Expr) -> List[Tuple[Optional[str], str, str, Any]]:
150
+ checks: List[Tuple[Optional[str], str, str, Any]] = []
151
+
152
+ def walk(node: Optional[Expr]) -> None:
153
+ if not node:
154
+ return
155
+ if node.kind == "binary" and node.op in ("=", "IN", "LIKE"):
156
+ left = node.left
157
+ right = node.right
158
+ if left and right:
159
+ if left.kind == "column" and right.kind == "literal":
160
+ if left.column_name:
161
+ checks.append((left.alias, self._normalize_name(left.column_name), node.op, right.value))
162
+ elif right.kind == "column" and left.kind == "literal":
163
+ if right.column_name:
164
+ checks.append((right.alias, self._normalize_name(right.column_name), node.op, left.value))
165
+ if node.kind == "binary":
166
+ walk(node.left)
167
+ walk(node.right)
168
+ elif node.kind == "func":
169
+ for arg in node.args:
170
+ walk(arg)
171
+ elif node.kind == "unary":
172
+ walk(node.expr)
173
+ elif node.kind == "case":
174
+ for when in node.whens:
175
+ walk(when.condition)
176
+ walk(when.result)
177
+ walk(node.else_expr)
178
+
179
+ walk(expr)
180
+ return checks
181
+
182
+ def _value_matches_stats(self, value: Any, stats: Dict[str, Any]) -> bool:
183
+ samples = stats.get("sample_values") or []
184
+ if not samples:
185
+ return True
186
+ if isinstance(value, str):
187
+ return value.lower() in {str(s).lower() for s in samples}
188
+ return value in samples
189
+
190
+ def _like_matches_stats(self, value: Any, stats: Dict[str, Any]) -> bool:
191
+ if not isinstance(value, str):
192
+ return False
193
+ samples = [str(s).lower() for s in (stats.get("sample_values") or [])]
194
+ synonyms = [str(s).lower() for s in (stats.get("synonyms") or [])]
195
+ candidates = samples + synonyms
196
+ if not candidates:
197
+ return True
198
+ val = value.lower()
199
+ return any(c in val or val in c for c in candidates)
200
+
201
+ def _condition_aliases(self, condition: Expr) -> Set[str]:
202
+ """Returns the table aliases referenced by a join condition."""
203
+ node = ValidationSqlVisitor().visit(condition)
204
+ return {col.table for col in node.find_all(exp.Column) if col.table}
205
+
206
+ def _resolve_plan_tables(
207
+ self, state: SubgraphExecutionState, plan: PlanModel
208
+ ) -> Tuple[Dict[str, Set[str]], Set[str], List[PipelineError]]:
209
+ """Resolves every plan table against the retrieved schema.
210
+
211
+ The resulting alias-to-column map is what is handed to sqlglot's
212
+ ``qualify()`` as the schema. Table existence is still checked here
213
+ because ``qualify()`` silently ignores relations that are absent from
214
+ the schema it is given.
215
+
216
+ Args:
217
+ state (SubgraphExecutionState): Current execution state containing relevant_tables.
218
+ plan (PlanModel): The plan containing table references.
219
+
220
+ Returns:
221
+ Tuple containing:
222
+ - alias_to_cols (Dict[str, Set[str]]): Map of alias to column names.
223
+ - plan_aliases (Set[str]): Set of aliases defined in the plan.
224
+ - errors (List[PipelineError]): List of errors if tables are missing.
225
+ """
226
+ alias_to_cols: Dict[str, Set[str]] = {}
227
+ plan_aliases: Set[str] = set()
228
+ errors: List[PipelineError] = []
229
+
230
+ simple_map: Dict[str, List[Any]] = {}
231
+ full_map: Dict[str, Any] = {}
232
+ for rt in state.relevant_tables:
233
+ rt_name = (rt.name or "").lower()
234
+ if not rt_name:
235
+ continue
236
+ simple_map.setdefault(rt_name.split(".")[-1], []).append(rt)
237
+ if "." in rt_name:
238
+ full_map[rt_name] = rt
239
+
240
+ for t in plan.tables:
241
+ plan_aliases.add(t.alias)
242
+
243
+ found_table = None
244
+ if t.schema_name or t.database:
245
+ found_table = full_map.get(
246
+ self._normalize_table_key(t.name, t.schema_name, t.database)
247
+ )
248
+
249
+ if not found_table:
250
+ candidates = simple_map.get((t.name or "").lower(), [])
251
+ if len(candidates) > 1:
252
+ errors.append(
253
+ PipelineError(
254
+ node="logical_validator",
255
+ message=(
256
+ f"Ambiguous table '{t.name}' across schemas; "
257
+ "plan must specify schema."
258
+ ),
259
+ severity=ErrorSeverity.ERROR,
260
+ error_code=ErrorCode.TABLE_NOT_FOUND,
261
+ )
262
+ )
263
+ continue
264
+ found_table = candidates[0] if candidates else None
265
+
266
+ if not found_table:
267
+ errors.append(
268
+ PipelineError(
269
+ node="logical_validator",
270
+ message=f"Table '{t.name}' not found in relevant tables.",
271
+ severity=ErrorSeverity.ERROR,
272
+ error_code=ErrorCode.TABLE_NOT_FOUND
273
+ )
274
+ )
275
+ continue
276
+
277
+ alias_to_cols[t.alias] = {
278
+ self._normalize_name(c.name) for c in found_table.columns
279
+ }
280
+
281
+ logger.debug("Validator alias map: %s", alias_to_cols)
282
+ return alias_to_cols, plan_aliases, errors
283
+
284
+ def _build_validation_query(
285
+ self, plan: PlanModel, aliases: List[str]
286
+ ) -> exp.Select:
287
+ """Builds a throw-away sqlglot query used purely for column resolution.
288
+
289
+ Every resolved plan table becomes a relation named after its alias, so
290
+ the sqlglot schema is keyed by alias and resolution scoping matches the
291
+ plan exactly. Tables are cross-joined rather than joined on their
292
+ conditions because the plan's alias-to-column visibility does not depend
293
+ on join structure; join conditions are folded into the WHERE clause so
294
+ that their columns are resolved too.
295
+ """
296
+ visitor = ValidationSqlVisitor()
297
+
298
+ selects = [visitor.visit(s.expr) for s in plan.select_items] or [exp.Star()]
299
+ query = exp.select(*selects)
300
+
301
+ query = query.from_(exp.Table(this=exp.Identifier(this=aliases[0], quoted=False)))
302
+ for alias in aliases[1:]:
303
+ query = query.join(
304
+ exp.Table(this=exp.Identifier(this=alias, quoted=False)),
305
+ join_type="CROSS",
306
+ )
307
+
308
+ for j in plan.joins:
309
+ query = query.where(visitor.visit(j.condition))
310
+ if plan.where:
311
+ query = query.where(visitor.visit(plan.where))
312
+ for g in plan.group_by:
313
+ query = query.group_by(visitor.visit(g.expr))
314
+ if plan.having:
315
+ query = query.having(visitor.visit(plan.having))
316
+ for o in plan.order_by:
317
+ query = query.order_by(visitor.visit(o.expr))
318
+
319
+ return query
320
+
321
+ def _describe_column_failure(
322
+ self, alias: str, column: str, alias_to_cols: Dict[str, Set[str]]
323
+ ) -> str:
324
+ """Turns an unresolvable column reference into actionable plan feedback.
325
+
326
+ sqlglot reports failures in terms of the SQL it was handed (``Unknown
327
+ column: x``, with a line/column offset). The planner never sees that
328
+ SQL, so the message is rewritten in terms of the plan's own aliases and
329
+ the schema the retriever supplied.
330
+ """
331
+ if alias and alias not in alias_to_cols:
332
+ known = ", ".join(sorted(alias_to_cols)) or "none"
333
+ return (
334
+ f"Column '{column}' uses undeclared alias '{alias}'. "
335
+ f"Declared table aliases: {known}."
336
+ )
337
+
338
+ if alias:
339
+ return (
340
+ f"Column '{column}' does not exist in table alias '{alias}'. "
341
+ f"Available columns: {self._format_columns(alias_to_cols[alias])}."
342
+ )
343
+
344
+ matches = sorted(
345
+ a for a, cols in alias_to_cols.items() if column.lower() in cols
346
+ )
347
+ if len(matches) > 1:
348
+ return (
349
+ f"Ambiguous column '{column}' referenced without alias. "
350
+ f"Qualify it with one of: {', '.join(matches)}."
351
+ )
352
+
353
+ available = {f"{a}.{c}" for a, cols in alias_to_cols.items() for c in cols}
354
+ return (
355
+ f"Column '{column}' not found in any relevant table. "
356
+ f"Available columns: {self._format_columns(available)}."
357
+ )
358
+
359
+ @staticmethod
360
+ def _format_columns(columns: Set[str]) -> str:
361
+ """Renders a bounded, deterministic list of column names for feedback."""
362
+ ordered = sorted(columns)
363
+ if not ordered:
364
+ return "none"
365
+ if len(ordered) > _MAX_HINTED_COLUMNS:
366
+ return ", ".join(ordered[:_MAX_HINTED_COLUMNS]) + ", ..."
367
+ return ", ".join(ordered)
368
+
369
+ def _validate_columns(
370
+ self, plan: PlanModel, alias_to_cols: Dict[str, Set[str]]
371
+ ) -> List[str]:
372
+ """Resolves every column reference in the plan via sqlglot's optimizer.
373
+
374
+ ``qualify()`` is the sole authority on whether a reference resolves; it
375
+ covers column existence, alias scoping and ambiguity. It fails on the
376
+ first bad reference, so when it does fail each distinct reference is
377
+ re-probed individually to report all of them at once.
378
+
379
+ Returns:
380
+ List[str]: Human-readable messages, one per unresolvable reference.
381
+ """
382
+ aliases = list(alias_to_cols)
383
+ if not aliases or not plan.select_items:
384
+ return []
385
+
386
+ schema = {a: {c: "UNKNOWN" for c in cols} for a, cols in alias_to_cols.items()}
387
+ query = self._build_validation_query(plan, aliases)
388
+
389
+ try:
390
+ qualify(query.copy(), schema=schema, validate_qualify_columns=True)
391
+ return []
392
+ except SqlglotError as exc:
393
+ logger.debug("sqlglot qualify rejected plan: %s", exc)
394
+ failure = exc
395
+
396
+ messages: List[str] = []
397
+ seen: Set[Tuple[str, str]] = set()
398
+ for column in query.find_all(exp.Column):
399
+ if isinstance(column.this, exp.Star):
400
+ continue
401
+ key = (column.table or "", column.name)
402
+ if key in seen:
403
+ continue
404
+ seen.add(key)
405
+
406
+ probe = exp.select(column.copy()).from_(
407
+ exp.Table(this=exp.Identifier(this=aliases[0], quoted=False))
408
+ )
409
+ for alias in aliases[1:]:
410
+ probe = probe.join(
411
+ exp.Table(this=exp.Identifier(this=alias, quoted=False)),
412
+ join_type="CROSS",
413
+ )
414
+ try:
415
+ qualify(probe, schema=schema, validate_qualify_columns=True)
416
+ except SqlglotError:
417
+ messages.append(
418
+ self._describe_column_failure(key[0], key[1], alias_to_cols)
419
+ )
420
+
421
+ if not messages:
422
+ messages.append(f"Plan columns could not be resolved: {failure}")
423
+ return messages
424
+
425
+ def _validate_ordinals(self, items: List[Any], label: str) -> Optional[PipelineError]:
426
+ """Checks if ordinals in a list of items are contiguous starting from 0."""
427
+ if not items:
428
+ return None
429
+
430
+ ords = [x.ordinal for x in items]
431
+ expected = list(range(len(items)))
432
+
433
+ if ords != expected:
434
+ return PipelineError(
435
+ node="logical_validator",
436
+ message=f"{label} ordinals must be contiguous 0..{len(items)-1}, found {ords}",
437
+ severity=ErrorSeverity.ERROR,
438
+ error_code=ErrorCode.INVALID_PLAN_STRUCTURE,
439
+ )
440
+ return None
441
+
442
+ def _alias_collision(self, plan: PlanModel) -> Optional[PipelineError]:
443
+ """Checks for duplicate table aliases in the plan."""
444
+ seen = set()
445
+ for t in plan.tables:
446
+ if t.alias in seen:
447
+ return PipelineError(
448
+ node="logical_validator",
449
+ message=f"Duplicate table alias '{t.alias}' in plan.",
450
+ severity=ErrorSeverity.ERROR,
451
+ error_code=ErrorCode.INVALID_PLAN_STRUCTURE,
452
+ )
453
+ seen.add(t.alias)
454
+ return None
455
+
456
+ def _validate_policy(self, state: SubgraphExecutionState) -> list[PipelineError]:
457
+ """Validates that the query adheres to access control policies.
458
+
459
+ Args:
460
+ state (SubgraphExecutionState): Execution state containing user_context.
461
+
462
+ Returns:
463
+ list[PipelineError]: Errors if unauthorized tables are accessed.
464
+ """
465
+ plan = state.ast_planner_response.plan if state.ast_planner_response else None
466
+ errors: list[PipelineError] = []
467
+
468
+ user_ctx = state.user_context
469
+ allowed_tables = self.rbac.get_allowed_tables(user_ctx)
470
+ role = ','.join(user_ctx.roles)
471
+
472
+ # Resolve Datasource ID for Namespacing
473
+ ds_id = state.sub_query.datasource_id if state.sub_query else None
474
+ if not ds_id:
475
+ # Fail Closed if we don't know the datasource (cannot enforce namespace)
476
+ return [
477
+ PipelineError(
478
+ node="logical_validator",
479
+ message="Security Enforcement Failed: No sub_query datasource_id in state.",
480
+ severity=ErrorSeverity.CRITICAL,
481
+ error_code=ErrorCode.SECURITY_VIOLATION
482
+ )
483
+ ]
484
+
485
+ logger.debug("Policy validation context: Role=%s, Allowed=%s", role, allowed_tables)
486
+
487
+ if "*" in allowed_tables:
488
+ return []
489
+
490
+ for t in plan.tables:
491
+ # STRICT Namespacing Logic
492
+ namespaced_name = f"{ds_id}.{t.name}"
493
+ ds_wildcard = f"{ds_id}.*"
494
+
495
+ # Check 1: Exact Match (e.g. "sales_db.orders")
496
+ if namespaced_name in allowed_tables:
497
+ continue
498
+
499
+ # Check 2: Datasource Wildcard (e.g. "sales_db.*")
500
+ if ds_wildcard in allowed_tables:
501
+ continue
502
+
503
+ # If no match -> Violation
504
+ errors.append(
505
+ PipelineError(
506
+ node="logical_validator",
507
+ message=f"Role '{role}' denied access to '{namespaced_name}'. Policy requires explicit 'datasource.table' allow.",
508
+ severity=ErrorSeverity.CRITICAL,
509
+ error_code=ErrorCode.SECURITY_VIOLATION,
510
+ )
511
+ )
512
+
513
+ return errors
514
+
515
+ def _validate_static(self, state: SubgraphExecutionState) -> list[PipelineError]:
516
+ """Performs static structure validation on the plan.
517
+
518
+ Checks:
519
+ - Query type allowed (READ only).
520
+ - Ordinal integrity.
521
+ - Alias uniqueness.
522
+ - Join alias validity.
523
+ - Column existence and scoping (via sqlglot's qualify optimizer).
524
+ """
525
+ plan: PlanModel = state.ast_planner_response.plan if state.ast_planner_response else None
526
+ errors: list[PipelineError] = []
527
+
528
+ if not plan.tables:
529
+ return [
530
+ PipelineError(
531
+ node="logical_validator",
532
+ message="Plan has no tables.",
533
+ severity=ErrorSeverity.ERROR,
534
+ error_code=ErrorCode.INVALID_PLAN_STRUCTURE,
535
+ )
536
+ ]
537
+
538
+ if plan.query_type != "READ":
539
+ return [
540
+ PipelineError(
541
+ node="logical_validator",
542
+ message=f"Query type '{plan.query_type}' not allowed.",
543
+ severity=ErrorSeverity.CRITICAL,
544
+ error_code=ErrorCode.SECURITY_VIOLATION,
545
+ )
546
+ ]
547
+
548
+ for label, group in [
549
+ ("tables", plan.tables),
550
+ ("joins", plan.joins),
551
+ ("select_items", plan.select_items),
552
+ ("group_by", plan.group_by),
553
+ ("order_by", plan.order_by),
554
+ ]:
555
+ err = self._validate_ordinals(group, label)
556
+ if err:
557
+ errors.append(err)
558
+
559
+ alias_err = self._alias_collision(plan)
560
+ if alias_err:
561
+ errors.append(alias_err)
562
+
563
+ if state.sub_query and state.sub_query.expected_schema:
564
+ expected_names = [c.name for c in state.sub_query.expected_schema if c.name]
565
+ actual_aliases = [s.alias for s in plan.select_items if s.alias]
566
+ if len(plan.select_items) != len(expected_names):
567
+ errors.append(
568
+ PipelineError(
569
+ node="logical_validator",
570
+ message=(
571
+ "Select item count must match expected_schema. "
572
+ f"Expected {len(expected_names)}, got {len(plan.select_items)}."
573
+ ),
574
+ severity=ErrorSeverity.ERROR,
575
+ error_code=ErrorCode.INVALID_PLAN_STRUCTURE,
576
+ )
577
+ )
578
+ if sorted(actual_aliases) != sorted(expected_names):
579
+ errors.append(
580
+ PipelineError(
581
+ node="logical_validator",
582
+ message=(
583
+ "Select aliases must match expected_schema names. "
584
+ f"Expected {expected_names}, got {actual_aliases}."
585
+ ),
586
+ severity=ErrorSeverity.ERROR,
587
+ error_code=ErrorCode.INVALID_PLAN_STRUCTURE,
588
+ )
589
+ )
590
+
591
+ alias_to_cols, plan_aliases, alias_errors = self._resolve_plan_tables(state, plan)
592
+ errors.extend(alias_errors)
593
+
594
+ table_to_cols, table_to_stats, relationships = self._build_allowed_schema(state)
595
+ alias_to_table: Dict[str, str] = {
596
+ t.alias: self._normalize_name(t.name) for t in plan.tables
597
+ }
598
+
599
+ for j in plan.joins:
600
+ if j.left_alias not in plan_aliases:
601
+ errors.append(
602
+ PipelineError(
603
+ node="logical_validator",
604
+ message=f"Join left alias '{j.left_alias}' not in plan tables.",
605
+ severity=ErrorSeverity.ERROR,
606
+ error_code=ErrorCode.JOIN_TABLE_NOT_IN_PLAN,
607
+ )
608
+ )
609
+
610
+ if j.right_alias not in plan_aliases:
611
+ errors.append(
612
+ PipelineError(
613
+ node="logical_validator",
614
+ message=f"Join right alias '{j.right_alias}' not in plan tables.",
615
+ severity=ErrorSeverity.ERROR,
616
+ error_code=ErrorCode.JOIN_TABLE_NOT_IN_PLAN,
617
+ )
618
+ )
619
+ join_aliases = self._condition_aliases(j.condition)
620
+ if j.left_alias not in join_aliases or j.right_alias not in join_aliases:
621
+ errors.append(
622
+ PipelineError(
623
+ node="logical_validator",
624
+ message=(
625
+ "Join condition must reference both "
626
+ f"'{j.left_alias}' and '{j.right_alias}'."
627
+ ),
628
+ severity=ErrorSeverity.ERROR,
629
+ error_code=ErrorCode.INVALID_PLAN_STRUCTURE,
630
+ )
631
+ )
632
+
633
+ join_pairs = self._extract_join_pairs(j.condition)
634
+ if not join_pairs:
635
+ errors.append(
636
+ PipelineError(
637
+ node="logical_validator",
638
+ message="Join condition must include an equality between join columns.",
639
+ severity=ErrorSeverity.ERROR,
640
+ error_code=ErrorCode.INVALID_PLAN_STRUCTURE,
641
+ )
642
+ )
643
+ else:
644
+ left_table = alias_to_table.get(j.left_alias, "")
645
+ right_table = alias_to_table.get(j.right_alias, "")
646
+ matched = False
647
+ for left_alias, left_col, right_alias, right_col in join_pairs:
648
+ if left_alias != j.left_alias or right_alias != j.right_alias:
649
+ continue
650
+ for rel in relationships:
651
+ from_table = self._normalize_name(rel.get("from_table"))
652
+ to_table = self._normalize_name(rel.get("to_table"))
653
+ from_cols = [self._normalize_name(c) for c in rel.get("from_columns") or []]
654
+ to_cols = [self._normalize_name(c) for c in rel.get("to_columns") or []]
655
+ if (
656
+ left_table == from_table
657
+ and right_table == to_table
658
+ and left_col in from_cols
659
+ and right_col in to_cols
660
+ ):
661
+ matched = True
662
+ break
663
+ if (
664
+ left_table == to_table
665
+ and right_table == from_table
666
+ and left_col in to_cols
667
+ and right_col in from_cols
668
+ ):
669
+ matched = True
670
+ break
671
+ if matched:
672
+ break
673
+ if not matched:
674
+ errors.append(
675
+ PipelineError(
676
+ node="logical_validator",
677
+ message="Join does not match any allowed relationship.",
678
+ severity=ErrorSeverity.ERROR,
679
+ error_code=ErrorCode.INVALID_PLAN_STRUCTURE,
680
+ )
681
+ )
682
+
683
+ column_messages = self._validate_columns(plan, alias_to_cols)
684
+
685
+ for expr in [plan.where, plan.having]:
686
+ if not expr:
687
+ continue
688
+ checks = self._extract_literal_checks(expr)
689
+ for alias, col_name, op, value in checks:
690
+ resolved_alias = alias
691
+ if not resolved_alias:
692
+ matches = [
693
+ a for a, cols in alias_to_cols.items() if col_name in cols
694
+ ]
695
+ if len(matches) == 1:
696
+ resolved_alias = matches[0]
697
+ if not resolved_alias:
698
+ continue
699
+ table_name = alias_to_table.get(resolved_alias, "")
700
+ stats = table_to_stats.get(table_name, {}).get(col_name)
701
+ if not stats:
702
+ continue
703
+ if op in ("=", "IN") and not self._value_matches_stats(value, stats):
704
+ errors.append(
705
+ PipelineError(
706
+ node="logical_validator",
707
+ message=(
708
+ f"Literal value '{value}' not found in stats for {table_name}.{col_name}."
709
+ ),
710
+ severity=ErrorSeverity.ERROR,
711
+ error_code=ErrorCode.INVALID_PLAN_STRUCTURE,
712
+ )
713
+ )
714
+ if op == "LIKE" and not self._like_matches_stats(value, stats):
715
+ errors.append(
716
+ PipelineError(
717
+ node="logical_validator",
718
+ message=(
719
+ f"LIKE pattern '{value}' is not derived from stats for {table_name}.{col_name}."
720
+ ),
721
+ severity=ErrorSeverity.ERROR,
722
+ error_code=ErrorCode.INVALID_PLAN_STRUCTURE,
723
+ )
724
+ )
725
+
726
+ column_severity = (
727
+ ErrorSeverity.ERROR if self.strict_columns else ErrorSeverity.WARNING
728
+ )
729
+ for msg in column_messages:
730
+ errors.append(
731
+ PipelineError(
732
+ node="logical_validator",
733
+ message=msg,
734
+ severity=column_severity,
735
+ error_code=ErrorCode.COLUMN_NOT_FOUND,
736
+ )
737
+ )
738
+
739
+ return errors
740
+
741
+
742
+
743
+ def __call__(self, state: SubgraphExecutionState) -> Dict[str, Any]:
744
+ """Executes the validation node.
745
+
746
+ Args:
747
+ state (SubgraphExecutionState): Current execution state.
748
+
749
+ Returns:
750
+ Dict[str, Any]: Validation results, including errors and reasoning.
751
+ """
752
+ node_name = "logical_validator"
753
+ errors: list[PipelineError] = []
754
+
755
+ try:
756
+ logger.debug("Logical Validator received plan:")
757
+ plan = state.ast_planner_response.plan if state.ast_planner_response else None
758
+ if plan:
759
+ logger.debug(plan.model_dump_json(indent=2))
760
+ else:
761
+ logger.warning("No plan to validate.")
762
+
763
+ if not plan:
764
+ return {
765
+ "logical_validator_response": LogicalValidatorResponse(
766
+ errors=[
767
+ PipelineError(
768
+ node=node_name,
769
+ message="Missing Plan",
770
+ severity=ErrorSeverity.CRITICAL,
771
+ error_code=ErrorCode.MISSING_PLAN,
772
+ )
773
+ ],
774
+ reasoning=[],
775
+ ),
776
+ "errors": [
777
+ PipelineError(
778
+ node=node_name,
779
+ message="Missing Plan",
780
+ severity=ErrorSeverity.CRITICAL,
781
+ error_code=ErrorCode.MISSING_PLAN,
782
+ )
783
+ ],
784
+ }
785
+
786
+ # Static validation is isolated so that a failure inside it can
787
+ # never skip policy enforcement. RBAC must run for every plan.
788
+ try:
789
+ errors.extend(self._validate_static(state))
790
+ except Exception as exc:
791
+ logger.exception("Static logical validation crashed")
792
+ errors.append(
793
+ PipelineError(
794
+ node=node_name,
795
+ message=f"Static logical validation crashed: {exc}",
796
+ severity=ErrorSeverity.ERROR,
797
+ error_code=ErrorCode.VALIDATOR_CRASH,
798
+ stack_trace=traceback.format_exc(),
799
+ )
800
+ )
801
+
802
+ errors.extend(self._validate_policy(state))
803
+
804
+ if any(e.severity in (ErrorSeverity.CRITICAL, ErrorSeverity.ERROR) for e in errors):
805
+ response = LogicalValidatorResponse(
806
+ errors=errors,
807
+ reasoning=[{"node": node_name, "content": [e.message for e in errors]}],
808
+ )
809
+ return {
810
+ "logical_validator_response": response,
811
+ "errors": errors,
812
+ "reasoning": response.reasoning,
813
+ }
814
+
815
+ reasoning = "Logical validation successful."
816
+
817
+ response = LogicalValidatorResponse(
818
+ errors=errors,
819
+ reasoning=[{"node": node_name, "content": reasoning}],
820
+ )
821
+ return {
822
+ "logical_validator_response": response,
823
+ "errors": errors,
824
+ "reasoning": response.reasoning,
825
+ }
826
+
827
+ except Exception as exc:
828
+ logger.exception("Logical Validator crashed")
829
+ error = PipelineError(
830
+ node=node_name,
831
+ message=f"Logical Validator crashed: {exc}",
832
+ severity=ErrorSeverity.ERROR,
833
+ error_code=ErrorCode.VALIDATOR_CRASH,
834
+ stack_trace=traceback.format_exc(),
835
+ )
836
+ return {
837
+ "logical_validator_response": LogicalValidatorResponse(errors=[error]),
838
+ "errors": [error],
839
+ }