sql-dialect-master 1.3.0__tar.gz → 1.4.0__tar.gz

{sql_dialect_master-1.3.0 → sql_dialect_master-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sql-dialect-master
-Version: 1.3.0
+Version: 1.4.0
 Summary: SQL Dialect Master - Multi-database SQL conversion engine
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
@@ -38,7 +38,7 @@ Dynamic: license-file
 
 Enterprise-grade multi-database SQL conversion engine supporting 12 database dialects.
 
-Current release: 1.3.0
+Current release: 1.4.0
 
 [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
@@ -177,6 +177,19 @@ print(result.sql)
 # Output: SELECT * FROM orders WHERE created_at >= DATE_SUB(CURRENT_DATE, 7)
 ```
 
+An optional request `schema` can ground both deterministic and LLM
+generation. It contains `tables` with optional `columns`, plus explicit
+`relationships`. Table and column references that the deterministic parser maps
+to the schema are matched case-insensitively; ambiguous mapped references and
+multi-table requests without a declared relationship fail closed instead of
+falling back to hardcoded table or column names. Empty column metadata means
+that the table's columns are unrestricted. Natural-language column names
+outside the deterministic mapping may be ignored; use `column_hints` or the
+LLM path for those.
+The response field `schema_used` is true when a successful deterministic or LLM
+generation was validated against the supplied schema. With no schema, the
+existing deterministic behavior is unchanged.
+
 #### Optional LLM (OpenRouter)
 
 `POST /api/nl2sql` can optionally call an OpenRouter-compatible chat-completions

{sql_dialect_master-1.3.0 → sql_dialect_master-1.4.0}/README.md

@@ -2,7 +2,7 @@
 
 Enterprise-grade multi-database SQL conversion engine supporting 12 database dialects.
 
-Current release: 1.3.0
+Current release: 1.4.0
 
 [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
@@ -141,6 +141,19 @@ print(result.sql)
 # Output: SELECT * FROM orders WHERE created_at >= DATE_SUB(CURRENT_DATE, 7)
 ```
 
+An optional request `schema` can ground both deterministic and LLM
+generation. It contains `tables` with optional `columns`, plus explicit
+`relationships`. Table and column references that the deterministic parser maps
+to the schema are matched case-insensitively; ambiguous mapped references and
+multi-table requests without a declared relationship fail closed instead of
+falling back to hardcoded table or column names. Empty column metadata means
+that the table's columns are unrestricted. Natural-language column names
+outside the deterministic mapping may be ignored; use `column_hints` or the
+LLM path for those.
+The response field `schema_used` is true when a successful deterministic or LLM
+generation was validated against the supplied schema. With no schema, the
+existing deterministic behavior is unchanged.
+
 #### Optional LLM (OpenRouter)
 
 `POST /api/nl2sql` can optionally call an OpenRouter-compatible chat-completions

{sql_dialect_master-1.3.0 → sql_dialect_master-1.4.0}/backend/api/main.py

@@ -53,6 +53,7 @@ from backend.core.functions_lookup import FunctionEncyclopedia
 from backend.core.nl2sql import NL2SQLGenerator, NL2SQLResult
 from backend.core.nl2sql_components.evidence import EvidenceItem, GenerationEvidence
 from backend.core.nl2sql_llm import LLMError, LLMN2SQLClient, validate_llm_sql
+from backend.core.nl2sql_schema import SchemaContext
 from backend.core.nl2sql_pipeline.orchestrator import SAFE_TABLE_HINT
 from backend.core.parser import SUPPORTED_DIALECTS, SQLParser
 from backend.core.semantic_diff import diff_sql_ast
@@ -468,12 +469,19 @@ class SchemaRelationship(BaseModel):
 
 class SchemaDefinition(BaseModel):
     tables: List[SchemaTable] = Field(default_factory=list, max_length=50)
-    relationships: List[SchemaRelationship] = Field(default_factory=list)
+    relationships: List[SchemaRelationship] = Field(default_factory=list, max_length=200)
 
     @model_validator(mode="after")
     def validate_total_columns(self):
         if sum(len(table.columns) for table in self.tables) > 1000:
             raise ValueError("Schema must contain at most 1000 columns in total")
+        table_names = {table.name.casefold() for table in self.tables}
+        for relationship in self.relationships:
+            if (
+                relationship.from_table.casefold() not in table_names
+                or relationship.to_table.casefold() not in table_names
+            ):
+                raise ValueError("Schema relationship references an unknown table")
         return self
 
 
@@ -979,6 +987,14 @@ async def generate_sql(request: NL2SQLRequest):
                 "dotted identifier"
             ),
         )
+    schema_payload = (
+        request.schema_definition.model_dump(exclude_none=True)
+        if request.schema_definition is not None else None
+    )
+    schema_context = (
+        SchemaContext.from_mapping(schema_payload)
+        if schema_payload is not None else None
+    )
     use_llm = (
         settings.nl2sql_llm_enabled
         and bool(request.text.strip())
@@ -987,18 +1003,25 @@ async def generate_sql(request: NL2SQLRequest):
         and nl2sql_generator._detect_operation(request.text.lower()) not in {"INSERT", "UPDATE", "DELETE"}
     )
     if not use_llm:
-        result = await _run_cpu_bound(
-            nl2sql_generator.generate,
-            request.text,
-            dialect,
-            request.table_hint,
-            request.column_hints,
-        )
+        if schema_context is None:
+            result = await _run_cpu_bound(
+                nl2sql_generator.generate,
+                request.text,
+                dialect,
+                request.table_hint,
+                request.column_hints,
+            )
+        else:
+            result = await _run_cpu_bound(
+                nl2sql_generator.generate,
+                request.text,
+                dialect,
+                request.table_hint,
+                request.column_hints,
+                schema_context,
+            )
     else:
-        schema = (
-            request.schema_definition.model_dump(exclude_none=True)
-            if request.schema_definition is not None else None
-        )
+        schema = schema_payload
         started = time.perf_counter()
         try:
             # Keep the legacy no-schema client call compatible with direct
@@ -1013,17 +1036,32 @@ async def generate_sql(request: NL2SQLRequest):
             record_nl2sql_llm_request(exc.reason, time.perf_counter() - started)
             if settings.nl2sql_llm_fallback:
                 record_nl2sql_llm_fallback()
-                result = copy.copy(await _run_cpu_bound(
-                    nl2sql_generator.generate,
-                    request.text,
-                    dialect,
-                    request.table_hint,
-                    request.column_hints,
-                ))
+                if schema_context is None:
+                    deterministic_result = await _run_cpu_bound(
+                        nl2sql_generator.generate,
+                        request.text,
+                        dialect,
+                        request.table_hint,
+                        request.column_hints,
+                    )
+                else:
+                    deterministic_result = await _run_cpu_bound(
+                        nl2sql_generator.generate,
+                        request.text,
+                        dialect,
+                        request.table_hint,
+                        request.column_hints,
+                        schema_context,
+                    )
+                result = copy.copy(deterministic_result)
                 result.provider = "deterministic"
                 result.model = None
                 result.fallback_reason = exc.reason
-                result.schema_used = False
+                result.schema_used = bool(
+                    schema_context is not None
+                    and deterministic_result.success
+                    and deterministic_result.schema_used
+                )
             else:
                 result = NL2SQLResult(
                     input_text=request.text,

{sql_dialect_master-1.3.0 → sql_dialect_master-1.4.0}/backend/core/metadata.py

@@ -17,8 +17,8 @@ Design rules:
 from __future__ import annotations
 
 # ── Release version facts (sole source of truth) ────────────────────────
-PROJECT_VERSION = "1.3.0"
-API_VERSION = "1.3.0"
+PROJECT_VERSION = "1.4.0"
+API_VERSION = "1.4.0"
 # Frontend v2 architecture — independent of the product release version.
 FRONTEND_ARCHITECTURE_VERSION = 2
 

{sql_dialect_master-1.3.0 → sql_dialect_master-1.4.0}/backend/core/nl2sql_llm.py

@@ -1,19 +1,18 @@
 """Opt-in OpenRouter-compatible generation with strict SQL output validation."""
 from __future__ import annotations
 
-from collections.abc import Iterable, Sequence
 from contextvars import ContextVar
 from dataclasses import dataclass
 import json
 import logging
 import re
-from typing import cast
 
 import httpx
 from sqlglot import exp
-from sqlglot.optimizer.scope import Scope, traverse_scope
+from sqlglot.optimizer.scope import traverse_scope
 
 from .config import AppSettings, settings
+from .nl2sql_schema import SchemaContext, validate_schema_references
 from .sql_statements import parse_single_sql
 
 
@@ -24,7 +23,6 @@ _ERROR_REASONS = frozenset({
 _VALIDATING_LLM_SQL = ContextVar("validating_llm_sql", default=False)
 _SQLGLOT_LOGGER = logging.getLogger("sqlglot")
 _SQL_FENCE = re.compile(r"```(?:sql)?[ \t]*\r?\n(.*?)\r?\n?```", re.IGNORECASE | re.DOTALL)
-_SCHEMA_IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_$]*")
 
 
 class LLMError(Exception):
@@ -48,118 +46,16 @@ class _LLMValidationLogFilter(logging.Filter):
         return not _VALIDATING_LLM_SQL.get()
 
 
-def _schema_reference_name(parts: Sequence[exp.Expr]) -> str:
-    # A quoted literal name such as "public.users" must not be confused
-    # with two separate schema/table identifiers, public.users.
-    if not parts or any(
-        not isinstance(part, exp.Identifier) or not _SCHEMA_IDENTIFIER.fullmatch(part.name)
-        for part in parts
-    ):
-        raise LLMError("invalid_sql")
-    return ".".join(part.name for part in parts).casefold()
-
-
 def _validate_schema_references(statement: exp.Expression, schema: dict) -> None:
-    """Validate physical sources and resolve aliases within their SQL scopes."""
-    table_columns = {
-        table["name"].casefold(): {column["name"].casefold() for column in table.get("columns", [])}
-        for table in schema.get("tables", [])
-    }
-    all_columns = set().union(*table_columns.values()) if table_columns else set()
-    scopes = traverse_scope(statement)
-    sources_by_scope: dict[int, dict[str, exp.Table | Scope]] = {}
-    for scope in scopes:
-        sources: dict[str, exp.Table | Scope] = {}
-        for name, (_, source) in scope.selected_sources.items():
-            alias = name.casefold()
-            if alias in sources:
-                raise LLMError("invalid_sql")
-            sources[alias] = source
-            if isinstance(source, exp.Table):
-                table_name = _schema_reference_name(source.parts)
-                if table_name not in table_columns:
-                    raise LLMError("invalid_sql")
-                # A fully qualified column must match the complete source name,
-                # and cannot resolve an unrelated schema through a shared basename.
-                if not source.alias:
-                    sources[table_name] = source
-        # SEMI/ANTI join sources are omitted by selected_sources, but still
-        # reference physical tables and must obey the same schema constraints.
-        for table in scope.tables:
-            table_source = scope.sources.get(table.alias_or_name)
-            if isinstance(table_source, exp.Table):
-                table_name = _schema_reference_name(table_source.parts)
-                if table_name not in table_columns:
-                    raise LLMError("invalid_sql")
-                sources.setdefault(table.alias_or_name.casefold(), table_source)
-        sources_by_scope[id(scope)] = sources
-
-    def resolve_source(scope: Scope, qualifier: str) -> exp.Table | Scope:
-        current: Scope | None = scope
-        while current is not None:
-            source = sources_by_scope.get(id(current), {}).get(qualifier)
-            if source is not None:
-                return source
-            if not current.can_be_correlated:
-                break
-            current = current.parent
-        raise LLMError("invalid_sql")
-
-    output_cache: dict[int, set[str] | None] = {}
-
-    def source_columns(
-        source: exp.Table | Scope, visiting: frozenset[int] = frozenset(),
-    ) -> set[str] | None:
-        if isinstance(source, exp.Table):
-            name = _schema_reference_name(source.parts)
-            return table_columns[name] or None
-        scope_id = id(source)
-        if scope_id in output_cache:
-            return output_cache[scope_id]
-        if scope_id in visiting:
-            # Recursive projections cannot be grounded without additional
-            # type/lineage analysis. Fail closed instead of guessing.
-            raise LLMError("invalid_sql")
-        visiting = visiting | {scope_id}
-        columns: set[str] | None
-        if source.outer_columns:
-            columns = {name.casefold() for name in source.outer_columns}
-        elif source.union_scopes:
-            columns = source_columns(source.union_scopes[0], visiting)
-        else:
-            columns = set()
-            # Projection expansion expects a query scope; other scope nodes
-            # retain the existing fail-closed behavior in validate_llm_sql.
-            for projection in cast(exp.Query, source.expression).selects:
-                if not projection.is_star:
-                    if projection.alias_or_name:
-                        columns.add(projection.alias_or_name.casefold())
-                    continue
-                projected_sources: Iterable[exp.Table | Scope]
-                if isinstance(projection, exp.Column):
-                    qualifier = _schema_reference_name(projection.parts[:-1])
-                    projected_sources = [resolve_source(source, qualifier)]
-                else:
-                    projected_sources = sources_by_scope[scope_id].values()
-                for projected in projected_sources:
-                    projected_columns = source_columns(projected, visiting)
-                    if projected_columns is None:
-                        output_cache[scope_id] = None
-                        return None
-                    columns.update(projected_columns)
-        output_cache[scope_id] = columns
-        return columns
-
-    for scope in scopes:
-        for column in scope.find_all(exp.Column):
-            if column.table:
-                qualifier = _schema_reference_name(column.parts[:-1])
-                source = resolve_source(scope, qualifier)
-                columns = source_columns(source)
-                if not column.is_star and columns is not None and column.name.casefold() not in columns:
-                    raise LLMError("invalid_sql")
-            elif all_columns and column.name.casefold() not in all_columns:
-                raise LLMError("invalid_sql")
+    """Compatibility wrapper for the shared schema reference validator."""
+    try:
+        validate_schema_references(
+            statement,
+            SchemaContext.from_mapping(schema),
+            scope_traverser=traverse_scope,
+        )
+    except Exception:
+        raise LLMError("invalid_sql") from None
 
 
 def validate_llm_sql(