polyglot-sql-chio 0.12.2__tar.gz → 0.13.0__tar.gz

{polyglot_sql_chio-0.12.2 → polyglot_sql_chio-0.13.0}/Cargo.lock

@@ -605,13 +605,14 @@ dependencies = [
 
 [[package]]
 name = "polyglot-sql"
-version = "0.12.2"
+version = "0.13.0"
 dependencies = [
  "criterion",
  "once_cell",
  "polyglot-sql-ast-derive",
  "polyglot-sql-function-catalogs",
  "pretty_assertions",
+ "regex",
  "serde",
  "serde_json",
  "stacker",
@@ -623,7 +624,7 @@ dependencies = [
 
 [[package]]
 name = "polyglot-sql-ast-derive"
-version = "0.12.2"
+version = "0.13.0"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -632,7 +633,7 @@ dependencies = [
 
 [[package]]
 name = "polyglot-sql-ffi"
-version = "0.12.2"
+version = "0.13.0"
 dependencies = [
  "cbindgen",
  "polyglot-sql",
@@ -642,11 +643,11 @@ dependencies = [
 
 [[package]]
 name = "polyglot-sql-function-catalogs"
-version = "0.12.2"
+version = "0.13.0"
 
 [[package]]
 name = "polyglot-sql-python"
-version = "0.12.2"
+version = "0.13.0"
 dependencies = [
  "polyglot-sql",
  "pyo3",
@@ -657,7 +658,7 @@ dependencies = [
 
 [[package]]
 name = "polyglot-sql-wasm"
-version = "0.12.2"
+version = "0.13.0"
 dependencies = [
  "console_error_panic_hook",
  "js-sys",

{polyglot_sql_chio-0.12.2 → polyglot_sql_chio-0.13.0}/Cargo.toml

@@ -6,7 +6,7 @@ exclude = [
 ]
 
 [workspace.package]
-version = "0.12.2"
+version = "0.13.0"
 edition = "2021"
 license = "MIT"
 authors = ["polyglot contributors"]

{polyglot_sql_chio-0.12.2 → polyglot_sql_chio-0.13.0}/PKG-INFO

@@ -1,15 +1,12 @@
 Metadata-Version: 2.4
 Name: polyglot-sql-chio
-Version: 0.12.2
+Version: 0.13.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Database
@@ -18,7 +15,7 @@ Summary: Rust-powered SQL transpiler for more than 30 SQL dialects. Parse, gener
 Keywords: sql,transpiler,parser,dialect,database
 Home-Page: https://github.com/tobilg/polyglot
 License: MIT
-Requires-Python: >=3.9
+Requires-Python: >=3.12
 Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 Project-URL: Documentation, https://polyglot.gh.tobilg.com
 Project-URL: Homepage, https://github.com/chio-labs/polyglot

{polyglot_sql_chio-0.12.2 → polyglot_sql_chio-0.13.0}/crates/polyglot-sql/Cargo.toml

@@ -28,7 +28,7 @@ generate = []
 transpile = ["generate"]
 builder = ["generate"]
 ast-tools = []
-semantic = ["ast-tools"]
+semantic = ["ast-tools", "function-catalog-duckdb", "dep:regex"]
 openlineage = ["semantic"]
 diff = ["generate"]
 planner = []
@@ -100,14 +100,15 @@ function-catalog-all-dialects = [
 ]
 
 [dependencies]
-polyglot-sql-ast-derive = { path = "../polyglot-sql-ast-derive", version = "0.12.2" }
+regex = { version = "1", optional = true }
+polyglot-sql-ast-derive = { path = "../polyglot-sql-ast-derive", version = "0.13.0" }
 serde = { workspace = true }
 serde_json = { workspace = true }
 thiserror = { workspace = true }
 unicode-segmentation = { workspace = true }
 stacker = { version = "0.1", optional = true }
 ts-rs = { version = "12.0", features = ["serde-compat"], optional = true }
-polyglot-sql-function-catalogs = { path = "../polyglot-sql-function-catalogs", version = "0.12.2", optional = true, default-features = false }
+polyglot-sql-function-catalogs = { path = "../polyglot-sql-function-catalogs", version = "0.13.0", optional = true, default-features = false }
 
 [dev-dependencies]
 pretty_assertions = "1.4"

{polyglot_sql_chio-0.12.2 → polyglot_sql_chio-0.13.0}/crates/polyglot-sql/README.md

@@ -25,12 +25,12 @@ By default, `polyglot-sql` enables the full public API. Parser-only consumers ca
 disable default features and opt into only the dialect parsers they need:
 
 ```toml
-polyglot-sql = { version = "0.12.2", default-features = false }
+polyglot-sql = { version = "0.13.0", default-features = false }
 ```
 
 ```toml
 polyglot-sql = {
-    version = "0.12.2",
+    version = "0.13.0",
     default-features = false,
     features = ["dialect-clickhouse"],
 }
@@ -44,14 +44,14 @@ Examples:
 ```toml
 # Parse and generate SQL for one dialect.
 polyglot-sql = {
-    version = "0.12.2",
+    version = "0.13.0",
     default-features = false,
     features = ["generate", "dialect-clickhouse"],
 }
 
 # Cross-dialect transpilation.
 polyglot-sql = {
-    version = "0.12.2",
+    version = "0.13.0",
     default-features = false,
     features = ["transpile", "dialect-clickhouse", "dialect-postgresql"],
 }

polyglot_sql_chio-0.13.0/crates/polyglot-sql/examples/validate_batch.rs

@@ -0,0 +1,22 @@
+//! Validate newline-delimited schema-aware requests from standard input.
+use polyglot_sql::validation::{validate_with_schema, SchemaValidationOptions, ValidationSchema};
+use polyglot_sql::DialectType;
+use serde::Deserialize;
+use std::io::{self, BufRead};
+
+#[derive(Deserialize)]
+struct Request {
+    sql: String,
+    dialect: String,
+    schema: ValidationSchema,
+    options: SchemaValidationOptions,
+}
+
+fn main() {
+    for line in io::stdin().lock().lines() {
+        let request: Request = serde_json::from_str(&line.unwrap()).unwrap();
+        let dialect: DialectType = request.dialect.parse().unwrap();
+        let result = validate_with_schema(&request.sql, dialect, &request.schema, &request.options);
+        println!("{}", serde_json::to_string(&result).unwrap());
+    }
+}

{polyglot_sql_chio-0.12.2 → polyglot_sql_chio-0.13.0}/crates/polyglot-sql/src/lineage.rs

@@ -714,13 +714,7 @@ fn expand_star_from_sources(
         let qual_normalized = normalize_cte_name(qual);
         for src in sources {
             if src.normalized == qual_normalized || src.alias.to_lowercase() == qual_normalized {
-                // Try CTE first
-                if let Some(cols) = resolved_ctes.get(&src.normalized) {
-                    expanded.extend(cols.iter().map(|c| (src.alias.clone(), c.clone())));
-                    return Some(expanded);
-                }
-                // Fall back to schema
-                if let Some(cols) = lookup_schema_columns(schema, &src.fq_name) {
+                if let Some(cols) = source_output_columns(src, resolved_ctes, schema) {
                     expanded.extend(cols.into_iter().map(|c| (src.alias.clone(), c)));
                     return Some(expanded);
                 }
@@ -735,15 +729,9 @@ fn expand_star_from_sources(
         // This matches sqlglot's behavior (raises SqlglotError when schema is missing).
         let mut any_expanded = false;
         for src in sources {
-            if let Some(cols) = resolved_ctes.get(&src.normalized) {
-                expanded.extend(cols.iter().map(|c| (src.alias.clone(), c.clone())));
-                any_expanded = true;
-            } else if let Some(cols) = lookup_schema_columns(schema, &src.fq_name) {
-                expanded.extend(cols.into_iter().map(|c| (src.alias.clone(), c)));
-                any_expanded = true;
-            } else {
-                return None;
-            }
+            let cols = source_output_columns(src, resolved_ctes, schema)?;
+            expanded.extend(cols.into_iter().map(|c| (src.alias.clone(), c)));
+            any_expanded = true;
         }
         if any_expanded {
             Some(expanded)
@@ -753,6 +741,57 @@ fn expand_star_from_sources(
     }
 }
 
+/// Output columns of a FROM/JOIN source, preferring resolved CTEs over the schema.
+fn source_output_columns(
+    src: &SourceInfo,
+    resolved_ctes: &HashMap<String, Vec<String>>,
+    schema: Option<&dyn Schema>,
+) -> Option<Vec<String>> {
+    match &src.relation {
+        SourceRelation::Plain => resolved_ctes
+            .get(&src.normalized)
+            .cloned()
+            .or_else(|| lookup_schema_columns(schema, &src.fq_name)),
+        SourceRelation::Pivot(pivot) if !pivot.unpivot => {
+            let input = pivot_input_columns(&pivot.this, resolved_ctes, schema)?;
+            let output = pivot_output_columns(pivot, &input, None);
+            (!output.is_empty()).then(|| output.into_iter().map(|(post, _)| post).collect())
+        }
+        SourceRelation::Unpivot(unpivot) => {
+            let input = pivot_input_columns(&unpivot.this, resolved_ctes, schema)?;
+            let output = crate::resolver::unpivot_output_columns(unpivot, input, None);
+            (!output.is_empty()).then_some(output)
+        }
+        SourceRelation::Pivot(_) => None,
+    }
+}
+
+fn pivot_input_columns(
+    input: &Expression,
+    resolved_ctes: &HashMap<String, Vec<String>>,
+    schema: Option<&dyn Schema>,
+) -> Option<Vec<String>> {
+    match input {
+        Expression::Table(table) => {
+            let cte_columns = (table.schema.is_none() && table.catalog.is_none())
+                .then(|| resolved_ctes.get(&normalize_cte_name(&table.name)).cloned())
+                .flatten();
+            cte_columns.or_else(|| {
+                let fq_name = [&table.catalog, &table.schema]
+                    .into_iter()
+                    .flatten()
+                    .map(|part| part.name.as_str())
+                    .chain(std::iter::once(table.name.name.as_str()))
+                    .collect::<Vec<_>>()
+                    .join(".");
+                lookup_schema_columns(schema, &fq_name)
+            })
+        }
+        Expression::Paren(paren) => pivot_input_columns(&paren.this, resolved_ctes, schema),
+        _ => None,
+    }
+}
+
 /// Look up column names for a table from the schema.
 fn lookup_schema_columns(schema: Option<&dyn Schema>, fq_name: &str) -> Option<Vec<String>> {
     let schema = schema?;
@@ -802,6 +841,16 @@ struct SourceInfo {
     normalized: String,
     /// Fully-qualified table name for schema lookup (e.g., "db.schema.table").
     fq_name: String,
+    /// Relation shape that determines how this source's output columns are derived.
+    relation: SourceRelation,
+}
+
+/// PIVOT and UNPIVOT change their input's columns, so their output cannot be
+/// read from the CTE or table that happens to share the source name.
+enum SourceRelation {
+    Plain,
+    Pivot(Box<crate::expressions::Pivot>),
+    Unpivot(Box<crate::expressions::Unpivot>),
 }
 
 /// Extract source info (alias, normalized CTE name, fully-qualified name) from a
@@ -816,6 +865,7 @@ fn get_select_sources(select: &Select) -> Vec<SourceInfo> {
                 quoted: alias.quoted,
                 normalized: normalize_cte_name(alias),
                 fq_name: alias.name.clone(),
+                relation: SourceRelation::Plain,
             }
         }
 
@@ -825,6 +875,7 @@ fn get_select_sources(select: &Select) -> Vec<SourceInfo> {
                 quoted: false,
                 normalized: alias.to_lowercase(),
                 fq_name: alias.to_string(),
+                relation: SourceRelation::Plain,
             }
         }
 
@@ -850,6 +901,7 @@ fn get_select_sources(select: &Select) -> Vec<SourceInfo> {
                     quoted: t.name.quoted,
                     normalized,
                     fq_name,
+                    relation: SourceRelation::Plain,
                 })
             }
             Expression::Subquery(s) => {
@@ -862,6 +914,7 @@ fn get_select_sources(select: &Select) -> Vec<SourceInfo> {
                     quoted: alias_identifier.quoted,
                     normalized,
                     fq_name,
+                    relation: SourceRelation::Plain,
                 })
             }
             Expression::Unnest(u) => u.alias.as_ref().map(virtual_source_info),
@@ -887,6 +940,7 @@ fn get_select_sources(select: &Select) -> Vec<SourceInfo> {
                     quoted: false,
                     normalized: alias.to_lowercase(),
                     fq_name: alias,
+                    relation: SourceRelation::Pivot(pivot.clone()),
                 })
             }
             Expression::Unpivot(unpivot) => {
@@ -899,6 +953,7 @@ fn get_select_sources(select: &Select) -> Vec<SourceInfo> {
                     quoted: false,
                     normalized: alias.to_lowercase(),
                     fq_name: alias,
+                    relation: SourceRelation::Unpivot(unpivot.clone()),
                 })
             }
             Expression::Paren(p) => extract_source(&p.this),

{polyglot_sql_chio-0.12.2 → polyglot_sql_chio-0.13.0}/crates/polyglot-sql/src/optimizer/annotate_types.rs

@@ -467,10 +467,7 @@ impl<'a> TypeAnnotator<'a> {
             Expression::Count(_) => Some(DataType::BigInt { length: None }),
             Expression::Sum(agg) => self.annotate_sum(&agg.this),
             Expression::SumIf(f) => self.annotate_sum(&f.this),
-            Expression::Avg(_) => Some(DataType::Double {
-                precision: None,
-                scale: None,
-            }),
+            Expression::Avg(agg) => self.annotate_avg(&agg.this),
             Expression::Min(agg) => self.annotate(&agg.this),
             Expression::Max(agg) => self.annotate(&agg.this),
             Expression::Median(agg) => {
@@ -1032,6 +1029,12 @@ impl<'a> TypeAnnotator<'a> {
             // WindowFunction
             Expression::WindowFunction(w) => {
                 self.annotate_in_place(&mut w.this);
+                for expression in &mut w.over.partition_by {
+                    self.annotate_in_place(expression);
+                }
+                for ordered in &mut w.over.order_by {
+                    self.annotate_in_place(&mut ordered.this);
+                }
             }
 
             // Subquery
@@ -1418,11 +1421,20 @@ impl<'a> TypeAnnotator<'a> {
         }
         let func_name = func.name.to_uppercase();
 
+        if self._dialect == Some(DialectType::PostgreSQL) && !func.quoted {
+            match func_name.as_str() {
+                "SUM" => return func.args.first().and_then(|arg| self.annotate_sum(arg)),
+                "AVG" => return func.args.first().and_then(|arg| self.annotate_avg(arg)),
+                _ => {}
+            }
+        }
+
         if self._dialect == Some(DialectType::DuckDB) && !func.quoted {
             match func_name.as_str() {
                 // Builder-created Function nodes use the same inference as the
                 // dedicated nodes produced when parsing these built-ins.
                 "SUM" => return func.args.first().and_then(|arg| self.annotate_sum(arg)),
+                "AVG" => return func.args.first().and_then(|arg| self.annotate_avg(arg)),
                 "COALESCE" | "IFNULL" => return self.coerce_arg_types(&func.args),
                 "DATE_TRUNC" => return self.annotate_duckdb_date_trunc(func),
                 "REGEXP_EXTRACT_ALL" => return self.annotate_duckdb_regexp_extract_all(func),
@@ -1469,6 +1481,36 @@ impl<'a> TypeAnnotator<'a> {
                 }
             }
             _ => {
+                if let Some(signature) = self._dialect.and_then(|dialect| {
+                    polyglot_sql_function_catalogs::types::type_signature(
+                        crate::validation::signatures::dialect_key(dialect),
+                        &func_name.to_lowercase(),
+                    )
+                }) {
+                    use polyglot_sql_function_catalogs::types::ReturnType;
+                    return match signature.returns {
+                        ReturnType::Average => {
+                            func.args.first().and_then(|arg| self.annotate_avg(arg))
+                        }
+                        ReturnType::Argument(index) => {
+                            func.args.get(index).and_then(|arg| self.annotate(arg))
+                        }
+                        ReturnType::Integer => Some(DataType::BigInt { length: None }),
+                        ReturnType::Numeric => Some(DataType::Double {
+                            precision: None,
+                            scale: None,
+                        }),
+                        ReturnType::String => Some(DataType::VarChar {
+                            length: None,
+                            parenthesized_length: false,
+                        }),
+                        ReturnType::Boolean => Some(DataType::Boolean),
+                        ReturnType::Timestamp => Some(DataType::Timestamp {
+                            precision: None,
+                            timezone: false,
+                        }),
+                    };
+                }
                 self.used_unknown_function_fallback = true;
                 func.args.first().and_then(|arg| self.annotate(arg))
             }
@@ -1477,7 +1519,7 @@ impl<'a> TypeAnnotator<'a> {
 
     /// Relation outputs cannot rely on the scalar annotator's legacy fallback
     /// that assigns an unknown function the type of its first argument.
-    pub(super) fn function_result_is_known(func: &Function, dialect: DialectType) -> bool {
+    pub(crate) fn function_result_is_known(func: &Function, dialect: DialectType) -> bool {
         let mut annotator = Self::new(None, Some(dialect));
         // Probe dispatch without traversing or cloning a query argument. The
         // original annotation already supplied the type for recognized rules.
@@ -1713,8 +1755,40 @@ impl<'a> TypeAnnotator<'a> {
     }
 
     /// Annotate SUM function - promotes to at least BigInt
+    fn annotate_avg(&mut self, arg: &Expression) -> Option<DataType> {
+        if matches!(
+            self._dialect,
+            Some(DialectType::PostgreSQL | DialectType::DuckDB)
+        ) {
+            if let Some(ty @ DataType::Interval { .. }) = self.annotate(arg) {
+                return Some(ty);
+            }
+        }
+        if self._dialect == Some(DialectType::DuckDB) {
+            match self.annotate(arg) {
+                Some(DataType::Date) => {
+                    return Some(DataType::Timestamp {
+                        precision: None,
+                        timezone: false,
+                    })
+                }
+                Some(ty @ (DataType::Timestamp { .. } | DataType::Time { .. })) => return Some(ty),
+                _ => {}
+            }
+        }
+        Some(DataType::Double {
+            precision: None,
+            scale: None,
+        })
+    }
+
     fn annotate_sum(&mut self, arg: &Expression) -> Option<DataType> {
         let arg_type = self.annotate(arg);
+        if self._dialect == Some(DialectType::PostgreSQL)
+            && matches!(arg_type, Some(DataType::Interval { .. }))
+        {
+            return arg_type;
+        }
         if self._dialect == Some(DialectType::DuckDB) {
             // These are DuckDB's bound aggregate result types, not the generic
             // integer-to-BIGINT promotion. UHUGEINT binds to the DOUBLE overload.

{polyglot_sql_chio-0.12.2 → polyglot_sql_chio-0.13.0}/crates/polyglot-sql/src/resolver.rs

@@ -315,7 +315,72 @@ impl<'a> Resolver<'a> {
         select
             .expressions
             .iter()
-            .filter_map(|expr| self.get_expression_alias(expr))
+            .flat_map(|expr| {
+                let Expression::Star(star) = expr else {
+                    return self
+                        .get_expression_alias(expr)
+                        .into_iter()
+                        .collect::<Vec<_>>();
+                };
+                let mut columns = Vec::new();
+                for source in select
+                    .from
+                    .iter()
+                    .flat_map(|from| &from.expressions)
+                    .chain(select.joins.iter().map(|join| &join.this))
+                {
+                    if let Some(qualifier) = &star.table {
+                        let name = match source {
+                            Expression::Table(table) => {
+                                Some(table.alias.as_ref().unwrap_or(&table.name))
+                            }
+                            Expression::Subquery(query) => query.alias.as_ref(),
+                            Expression::Alias(alias) => Some(&alias.alias),
+                            _ => None,
+                        };
+                        if !name.is_some_and(|name| {
+                            crate::set_operation::identifier_key(name, self.dialect)
+                                == crate::set_operation::identifier_key(qualifier, self.dialect)
+                        }) {
+                            continue;
+                        }
+                    }
+                    let input = self
+                        .get_source_columns_for_expression(source)
+                        .unwrap_or_default();
+                    if input.is_empty() {
+                        columns.push("*".to_owned());
+                    } else {
+                        columns.extend(input);
+                    }
+                }
+                if columns.is_empty() {
+                    columns.push("*".to_owned());
+                }
+                let column_key = |column: &str| {
+                    crate::set_operation::identifier_key(
+                        &crate::binding::schema_identifier(column),
+                        self.dialect,
+                    )
+                };
+                columns.retain(|column| {
+                    !star.except.iter().flatten().any(|name| {
+                        crate::set_operation::identifier_key(name, self.dialect)
+                            == column_key(column)
+                    })
+                });
+                for column in &mut columns {
+                    if let Some((_, replacement)) =
+                        star.rename.iter().flatten().find(|(name, _)| {
+                            crate::set_operation::identifier_key(name, self.dialect)
+                                == column_key(column)
+                        })
+                    {
+                        *column = replacement.name.clone();
+                    }
+                }
+                columns
+            })
             .collect()
     }
 
@@ -387,29 +452,7 @@ impl<'a> Resolver<'a> {
 
     fn get_unpivot_output_columns(&self, unpivot: &crate::expressions::Unpivot) -> Vec<String> {
         let pre_columns = self.get_source_output_columns(&unpivot.this);
-        if pre_columns.is_empty() || pre_columns.iter().any(|column| column == "*") {
-            return Vec::new();
-        }
-
-        let input_columns: HashSet<String> = unpivot
-            .columns
-            .iter()
-            .flat_map(expression_column_names)
-            .map(|column| normalize_column_name(&column, self.dialect))
-            .collect();
-        let mut columns: Vec<String> = pre_columns
-            .into_iter()
-            .filter(|column| !input_columns.contains(&normalize_column_name(column, self.dialect)))
-            .collect();
-        columns.push(unpivot.name_column.name.clone());
-        columns.push(unpivot.value_column.name.clone());
-        columns.extend(
-            unpivot
-                .extra_value_columns
-                .iter()
-                .map(|column| column.name.clone()),
-        );
-        apply_alias_columns(columns, &unpivot.alias_columns)
+        unpivot_output_columns(unpivot, pre_columns, self.dialect)
     }
 
     fn get_source_output_columns(&self, source: &Expression) -> Vec<String> {
@@ -657,6 +700,38 @@ fn normalize_column_name(name: &str, dialect: Option<DialectType>) -> String {
     normalize_name(name, dialect, false, true)
 }
 
+/// UNPIVOT output: unconsumed input columns, then name and value columns.
+/// Returns no columns when the input columns are unknown.
+pub(crate) fn unpivot_output_columns(
+    unpivot: &crate::expressions::Unpivot,
+    pre_columns: Vec<String>,
+    dialect: Option<DialectType>,
+) -> Vec<String> {
+    if pre_columns.is_empty() || pre_columns.iter().any(|column| column == "*") {
+        return Vec::new();
+    }
+
+    let input_columns: HashSet<String> = unpivot
+        .columns
+        .iter()
+        .flat_map(expression_column_names)
+        .map(|column| normalize_column_name(&column, dialect))
+        .collect();
+    let mut columns: Vec<String> = pre_columns
+        .into_iter()
+        .filter(|column| !input_columns.contains(&normalize_column_name(column, dialect)))
+        .collect();
+    columns.push(unpivot.name_column.name.clone());
+    columns.push(unpivot.value_column.name.clone());
+    columns.extend(
+        unpivot
+            .extra_value_columns
+            .iter()
+            .map(|column| column.name.clone()),
+    );
+    apply_alias_columns(columns, &unpivot.alias_columns)
+}
+
 fn apply_alias_columns(mut columns: Vec<String>, alias_columns: &[Identifier]) -> Vec<String> {
     for (idx, alias) in alias_columns.iter().enumerate() {
         if let Some(column) = columns.get_mut(idx) {