polyglot-sql 0.6.3__tar.gz → 0.8.0__tar.gz

{polyglot_sql-0.6.3 → polyglot_sql-0.8.0}/Cargo.lock

@@ -605,7 +605,7 @@ dependencies = [
 
 [[package]]
 name = "polyglot-sql"
-version = "0.6.3"
+version = "0.8.0"
 dependencies = [
  "criterion",
  "once_cell",
@@ -623,7 +623,7 @@ dependencies = [
 
 [[package]]
 name = "polyglot-sql-ast-derive"
-version = "0.6.3"
+version = "0.8.0"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -632,7 +632,7 @@ dependencies = [
 
 [[package]]
 name = "polyglot-sql-ffi"
-version = "0.6.3"
+version = "0.8.0"
 dependencies = [
  "cbindgen",
  "polyglot-sql",
@@ -642,11 +642,11 @@ dependencies = [
 
 [[package]]
 name = "polyglot-sql-function-catalogs"
-version = "0.6.3"
+version = "0.8.0"
 
 [[package]]
 name = "polyglot-sql-python"
-version = "0.6.3"
+version = "0.8.0"
 dependencies = [
  "polyglot-sql",
  "pyo3",
@@ -657,7 +657,7 @@ dependencies = [
 
 [[package]]
 name = "polyglot-sql-wasm"
-version = "0.6.3"
+version = "0.8.0"
 dependencies = [
  "console_error_panic_hook",
  "js-sys",

{polyglot_sql-0.6.3 → polyglot_sql-0.8.0}/Cargo.toml

@@ -6,7 +6,7 @@ exclude = [
 ]
 
 [workspace.package]
-version = "0.6.3"
+version = "0.8.0"
 edition = "2021"
 license = "MIT"
 authors = ["polyglot contributors"]

{polyglot_sql-0.6.3 → polyglot_sql-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polyglot-sql
-Version: 0.6.3
+Version: 0.8.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
@@ -177,6 +177,13 @@ Function-like projections may include `transformFunction` with the function
 name, literal arguments, and column arguments, for example for
 `DATE_TRUNC('month', created_at)`.
 
+Each `analysis["setOperations"][...]["branches"]` entry includes a `role` of
+`"value"` or `"filter"`. Lineage results attach optional `set_branch` metadata
+to immediate set-operation branch roots with the `operator`, original
+zero-based `ordinal`, and `all` flag; omitted branches do not renumber the
+surviving nodes. In OpenLineage output, `EXCEPT` and `INTERSECT` right-hand
+inputs are emitted as indirect `FILTER` dependencies.
+
 Validation schema dictionaries use:
 
 ```python
@@ -222,6 +229,11 @@ All functions are exported from `polyglot_sql`.
 - `validate(sql: str, dialect: str = "generic", *, strict_syntax: bool = False, semantic: bool = False) -> ValidationResult`
 - `optimize(sql: str, dialect: str = "generic") -> str`
 - `lineage(column: str, sql: str, dialect: str = "generic") -> dict`
+- `lineage_at(ordinal: int, sql: str, dialect: str = "generic") -> dict`
+- `lineage_at_with_schema(ordinal: int, sql: str, schema: dict, dialect: str = "generic") -> dict`
+- `lineage_with_schema(column: str, sql: str, schema: dict, dialect: str = "generic") -> dict`
+- `output_columns(sql: str, dialect: str = "generic") -> dict`
+- `output_columns_with_schema(sql: str, schema: dict, dialect: str = "generic") -> dict`
 - `source_tables(column: str, sql: str, dialect: str = "generic") -> list[str]`
 - `analyze_query(sql: str, options: dict | None = None, dialect: str = "generic") -> dict`
 - `openlineage_column_lineage(sql: str, options: dict) -> dict`
@@ -246,6 +258,7 @@ Exception hierarchy:
 - `GenerateError`
 - `TranspileError`
 - `ValidationError`
+- `ColumnResolutionError` (`reason`, `column`, and `ordinal` attributes)
 
 Unknown dialect names raise built-in `ValueError`.
 

{polyglot_sql-0.6.3/crates/polyglot-sql-python → polyglot_sql-0.8.0}/README.md

@@ -150,6 +150,13 @@ Function-like projections may include `transformFunction` with the function
 name, literal arguments, and column arguments, for example for
 `DATE_TRUNC('month', created_at)`.
 
+Each `analysis["setOperations"][...]["branches"]` entry includes a `role` of
+`"value"` or `"filter"`. Lineage results attach optional `set_branch` metadata
+to immediate set-operation branch roots with the `operator`, original
+zero-based `ordinal`, and `all` flag; omitted branches do not renumber the
+surviving nodes. In OpenLineage output, `EXCEPT` and `INTERSECT` right-hand
+inputs are emitted as indirect `FILTER` dependencies.
+
 Validation schema dictionaries use:
 
 ```python
@@ -195,6 +202,11 @@ All functions are exported from `polyglot_sql`.
 - `validate(sql: str, dialect: str = "generic", *, strict_syntax: bool = False, semantic: bool = False) -> ValidationResult`
 - `optimize(sql: str, dialect: str = "generic") -> str`
 - `lineage(column: str, sql: str, dialect: str = "generic") -> dict`
+- `lineage_at(ordinal: int, sql: str, dialect: str = "generic") -> dict`
+- `lineage_at_with_schema(ordinal: int, sql: str, schema: dict, dialect: str = "generic") -> dict`
+- `lineage_with_schema(column: str, sql: str, schema: dict, dialect: str = "generic") -> dict`
+- `output_columns(sql: str, dialect: str = "generic") -> dict`
+- `output_columns_with_schema(sql: str, schema: dict, dialect: str = "generic") -> dict`
 - `source_tables(column: str, sql: str, dialect: str = "generic") -> list[str]`
 - `analyze_query(sql: str, options: dict | None = None, dialect: str = "generic") -> dict`
 - `openlineage_column_lineage(sql: str, options: dict) -> dict`
@@ -219,6 +231,7 @@ Exception hierarchy:
 - `GenerateError`
 - `TranspileError`
 - `ValidationError`
+- `ColumnResolutionError` (`reason`, `column`, and `ordinal` attributes)
 
 Unknown dialect names raise built-in `ValueError`.
 

{polyglot_sql-0.6.3 → polyglot_sql-0.8.0}/crates/polyglot-sql/Cargo.toml

@@ -100,14 +100,14 @@ function-catalog-all-dialects = [
 ]
 
 [dependencies]
-polyglot-sql-ast-derive = { path = "../polyglot-sql-ast-derive", version = "0.6.3" }
+polyglot-sql-ast-derive = { path = "../polyglot-sql-ast-derive", version = "0.8.0" }
 serde = { workspace = true }
 serde_json = { workspace = true }
 thiserror = { workspace = true }
 unicode-segmentation = { workspace = true }
 stacker = { version = "0.1", optional = true }
 ts-rs = { version = "12.0", features = ["serde-compat"], optional = true }
-polyglot-sql-function-catalogs = { path = "../polyglot-sql-function-catalogs", version = "0.6.3", optional = true, default-features = false }
+polyglot-sql-function-catalogs = { path = "../polyglot-sql-function-catalogs", version = "0.8.0", optional = true, default-features = false }
 
 [dev-dependencies]
 pretty_assertions = "1.4"

{polyglot_sql-0.6.3 → polyglot_sql-0.8.0}/crates/polyglot-sql/README.md

@@ -25,12 +25,12 @@ By default, `polyglot-sql` enables the full public API. Parser-only consumers ca
 disable default features and opt into only the dialect parsers they need:
 
 ```toml
-polyglot-sql = { version = "0.6.3", default-features = false }
+polyglot-sql = { version = "0.8.0", default-features = false }
 ```
 
 ```toml
 polyglot-sql = {
-    version = "0.5",
+    version = "0.8.0",
     default-features = false,
     features = ["dialect-clickhouse"],
 }
@@ -44,14 +44,14 @@ Examples:
 ```toml
 # Parse and generate SQL for one dialect.
 polyglot-sql = {
-    version = "0.5",
+    version = "0.8.0",
     default-features = false,
     features = ["generate", "dialect-clickhouse"],
 }
 
 # Cross-dialect transpilation.
 polyglot-sql = {
-    version = "0.5",
+    version = "0.8.0",
     default-features = false,
     features = ["transpile", "dialect-clickhouse", "dialect-postgresql"],
 }
@@ -332,6 +332,13 @@ and CTE output columns. `star_projections` reports the original top-level star
 projection index, optional table qualifier, and schema-expanded columns when
 known. Each projection also includes conservative `nullability`:
 `non_null`, `nullable`, or `unknown`.
+Each `set_operations` branch includes a semantic `role`: both `UNION` branches
+are `value`, while the right branch of `EXCEPT` and `INTERSECT` is `filter`.
+
+Lineage nodes at immediate set-operation branch roots expose optional
+`set_branch` metadata with the operator, original zero-based branch ordinal,
+and `ALL` flag. The ordinal remains stable when another branch cannot be
+resolved.
 
 ```rust
 use polyglot_sql::{

{polyglot_sql-0.6.3 → polyglot_sql-0.8.0}/crates/polyglot-sql/src/dialects/duckdb.rs

@@ -15,7 +15,7 @@ use crate::expressions::{
     AggFunc, Alias, BinaryOp, Case, Cast, CeilFunc, Column, DataType, Expression, Function,
     Identifier, Interval, IntervalUnit, IntervalUnitSpec, IsNull, JSONPath, JSONPathKey,
     JSONPathRoot, JSONPathSubscript, JsonExtractFunc, Literal, Null, Paren, Struct, Subquery,
-    SubstringFunc, UnaryFunc, UnaryOp, VarArgFunc, WindowFunction,
+    SubstringFunc, UnaryFunc, UnaryOp, Unhex, VarArgFunc, WindowFunction,
 };
 #[cfg(feature = "generate")]
 use crate::generator::GeneratorConfig;
@@ -1687,6 +1687,14 @@ impl DuckDBDialect {
     fn transform_function(&self, f: Function) -> Result<Expression> {
         let name_upper = f.name.to_uppercase();
         match name_upper.as_str() {
+            "FROM_HEX" if f.args.len() == 1 => {
+                let mut args = f.args;
+                Ok(Expression::Unhex(Box::new(Unhex {
+                    this: Box::new(args.remove(0)),
+                    expression: None,
+                })))
+            }
+
             // IFNULL -> COALESCE
             "IFNULL" if f.args.len() == 2 => Ok(Expression::Coalesce(Box::new(VarArgFunc {
                 original_name: None,
@@ -1968,14 +1976,6 @@ impl DuckDBDialect {
                 f.args,
             )))),
 
-            // NOW -> CURRENT_TIMESTAMP
-            "NOW" => Ok(Expression::CurrentTimestamp(
-                crate::expressions::CurrentTimestamp {
-                    precision: None,
-                    sysdate: false,
-                },
-            )),
-
             // GETDATE -> CURRENT_TIMESTAMP
             "GETDATE" => Ok(Expression::CurrentTimestamp(
                 crate::expressions::CurrentTimestamp {

{polyglot_sql-0.6.3 → polyglot_sql-0.8.0}/crates/polyglot-sql/src/dialects/mod.rs

@@ -580,6 +580,21 @@ fn presto_to_java_format(fmt: &str) -> String {
         .replace("%M", "MMMM")
 }
 
+#[cfg(feature = "transpile")]
+fn presto_to_java_parse_format(fmt: &str) -> String {
+    fmt.replace("%Y", "yyyy")
+        .replace("%m", "M")
+        .replace("%d", "d")
+        .replace("%H", "H")
+        .replace("%i", "m")
+        .replace("%S", "s")
+        .replace("%s", "s")
+        .replace("%y", "yy")
+        .replace("%T", "H:m:s")
+        .replace("%F", "yyyy-M-d")
+        .replace("%M", "MMMM")
+}
+
 #[cfg(feature = "transpile")]
 fn normalize_presto_format(fmt: &str) -> String {
     fmt.replace("%H:%i:%S", "%T").replace("%H:%i:%s", "%T")

{polyglot_sql-0.6.3 → polyglot_sql-0.8.0}/crates/polyglot-sql/src/dialects/normalization/aggregates.rs

@@ -632,12 +632,9 @@ pub(super) fn rewrite(
 
             Action::ArrayAggToCollectList => {
                 // ARRAY_AGG(x ORDER BY ...) -> COLLECT_LIST(x) for Hive/Spark
-                // Python sqlglot Hive.arrayagg_sql strips ORDER BY for simple cases
-                // but preserves it when DISTINCT/IGNORE NULLS/LIMIT are present
                 match e {
                     Expression::AggregateFunction(mut af) => {
-                        let is_simple =
-                            !af.distinct && af.ignore_nulls.is_none() && af.limit.is_none();
+                        let preserves_order = af.limit.is_some();
                         let args = if af.args.is_empty() {
                             vec![]
                         } else {
@@ -645,24 +642,23 @@ pub(super) fn rewrite(
                         };
                         af.name = "COLLECT_LIST".to_string();
                         af.args = args;
-                        if is_simple {
+                        if !preserves_order {
                             af.order_by = Vec::new();
                         }
                         Ok(Expression::AggregateFunction(af))
                     }
                     Expression::ArrayAgg(agg) => {
-                        let is_simple =
-                            !agg.distinct && agg.ignore_nulls.is_none() && agg.limit.is_none();
+                        let preserves_order = agg.limit.is_some();
                         Ok(Expression::AggregateFunction(Box::new(
                             crate::expressions::AggregateFunction {
                                 name: "COLLECT_LIST".to_string(),
                                 args: vec![agg.this.clone()],
                                 distinct: agg.distinct,
                                 filter: agg.filter.clone(),
-                                order_by: if is_simple {
-                                    Vec::new()
-                                } else {
+                                order_by: if preserves_order {
                                     agg.order_by.clone()
+                                } else {
+                                    Vec::new()
                                 },
                                 limit: agg.limit.clone(),
                                 ignore_nulls: agg.ignore_nulls,
@@ -716,14 +712,19 @@ pub(super) fn rewrite(
             }
 
             Action::ArrayAggIgnoreNullsDuckDB => {
-                // ARRAY_AGG(x IGNORE NULLS ORDER BY a, b DESC) -> ARRAY_AGG(x ORDER BY a NULLS FIRST, b DESC)
-                // Strip IGNORE NULLS, add NULLS FIRST to first ORDER BY column
+                // ARRAY_AGG(x IGNORE NULLS ...) -> ARRAY_AGG(x ...) FILTER(WHERE x IS NOT NULL)
+                // DuckDB also needs NULLS FIRST on the first inline ordering expression.
                 let mut agg = if let Expression::ArrayAgg(a) = e {
                     *a
                 } else {
                     unreachable!("action only triggered for ArrayAgg expressions")
                 };
-                agg.ignore_nulls = None; // Strip IGNORE NULLS
+                agg.ignore_nulls = None;
+                agg.filter = Some(Expression::IsNull(Box::new(IsNull {
+                    this: agg.this.clone(),
+                    not: true,
+                    postfix_form: false,
+                })));
                 if !agg.order_by.is_empty() {
                     agg.order_by[0].nulls_first = Some(true);
                 }
@@ -858,11 +859,19 @@ pub(super) fn rewrite(
 
             Action::AggFilterToIff => {
                 // AggFunc.filter -> IFF wrapping: AVG(x) FILTER(WHERE cond) -> AVG(IFF(cond, x, NULL))
+                let prefix_not_null = |condition: Expression| match condition {
+                    Expression::IsNull(mut is_null) if is_null.not => {
+                        is_null.postfix_form = true;
+                        Expression::IsNull(is_null)
+                    }
+                    other => other,
+                };
                 // Helper macro to handle the common AggFunc case
                 macro_rules! handle_agg_filter_to_iff {
                     ($variant:ident, $agg:expr) => {{
                         let mut agg = $agg;
                         if let Some(filter_cond) = agg.filter.take() {
+                            let filter_cond = prefix_not_null(filter_cond);
                             let iff_call = Expression::Function(Box::new(Function::new(
                                 "IFF".to_string(),
                                 vec![filter_cond, agg.this.clone(), Expression::Null(Null)],
@@ -878,7 +887,24 @@ pub(super) fn rewrite(
                     Expression::Sum(agg) => handle_agg_filter_to_iff!(Sum, agg),
                     Expression::Min(agg) => handle_agg_filter_to_iff!(Min, agg),
                     Expression::Max(agg) => handle_agg_filter_to_iff!(Max, agg),
-                    Expression::ArrayAgg(agg) => handle_agg_filter_to_iff!(ArrayAgg, agg),
+                    Expression::ArrayAgg(mut agg) => {
+                        if let Some(filter_cond) = agg.filter.take() {
+                            let filter_cond = prefix_not_null(filter_cond);
+                            agg.this = Expression::Function(Box::new(Function::new(
+                                "IFF".to_string(),
+                                vec![filter_cond, agg.this, Expression::Null(Null)],
+                            )));
+                        }
+                        if agg.order_by.is_empty() {
+                            Ok(Expression::ArrayAgg(agg))
+                        } else {
+                            let order_by = std::mem::take(&mut agg.order_by);
+                            Ok(Expression::WithinGroup(Box::new(WithinGroup {
+                                this: Expression::ArrayAgg(agg),
+                                order_by,
+                            })))
+                        }
+                    }
                     Expression::CountIf(agg) => handle_agg_filter_to_iff!(CountIf, agg),
                     Expression::Stddev(agg) => handle_agg_filter_to_iff!(Stddev, agg),
                     Expression::StddevPop(agg) => handle_agg_filter_to_iff!(StddevPop, agg),
@@ -896,7 +922,21 @@ pub(super) fn rewrite(
                     }
                     Expression::Count(mut c) => {
                         if let Some(filter_cond) = c.filter.take() {
-                            if let Some(ref this_expr) = c.this {
+                            let filter_cond = prefix_not_null(filter_cond);
+                            let is_star = c.star || matches!(c.this, Some(Expression::Star(_)));
+                            if is_star {
+                                return Ok(Expression::CountIf(Box::new(AggFunc {
+                                    this: filter_cond,
+                                    distinct: false,
+                                    filter: None,
+                                    order_by: Vec::new(),
+                                    name: None,
+                                    ignore_nulls: None,
+                                    having_max: None,
+                                    limit: None,
+                                    inferred_type: None,
+                                })));
+                            } else if let Some(ref this_expr) = c.this {
                                 let iff_call = Expression::Function(Box::new(Function::new(
                                     "IFF".to_string(),
                                     vec![filter_cond, this_expr.clone(), Expression::Null(Null)],
@@ -906,6 +946,18 @@ pub(super) fn rewrite(
                         }
                         Ok(Expression::Count(c))
                     }
+                    Expression::AggregateFunction(mut af) => {
+                        if let Some(filter_cond) = af.filter.take() {
+                            let filter_cond = prefix_not_null(filter_cond);
+                            if let Some(first_arg) = af.args.first_mut() {
+                                *first_arg = Expression::Function(Box::new(Function::new(
+                                    "IFF".to_string(),
+                                    vec![filter_cond, first_arg.clone(), Expression::Null(Null)],
+                                )));
+                            }
+                        }
+                        Ok(Expression::AggregateFunction(af))
+                    }
                     other => Ok(other),
                 }
             }

{polyglot_sql-0.6.3 → polyglot_sql-0.8.0}/crates/polyglot-sql/src/dialects/normalization/collections.rs

@@ -862,7 +862,11 @@ pub(super) fn rewrite(
                         }
 
                         let seq = Expression::Function(Box::new(Function::new(
-                            "SEQUENCE".to_string(),
+                            if matches!(target, DialectType::BigQuery) {
+                                "GENERATE_ARRAY".to_string()
+                            } else {
+                                "SEQUENCE".to_string()
+                            },
                             seq_args,
                         )));