polyglot-sql 0.7.0__tar.gz → 0.8.0__tar.gz

{polyglot_sql-0.7.0 → polyglot_sql-0.8.0}/Cargo.lock

@@ -605,7 +605,7 @@ dependencies = [
 
 [[package]]
 name = "polyglot-sql"
-version = "0.7.0"
+version = "0.8.0"
 dependencies = [
  "criterion",
  "once_cell",
@@ -623,7 +623,7 @@ dependencies = [
 
 [[package]]
 name = "polyglot-sql-ast-derive"
-version = "0.7.0"
+version = "0.8.0"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -632,7 +632,7 @@ dependencies = [
 
 [[package]]
 name = "polyglot-sql-ffi"
-version = "0.7.0"
+version = "0.8.0"
 dependencies = [
  "cbindgen",
  "polyglot-sql",
@@ -642,11 +642,11 @@ dependencies = [
 
 [[package]]
 name = "polyglot-sql-function-catalogs"
-version = "0.7.0"
+version = "0.8.0"
 
 [[package]]
 name = "polyglot-sql-python"
-version = "0.7.0"
+version = "0.8.0"
 dependencies = [
  "polyglot-sql",
  "pyo3",
@@ -657,7 +657,7 @@ dependencies = [
 
 [[package]]
 name = "polyglot-sql-wasm"
-version = "0.7.0"
+version = "0.8.0"
 dependencies = [
  "console_error_panic_hook",
  "js-sys",

{polyglot_sql-0.7.0 → polyglot_sql-0.8.0}/Cargo.toml

@@ -6,7 +6,7 @@ exclude = [
 ]
 
 [workspace.package]
-version = "0.7.0"
+version = "0.8.0"
 edition = "2021"
 license = "MIT"
 authors = ["polyglot contributors"]

{polyglot_sql-0.7.0 → polyglot_sql-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polyglot-sql
-Version: 0.7.0
+Version: 0.8.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
@@ -177,6 +177,13 @@ Function-like projections may include `transformFunction` with the function
 name, literal arguments, and column arguments, for example for
 `DATE_TRUNC('month', created_at)`.
 
+Each `analysis["setOperations"][...]["branches"]` entry includes a `role` of
+`"value"` or `"filter"`. Lineage results attach optional `set_branch` metadata
+to immediate set-operation branch roots with the `operator`, original
+zero-based `ordinal`, and `all` flag; omitted branches do not renumber the
+surviving nodes. In OpenLineage output, `EXCEPT` and `INTERSECT` right-hand
+inputs are emitted as indirect `FILTER` dependencies.
+
 Validation schema dictionaries use:
 
 ```python

{polyglot_sql-0.7.0/crates/polyglot-sql-python → polyglot_sql-0.8.0}/README.md

@@ -150,6 +150,13 @@ Function-like projections may include `transformFunction` with the function
 name, literal arguments, and column arguments, for example for
 `DATE_TRUNC('month', created_at)`.
 
+Each `analysis["setOperations"][...]["branches"]` entry includes a `role` of
+`"value"` or `"filter"`. Lineage results attach optional `set_branch` metadata
+to immediate set-operation branch roots with the `operator`, original
+zero-based `ordinal`, and `all` flag; omitted branches do not renumber the
+surviving nodes. In OpenLineage output, `EXCEPT` and `INTERSECT` right-hand
+inputs are emitted as indirect `FILTER` dependencies.
+
 Validation schema dictionaries use:
 
 ```python

{polyglot_sql-0.7.0 → polyglot_sql-0.8.0}/crates/polyglot-sql/Cargo.toml

@@ -100,14 +100,14 @@ function-catalog-all-dialects = [
 ]
 
 [dependencies]
-polyglot-sql-ast-derive = { path = "../polyglot-sql-ast-derive", version = "0.7.0" }
+polyglot-sql-ast-derive = { path = "../polyglot-sql-ast-derive", version = "0.8.0" }
 serde = { workspace = true }
 serde_json = { workspace = true }
 thiserror = { workspace = true }
 unicode-segmentation = { workspace = true }
 stacker = { version = "0.1", optional = true }
 ts-rs = { version = "12.0", features = ["serde-compat"], optional = true }
-polyglot-sql-function-catalogs = { path = "../polyglot-sql-function-catalogs", version = "0.7.0", optional = true, default-features = false }
+polyglot-sql-function-catalogs = { path = "../polyglot-sql-function-catalogs", version = "0.8.0", optional = true, default-features = false }
 
 [dev-dependencies]
 pretty_assertions = "1.4"

{polyglot_sql-0.7.0 → polyglot_sql-0.8.0}/crates/polyglot-sql/README.md

@@ -25,12 +25,12 @@ By default, `polyglot-sql` enables the full public API. Parser-only consumers ca
 disable default features and opt into only the dialect parsers they need:
 
 ```toml
-polyglot-sql = { version = "0.7.0", default-features = false }
+polyglot-sql = { version = "0.8.0", default-features = false }
 ```
 
 ```toml
 polyglot-sql = {
-    version = "0.5",
+    version = "0.8.0",
     default-features = false,
     features = ["dialect-clickhouse"],
 }
@@ -44,14 +44,14 @@ Examples:
 ```toml
 # Parse and generate SQL for one dialect.
 polyglot-sql = {
-    version = "0.5",
+    version = "0.8.0",
     default-features = false,
     features = ["generate", "dialect-clickhouse"],
 }
 
 # Cross-dialect transpilation.
 polyglot-sql = {
-    version = "0.5",
+    version = "0.8.0",
     default-features = false,
     features = ["transpile", "dialect-clickhouse", "dialect-postgresql"],
 }
@@ -332,6 +332,13 @@ and CTE output columns. `star_projections` reports the original top-level star
 projection index, optional table qualifier, and schema-expanded columns when
 known. Each projection also includes conservative `nullability`:
 `non_null`, `nullable`, or `unknown`.
+Each `set_operations` branch includes a semantic `role`: both `UNION` branches
+are `value`, while the right branch of `EXCEPT` and `INTERSECT` is `filter`.
+
+Lineage nodes at immediate set-operation branch roots expose optional
+`set_branch` metadata with the operator, original zero-based branch ordinal,
+and `ALL` flag. The ordinal remains stable when another branch cannot be
+resolved.
 
 ```rust
 use polyglot_sql::{

{polyglot_sql-0.7.0 → polyglot_sql-0.8.0}/crates/polyglot-sql/src/lib.rs

@@ -98,7 +98,7 @@ pub use helper::{
 #[cfg(feature = "semantic")]
 pub use lineage::{
     lineage, lineage_at, lineage_at_with_schema, lineage_with_schema, output_columns,
-    output_columns_with_schema, LineageNode, OutputColumn, QueryOutput,
+    output_columns_with_schema, LineageNode, OutputColumn, QueryOutput, SetBranch, SetOperator,
 };
 #[cfg(feature = "semantic")]
 pub use optimizer::{
@@ -109,8 +109,8 @@ pub use parser::Parser;
 pub use query_analysis::{
     analyze_query, AnalyzeQueryOptions, ColumnReferenceFact, CteFact, ProjectionFact,
     ProjectionNullability, QueryAnalysis, QueryShape, ReferenceConfidence, RelationFact,
-    SetOperationBranchFact, SetOperationFact, StarProjectionFact, TransformFunctionFact,
-    TransformKind,
+    SetOperationBranchFact, SetOperationBranchRole, SetOperationFact, StarProjectionFact,
+    TransformFunctionFact, TransformKind,
 };
 #[cfg(feature = "semantic")]
 pub use resolver::{is_column_ambiguous, resolve_column, Resolver, ResolverError, ResolverResult};

{polyglot_sql-0.7.0 → polyglot_sql-0.8.0}/crates/polyglot-sql/src/lineage.rs

@@ -55,6 +55,27 @@ pub enum OutputColumn {
     },
 }
 
+/// The set operator that owns an immediate lineage branch.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum SetOperator {
+    Union,
+    Intersect,
+    Except,
+}
+
+/// Metadata describing a lineage node's position in an immediate set operation.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct SetBranch {
+    /// The set operator that owns this branch.
+    pub operator: SetOperator,
+    /// The zero-based branch ordinal within the immediate binary operation.
+    pub ordinal: usize,
+    /// Whether the immediate set operation uses `ALL` semantics.
+    pub all: bool,
+}
+
 /// A node in the column lineage graph
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct LineageNode {
@@ -73,6 +94,9 @@ pub struct LineageNode {
     /// User-written source alias when different from canonical source name.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub source_alias: Option<String>,
+    /// Immediate set-operation branch metadata, when this node is a branch root.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub set_branch: Option<SetBranch>,
     /// Optional reference node name (e.g., for CTEs)
     pub reference_node_name: String,
 }
@@ -88,6 +112,7 @@ impl LineageNode {
             source_name: String::new(),
             source_kind: SourceKind::Unknown,
             source_alias: None,
+            set_branch: None,
             reference_node_name: String::new(),
         }
     }
@@ -1314,6 +1339,27 @@ fn query_expressions_in_scope(expr: &Expression) -> Vec<&Expression> {
 // Set operation handling
 // ---------------------------------------------------------------------------
 
+fn set_branch_metadata(expression: &Expression, ordinal: usize) -> SetBranch {
+    match expression {
+        Expression::Union(union) => SetBranch {
+            operator: SetOperator::Union,
+            ordinal,
+            all: union.all,
+        },
+        Expression::Intersect(intersect) => SetBranch {
+            operator: SetOperator::Intersect,
+            ordinal,
+            all: intersect.all,
+        },
+        Expression::Except(except) => SetBranch {
+            operator: SetOperator::Except,
+            ordinal,
+            all: except.all,
+        },
+        _ => unreachable!("set-operation metadata requires a set operation"),
+    }
+}
+
 fn handle_set_operation(
     column: &ColumnRef<'_>,
     context: &LineageScopeContext,
@@ -1361,7 +1407,9 @@ fn handle_set_operation(
 
     // Recurse into each set-operation branch. Resolution failures are branch-local,
     // but genuine parser/internal errors must not be silently discarded.
-    for &branch_scope_id in &context.indexed(scope_id).union_scopes {
+    for (branch_ordinal, &branch_scope_id) in
+        context.indexed(scope_id).union_scopes.iter().enumerate()
+    {
         let branch_column = if trace_wildcard_by_name {
             match column {
                 ColumnRef::Name(name) => ColumnRef::Name(name),
@@ -1383,7 +1431,10 @@ fn handle_set_operation(
             ancestor_cte_scopes,
             depth + 1,
         ) {
-            Ok(child) => node.downstream.push(child),
+            Ok(mut child) => {
+                child.set_branch = Some(set_branch_metadata(scope_expr, branch_ordinal));
+                node.downstream.push(child);
+            }
             Err(Error::ColumnResolution { reason, .. }) => {
                 resolution_failure = Some(merge_resolution_reason(resolution_failure, reason));
             }
@@ -4224,6 +4275,7 @@ select col_a from unioned";
             source_name: String::new(),
             source_kind: SourceKind::Unknown,
             source_alias: None,
+            set_branch: None,
             reference_node_name: String::new(),
         };
 
@@ -5488,6 +5540,31 @@ FROM t JOIN UNNEST(t.items) AS item ON TRUE
 
         assert_lineage_contains(&node, "known_source.known_second");
         assert_eq!(node.downstream.len(), 1);
+        assert_eq!(
+            node.downstream[0].set_branch,
+            Some(SetBranch {
+                operator: SetOperator::Union,
+                ordinal: 1,
+                all: true,
+            })
+        );
+
+        let expr = parse(
+            "SELECT known_first, known_second FROM known_source EXCEPT \
+             SELECT * FROM unknown_source",
+        );
+        let node = lineage_at(1, &expr, None, false).expect("left-only ordinal lineage");
+
+        assert_lineage_contains(&node, "known_source.known_second");
+        assert_eq!(node.downstream.len(), 1);
+        assert_eq!(
+            node.downstream[0].set_branch,
+            Some(SetBranch {
+                operator: SetOperator::Except,
+                ordinal: 0,
+                all: false,
+            })
+        );
     }
 
     #[test]
@@ -5519,16 +5596,38 @@ FROM t JOIN UNNEST(t.items) AS item ON TRUE
 
     #[test]
     fn test_issue_383_lineage_at_supports_all_set_operations() {
-        for operator in ["UNION ALL", "INTERSECT", "EXCEPT"] {
+        for (sql_operator, operator, all) in [
+            ("UNION ALL", SetOperator::Union, true),
+            ("INTERSECT", SetOperator::Intersect, false),
+            ("EXCEPT", SetOperator::Except, false),
+        ] {
             let expr = parse(&format!(
-                "SELECT left_value AS left_name FROM left_source {operator} \
+                "SELECT left_value AS left_name FROM left_source {sql_operator} \
                  SELECT right_value AS right_name FROM right_source"
             ));
             let node = lineage_at(0, &expr, None, false).expect("ordinal lineage");
             assert_eq!(
                 node.downstream.len(),
                 2,
-                "expected both branches for {operator}"
+                "expected both branches for {sql_operator}"
+            );
+            assert_eq!(
+                node.downstream
+                    .iter()
+                    .map(|child| child.set_branch)
+                    .collect::<Vec<_>>(),
+                vec![
+                    Some(SetBranch {
+                        operator,
+                        ordinal: 0,
+                        all,
+                    }),
+                    Some(SetBranch {
+                        operator,
+                        ordinal: 1,
+                        all,
+                    }),
+                ]
             );
         }
     }

{polyglot_sql-0.7.0 → polyglot_sql-0.8.0}/crates/polyglot-sql/src/openlineage.rs

@@ -6,7 +6,7 @@
 
 use crate::dialects::{Dialect, DialectType};
 use crate::expressions::*;
-use crate::lineage::{self, LineageNode};
+use crate::lineage::{self, LineageNode, SetOperator};
 use crate::schema::Schema;
 use crate::scope::SourceKind;
 use crate::traversal::ExpressionWalk;
@@ -180,6 +180,13 @@ struct OutputField {
 struct TerminalField {
     table: String,
     field: String,
+    dependency: TerminalDependency,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+enum TerminalDependency {
+    Direct,
+    Filter,
 }
 
 /// Produce a standalone OpenLineage columnLineage facet plus inferred datasets.
@@ -386,6 +393,23 @@ fn analyze_statement(
                 output_column_names: Vec::new(),
             })
         }
+        Expression::Union(_) | Expression::Intersect(_) | Expression::Except(_) => {
+            let output = options.output_dataset.clone().ok_or_else(|| {
+                Error::parse(
+                    "OpenLineage outputDataset is required for set-operation queries",
+                    0,
+                    0,
+                    0,
+                    0,
+                )
+            })?;
+            Ok(StatementAnalysis {
+                query: expr.clone(),
+                inputs: collect_input_datasets(expr, options, Some(&output), warnings)?,
+                output,
+                output_column_names: Vec::new(),
+            })
+        }
         Expression::Insert(insert) => {
             let output = dataset_from_table_ref(&insert.table, options)?;
             let query = insert.query.clone().ok_or_else(|| {
@@ -494,6 +518,7 @@ fn input_fields_for_output(
             vec![TerminalField {
                 table: table.clone(),
                 field: output_field.lineage_name.clone(),
+                dependency: TerminalDependency::Direct,
             }],
             "IDENTITY",
             Some(format!("SELECT {}", output_field.lineage_name)),
@@ -534,7 +559,7 @@ fn input_fields_for_output(
     };
 
     let mut terminals = BTreeSet::new();
-    collect_terminal_fields(&node, &mut terminals);
+    collect_terminal_fields(&node, TerminalDependency::Direct, &mut terminals);
     let terminals: Vec<TerminalField> = terminals.into_iter().collect();
 
     if terminals.is_empty() {
@@ -580,7 +605,8 @@ fn terminal_fields_to_openlineage(
     options: &OpenLineageOptions,
     warnings: &mut Vec<OpenLineageWarning>,
 ) -> Result<Vec<OpenLineageInputField>> {
-    let mut result = Vec::new();
+    let mut grouped =
+        BTreeMap::<(String, String, String), BTreeSet<OpenLineageTransformation>>::new();
     for terminal in terminals {
         let dataset = dataset_from_table_name(&terminal.table, options).map_err(|err| {
             warnings.push(OpenLineageWarning::new(
@@ -592,19 +618,31 @@ fn terminal_fields_to_openlineage(
             ));
             err
         })?;
-        result.push(OpenLineageInputField {
-            namespace: dataset.namespace,
-            name: dataset.name,
-            field: terminal.field,
-            transformations: vec![OpenLineageTransformation {
-                type_: "DIRECT".to_string(),
-                subtype: subtype.to_string(),
+        let (type_, transformation_subtype) = match terminal.dependency {
+            TerminalDependency::Direct => ("DIRECT", subtype),
+            TerminalDependency::Filter => ("INDIRECT", "FILTER"),
+        };
+        grouped
+            .entry((dataset.namespace, dataset.name, terminal.field))
+            .or_default()
+            .insert(OpenLineageTransformation {
+                type_: type_.to_string(),
+                subtype: transformation_subtype.to_string(),
                 description: description.clone(),
                 masking: Some(false),
-            }],
-        });
+            });
     }
-    Ok(result)
+    Ok(grouped
+        .into_iter()
+        .map(
+            |((namespace, name, field), transformations)| OpenLineageInputField {
+                namespace,
+                name,
+                field,
+                transformations: transformations.into_iter().collect(),
+            },
+        )
+        .collect())
 }
 
 fn transformation_subtype(expr: Option<&Expression>, terminals: &[TerminalField]) -> &'static str {
@@ -615,7 +653,11 @@ fn transformation_subtype(expr: Option<&Expression>, terminals: &[TerminalField]
     if expression_contains_aggregate(unaliased) {
         return "AGGREGATION";
     }
-    if terminals.len() == 1 {
+    let distinct_fields = terminals
+        .iter()
+        .map(|terminal| (&terminal.table, &terminal.field))
+        .collect::<BTreeSet<_>>();
+    if distinct_fields.len() == 1 {
         if let Expression::Column(col) = unaliased {
             if col.name.name == terminals[0].field {
                 return "IDENTITY";
@@ -625,7 +667,23 @@ fn transformation_subtype(expr: Option<&Expression>, terminals: &[TerminalField]
     "TRANSFORMATION"
 }
 
-fn collect_terminal_fields(node: &LineageNode, terminals: &mut BTreeSet<TerminalField>) {
+fn collect_terminal_fields(
+    node: &LineageNode,
+    inherited_dependency: TerminalDependency,
+    terminals: &mut BTreeSet<TerminalField>,
+) {
+    let dependency = if inherited_dependency == TerminalDependency::Filter
+        || matches!(
+            node.set_branch,
+            Some(branch)
+                if branch.ordinal == 1
+                    && matches!(branch.operator, SetOperator::Intersect | SetOperator::Except)
+        ) {
+        TerminalDependency::Filter
+    } else {
+        TerminalDependency::Direct
+    };
+
     if node.downstream.is_empty() {
         if node.source_kind == SourceKind::Virtual {
             return;
@@ -642,6 +700,7 @@ fn collect_terminal_fields(node: &LineageNode, terminals: &mut BTreeSet<Terminal
                 terminals.insert(TerminalField {
                     table,
                     field: column.name.name.clone(),
+                    dependency,
                 });
             }
         }
@@ -649,7 +708,7 @@ fn collect_terminal_fields(node: &LineageNode, terminals: &mut BTreeSet<Terminal
     }
 
     for child in &node.downstream {
-        collect_terminal_fields(child, terminals);
+        collect_terminal_fields(child, dependency, terminals);
     }
 }
 
@@ -1119,6 +1178,83 @@ mod tests {
         assert_eq!(field.input_fields[0].transformations[0].subtype, "IDENTITY");
     }
 
+    #[test]
+    fn emits_set_operation_value_and_filter_dependencies() {
+        let union = openlineage_column_lineage(
+            "SELECT a FROM left_table UNION ALL SELECT b FROM right_table",
+            &options(),
+        )
+        .expect("union lineage");
+        let union_field = union.facet.fields.get("a").expect("union field");
+        assert_eq!(union_field.input_fields.len(), 2);
+        assert!(union_field.input_fields.iter().all(|input| input
+            .transformations
+            .iter()
+            .all(|transformation| transformation.type_ == "DIRECT")));
+
+        for operator in ["EXCEPT", "INTERSECT"] {
+            let result = openlineage_column_lineage(
+                &format!("SELECT a FROM left_table {operator} SELECT b FROM right_table"),
+                &options(),
+            )
+            .unwrap_or_else(|error| panic!("{operator} lineage failed: {error}"));
+            let field = result.facet.fields.get("a").expect("output field");
+            let left = field
+                .input_fields
+                .iter()
+                .find(|input| input.name == "left_table")
+                .expect("left input");
+            let right = field
+                .input_fields
+                .iter()
+                .find(|input| input.name == "right_table")
+                .expect("right input");
+            assert!(left
+                .transformations
+                .iter()
+                .all(|transformation| transformation.type_ == "DIRECT"));
+            assert!(right.transformations.iter().any(|transformation| {
+                transformation.type_ == "INDIRECT" && transformation.subtype == "FILTER"
+            }));
+        }
+
+        let merged = openlineage_column_lineage(
+            "SELECT a FROM shared_table EXCEPT SELECT a FROM shared_table",
+            &options(),
+        )
+        .expect("merged dependency lineage");
+        let merged_field = merged.facet.fields.get("a").expect("merged field");
+        assert_eq!(merged_field.input_fields.len(), 1);
+        assert!(merged_field.input_fields[0]
+            .transformations
+            .iter()
+            .any(|transformation| transformation.type_ == "DIRECT"));
+        assert!(merged_field.input_fields[0]
+            .transformations
+            .iter()
+            .any(|transformation| {
+                transformation.type_ == "INDIRECT" && transformation.subtype == "FILTER"
+            }));
+
+        let nested = openlineage_column_lineage(
+            "SELECT a FROM left_table EXCEPT \
+             (SELECT b FROM right_table UNION ALL SELECT c FROM third_table)",
+            &options(),
+        )
+        .expect("nested set-operation lineage");
+        let nested_field = nested.facet.fields.get("a").expect("nested field");
+        for table in ["right_table", "third_table"] {
+            let input = nested_field
+                .input_fields
+                .iter()
+                .find(|input| input.name == table)
+                .unwrap_or_else(|| panic!("missing nested input {table}"));
+            assert!(input.transformations.iter().any(|transformation| {
+                transformation.type_ == "INDIRECT" && transformation.subtype == "FILTER"
+            }));
+        }
+    }
+
     #[test]
     fn emits_column_lineage_for_prepared_statement_body() {
         let result = openlineage_column_lineage(