dlin-cli 0.2.3__tar.gz → 0.2.4rc1__tar.gz

{dlin_cli-0.2.3 → dlin_cli-0.2.4rc1}/Cargo.lock

@@ -87,9 +87,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.102"
+version = "1.0.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
+checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
 
 [[package]]
 name = "arraydeque"
@@ -139,9 +139,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
 
 [[package]]
 name = "clap"
-version = "4.6.1"
+version = "4.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
+checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -149,9 +149,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.6.0"
+version = "4.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
+checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b"
 dependencies = [
  "anstream",
  "anstyle",
@@ -161,14 +161,14 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "4.6.1"
+version = "4.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"
+checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061"
 dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 3.0.3",
 ]
 
 [[package]]
@@ -230,7 +230,7 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
 
 [[package]]
 name = "dlin"
-version = "0.2.3"
+version = "0.2.4-rc.1"
 dependencies = [
  "anyhow",
  "clap",
@@ -247,7 +247,7 @@ dependencies = [
 
 [[package]]
 name = "dlin-core"
-version = "0.2.3"
+version = "0.2.4-rc.1"
 dependencies = [
  "anyhow",
  "clap",
@@ -266,7 +266,7 @@ dependencies = [
  "serde_json",
  "serial_test",
  "tempfile",
- "thiserror 2.0.18",
+ "thiserror 2.0.19",
  "walkdir",
 ]
 
@@ -385,9 +385,9 @@ dependencies = [
 
 [[package]]
 name = "globset"
-version = "0.4.18"
+version = "0.4.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52dfc19153a48bde0cbd630453615c8151bce3a5adfac7a0aebfbf0a1e1f57e3"
+checksum = "e47d37d2ae4464254884b60ab7071be2b876a9c35b696bd018ddcc76847309cd"
 dependencies = [
  "aho-corasick",
  "bstr",
@@ -398,9 +398,9 @@ dependencies = [
 
 [[package]]
 name = "granit-parser"
-version = "0.0.3"
+version = "0.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f50ba32164f9e098d5da618776a32afbb32270adcbe3d3d006107dae11e37c91"
+checksum = "d03f81ad4732830d85cfd417a9f62cde6dadda4354d37d078a6084a19560aa2d"
 dependencies = [
  "arraydeque",
  "smallvec",
@@ -439,9 +439,9 @@ dependencies = [
 
 [[package]]
 name = "insta"
-version = "1.47.2"
+version = "1.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b4a6248eb93a4401ed2f37dfe8ea592d3cf05b7cf4f8efa867b6895af7e094e"
+checksum = "86f0f8fee8c926415c58d6ae43a08523a26faccb2323f5e6b644fe7dd4ef6b82"
 dependencies = [
  "console",
  "once_cell",
@@ -473,9 +473,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.186"
+version = "0.2.189"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
 
 [[package]]
 name = "linux-raw-sys"
@@ -512,9 +512,9 @@ checksum = "38d1115007560874e373613744c6fba374c17688327a71c1476d1a5954cc857b"
 
 [[package]]
 name = "minijinja"
-version = "2.20.0"
+version = "2.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2929e494b2280e1e18959bb2e121da03347ae896896fdfaceaab43c88a02803f"
+checksum = "cb3d648e68cea56d9858d535ee28f9538404e2dd8cb08ed0bd05dca379477f39"
 dependencies = [
  "memo-map",
  "serde",
@@ -597,16 +597,28 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
 
 [[package]]
 name = "polyglot-sql"
-version = "0.4.4"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0d5ea9283f03d3b6c32a230e60e7c6bc854e5bec83c399c75cad02bf46e9622"
+checksum = "b54acea84522c6c51f6cbfa56600097cd8c6c5583a70e67f2787b6f70c3dd956"
 dependencies = [
+ "polyglot-sql-ast-derive",
  "serde",
  "serde_json",
  "thiserror 1.0.69",
  "unicode-segmentation",
 ]
 
+[[package]]
+name = "polyglot-sql-ast-derive"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c19af779599f7bf0ec303044dbb8e1c3f6c5b4dca29ea703b033519860104031"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+]
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.106"
@@ -725,9 +737,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
 [[package]]
 name = "serde"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
 dependencies = [
  "serde_core",
  "serde_derive",
@@ -735,9 +747,9 @@ dependencies = [
 
 [[package]]
 name = "serde-saphyr"
-version = "0.0.27"
+version = "0.0.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5897b4c3faadadd35fdb6689f015641f3bc481d5adaaac56231ea15aeb243db3"
+checksum = "7bd22781911de0ca6debda95f073c8f18bec65d1a94f1fa9573f3102e514cea4"
 dependencies = [
  "ahash",
  "annotate-snippets",
@@ -747,36 +759,36 @@ dependencies = [
  "granit-parser",
  "nohash-hasher",
  "num-traits",
- "serde",
+ "serde_core",
  "smallvec",
  "zmij",
 ]
 
 [[package]]
 name = "serde_core"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 3.0.3",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.150"
+version = "1.0.151"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
+checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
 dependencies = [
  "itoa",
  "memchr",
@@ -807,7 +819,7 @@ checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.117",
 ]
 
 [[package]]
@@ -845,6 +857,17 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "syn"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "tempfile"
 version = "3.27.0"
@@ -869,11 +892,11 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "2.0.18"
+version = "2.0.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
+checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9"
 dependencies = [
- "thiserror-impl 2.0.18",
+ "thiserror-impl 2.0.19",
 ]
 
 [[package]]
@@ -884,18 +907,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.117",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "2.0.18"
+version = "2.0.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
+checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 3.0.3",
 ]
 
 [[package]]
@@ -979,7 +1002,7 @@ dependencies = [
  "bumpalo",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.117",
  "wasm-bindgen-shared",
 ]
 
@@ -1039,7 +1062,7 @@ checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.117",
 ]
 
 [[package]]

{dlin_cli-0.2.3 → dlin_cli-0.2.4rc1}/Cargo.toml

@@ -3,7 +3,7 @@ members = ["crates/*"]
 resolver = "3"
 
 [workspace.package]
-version = "0.2.3"
+version = "0.2.4-rc.1"
 edition = "2024"
 license = "MIT"
 repository = "https://github.com/eitsupi/dlin"
@@ -13,7 +13,7 @@ homepage = "https://github.com/eitsupi/dlin"
 petgraph = { version = "0.8", features = ["serde-1"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
-serde-saphyr = "0.0.27"
+serde-saphyr = "0.0.29"
 regex = "1"
 walkdir = "2"
 colored = "3"
@@ -23,7 +23,7 @@ indexmap = "2"
 minijinja = "2"
 rayon = "1"
 globset = "0.4"
-polyglot-sql = { version = "0.4.4", default-features = false, features = ["all-dialects", "semantic"] }
+polyglot-sql = { version = "0.6.3", default-features = false, features = ["all-dialects", "semantic"] }
 path-slash = "0.2.1"
 clap = { version = "4", features = ["derive", "env"] }
 libc = "0.2"
@@ -34,7 +34,7 @@ insta = "1"
 serial_test = "3.4.0"
 
 # internal
-dlin-core = { version = "0.2.3", path = "crates/dlin-core" }
+dlin-core = { version = "0.2.4-rc.1", path = "crates/dlin-core" }
 
 [workspace.lints.rust]
 unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tarpaulin_include)'] }

{dlin_cli-0.2.3 → dlin_cli-0.2.4rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dlin-cli
-Version: 0.2.3
+Version: 0.2.4rc1
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers

{dlin_cli-0.2.3 → dlin_cli-0.2.4rc1}/crates/dlin-core/src/graph/column_lineage/mod.rs

@@ -11,6 +11,7 @@ mod cross_model;
 mod impact;
 mod schema;
 mod single_model;
+mod star_guard;
 #[cfg(test)]
 mod tests;
 mod types;
@@ -24,7 +25,8 @@ pub use impact::{
     compute_column_impact_with_manifest_path,
 };
 use schema::{build_yaml_schema_for_node, compute_manifest_columns_hash, infer_output_columns};
-use single_model::{has_unresolved_stars, prepare_lineage_context_with_expr, run_column_lineage};
+use single_model::{prepare_lineage_context_with_expr, run_column_lineage};
+use star_guard::has_unresolved_stars;
 pub use types::{
     ColumnLineageEntry, ColumnLineageError, ColumnLineageErrorKind, ColumnSource,
     ModelColumnLineage, TransformationType,

{dlin_cli-0.2.3 → dlin_cli-0.2.4rc1}/crates/dlin-core/src/graph/column_lineage/single_model.rs

@@ -1,14 +1,18 @@
+use polyglot_sql::lineage::LineageNode;
+use polyglot_sql::scope::SourceKind;
 use polyglot_sql::{DialectType, Expression};
 
 use crate::parser::manifest::Manifest;
 
 use super::schema::build_schema_from_manifest;
+use super::star_guard::StarGuard;
 use super::{ColumnSource, TransformationType};
 
 pub(super) struct LineageContext {
     pub(super) expanded_expr: Expression,
     schema: Option<polyglot_sql::MappingSchema>,
     dialect: DialectType,
+    star_guard: StarGuard,
 }
 
 pub(super) fn prepare_lineage_context(
@@ -31,7 +35,6 @@ pub(super) fn prepare_lineage_context_with_expr(
         Some(e) => e.clone(),
         None => polyglot_sql::parse_one(compiled_code, dialect).map_err(|e| format!("{}", e))?,
     };
-
     let schema = build_schema_from_manifest(manifest, node, dialect);
 
     let mut expanded_expr = expr;
@@ -39,11 +42,12 @@ pub(super) fn prepare_lineage_context_with_expr(
         &mut expanded_expr,
         schema.as_ref().map(|s| s as &dyn polyglot_sql::Schema),
     );
-
+    let star_guard = StarGuard::new(&expanded_expr, Some(dialect));
     Ok(LineageContext {
         expanded_expr,
         schema,
         dialect,
+        star_guard,
     })
 }
 
@@ -51,54 +55,91 @@ pub(super) fn run_column_lineage(
     col_name: &str,
     ctx: &LineageContext,
 ) -> Result<ColumnLineageResult, String> {
+    let modifier_expr =
+        ctx.star_guard
+            .materialize_star_modifier(&ctx.expanded_expr, col_name, ctx.dialect);
+    let lineage_expr = modifier_expr.as_ref().unwrap_or(&ctx.expanded_expr);
+    let lineage_result = lineage_node(col_name, lineage_expr, ctx);
+
+    match lineage_result {
+        Ok(node) => {
+            if ctx.star_guard.rejects(&node) {
+                Err(format!("Cannot find column '{}' in query", col_name))
+            } else {
+                Ok(extract_leaf_sources(&node))
+            }
+        }
+        Err(e) => explicit_set_operand_lineage(col_name, lineage_expr, ctx, &e)
+            .ok_or_else(|| format_lineage_error(&e)),
+    }
+}
+
+fn lineage_node(
+    col_name: &str,
+    expr: &Expression,
+    ctx: &LineageContext,
+) -> Result<LineageNode, polyglot_sql::Error> {
     let dialect = Some(ctx.dialect);
-    let lineage_result = if let Some(ref s) = ctx.schema {
+    if let Some(ref schema) = ctx.schema {
         polyglot_sql::lineage::lineage_with_schema(
             col_name,
-            &ctx.expanded_expr,
-            Some(s as &dyn polyglot_sql::Schema),
+            expr,
+            Some(schema as &dyn polyglot_sql::Schema),
             dialect,
             false,
         )
-        .or_else(|_| polyglot_sql::lineage::lineage(col_name, &ctx.expanded_expr, dialect, false))
+        .or_else(|_| polyglot_sql::lineage::lineage(col_name, expr, dialect, false))
     } else {
-        polyglot_sql::lineage::lineage(col_name, &ctx.expanded_expr, dialect, false)
-    };
+        polyglot_sql::lineage::lineage(col_name, expr, dialect, false)
+    }
+}
 
-    match lineage_result {
-        Ok(node) => Ok(extract_leaf_sources(&node)),
-        Err(e) => Err(format_lineage_error(&e)),
+// TODO: this locates an operand's projection by ordinal (see
+// `StarGuard::explicit_set_operands`) but then re-derives lineage by looking that
+// projection's output *name* back up via `lineage_node`, instead of tracing the
+// already-located projection directly. Unaliased expressions have no name to look
+// up (e.g. `fee * 2` in `SELECT other.*, fee FROM a UNION ALL SELECT x, fee * 2
+// FROM b`) and are silently dropped; duplicate output names (`SELECT a.id, b.id`)
+// make the lookup ambiguous and can retrace the wrong projection. Redesign: keep
+// projection-list position and output ordinal as distinct types and trace
+// positionally instead of round-tripping through a name.
+fn explicit_set_operand_lineage(
+    col_name: &str,
+    expr: &Expression,
+    ctx: &LineageContext,
+    original_error: &polyglot_sql::Error,
+) -> Option<ColumnLineageResult> {
+    let error = original_error.to_string();
+    if !error.contains("Cannot find column") || !error.contains("in set operation") {
+        return None;
     }
+
+    // A set's leftmost star can make its output names unknowable to polyglot-sql.
+    // Trace only intact operands that explicitly project the requested ordinal;
+    // unresolved star operands contribute no source instead of a guessed one.
+    let operands = StarGuard::explicit_set_operands(expr, col_name, Some(ctx.dialect))?;
+    let mut results = operands.into_iter().filter_map(|(operand, operand_name)| {
+        let node = lineage_node(operand_name, operand, ctx).ok()?;
+        (!ctx.star_guard.rejects(&node)).then(|| extract_leaf_sources(&node))
+    });
+    let first = results.next()?;
+    Some(results.fold(first, merge_lineage_results))
 }
 
-pub(super) fn has_unresolved_stars(expr: &Expression) -> bool {
-    match expr {
-        Expression::Select(select) => {
-            let outer_has_star = select.expressions.iter().any(|e| {
-                matches!(e, Expression::Star(_))
-                    || matches!(e, Expression::Column(c) if c.name.name == "*")
-            });
-            if outer_has_star {
-                return true;
-            }
-            if let Some(with) = &select.with
-                && with.ctes.iter().any(|cte| has_unresolved_stars(&cte.this))
-            {
-                return true;
-            }
-            if let Some(from) = &select.from
-                && from.expressions.iter().any(has_unresolved_stars)
-            {
-                return true;
-            }
-            if select.joins.iter().any(|j| has_unresolved_stars(&j.this)) {
-                return true;
-            }
-            false
-        }
-        Expression::Subquery(subq) => has_unresolved_stars(&subq.this),
-        _ => false,
+fn merge_lineage_results(
+    mut left: ColumnLineageResult,
+    right: ColumnLineageResult,
+) -> ColumnLineageResult {
+    if left.transformation == TransformationType::Direct
+        && right.transformation != TransformationType::Direct
+    {
+        left.transformation = right.transformation;
     }
+    left.sources.extend(right.sources);
+    left.sources
+        .sort_by(|a, b| (&a.table, &a.column).cmp(&(&b.table, &b.column)));
+    left.sources.dedup();
+    left
 }
 
 pub(super) fn format_lineage_error(e: &polyglot_sql::Error) -> String {
@@ -118,7 +159,7 @@ pub(super) fn format_lineage_error(e: &polyglot_sql::Error) -> String {
     }
 }
 
-fn classify_transformation(node: &polyglot_sql::lineage::LineageNode) -> TransformationType {
+fn classify_transformation(node: &LineageNode) -> TransformationType {
     let t = classify_expression(&node.expression);
     if t != TransformationType::Direct || node.downstream.is_empty() {
         return t;
@@ -135,8 +176,7 @@ fn classify_transformation(node: &polyglot_sql::lineage::LineageNode) -> Transfo
     TransformationType::Direct
 }
 
-fn classify_expression(expr: &polyglot_sql::Expression) -> TransformationType {
-    use polyglot_sql::Expression;
+fn classify_expression(expr: &Expression) -> TransformationType {
     match expr {
         Expression::Column(_) | Expression::Identifier(_) => TransformationType::Direct,
         Expression::Alias(alias) => classify_expression(&alias.this),
@@ -169,7 +209,7 @@ pub(super) struct ColumnLineageResult {
     pub(super) transformation: TransformationType,
 }
 
-fn extract_leaf_sources(node: &polyglot_sql::lineage::LineageNode) -> ColumnLineageResult {
+fn extract_leaf_sources(node: &LineageNode) -> ColumnLineageResult {
     let transformation = classify_transformation(node);
     let mut sources = Vec::new();
     collect_leaves(node, &mut sources);
@@ -181,9 +221,9 @@ fn extract_leaf_sources(node: &polyglot_sql::lineage::LineageNode) -> ColumnLine
     }
 }
 
-fn collect_leaves(node: &polyglot_sql::lineage::LineageNode, sources: &mut Vec<ColumnSource>) {
+fn collect_leaves(node: &LineageNode, sources: &mut Vec<ColumnSource>) {
     if node.downstream.is_empty() {
-        if node.source_kind == polyglot_sql::scope::SourceKind::Virtual {
+        if node.source_kind == SourceKind::Virtual {
             return;
         }
         let name = &node.name;
@@ -204,7 +244,7 @@ fn collect_leaves(node: &polyglot_sql::lineage::LineageNode, sources: &mut Vec<C
             });
         } else {
             sources.push(ColumnSource {
-                table: String::new(),
+                table: node.source_name.clone(),
                 column: name.to_string(),
                 model_path: vec![],
             });