dlin-cli 0.2.4b1__tar.gz → 0.2.4rc1__tar.gz

{dlin_cli-0.2.4b1 → dlin_cli-0.2.4rc1}/Cargo.lock

@@ -87,9 +87,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.103"
+version = "1.0.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3"
+checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
 
 [[package]]
 name = "arraydeque"
@@ -139,9 +139,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
 
 [[package]]
 name = "clap"
-version = "4.6.1"
+version = "4.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
+checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -149,9 +149,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.6.0"
+version = "4.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
+checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b"
 dependencies = [
  "anstream",
  "anstyle",
@@ -161,14 +161,14 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "4.6.1"
+version = "4.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"
+checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061"
 dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 3.0.3",
 ]
 
 [[package]]
@@ -230,7 +230,7 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
 
 [[package]]
 name = "dlin"
-version = "0.2.4-beta.1"
+version = "0.2.4-rc.1"
 dependencies = [
  "anyhow",
  "clap",
@@ -247,7 +247,7 @@ dependencies = [
 
 [[package]]
 name = "dlin-core"
-version = "0.2.4-beta.1"
+version = "0.2.4-rc.1"
 dependencies = [
  "anyhow",
  "clap",
@@ -266,7 +266,7 @@ dependencies = [
  "serde_json",
  "serial_test",
  "tempfile",
- "thiserror 2.0.18",
+ "thiserror 2.0.19",
  "walkdir",
 ]
 
@@ -385,9 +385,9 @@ dependencies = [
 
 [[package]]
 name = "globset"
-version = "0.4.18"
+version = "0.4.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52dfc19153a48bde0cbd630453615c8151bce3a5adfac7a0aebfbf0a1e1f57e3"
+checksum = "e47d37d2ae4464254884b60ab7071be2b876a9c35b696bd018ddcc76847309cd"
 dependencies = [
  "aho-corasick",
  "bstr",
@@ -473,9 +473,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.186"
+version = "0.2.189"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
 
 [[package]]
 name = "linux-raw-sys"
@@ -597,9 +597,9 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
 
 [[package]]
 name = "polyglot-sql"
-version = "0.6.2"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "009cbd62f8c82de46697221ee66b0cf3a6dacbda16f8c5a2ce25853940073bd5"
+checksum = "b54acea84522c6c51f6cbfa56600097cd8c6c5583a70e67f2787b6f70c3dd956"
 dependencies = [
  "polyglot-sql-ast-derive",
  "serde",
@@ -610,13 +610,13 @@ dependencies = [
 
 [[package]]
 name = "polyglot-sql-ast-derive"
-version = "0.6.2"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3250a6ffcd1d39670eb532e2b1877ad430799c49cb91b243d63c9425b868161"
+checksum = "c19af779599f7bf0ec303044dbb8e1c3f6c5b4dca29ea703b033519860104031"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.117",
 ]
 
 [[package]]
@@ -737,9 +737,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
 [[package]]
 name = "serde"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
 dependencies = [
  "serde_core",
  "serde_derive",
@@ -766,29 +766,29 @@ dependencies = [
 
 [[package]]
 name = "serde_core"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 3.0.3",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.150"
+version = "1.0.151"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
+checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
 dependencies = [
  "itoa",
  "memchr",
@@ -819,7 +819,7 @@ checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.117",
 ]
 
 [[package]]
@@ -857,6 +857,17 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "syn"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "tempfile"
 version = "3.27.0"
@@ -881,11 +892,11 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "2.0.18"
+version = "2.0.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
+checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9"
 dependencies = [
- "thiserror-impl 2.0.18",
+ "thiserror-impl 2.0.19",
 ]
 
 [[package]]
@@ -896,18 +907,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.117",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "2.0.18"
+version = "2.0.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
+checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 3.0.3",
 ]
 
 [[package]]
@@ -991,7 +1002,7 @@ dependencies = [
  "bumpalo",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.117",
  "wasm-bindgen-shared",
 ]
 
@@ -1051,7 +1062,7 @@ checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.117",
 ]
 
 [[package]]

{dlin_cli-0.2.4b1 → dlin_cli-0.2.4rc1}/Cargo.toml

@@ -3,7 +3,7 @@ members = ["crates/*"]
 resolver = "3"
 
 [workspace.package]
-version = "0.2.4-beta.1"
+version = "0.2.4-rc.1"
 edition = "2024"
 license = "MIT"
 repository = "https://github.com/eitsupi/dlin"
@@ -23,7 +23,7 @@ indexmap = "2"
 minijinja = "2"
 rayon = "1"
 globset = "0.4"
-polyglot-sql = { version = "0.6.2", default-features = false, features = ["all-dialects", "semantic"] }
+polyglot-sql = { version = "0.6.3", default-features = false, features = ["all-dialects", "semantic"] }
 path-slash = "0.2.1"
 clap = { version = "4", features = ["derive", "env"] }
 libc = "0.2"
@@ -34,7 +34,7 @@ insta = "1"
 serial_test = "3.4.0"
 
 # internal
-dlin-core = { version = "0.2.4-beta.1", path = "crates/dlin-core" }
+dlin-core = { version = "0.2.4-rc.1", path = "crates/dlin-core" }
 
 [workspace.lints.rust]
 unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tarpaulin_include)'] }

{dlin_cli-0.2.4b1 → dlin_cli-0.2.4rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dlin-cli
-Version: 0.2.4b1
+Version: 0.2.4rc1
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers

{dlin_cli-0.2.4b1 → dlin_cli-0.2.4rc1}/crates/dlin-core/src/graph/column_lineage/mod.rs

@@ -11,6 +11,7 @@ mod cross_model;
 mod impact;
 mod schema;
 mod single_model;
+mod star_guard;
 #[cfg(test)]
 mod tests;
 mod types;
@@ -24,7 +25,8 @@ pub use impact::{
     compute_column_impact_with_manifest_path,
 };
 use schema::{build_yaml_schema_for_node, compute_manifest_columns_hash, infer_output_columns};
-use single_model::{has_unresolved_stars, prepare_lineage_context_with_expr, run_column_lineage};
+use single_model::{prepare_lineage_context_with_expr, run_column_lineage};
+use star_guard::has_unresolved_stars;
 pub use types::{
     ColumnLineageEntry, ColumnLineageError, ColumnLineageErrorKind, ColumnSource,
     ModelColumnLineage, TransformationType,

dlin_cli-0.2.4rc1/crates/dlin-core/src/graph/column_lineage/single_model.rs

@@ -0,0 +1,257 @@
+use polyglot_sql::lineage::LineageNode;
+use polyglot_sql::scope::SourceKind;
+use polyglot_sql::{DialectType, Expression};
+
+use crate::parser::manifest::Manifest;
+
+use super::schema::build_schema_from_manifest;
+use super::star_guard::StarGuard;
+use super::{ColumnSource, TransformationType};
+
+pub(super) struct LineageContext {
+    pub(super) expanded_expr: Expression,
+    schema: Option<polyglot_sql::MappingSchema>,
+    dialect: DialectType,
+    star_guard: StarGuard,
+}
+
+pub(super) fn prepare_lineage_context(
+    compiled_code: &str,
+    manifest: &Manifest,
+    node: &crate::parser::manifest::ManifestNode,
+    dialect: DialectType,
+) -> Result<LineageContext, String> {
+    prepare_lineage_context_with_expr(compiled_code, manifest, node, dialect, None)
+}
+
+pub(super) fn prepare_lineage_context_with_expr(
+    compiled_code: &str,
+    manifest: &Manifest,
+    node: &crate::parser::manifest::ManifestNode,
+    dialect: DialectType,
+    parsed_expr: Option<&Expression>,
+) -> Result<LineageContext, String> {
+    let expr = match parsed_expr {
+        Some(e) => e.clone(),
+        None => polyglot_sql::parse_one(compiled_code, dialect).map_err(|e| format!("{}", e))?,
+    };
+    let schema = build_schema_from_manifest(manifest, node, dialect);
+
+    let mut expanded_expr = expr;
+    polyglot_sql::lineage::expand_cte_stars(
+        &mut expanded_expr,
+        schema.as_ref().map(|s| s as &dyn polyglot_sql::Schema),
+    );
+    let star_guard = StarGuard::new(&expanded_expr, Some(dialect));
+    Ok(LineageContext {
+        expanded_expr,
+        schema,
+        dialect,
+        star_guard,
+    })
+}
+
+pub(super) fn run_column_lineage(
+    col_name: &str,
+    ctx: &LineageContext,
+) -> Result<ColumnLineageResult, String> {
+    let modifier_expr =
+        ctx.star_guard
+            .materialize_star_modifier(&ctx.expanded_expr, col_name, ctx.dialect);
+    let lineage_expr = modifier_expr.as_ref().unwrap_or(&ctx.expanded_expr);
+    let lineage_result = lineage_node(col_name, lineage_expr, ctx);
+
+    match lineage_result {
+        Ok(node) => {
+            if ctx.star_guard.rejects(&node) {
+                Err(format!("Cannot find column '{}' in query", col_name))
+            } else {
+                Ok(extract_leaf_sources(&node))
+            }
+        }
+        Err(e) => explicit_set_operand_lineage(col_name, lineage_expr, ctx, &e)
+            .ok_or_else(|| format_lineage_error(&e)),
+    }
+}
+
+fn lineage_node(
+    col_name: &str,
+    expr: &Expression,
+    ctx: &LineageContext,
+) -> Result<LineageNode, polyglot_sql::Error> {
+    let dialect = Some(ctx.dialect);
+    if let Some(ref schema) = ctx.schema {
+        polyglot_sql::lineage::lineage_with_schema(
+            col_name,
+            expr,
+            Some(schema as &dyn polyglot_sql::Schema),
+            dialect,
+            false,
+        )
+        .or_else(|_| polyglot_sql::lineage::lineage(col_name, expr, dialect, false))
+    } else {
+        polyglot_sql::lineage::lineage(col_name, expr, dialect, false)
+    }
+}
+
+// TODO: this locates an operand's projection by ordinal (see
+// `StarGuard::explicit_set_operands`) but then re-derives lineage by looking that
+// projection's output *name* back up via `lineage_node`, instead of tracing the
+// already-located projection directly. Unaliased expressions have no name to look
+// up (e.g. `fee * 2` in `SELECT other.*, fee FROM a UNION ALL SELECT x, fee * 2
+// FROM b`) and are silently dropped; duplicate output names (`SELECT a.id, b.id`)
+// make the lookup ambiguous and can retrace the wrong projection. Redesign: keep
+// projection-list position and output ordinal as distinct types and trace
+// positionally instead of round-tripping through a name.
+fn explicit_set_operand_lineage(
+    col_name: &str,
+    expr: &Expression,
+    ctx: &LineageContext,
+    original_error: &polyglot_sql::Error,
+) -> Option<ColumnLineageResult> {
+    let error = original_error.to_string();
+    if !error.contains("Cannot find column") || !error.contains("in set operation") {
+        return None;
+    }
+
+    // A set's leftmost star can make its output names unknowable to polyglot-sql.
+    // Trace only intact operands that explicitly project the requested ordinal;
+    // unresolved star operands contribute no source instead of a guessed one.
+    let operands = StarGuard::explicit_set_operands(expr, col_name, Some(ctx.dialect))?;
+    let mut results = operands.into_iter().filter_map(|(operand, operand_name)| {
+        let node = lineage_node(operand_name, operand, ctx).ok()?;
+        (!ctx.star_guard.rejects(&node)).then(|| extract_leaf_sources(&node))
+    });
+    let first = results.next()?;
+    Some(results.fold(first, merge_lineage_results))
+}
+
+fn merge_lineage_results(
+    mut left: ColumnLineageResult,
+    right: ColumnLineageResult,
+) -> ColumnLineageResult {
+    if left.transformation == TransformationType::Direct
+        && right.transformation != TransformationType::Direct
+    {
+        left.transformation = right.transformation;
+    }
+    left.sources.extend(right.sources);
+    left.sources
+        .sort_by(|a, b| (&a.table, &a.column).cmp(&(&b.table, &b.column)));
+    left.sources.dedup();
+    left
+}
+
+pub(super) fn format_lineage_error(e: &polyglot_sql::Error) -> String {
+    let msg = e.to_string();
+    if let Some(rest) = msg
+        .strip_prefix("Parse error at line 0, column 0: ")
+        .or_else(|| msg.strip_prefix("Syntax error at line 0, column 0: "))
+    {
+        format!("lineage failed: {}", rest)
+    } else if msg.starts_with("Internal error: ") {
+        format!(
+            "lineage failed: {}",
+            msg.strip_prefix("Internal error: ").unwrap()
+        )
+    } else {
+        msg
+    }
+}
+
+fn classify_transformation(node: &LineageNode) -> TransformationType {
+    let t = classify_expression(&node.expression);
+    if t != TransformationType::Direct || node.downstream.is_empty() {
+        return t;
+    }
+    // Direct pass-through: traverse downstream to find the actual transformation type.
+    // This handles the case where a CTE computes an expression and a later SELECT
+    // references the result by name (which looks like Column → Direct at the surface).
+    for child in &node.downstream {
+        let child_t = classify_transformation(child);
+        if child_t != TransformationType::Direct {
+            return child_t;
+        }
+    }
+    TransformationType::Direct
+}
+
+fn classify_expression(expr: &Expression) -> TransformationType {
+    match expr {
+        Expression::Column(_) | Expression::Identifier(_) => TransformationType::Direct,
+        Expression::Alias(alias) => classify_expression(&alias.this),
+        Expression::Count(_)
+        | Expression::Sum(_)
+        | Expression::Avg(_)
+        | Expression::Min(_)
+        | Expression::Max(_) => TransformationType::Aggregation,
+        Expression::Cast(_) => TransformationType::Cast,
+        Expression::Case(_) => TransformationType::Conditional,
+        Expression::Add(_) | Expression::Sub(_) | Expression::Mul(_) | Expression::Div(_) => {
+            TransformationType::Expression
+        }
+        Expression::Anonymous(_) | Expression::Coalesce(_) | Expression::NullIf(_) => {
+            TransformationType::Expression
+        }
+        // Generic function calls (covers CONCAT, NULLIF, SPLIT, etc. parsed as Function)
+        Expression::Function(_) => TransformationType::Expression,
+        // Specialized scalar function variants with their own AST types
+        Expression::Upper(_)
+        | Expression::Lower(_)
+        | Expression::Length(_)
+        | Expression::Concat(_) => TransformationType::Expression,
+        _ => TransformationType::Unknown,
+    }
+}
+
+pub(super) struct ColumnLineageResult {
+    pub(super) sources: Vec<ColumnSource>,
+    pub(super) transformation: TransformationType,
+}
+
+fn extract_leaf_sources(node: &LineageNode) -> ColumnLineageResult {
+    let transformation = classify_transformation(node);
+    let mut sources = Vec::new();
+    collect_leaves(node, &mut sources);
+    sources.sort_by(|a, b| (&a.table, &a.column).cmp(&(&b.table, &b.column)));
+    sources.dedup();
+    ColumnLineageResult {
+        sources,
+        transformation,
+    }
+}
+
+fn collect_leaves(node: &LineageNode, sources: &mut Vec<ColumnSource>) {
+    if node.downstream.is_empty() {
+        if node.source_kind == SourceKind::Virtual {
+            return;
+        }
+        let name = &node.name;
+        if let Some((alias, column)) = name.rsplit_once('.') {
+            // polyglot-sql puts the SQL alias in node.name (e.g. "c.customer_id") and the
+            // actual table name in source_name (e.g. "stg_customers"). Use source_name
+            // when it genuinely differs from the alias, including fully-qualified forms
+            // (e.g. "shop.main.stg_orders" when the SQL uses the short name "stg_orders").
+            let table = if !node.source_name.is_empty() && node.source_name != alias {
+                node.source_name.as_str()
+            } else {
+                alias
+            };
+            sources.push(ColumnSource {
+                table: table.to_string(),
+                column: column.to_string(),
+                model_path: vec![],
+            });
+        } else {
+            sources.push(ColumnSource {
+                table: node.source_name.clone(),
+                column: name.to_string(),
+                model_path: vec![],
+            });
+        }
+    } else {
+        for child in &node.downstream {
+            collect_leaves(child, sources);
+        }
+    }
+}