dlin-cli 0.2.3__tar.gz → 0.2.4b1__tar.gz

{dlin_cli-0.2.3 → dlin_cli-0.2.4b1}/Cargo.lock

@@ -87,9 +87,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.102"
+version = "1.0.103"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
+checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3"
 
 [[package]]
 name = "arraydeque"
@@ -230,7 +230,7 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
 
 [[package]]
 name = "dlin"
-version = "0.2.3"
+version = "0.2.4-beta.1"
 dependencies = [
  "anyhow",
  "clap",
@@ -247,7 +247,7 @@ dependencies = [
 
 [[package]]
 name = "dlin-core"
-version = "0.2.3"
+version = "0.2.4-beta.1"
 dependencies = [
  "anyhow",
  "clap",
@@ -398,9 +398,9 @@ dependencies = [
 
 [[package]]
 name = "granit-parser"
-version = "0.0.3"
+version = "0.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f50ba32164f9e098d5da618776a32afbb32270adcbe3d3d006107dae11e37c91"
+checksum = "d03f81ad4732830d85cfd417a9f62cde6dadda4354d37d078a6084a19560aa2d"
 dependencies = [
  "arraydeque",
  "smallvec",
@@ -439,9 +439,9 @@ dependencies = [
 
 [[package]]
 name = "insta"
-version = "1.47.2"
+version = "1.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b4a6248eb93a4401ed2f37dfe8ea592d3cf05b7cf4f8efa867b6895af7e094e"
+checksum = "86f0f8fee8c926415c58d6ae43a08523a26faccb2323f5e6b644fe7dd4ef6b82"
 dependencies = [
  "console",
  "once_cell",
@@ -512,9 +512,9 @@ checksum = "38d1115007560874e373613744c6fba374c17688327a71c1476d1a5954cc857b"
 
 [[package]]
 name = "minijinja"
-version = "2.20.0"
+version = "2.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2929e494b2280e1e18959bb2e121da03347ae896896fdfaceaab43c88a02803f"
+checksum = "cb3d648e68cea56d9858d535ee28f9538404e2dd8cb08ed0bd05dca379477f39"
 dependencies = [
  "memo-map",
  "serde",
@@ -597,16 +597,28 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
 
 [[package]]
 name = "polyglot-sql"
-version = "0.4.4"
+version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0d5ea9283f03d3b6c32a230e60e7c6bc854e5bec83c399c75cad02bf46e9622"
+checksum = "009cbd62f8c82de46697221ee66b0cf3a6dacbda16f8c5a2ce25853940073bd5"
 dependencies = [
+ "polyglot-sql-ast-derive",
  "serde",
  "serde_json",
  "thiserror 1.0.69",
  "unicode-segmentation",
 ]
 
+[[package]]
+name = "polyglot-sql-ast-derive"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3250a6ffcd1d39670eb532e2b1877ad430799c49cb91b243d63c9425b868161"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.106"
@@ -735,9 +747,9 @@ dependencies = [
 
 [[package]]
 name = "serde-saphyr"
-version = "0.0.27"
+version = "0.0.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5897b4c3faadadd35fdb6689f015641f3bc481d5adaaac56231ea15aeb243db3"
+checksum = "7bd22781911de0ca6debda95f073c8f18bec65d1a94f1fa9573f3102e514cea4"
 dependencies = [
  "ahash",
  "annotate-snippets",
@@ -747,7 +759,7 @@ dependencies = [
  "granit-parser",
  "nohash-hasher",
  "num-traits",
- "serde",
+ "serde_core",
  "smallvec",
  "zmij",
 ]

{dlin_cli-0.2.3 → dlin_cli-0.2.4b1}/Cargo.toml

@@ -3,7 +3,7 @@ members = ["crates/*"]
 resolver = "3"
 
 [workspace.package]
-version = "0.2.3"
+version = "0.2.4-beta.1"
 edition = "2024"
 license = "MIT"
 repository = "https://github.com/eitsupi/dlin"
@@ -13,7 +13,7 @@ homepage = "https://github.com/eitsupi/dlin"
 petgraph = { version = "0.8", features = ["serde-1"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
-serde-saphyr = "0.0.27"
+serde-saphyr = "0.0.29"
 regex = "1"
 walkdir = "2"
 colored = "3"
@@ -23,7 +23,7 @@ indexmap = "2"
 minijinja = "2"
 rayon = "1"
 globset = "0.4"
-polyglot-sql = { version = "0.4.4", default-features = false, features = ["all-dialects", "semantic"] }
+polyglot-sql = { version = "0.6.2", default-features = false, features = ["all-dialects", "semantic"] }
 path-slash = "0.2.1"
 clap = { version = "4", features = ["derive", "env"] }
 libc = "0.2"
@@ -34,7 +34,7 @@ insta = "1"
 serial_test = "3.4.0"
 
 # internal
-dlin-core = { version = "0.2.3", path = "crates/dlin-core" }
+dlin-core = { version = "0.2.4-beta.1", path = "crates/dlin-core" }
 
 [workspace.lints.rust]
 unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tarpaulin_include)'] }

{dlin_cli-0.2.3 → dlin_cli-0.2.4b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dlin-cli
-Version: 0.2.3
+Version: 0.2.4b1
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers

dlin_cli-0.2.4b1/crates/dlin-core/src/graph/column_lineage/single_model.rs

@@ -0,0 +1,450 @@
+use polyglot_sql::lineage::LineageNode;
+use polyglot_sql::scope::SourceKind;
+use polyglot_sql::{DialectType, Expression};
+
+use crate::parser::manifest::Manifest;
+
+use super::schema::build_schema_from_manifest;
+use super::{ColumnSource, TransformationType};
+
+pub(super) struct LineageContext {
+    pub(super) expanded_expr: Expression,
+    schema: Option<polyglot_sql::MappingSchema>,
+    dialect: DialectType,
+}
+
+pub(super) fn prepare_lineage_context(
+    compiled_code: &str,
+    manifest: &Manifest,
+    node: &crate::parser::manifest::ManifestNode,
+    dialect: DialectType,
+) -> Result<LineageContext, String> {
+    prepare_lineage_context_with_expr(compiled_code, manifest, node, dialect, None)
+}
+
+pub(super) fn prepare_lineage_context_with_expr(
+    compiled_code: &str,
+    manifest: &Manifest,
+    node: &crate::parser::manifest::ManifestNode,
+    dialect: DialectType,
+    parsed_expr: Option<&Expression>,
+) -> Result<LineageContext, String> {
+    let expr = match parsed_expr {
+        Some(e) => e.clone(),
+        None => polyglot_sql::parse_one(compiled_code, dialect).map_err(|e| format!("{}", e))?,
+    };
+
+    let schema = build_schema_from_manifest(manifest, node, dialect);
+
+    let mut expanded_expr = expr;
+    polyglot_sql::lineage::expand_cte_stars(
+        &mut expanded_expr,
+        schema.as_ref().map(|s| s as &dyn polyglot_sql::Schema),
+    );
+
+    Ok(LineageContext {
+        expanded_expr,
+        schema,
+        dialect,
+    })
+}
+
+pub(super) fn run_column_lineage(
+    col_name: &str,
+    ctx: &LineageContext,
+) -> Result<ColumnLineageResult, String> {
+    let dialect = Some(ctx.dialect);
+    let lineage_result = if let Some(ref s) = ctx.schema {
+        polyglot_sql::lineage::lineage_with_schema(
+            col_name,
+            &ctx.expanded_expr,
+            Some(s as &dyn polyglot_sql::Schema),
+            dialect,
+            false,
+        )
+        .or_else(|_| polyglot_sql::lineage::lineage(col_name, &ctx.expanded_expr, dialect, false))
+    } else {
+        polyglot_sql::lineage::lineage(col_name, &ctx.expanded_expr, dialect, false)
+    };
+
+    match lineage_result {
+        Ok(node) => {
+            if lineage_depends_on_unresolved_star(&node, true, false, None, Some(ctx.dialect)) {
+                Err(format!("Cannot find column '{}' in query", col_name))
+            } else {
+                Ok(extract_leaf_sources(&node))
+            }
+        }
+        Err(e) => Err(format_lineage_error(&e)),
+    }
+}
+
+/// Return whether lineage for this column relied on polyglot-sql's schema-less
+/// star passthrough. A star elsewhere in the query is not sufficient: explicit
+/// output expressions must remain traceable independently.
+fn lineage_depends_on_unresolved_star(
+    node: &LineageNode,
+    is_root: bool,
+    is_set_operation_branch: bool,
+    set_operation_ordinal: Option<usize>,
+    dialect: Option<DialectType>,
+) -> bool {
+    let column_name = node
+        .reference_node_name
+        .rsplit_once('.')
+        .map_or(node.name.as_str(), |(_, name)| name);
+
+    let is_set_operation = matches!(
+        &node.source,
+        Expression::Union(_) | Expression::Intersect(_) | Expression::Except(_)
+    );
+    let set_operation_ordinal = if is_set_operation {
+        column_to_ordinal(&node.source, column_name, dialect)
+            .or_else(|| synthetic_ordinal(column_name))
+    } else {
+        set_operation_ordinal
+    };
+    if is_set_operation
+        && set_operation_ordinal
+            .is_some_and(|ordinal| set_operation_has_star_dependent_branch(&node.source, ordinal))
+    {
+        return true;
+    }
+    let crosses_query_boundary =
+        matches!(node.source_kind, SourceKind::Cte | SourceKind::DerivedTable);
+    let source_has_star = has_unresolved_stars(&node.source);
+    let source_has_explicit_column = select_has_explicit_output(&node.source, column_name, dialect);
+    let branch_projection_checked = is_set_operation_branch
+        && !crosses_query_boundary
+        && matches!(&node.source, Expression::Select(_))
+        && set_operation_ordinal.is_some();
+    let branch_projection_is_star = branch_projection_checked
+        && set_operation_ordinal
+            .is_some_and(|ordinal| projection_ordinal_depends_on_star(&node.source, ordinal));
+
+    if is_root {
+        if matches!(&node.expression, Expression::Column(c) if c.table.is_some())
+            && source_has_star
+            && !source_has_explicit_column
+        {
+            return true;
+        }
+    } else if branch_projection_is_star
+        || (!branch_projection_checked
+            && !is_set_operation
+            && (is_set_operation_branch
+                || matches!(
+                    &node.source,
+                    Expression::Select(_)
+                        | Expression::Union(_)
+                        | Expression::Intersect(_)
+                        | Expression::Except(_)
+                        | Expression::Subquery(_)
+                        | Expression::Cte(_)
+                        | Expression::Paren(_)
+                ))
+            && source_has_star
+            && !source_has_explicit_column)
+    {
+        return true;
+    }
+
+    let child_set_operation_ordinal = if is_set_operation || !crosses_query_boundary {
+        set_operation_ordinal
+    } else {
+        None
+    };
+    node.downstream.iter().any(|child| {
+        lineage_depends_on_unresolved_star(
+            child,
+            false,
+            is_set_operation_branch || is_set_operation,
+            child_set_operation_ordinal,
+            dialect,
+        )
+    })
+}
+
+/// Find the positional index of a column name in a set operation's first SELECT branch.
+fn column_to_ordinal(
+    set_operation_expr: &Expression,
+    column_name: &str,
+    dialect: Option<DialectType>,
+) -> Option<usize> {
+    let normalized_column_name = polyglot_sql::normalize_name(column_name, dialect, false, true);
+    let mut expr = set_operation_expr;
+
+    loop {
+        match expr {
+            Expression::Union(union) => expr = &union.left,
+            Expression::Intersect(intersect) => expr = &intersect.left,
+            Expression::Except(except) => expr = &except.left,
+            Expression::Annotated(annotated) => expr = &annotated.this,
+            Expression::Subquery(subquery) => expr = &subquery.this,
+            Expression::Cte(cte) => expr = &cte.this,
+            Expression::Paren(paren) => expr = &paren.this,
+            Expression::Select(select) => {
+                return select.expressions.iter().position(|expr| {
+                    explicit_output_name(expr).is_some_and(|name| {
+                        polyglot_sql::normalize_name(name, dialect, false, true)
+                            == normalized_column_name
+                    })
+                });
+            }
+            _ => return None,
+        }
+    }
+}
+
+/// Polyglot names an index-resolved set-operation node `_N`.
+fn synthetic_ordinal(column_name: &str) -> Option<usize> {
+    column_name.strip_prefix('_')?.parse().ok()
+}
+
+fn projection_ordinal_depends_on_star(expr: &Expression, ordinal: usize) -> bool {
+    match expr {
+        Expression::Annotated(annotated) => {
+            projection_ordinal_depends_on_star(&annotated.this, ordinal)
+        }
+        Expression::Select(select) => select
+            .expressions
+            .iter()
+            .position(expression_is_star)
+            .is_some_and(|first_star_ordinal| ordinal >= first_star_ordinal),
+        _ => false,
+    }
+}
+
+fn set_operation_has_star_dependent_branch(expr: &Expression, ordinal: usize) -> bool {
+    match expr {
+        Expression::Union(union) => {
+            set_operation_has_star_dependent_branch(&union.left, ordinal)
+                || set_operation_has_star_dependent_branch(&union.right, ordinal)
+        }
+        Expression::Intersect(intersect) => {
+            set_operation_has_star_dependent_branch(&intersect.left, ordinal)
+                || set_operation_has_star_dependent_branch(&intersect.right, ordinal)
+        }
+        Expression::Except(except) => {
+            set_operation_has_star_dependent_branch(&except.left, ordinal)
+                || set_operation_has_star_dependent_branch(&except.right, ordinal)
+        }
+        Expression::Annotated(annotated) => {
+            set_operation_has_star_dependent_branch(&annotated.this, ordinal)
+        }
+        Expression::Subquery(subquery) => {
+            set_operation_has_star_dependent_branch(&subquery.this, ordinal)
+        }
+        Expression::Cte(cte) => set_operation_has_star_dependent_branch(&cte.this, ordinal),
+        Expression::Paren(paren) => set_operation_has_star_dependent_branch(&paren.this, ordinal),
+        Expression::Select(_) => projection_ordinal_depends_on_star(expr, ordinal),
+        _ => false,
+    }
+}
+
+fn select_has_explicit_output(
+    expr: &Expression,
+    column_name: &str,
+    dialect: Option<DialectType>,
+) -> bool {
+    let normalized_column_name = polyglot_sql::normalize_name(column_name, dialect, false, true);
+
+    match expr {
+        Expression::Annotated(annotated) => {
+            select_has_explicit_output(&annotated.this, column_name, dialect)
+        }
+        Expression::Select(select) => select.expressions.iter().any(|expr| {
+            let output_name = explicit_output_name(expr);
+            output_name.is_some_and(|name| {
+                name != "*"
+                    && polyglot_sql::normalize_name(name, dialect, false, true)
+                        == normalized_column_name
+            })
+        }),
+        Expression::Union(union) => select_has_explicit_output(&union.left, column_name, dialect),
+        Expression::Intersect(intersect) => {
+            select_has_explicit_output(&intersect.left, column_name, dialect)
+        }
+        Expression::Except(except) => {
+            select_has_explicit_output(&except.left, column_name, dialect)
+        }
+        Expression::Subquery(subquery) => {
+            select_has_explicit_output(&subquery.this, column_name, dialect)
+        }
+        Expression::Cte(cte) => select_has_explicit_output(&cte.this, column_name, dialect),
+        Expression::Paren(paren) => select_has_explicit_output(&paren.this, column_name, dialect),
+        _ => false,
+    }
+}
+
+fn explicit_output_name(expr: &Expression) -> Option<&str> {
+    match expr {
+        Expression::Annotated(annotated) => explicit_output_name(&annotated.this),
+        Expression::Alias(alias) => Some(alias.alias.name.as_str()),
+        Expression::Column(column) => Some(column.name.name.as_str()),
+        Expression::Identifier(identifier) => Some(identifier.name.as_str()),
+        _ => None,
+    }
+}
+
+pub(super) fn has_unresolved_stars(expr: &Expression) -> bool {
+    match expr {
+        Expression::Select(select) => {
+            let outer_has_star = select.expressions.iter().any(expression_is_star);
+            if outer_has_star {
+                return true;
+            }
+            if let Some(with) = &select.with
+                && with.ctes.iter().any(|cte| has_unresolved_stars(&cte.this))
+            {
+                return true;
+            }
+            if let Some(from) = &select.from
+                && from.expressions.iter().any(has_unresolved_stars)
+            {
+                return true;
+            }
+            if select.joins.iter().any(|j| has_unresolved_stars(&j.this)) {
+                return true;
+            }
+            false
+        }
+        Expression::Subquery(subq) => has_unresolved_stars(&subq.this),
+        Expression::Cte(cte) => has_unresolved_stars(&cte.this),
+        Expression::Annotated(annotated) => has_unresolved_stars(&annotated.this),
+        Expression::Paren(paren) => has_unresolved_stars(&paren.this),
+        Expression::Union(union) => {
+            has_unresolved_stars(&union.left) || has_unresolved_stars(&union.right)
+        }
+        Expression::Intersect(intersect) => {
+            has_unresolved_stars(&intersect.left) || has_unresolved_stars(&intersect.right)
+        }
+        Expression::Except(except) => {
+            has_unresolved_stars(&except.left) || has_unresolved_stars(&except.right)
+        }
+        _ => false,
+    }
+}
+
+fn expression_is_star(expr: &Expression) -> bool {
+    match expr {
+        Expression::Star(_) => true,
+        Expression::Column(column) => column.name.name == "*",
+        Expression::Annotated(annotated) => expression_is_star(&annotated.this),
+        _ => false,
+    }
+}
+
+pub(super) fn format_lineage_error(e: &polyglot_sql::Error) -> String {
+    let msg = e.to_string();
+    if let Some(rest) = msg
+        .strip_prefix("Parse error at line 0, column 0: ")
+        .or_else(|| msg.strip_prefix("Syntax error at line 0, column 0: "))
+    {
+        format!("lineage failed: {}", rest)
+    } else if msg.starts_with("Internal error: ") {
+        format!(
+            "lineage failed: {}",
+            msg.strip_prefix("Internal error: ").unwrap()
+        )
+    } else {
+        msg
+    }
+}
+
+fn classify_transformation(node: &LineageNode) -> TransformationType {
+    let t = classify_expression(&node.expression);
+    if t != TransformationType::Direct || node.downstream.is_empty() {
+        return t;
+    }
+    // Direct pass-through: traverse downstream to find the actual transformation type.
+    // This handles the case where a CTE computes an expression and a later SELECT
+    // references the result by name (which looks like Column → Direct at the surface).
+    for child in &node.downstream {
+        let child_t = classify_transformation(child);
+        if child_t != TransformationType::Direct {
+            return child_t;
+        }
+    }
+    TransformationType::Direct
+}
+
+fn classify_expression(expr: &Expression) -> TransformationType {
+    match expr {
+        Expression::Column(_) | Expression::Identifier(_) => TransformationType::Direct,
+        Expression::Alias(alias) => classify_expression(&alias.this),
+        Expression::Count(_)
+        | Expression::Sum(_)
+        | Expression::Avg(_)
+        | Expression::Min(_)
+        | Expression::Max(_) => TransformationType::Aggregation,
+        Expression::Cast(_) => TransformationType::Cast,
+        Expression::Case(_) => TransformationType::Conditional,
+        Expression::Add(_) | Expression::Sub(_) | Expression::Mul(_) | Expression::Div(_) => {
+            TransformationType::Expression
+        }
+        Expression::Anonymous(_) | Expression::Coalesce(_) | Expression::NullIf(_) => {
+            TransformationType::Expression
+        }
+        // Generic function calls (covers CONCAT, NULLIF, SPLIT, etc. parsed as Function)
+        Expression::Function(_) => TransformationType::Expression,
+        // Specialized scalar function variants with their own AST types
+        Expression::Upper(_)
+        | Expression::Lower(_)
+        | Expression::Length(_)
+        | Expression::Concat(_) => TransformationType::Expression,
+        _ => TransformationType::Unknown,
+    }
+}
+
+pub(super) struct ColumnLineageResult {
+    pub(super) sources: Vec<ColumnSource>,
+    pub(super) transformation: TransformationType,
+}
+
+fn extract_leaf_sources(node: &LineageNode) -> ColumnLineageResult {
+    let transformation = classify_transformation(node);
+    let mut sources = Vec::new();
+    collect_leaves(node, &mut sources);
+    sources.sort_by(|a, b| (&a.table, &a.column).cmp(&(&b.table, &b.column)));
+    sources.dedup();
+    ColumnLineageResult {
+        sources,
+        transformation,
+    }
+}
+
+fn collect_leaves(node: &LineageNode, sources: &mut Vec<ColumnSource>) {
+    if node.downstream.is_empty() {
+        if node.source_kind == SourceKind::Virtual {
+            return;
+        }
+        let name = &node.name;
+        if let Some((alias, column)) = name.rsplit_once('.') {
+            // polyglot-sql puts the SQL alias in node.name (e.g. "c.customer_id") and the
+            // actual table name in source_name (e.g. "stg_customers"). Use source_name
+            // when it genuinely differs from the alias, including fully-qualified forms
+            // (e.g. "shop.main.stg_orders" when the SQL uses the short name "stg_orders").
+            let table = if !node.source_name.is_empty() && node.source_name != alias {
+                node.source_name.as_str()
+            } else {
+                alias
+            };
+            sources.push(ColumnSource {
+                table: table.to_string(),
+                column: column.to_string(),
+                model_path: vec![],
+            });
+        } else {
+            sources.push(ColumnSource {
+                table: String::new(),
+                column: name.to_string(),
+                model_path: vec![],
+            });
+        }
+    } else {
+        for child in &node.downstream {
+            collect_leaves(child, sources);
+        }
+    }
+}