dlin-cli 0.2.4rc1__tar.gz → 0.2.4rc2__tar.gz

{dlin_cli-0.2.4rc1 → dlin_cli-0.2.4rc2}/Cargo.lock

@@ -230,7 +230,7 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
 
 [[package]]
 name = "dlin"
-version = "0.2.4-rc.1"
+version = "0.2.4-rc.2"
 dependencies = [
  "anyhow",
  "clap",
@@ -247,7 +247,7 @@ dependencies = [
 
 [[package]]
 name = "dlin-core"
-version = "0.2.4-rc.1"
+version = "0.2.4-rc.2"
 dependencies = [
  "anyhow",
  "clap",
@@ -597,9 +597,9 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
 
 [[package]]
 name = "polyglot-sql"
-version = "0.6.3"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b54acea84522c6c51f6cbfa56600097cd8c6c5583a70e67f2787b6f70c3dd956"
+checksum = "6b7dd6da530087bffbf2b8b791914093b669f70eac07dbc66cf08f6954a1902e"
 dependencies = [
  "polyglot-sql-ast-derive",
  "serde",
@@ -610,9 +610,9 @@ dependencies = [
 
 [[package]]
 name = "polyglot-sql-ast-derive"
-version = "0.6.3"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c19af779599f7bf0ec303044dbb8e1c3f6c5b4dca29ea703b033519860104031"
+checksum = "c72cee7fdad5082a0cc73d7a9064914311fcd095b63bf5d85bd4c5b8dd89249e"
 dependencies = [
  "proc-macro2",
  "quote",

{dlin_cli-0.2.4rc1 → dlin_cli-0.2.4rc2}/Cargo.toml

@@ -3,7 +3,7 @@ members = ["crates/*"]
 resolver = "3"
 
 [workspace.package]
-version = "0.2.4-rc.1"
+version = "0.2.4-rc.2"
 edition = "2024"
 license = "MIT"
 repository = "https://github.com/eitsupi/dlin"
@@ -23,7 +23,7 @@ indexmap = "2"
 minijinja = "2"
 rayon = "1"
 globset = "0.4"
-polyglot-sql = { version = "0.6.3", default-features = false, features = ["all-dialects", "semantic"] }
+polyglot-sql = { version = "0.8.0", default-features = false, features = ["all-dialects", "semantic"] }
 path-slash = "0.2.1"
 clap = { version = "4", features = ["derive", "env"] }
 libc = "0.2"
@@ -34,7 +34,7 @@ insta = "1"
 serial_test = "3.4.0"
 
 # internal
-dlin-core = { version = "0.2.4-rc.1", path = "crates/dlin-core" }
+dlin-core = { version = "0.2.4-rc.2", path = "crates/dlin-core" }
 
 [workspace.lints.rust]
 unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tarpaulin_include)'] }

{dlin_cli-0.2.4rc1 → dlin_cli-0.2.4rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dlin-cli
-Version: 0.2.4rc1
+Version: 0.2.4rc2
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers

{dlin_cli-0.2.4rc1 → dlin_cli-0.2.4rc2}/crates/dlin-core/src/graph/column_lineage/cross_model.rs

@@ -5,6 +5,7 @@ use polyglot_sql::DialectType;
 
 use crate::parser::manifest::Manifest;
 
+use super::manifest_schema::make_fq_table_name;
 use super::{
     ColumnLineageCache, ColumnLineageError, ColumnSource, ModelColumnLineage, TransformationType,
     find_model_by_name,
@@ -254,11 +255,3 @@ fn compute_single_column_lineage(
         .ok()
         .map(|r| (r.sources, r.transformation))
 }
-
-fn make_fq_table_name(database: Option<&str>, schema: Option<&str>, name: &str) -> String {
-    match (database, schema) {
-        (Some(db), Some(s)) => format!("{}.{}.{}", db, s, name),
-        (None, Some(s)) => format!("{}.{}", s, name),
-        _ => name.to_string(),
-    }
-}

dlin_cli-0.2.4rc2/crates/dlin-core/src/graph/column_lineage/manifest_schema.rs

@@ -0,0 +1,302 @@
+//! Manifest helpers that are safe to use for ordinal-aware analysis.
+//!
+//! The legacy YAML schema builders remain in `schema.rs`.  The builder in this
+//! module has a stricter contract: only a compiled query whose output order is
+//! proven by polyglot-sql may become a `MappingSchema` relation.
+
+use std::collections::HashSet;
+
+use polyglot_sql::{DialectType, MappingSchema, Schema};
+
+use crate::parser::cache::hash_str;
+use crate::parser::manifest::{Manifest, ManifestNode};
+
+use super::semantic_model::build_query_model_from_sql;
+
+pub(super) fn make_fq_table_name(
+    database: Option<&str>,
+    schema: Option<&str>,
+    name: &str,
+) -> String {
+    match (database, schema) {
+        (Some(db), Some(s)) => format!("{}.{}.{}", db, s, name),
+        (None, Some(s)) => format!("{}.{}", s, name),
+        _ => name.to_string(),
+    }
+}
+
+/// Build a schema only from dependency models with a proven SQL output order.
+///
+/// Manifest YAML columns are intentionally not consulted for the relation's
+/// column order (or for adding a relation).  A source, YAML-only model,
+/// unnamed output, duplicate output name, or unresolved wildcard is omitted.
+pub fn build_proven_ordered_schema(
+    manifest: &Manifest,
+    node: &ManifestNode,
+    dialect: DialectType,
+) -> Option<MappingSchema> {
+    let mut schema = MappingSchema::with_dialect(dialect);
+    let mut has_entries = false;
+
+    for dependency_id in &node.depends_on.nodes {
+        let Some(dependency) = manifest.nodes.get(dependency_id) else {
+            // Sources and other YAML-only relations do not prove an ordinal
+            // schema for this stage.
+            continue;
+        };
+        let Some(sql) = dependency.compiled_code.as_deref() else {
+            continue;
+        };
+        let Ok(query_model) = build_query_model_from_sql(sql, dialect, None) else {
+            continue;
+        };
+        let Some(column_names) = query_model.outputs.proven_ordered_named_outputs() else {
+            continue;
+        };
+
+        let columns = column_names
+            .into_iter()
+            .map(|name| (name, polyglot_sql::expressions::DataType::Unknown))
+            .collect::<Vec<_>>();
+        let qualified_name = make_fq_table_name(
+            dependency.database.as_deref(),
+            dependency.schema.as_deref(),
+            &dependency.name,
+        );
+        if schema.add_table(&qualified_name, &columns, None).is_ok() {
+            has_entries = true;
+        }
+        if qualified_name != dependency.name {
+            let _ = schema.add_table(&dependency.name, &columns, None);
+        }
+    }
+
+    has_entries.then_some(schema)
+}
+
+pub(super) fn compute_manifest_columns_hash(manifest: &Manifest, node: &ManifestNode) -> u64 {
+    let mut visited = HashSet::new();
+    hash_node_columns_transitive(manifest, node, &mut visited)
+}
+
+fn hash_node_columns_transitive(
+    manifest: &Manifest,
+    node: &ManifestNode,
+    visited: &mut HashSet<String>,
+) -> u64 {
+    let mut parts = Vec::new();
+
+    let mut own_cols = node.columns.keys().collect::<Vec<_>>();
+    own_cols.sort_unstable();
+    for column in own_cols {
+        parts.push(column.clone());
+    }
+    if let Some(code) = &node.compiled_code {
+        parts.push(format!("sql:{}", hash_str(code)));
+    }
+    parts.push("|".to_string());
+
+    let mut dependency_ids = node.depends_on.nodes.iter().collect::<Vec<_>>();
+    dependency_ids.sort_unstable();
+    for dependency_id in dependency_ids {
+        parts.push(dependency_id.clone());
+        if !visited.insert(dependency_id.clone()) {
+            continue;
+        }
+        if let Some(dependency) = manifest.nodes.get(dependency_id) {
+            let dependency_hash = hash_node_columns_transitive(manifest, dependency, visited);
+            parts.push(format!("node:{}", dependency_hash));
+        } else if let Some(source) = manifest.sources.get(dependency_id) {
+            let mut columns = source.columns.keys().collect::<Vec<_>>();
+            columns.sort_unstable();
+            for column in columns {
+                parts.push(column.clone());
+            }
+            if let Some(database) = &source.database {
+                parts.push(format!("db:{}", database));
+            }
+            if let Some(schema) = &source.schema {
+                parts.push(format!("schema:{}", schema));
+            }
+            if let Some(identifier) = &source.identifier {
+                parts.push(format!("id:{}", identifier));
+            }
+        }
+    }
+
+    hash_str(&parts.join("\0"))
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+
+    use super::*;
+    use crate::parser::manifest::{DependsOn, ManifestColumn, ManifestConfig};
+
+    fn node(
+        unique_id: &str,
+        name: &str,
+        compiled_code: Option<&str>,
+        columns: &[&str],
+        dependencies: &[&str],
+    ) -> ManifestNode {
+        let columns = columns
+            .iter()
+            .map(|name| {
+                (
+                    (*name).to_string(),
+                    ManifestColumn {
+                        name: (*name).to_string(),
+                    },
+                )
+            })
+            .collect();
+        ManifestNode {
+            unique_id: unique_id.to_string(),
+            name: name.to_string(),
+            resource_type: "model".to_string(),
+            depends_on: DependsOn {
+                nodes: dependencies.iter().map(|id| (*id).to_string()).collect(),
+            },
+            config: ManifestConfig::default(),
+            description: None,
+            path: None,
+            original_file_path: None,
+            columns,
+            compiled_code: compiled_code.map(str::to_string),
+            database: None,
+            schema: None,
+        }
+    }
+
+    fn manifest(nodes: Vec<ManifestNode>) -> Manifest {
+        Manifest {
+            nodes: nodes
+                .into_iter()
+                .map(|node| (node.unique_id.clone(), node))
+                .collect::<HashMap<_, _>>(),
+            ..Manifest::default()
+        }
+    }
+
+    #[test]
+    fn yaml_map_order_does_not_change_proven_ordinal_schema() {
+        let upstream_id = "model.test.upstream";
+        let downstream_id = "model.test.downstream";
+        let upstream_sql = Some("SELECT first_col, second_col FROM source_table");
+        let downstream = node(
+            downstream_id,
+            "downstream",
+            Some("SELECT first_col FROM upstream"),
+            &[],
+            &[upstream_id],
+        );
+
+        let first = build_proven_ordered_schema(
+            &manifest(vec![
+                node(
+                    upstream_id,
+                    "upstream",
+                    upstream_sql,
+                    &["first_col", "second_col"],
+                    &[],
+                ),
+                node(
+                    downstream_id,
+                    "downstream",
+                    Some("SELECT first_col FROM upstream"),
+                    &[],
+                    &[upstream_id],
+                ),
+            ]),
+            &downstream,
+            DialectType::Generic,
+        )
+        .unwrap();
+        assert_eq!(
+            first.column_names("upstream").unwrap(),
+            vec!["first_col", "second_col"]
+        );
+        // The second manifest has the same SQL but a different YAML map order.
+        // Rebuild the node reference from that manifest so only the SQL proof
+        // participates in the schema.
+        let second_manifest = manifest(vec![
+            node(
+                upstream_id,
+                "upstream",
+                upstream_sql,
+                &["second_col", "first_col"],
+                &[],
+            ),
+            node(
+                downstream_id,
+                "downstream",
+                Some("SELECT first_col FROM upstream"),
+                &[],
+                &[upstream_id],
+            ),
+        ]);
+        let second = build_proven_ordered_schema(
+            &second_manifest,
+            second_manifest.nodes.get(downstream_id).unwrap(),
+            DialectType::Generic,
+        )
+        .unwrap();
+        assert_eq!(
+            second.column_names("upstream").unwrap(),
+            first.column_names("upstream").unwrap()
+        );
+    }
+
+    #[test]
+    fn only_proven_compiled_named_outputs_enter_the_schema() {
+        let good_id = "model.test.good";
+        let unnamed_id = "model.test.unnamed";
+        let star_id = "model.test.star";
+        let downstream = node(
+            "model.test.downstream",
+            "downstream",
+            Some("SELECT value FROM good"),
+            &[],
+            &[good_id, unnamed_id, star_id],
+        );
+        let manifest = manifest(vec![
+            node(
+                good_id,
+                "good",
+                Some("SELECT first_col, second_col FROM t"),
+                &[],
+                &[],
+            ),
+            node(
+                unnamed_id,
+                "unnamed",
+                Some("SELECT value * 2 FROM t"),
+                &["yaml_name"],
+                &[],
+            ),
+            node(
+                star_id,
+                "star",
+                Some("SELECT * FROM unknown"),
+                &["yaml_name"],
+                &[],
+            ),
+            downstream,
+        ]);
+
+        let schema = build_proven_ordered_schema(
+            &manifest,
+            manifest.nodes.get("model.test.downstream").unwrap(),
+            DialectType::Generic,
+        )
+        .unwrap();
+        assert_eq!(
+            schema.column_names("good").unwrap(),
+            vec!["first_col", "second_col"]
+        );
+        assert!(schema.column_names("unnamed").is_err());
+        assert!(schema.column_names("star").is_err());
+    }
+}

{dlin_cli-0.2.4rc1 → dlin_cli-0.2.4rc2}/crates/dlin-core/src/graph/column_lineage/mod.rs

@@ -1,7 +1,7 @@
 use std::collections::HashSet;
 use std::path::Path;
 
-use polyglot_sql::DialectType;
+use polyglot_sql::{ColumnResolutionReason, DialectType, Error};
 use rayon::prelude::*;
 
 use crate::parser::manifest::Manifest;
@@ -9,7 +9,9 @@ use crate::parser::manifest::Manifest;
 mod cache;
 mod cross_model;
 mod impact;
+pub mod manifest_schema;
 mod schema;
+pub mod semantic_model;
 mod single_model;
 mod star_guard;
 #[cfg(test)]
@@ -24,7 +26,8 @@ pub use impact::{
     ColumnImpactReport, ImpactedColumn, compute_column_impact,
     compute_column_impact_with_manifest_path,
 };
-use schema::{build_yaml_schema_for_node, compute_manifest_columns_hash, infer_output_columns};
+use manifest_schema::compute_manifest_columns_hash;
+use schema::{build_yaml_schema_for_node, infer_output_columns};
 use single_model::{prepare_lineage_context_with_expr, run_column_lineage};
 use star_guard::has_unresolved_stars;
 pub use types::{
@@ -176,14 +179,19 @@ pub fn compute_column_lineage_with_manifest_path(
                 sources: result.sources,
             }),
             Err(e) => {
-                let hint = (has_star_columns && e.contains("Cannot find column")).then(|| {
-                    "column may be introduced via SELECT * that could not be expanded; \
+                let hint =
+                    (has_star_columns && is_indeterminate_column_resolution(&e)).then(|| {
+                        "column may be introduced via SELECT * that could not be expanded; \
                      define upstream columns in the model YAML to enable full resolution"
-                        .to_string()
-                });
+                            .to_string()
+                    });
                 Err(ColumnLineageError {
                     kind: ColumnLineageErrorKind::ColumnNotFound,
-                    what: format!("column '{}': {}", col_name, e),
+                    what: format!(
+                        "column '{}': {}",
+                        col_name,
+                        single_model::format_lineage_error(&e)
+                    ),
                     why: None,
                     hint,
                 })
@@ -220,6 +228,16 @@ pub fn compute_column_lineage_with_manifest_path(
     result
 }
 
+fn is_indeterminate_column_resolution(error: &Error) -> bool {
+    matches!(
+        error,
+        Error::ColumnResolution {
+            reason: ColumnResolutionReason::Indeterminate,
+            ..
+        }
+    )
+}
+
 pub(super) fn find_model_by_name<'a>(
     manifest: &'a Manifest,
     name: &str,

{dlin_cli-0.2.4rc1 → dlin_cli-0.2.4rc2}/crates/dlin-core/src/graph/column_lineage/schema.rs

@@ -2,16 +2,9 @@ use std::collections::HashSet;
 
 use polyglot_sql::{DialectType, Expression, Schema};
 
-use crate::parser::cache::hash_str;
 use crate::parser::manifest::Manifest;
 
-fn make_fq_table_name(database: Option<&str>, schema: Option<&str>, name: &str) -> String {
-    match (database, schema) {
-        (Some(db), Some(s)) => format!("{}.{}.{}", db, s, name),
-        (None, Some(s)) => format!("{}.{}", s, name),
-        _ => name.to_string(),
-    }
-}
+use super::manifest_schema::make_fq_table_name;
 
 pub(super) fn build_schema_from_manifest(
     manifest: &Manifest,
@@ -190,60 +183,3 @@ pub(super) fn infer_output_columns(
         schema.map(|s| s as &dyn polyglot_sql::Schema),
     )
 }
-
-pub(super) fn compute_manifest_columns_hash(
-    manifest: &Manifest,
-    node: &crate::parser::manifest::ManifestNode,
-) -> u64 {
-    let mut visited: HashSet<String> = HashSet::new();
-    hash_node_columns_transitive(manifest, node, &mut visited)
-}
-
-fn hash_node_columns_transitive(
-    manifest: &Manifest,
-    node: &crate::parser::manifest::ManifestNode,
-    visited: &mut HashSet<String>,
-) -> u64 {
-    let mut parts: Vec<String> = Vec::new();
-
-    let mut own_cols: Vec<&String> = node.columns.keys().collect();
-    own_cols.sort();
-    for col in own_cols {
-        parts.push(col.clone());
-    }
-    if let Some(code) = &node.compiled_code {
-        parts.push(format!("sql:{}", hash_str(code)));
-    }
-    parts.push("|".to_string());
-
-    let mut dep_ids: Vec<&String> = node.depends_on.nodes.iter().collect();
-    dep_ids.sort();
-    for dep_id in dep_ids {
-        parts.push(dep_id.clone());
-        if visited.contains(dep_id) {
-            continue;
-        }
-        visited.insert(dep_id.clone());
-        if let Some(dep_node) = manifest.nodes.get(dep_id) {
-            let dep_hash = hash_node_columns_transitive(manifest, dep_node, visited);
-            parts.push(format!("node:{}", dep_hash));
-        } else if let Some(dep_source) = manifest.sources.get(dep_id) {
-            let mut cols: Vec<&String> = dep_source.columns.keys().collect();
-            cols.sort();
-            for col in cols {
-                parts.push(col.clone());
-            }
-            if let Some(db) = &dep_source.database {
-                parts.push(format!("db:{}", db));
-            }
-            if let Some(s) = &dep_source.schema {
-                parts.push(format!("schema:{}", s));
-            }
-            if let Some(id) = &dep_source.identifier {
-                parts.push(format!("id:{}", id));
-            }
-        }
-    }
-
-    hash_str(&parts.join("\0"))
-}