recce-nightly 1.10.0.20250625__py3-none-any.whl → 1.30.0.20251221__py3-none-any.whl

tests/adapter/dbt_adapter/test_dbt_cll.py

@@ -1,384 +0,0 @@
-from recce.adapter.dbt_adapter import DbtAdapter
-from recce.models.types import CllData
-
-
-def assert_column(result: CllData, node_name, column_name, transformation_type, depends_on):
-    column_id = f"{node_name}_{column_name}"
-    entry = result.columns.get(column_id)
-    assert entry is not None, f"Column {column_id} not found in result"
-    assert (
-        entry.transformation_type == transformation_type
-    ), f"Column {column_name} type mismatch: expected {transformation_type}, got {entry.transformation_type}"
-    parents = result.parent_map.get(column_id)
-
-    assert len(parents) == len(depends_on), "depends_on length mismatch"
-    for i in range(len(depends_on)):
-        node, column = depends_on[i]
-        parent_column_id = f"{node}_{column}"
-
-        assert parent_column_id in parents, f"Column {parent_column_id} not found in {column_id}'s parent list"
-
-
-def assert_model(result: CllData, node_name, depends_on):
-    assert result.nodes.get(node_name) is not None, f"Node {node_name} not found in result"
-    parent_map = result.parent_map.get(node_name)
-    assert parent_map is not None, f"Parent map {node_name} not found in result"
-    # assert len(parent_map) == len(depends_on), "depends_on length mismatch"
-    for i in range(len(depends_on)):
-        node, column = depends_on[i]
-        column_id = f"{node}_{column}"
-
-        assert column_id in parent_map, f"Parent map {node_name} does not contain {column_id}"
-
-
-def test_cll_basic(dbt_test_helper):
-    dbt_test_helper.create_model(
-        "model1", unique_id="model.model1", curr_sql="select 1 as c", curr_columns={"c": "int"}
-    )
-    dbt_test_helper.create_model(
-        "model2",
-        unique_id="model.model2",
-        curr_sql='select c from {{ ref("model1") }} where c > 0',
-        curr_columns={"c": "int"},
-        depends_on=["model.model1"],
-    )
-    dbt_test_helper.create_model(
-        "model3",
-        unique_id="model.model3",
-        curr_sql='select c from {{ ref("recce_test", "model1") }} where c > 0',
-        curr_columns={"c": "int"},
-        depends_on=["model.model1"],
-    )
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-
-    result = adapter.get_cll("model.model2", no_filter=True)
-    assert_model(result, "model.model2", [("model.model1", "c")])
-    assert_column(result, "model.model2", "c", "passthrough", [("model.model1", "c")])
-
-    result = adapter.get_cll("model.model3", no_filter=True)
-    assert_model(result, "model.model3", [("model.model1", "c")])
-    assert_column(result, "model.model3", "c", "passthrough", [("model.model1", "c")])
-
-
-def test_cll_table_alisa(dbt_test_helper):
-    def patch_node(node):
-        node["alias"] = "model1_alias"
-
-    dbt_test_helper.create_model(
-        "model1", unique_id="model.model1", curr_sql="select 1 as c", curr_columns={"c": "int"}, patch_func=patch_node
-    )
-    dbt_test_helper.create_model(
-        "model2",
-        unique_id="model.model2",
-        curr_sql='select c from {{ ref("model1") }}',
-        curr_columns={"c": "int"},
-        depends_on=["model.model1"],
-    )
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-    result = adapter.get_cll("model.model1", no_filter=True)
-    assert_column(result, "model.model2", "c", "passthrough", [("model.model1", "c")])
-
-
-def test_seed(dbt_test_helper):
-    csv_data_curr = """
-    customer_id,name,age
-    1,Alice,30
-    2,Bob,25
-    3,Charlie,35
-    """
-
-    dbt_test_helper.create_model(
-        "seed1",
-        unique_id="seed.seed1",
-        curr_csv=csv_data_curr,
-        curr_columns={"customer_id": "varchar", "name": "varchar", "age": "int"},
-        resource_type="seed",
-    )
-    dbt_test_helper.create_model(
-        "model1",
-        unique_id="model.model1",
-        curr_sql='select customer_id from {{ ref("seed1") }} where age > 0',
-        curr_columns={"customer_id": "varchar"},
-        depends_on=["seed.seed1"],
-    )
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-
-    result = adapter.get_cll("model.model1", no_filter=True)
-    assert_model(result, "seed.seed1", [])
-    assert_column(result, "seed.seed1", "customer_id", "source", [])
-    assert_model(result, "model.model1", [("seed.seed1", "age")])
-    assert_column(result, "model.model1", "customer_id", "passthrough", [("seed.seed1", "customer_id")])
-
-
-def test_python_model(dbt_test_helper):
-    def python_node(node):
-        node["language"] = "python"
-
-    csv_data_curr = """
-    customer_id,name,age
-    1,Alice,30
-    2,Bob,25
-    3,Charlie,35
-    """
-    dbt_test_helper.create_model(
-        "model1", curr_csv=csv_data_curr, curr_columns={"customer_id": "varchar", "name": "varchar", "age": "int"}
-    )
-    dbt_test_helper.create_model(
-        "model2",
-        curr_csv=csv_data_curr,
-        curr_columns={"customer_id": "varchar", "name": "varchar", "age": "int"},
-        depends_on=["model1"],
-        patch_func=python_node,
-    )
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-    assert not adapter.is_python_model("model1")
-    assert adapter.is_python_model("model2")
-
-    result = adapter.get_cll("model1", no_filter=True)
-    assert_column(result, "model2", "customer_id", "unknown", [])
-
-
-def test_source(dbt_test_helper):
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    dbt_test_helper.create_source(
-        "source1",
-        "table1",
-        unique_id="source.source1.table1",
-        curr_csv=csv_data_curr,
-        curr_columns={"customer_id": "varchar", "name": "varchar", "age": "int"},
-    )
-    dbt_test_helper.create_model(
-        "model1",
-        unique_id="model.model1",
-        curr_sql='select customer_id from {{ source("source1", "table1") }}',
-        curr_columns={"customer_id": "int"},
-        depends_on=["source.source1.table1"],
-    )
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-    result = adapter.get_cll("source.source1.table1", no_filter=True)
-    assert_column(result, "source.source1.table1", "customer_id", "source", [])
-    result = adapter.get_cll("model.model1", no_filter=True)
-    assert_column(result, "model.model1", "customer_id", "passthrough", [("source.source1.table1", "customer_id")])
-
-
-def test_parse_error(dbt_test_helper):
-    dbt_test_helper.create_model("model1", curr_sql="select 1 as c", curr_columns={"c": "int"})
-    dbt_test_helper.create_model("model2", curr_sql="this is not a valid sql", curr_columns={"c": "int"})
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-    result = adapter.get_cll("model2", no_filter=True)
-    assert_column(result, "model2", "c", "unknown", [])
-
-
-def test_model_without_catalog(dbt_test_helper):
-    dbt_test_helper.create_model("model1", curr_sql="select 1 as c")
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-    result = adapter.get_cll("model1", no_filter=True)
-    assert not result.nodes["model1"].columns
-
-
-def assert_lineage_model(cll_data: CllData, nodes):
-    assert len(nodes) == len(cll_data.nodes), "Model count mismatch"
-    for node in nodes:
-        assert node in cll_data.nodes, f"Model {node} not found in lineage"
-
-
-def assert_lineage_column(cll_data: CllData, columns):
-    assert len(columns) == len(cll_data.columns), "Column count mismatch"
-    for column in columns:
-        column_key = f"{column[0]}_{column[1]}"
-        assert column_key in cll_data.columns, f"Column {column} not found in lineage"
-        assert column[0] == cll_data.columns[column_key].table_id, f"Column {column[0]} node mismatch"
-        assert column[1] == cll_data.columns[column_key].name, f"Column {column[1]} name mismatch"
-
-
-def test_cll_column_filter(dbt_test_helper):
-    dbt_test_helper.create_model(
-        "model1", unique_id="model.model1", curr_sql="select 1 as c", curr_columns={"c": "int"}
-    )
-    dbt_test_helper.create_model(
-        "model2",
-        unique_id="model.model2",
-        curr_sql='select c, 2025 as y from {{ ref("model1") }}',
-        curr_columns={"c": "int", "y": "int"},
-        depends_on=["model.model1"],
-    )
-    dbt_test_helper.create_model(
-        "model3",
-        unique_id="model.model3",
-        curr_sql='select c from {{ ref("model2") }} where y < 2025',
-        curr_columns={"c": "int"},
-        depends_on=["model.model2"],
-    )
-    dbt_test_helper.create_model(
-        "model4",
-        unique_id="model.model4",
-        curr_sql='select y from {{ ref("model2") }}',
-        curr_columns={"y": "int"},
-        depends_on=["model.model2"],
-    )
-
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-
-    result = adapter.get_cll("model.model2", "c")
-    assert_lineage_model(result, [])
-    assert_lineage_column(result, [("model.model1", "c"), ("model.model2", "c"), ("model.model3", "c")])
-
-    result = adapter.get_cll("model.model2", "y")
-    assert_lineage_model(result, ["model.model3"])
-    assert_lineage_column(result, [("model.model2", "y"), ("model.model4", "y")])
-
-    result = adapter.get_cll("model.model3", "c")
-    assert_lineage_model(result, [])
-    assert_lineage_column(result, [("model.model1", "c"), ("model.model2", "c"), ("model.model3", "c")])
-
-
-def test_impact_radius_nodes(dbt_test_helper):
-    # non-breaking
-    dbt_test_helper.create_model(
-        "model1",
-        unique_id="model.model1",
-        curr_sql="select 1 as c",
-        base_sql="select 1 as c --- non-breaking",
-        curr_columns={"c": "int"},
-        base_columns={"c": "int"},
-    )
-    # breaking
-    dbt_test_helper.create_model(
-        "model2",
-        unique_id="model.model2",
-        curr_sql='select c, 2025 as y from {{ ref("model1") }}',
-        base_sql='select c, 2025 as y from {{ ref("model1") }} where c > 0 --- breaking',
-        curr_columns={"c": "int", "y": "int"},
-        base_columns={"c": "int", "y": "int"},
-        depends_on=["model.model1"],
-    )
-    dbt_test_helper.create_model(
-        "model3",
-        unique_id="model.model3",
-        curr_sql='select c from {{ ref("model2") }} where y < 2025',
-        base_sql='select c from {{ ref("model2") }} where y < 2025',
-        curr_columns={"c": "int"},
-        base_columns={"c": "int"},
-        depends_on=["model.model2"],
-    )
-    dbt_test_helper.create_model(
-        "model4",
-        unique_id="model.model4",
-        curr_sql='select y from {{ ref("model2") }}',
-        base_sql='select y from {{ ref("model2") }}',
-        curr_columns={"y": "int"},
-        base_columns={"y": "int"},
-        depends_on=["model.model2"],
-    )
-
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-
-    # breaking
-    result = adapter.get_impacted_nodes("model.model2")
-    assert_lineage_model(result, ["model.model2", "model.model3", "model.model4"])
-
-    # non-breaking
-    result = adapter.get_impacted_nodes("model.model1")
-    assert_lineage_model(result, [])
-
-
-def test_impact_radius_columns(dbt_test_helper):
-    # added column
-    dbt_test_helper.create_model(
-        "model1",
-        unique_id="model.model1",
-        curr_sql="select 1 as c, 2 as d --- add d",
-        base_sql="select 1 as c",
-        curr_columns={"c": "int", "d": "int"},
-        base_columns={"c": "int"},
-    )
-    # modified column
-    dbt_test_helper.create_model(
-        "model2",
-        unique_id="model.model2",
-        curr_sql='select c, 2024 as y from {{ ref("model1") }} --- modify y',
-        base_sql='select c, 2025 as y from {{ ref("model1") }}',
-        curr_columns={"c": "int", "y": "int"},
-        base_columns={"c": "int", "y": "int"},
-        depends_on=["model.model1"],
-    )
-    dbt_test_helper.create_model(
-        "model3",
-        unique_id="model.model3",
-        curr_sql='select c from {{ ref("model2") }} where y < 2025',
-        base_sql='select c from {{ ref("model2") }} where y < 2025',
-        curr_columns={"c": "int"},
-        base_columns={"c": "int"},
-        depends_on=["model.model2"],
-    )
-    dbt_test_helper.create_model(
-        "model4",
-        unique_id="model.model4",
-        curr_sql='select y from {{ ref("model2") }}',
-        base_sql='select y from {{ ref("model2") }}',
-        curr_columns={"y": "int"},
-        base_columns={"y": "int"},
-        depends_on=["model.model2"],
-    )
-
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-
-    result = adapter.get_impacted_cll("model.model2")
-    assert_lineage_model(result, ["model.model3"])
-    assert_lineage_column(result, [("model.model2", "y"), ("model.model4", "y")])
-
-    result = adapter.get_impacted_cll("model.model1")
-    assert_lineage_model(result, [])
-    assert_lineage_column(result, [("model.model1", "d")])
-
-
-def test_impact_radius(dbt_test_helper):
-    # added column
-    dbt_test_helper.create_model(
-        "model1",
-        unique_id="model.model1",
-        curr_sql="select 1 as c, 2 as d --- add d",
-        base_sql="select 1 as c",
-        curr_columns={"c": "int", "d": "int"},
-        base_columns={"c": "int"},
-    )
-    # breaking, added column, modified column
-    dbt_test_helper.create_model(
-        "model2",
-        unique_id="model.model2",
-        curr_sql='select c, 2024 as y, d from {{ ref("model1") }} --- modify y, add d',
-        base_sql='select c, 2025 as y from {{ ref("model1") }} where c > 0 --- breaking',
-        curr_columns={"c": "int", "y": "int", "d": "int"},
-        base_columns={"c": "int", "y": "int"},
-        depends_on=["model.model1"],
-    )
-    dbt_test_helper.create_model(
-        "model3",
-        unique_id="model.model3",
-        curr_sql='select c from {{ ref("model2") }} where y < 2025',
-        base_sql='select c from {{ ref("model2") }} where y < 2025',
-        curr_columns={"c": "int"},
-        base_columns={"c": "int"},
-        depends_on=["model.model2"],
-    )
-    dbt_test_helper.create_model(
-        "model4",
-        unique_id="model.model4",
-        curr_sql='select y from {{ ref("model2") }}',
-        base_sql='select y from {{ ref("model2") }}',
-        curr_columns={"y": "int"},
-        base_columns={"y": "int"},
-        depends_on=["model.model2"],
-    )
-
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-
-    result = adapter.get_impact_radius("model.model2")
-    assert_lineage_model(result, ["model.model2", "model.model3", "model.model4"])
-    assert_lineage_column(result, [("model.model2", "d"), ("model.model2", "y"), ("model.model4", "y")])

tests/adapter/dbt_adapter/test_selector.py

@@ -1,202 +0,0 @@
-from recce.adapter.dbt_adapter import DbtAdapter
-
-
-def test_select(dbt_test_helper):
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,35
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    dbt_test_helper.create_model("customers_1", csv_data_base, csv_data_curr)
-    dbt_test_helper.create_model("customers_2", csv_data_base, csv_data_base, depends_on=["customers_1"])
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-
-    # Test methods
-    node_ids = adapter.select_nodes("customers_1")
-    assert len(node_ids) == 1
-    node_ids = adapter.select_nodes("resource_type:model")
-    assert len(node_ids) == 2
-    node_ids = adapter.select_nodes("tag:test_tag")
-    assert len(node_ids) == 2
-    node_ids = adapter.select_nodes("tag:test_tag2")
-    assert len(node_ids) == 0
-    node_ids = adapter.select_nodes("config.materialized:incremental")
-    assert len(node_ids) == 0
-    node_ids = adapter.select_nodes("config.materialized:table")
-    assert len(node_ids) == 2
-
-    # Test state method
-    node_ids = adapter.select_nodes("state:modified")
-    assert len(node_ids) == 1
-
-    # Test set operation
-    node_ids = adapter.select_nodes("customers_1 customers_2")
-    assert len(node_ids) == 2
-    node_ids = adapter.select_nodes("customers_1,customers_2")
-    assert len(node_ids) == 0
-    node_ids = adapter.select_nodes("config.materialized:table,tag:test_tag")
-    assert len(node_ids) == 2
-    node_ids = adapter.select_nodes("config.materialized:table,tag:test_tag2")
-    assert len(node_ids) == 0
-    node_ids = adapter.select_nodes(exclude="customers_1")
-    assert len(node_ids) == 1
-    node_ids = adapter.select_nodes("customers_1", exclude="customers_2")
-    assert len(node_ids) == 1
-
-    # Test graph operation
-    node_ids = adapter.select_nodes("state:modified+")
-    assert len(node_ids) == 2
-    node_ids = adapter.select_nodes("+state:modified")
-    assert len(node_ids) == 1
-
-    # Test resource type: snapshot
-    dbt_test_helper.create_snapshot("snapshot_1", csv_data_base, csv_data_curr)
-    dbt_test_helper.create_model("use_snapshot", csv_data_base, csv_data_base, depends_on=["snapshot_1"])
-
-    node_ids = adapter.select_nodes("resource_type:snapshot")
-    assert len(node_ids) == 1
-
-    node_ids = adapter.select_nodes("resource_type:snapshot+")
-    assert len(node_ids) == 2
-
-    node_ids = adapter.select_nodes("state:modified,resource_type:snapshot")
-    assert len(node_ids) == 1
-
-
-def test_select_removed_by_graph(dbt_test_helper):
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,35
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    dbt_test_helper.create_model("customers_1", csv_data_base, csv_data_curr)
-    dbt_test_helper.create_model("customers_2", base_csv=csv_data_base, depends_on=["customers_1"])
-    dbt_test_helper.create_model("customers_3", curr_csv=csv_data_curr, depends_on=["customers_1"])
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-
-    # Test graph operation
-    node_ids = adapter.select_nodes("customers_1+")
-    assert len(node_ids) == 3
-    node_ids = adapter.select_nodes("+customers_2")
-    assert len(node_ids) == 2
-    node_ids = adapter.select_nodes("+customers_3")
-    assert len(node_ids) == 2
-
-
-def test_select_source_removed_by_graph(dbt_test_helper):
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,35
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    dbt_test_helper.create_model("customers_2", base_csv=csv_data_base)
-    dbt_test_helper.create_model("customers_3", curr_csv=csv_data_curr)
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-
-    # Test graph operation
-    node_ids = adapter.select_nodes("state:modified")
-    assert len(node_ids) == 2
-
-
-def test_select_with_disabled(dbt_test_helper):
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,35
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    dbt_test_helper.create_model("customers_1", csv_data_base, csv_data_curr)
-    dbt_test_helper.create_model("customers_2", csv_data_base, csv_data_curr, disabled=True)
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-
-    # Test graph operation
-    node_ids = adapter.select_nodes("customers_1+")
-    assert len(node_ids) == 1
-
-
-def test_select_with_pacakage_mode_include_exclude(dbt_test_helper):
-    csv_data_curr = """
-            customer_id,name,age
-            1,Alice,30
-            2,Bob,25
-            3,Charlie,35
-            """
-
-    csv_data_base = """
-            customer_id,name,age
-            1,Alice,35
-            2,Bob,25
-            3,Charlie,35
-            """
-
-    """
-    The diagram of the models:
-    customers_1 ── customers_2 ── customers_3(*) ├── other_package.customers_4
-                                                 └── customers_5
-
-    Only customers_3 is changed
-    """
-    dbt_test_helper.create_model("customers_1", csv_data_base, csv_data_base)
-    dbt_test_helper.create_model("customers_2", csv_data_base, csv_data_base, depends_on=["customers_1"])
-    dbt_test_helper.create_model("customers_3", csv_data_base, csv_data_curr, depends_on=["customers_2"])
-    dbt_test_helper.create_model(
-        "customers_4", csv_data_base, csv_data_base, depends_on=["customers_3"], package_name="other_package"
-    )
-    dbt_test_helper.create_model("customers_5", csv_data_base, csv_data_base, depends_on=["customers_3"])
-
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-
-    node_ids = adapter.select_nodes(packages=["other_package"])
-    assert len(node_ids) == 1
-
-    node_ids = adapter.select_nodes(view_mode="changed_models")
-    assert len(node_ids) == 4
-
-    node_ids = adapter.select_nodes(view_mode="changed_models", packages=["other_package"])
-    assert len(node_ids) == 1
-
-    node_ids = adapter.select_nodes(view_mode="changed_models", packages=["other_package"], exclude="customers_4")
-    assert len(node_ids) == 0
-    node_ids = adapter.select_nodes(view_mode="changed_models", packages=["other_package"], select="customers_1+")
-    assert len(node_ids) == 1
-    node_ids = adapter.select_nodes(view_mode="changed_models", select="+customers_5")
-    assert len(node_ids) == 3
-    node_ids = adapter.select_nodes(view_mode="all", select="+customers_5")
-    assert len(node_ids) == 4
-    node_ids = adapter.select_nodes(select="+customers_5")
-    assert len(node_ids) == 4

tests/tasks/conftest.py

@@ -1,4 +0,0 @@
-# Share the fixtures for the tasks tests
-
-# noinspection PyUnresolvedReferences
-from tests.adapter.dbt_adapter.conftest import dbt_test_helper  # noqa: F401