recce-nightly 0.57.0.20250309__py3-none-any.whl → 0.57.0.20250311__py3-none-any.whl

tests/adapter/dbt_adapter/test_dbt_cll.py

@@ -0,0 +1,100 @@
+from recce.adapter.dbt_adapter import DbtAdapter
+
+
+def test_cll_basic(dbt_test_helper):
+    dbt_test_helper.create_model("model1", curr_sql="select 1 as c", curr_columns={"c": "int"})
+    dbt_test_helper.create_model("model2", curr_sql='select c from {{ ref("model1") }}', curr_columns={"c": "int"})
+    adapter: DbtAdapter = dbt_test_helper.context.adapter
+    result = adapter.get_lineage()
+    assert result['nodes']['model2']['columns']['c']['depends_on'][0].column == 'c'
+    assert result['nodes']['model2']['columns']['c']['depends_on'][0].node == 'model1'
+
+
+def test_cll_table_alisa(dbt_test_helper):
+    def patch_node(node):
+        node['alias'] = 'model1_alias'
+
+    dbt_test_helper.create_model("model1", curr_sql="select 1 as c", curr_columns={"c": "int"}, patch_func=patch_node)
+    dbt_test_helper.create_model("model2", curr_sql='select c from {{ ref("model1") }}', curr_columns={"c": "int"})
+    adapter: DbtAdapter = dbt_test_helper.context.adapter
+    result = adapter.get_lineage()
+    assert result['nodes']['model2']['columns']['c']['depends_on'][0].column == 'c'
+    assert result['nodes']['model2']['columns']['c']['depends_on'][0].node == 'model1'
+
+
+def test_seed(dbt_test_helper):
+    csv_data_curr = """
+    customer_id,name,age
+    1,Alice,30
+    2,Bob,25
+    3,Charlie,35
+    """
+
+    dbt_test_helper.create_model("seed1",
+                                 curr_csv=csv_data_curr,
+                                 curr_columns={"customer_id": "varchar", "name": "varchar", "age": "int"},
+                                 resource_type="seed")
+    adapter: DbtAdapter = dbt_test_helper.context.adapter
+    result = adapter.get_lineage()
+
+    assert result['nodes']['seed1']['columns']['customer_id']['transformation_type'] == 'source'
+    assert len(result['nodes']['seed1']['columns']['customer_id']['depends_on']) == 0
+
+
+def test_python_model(dbt_test_helper):
+    def python_node(node):
+        node['language'] = 'python'
+
+    csv_data_curr = """
+    customer_id,name,age
+    1,Alice,30
+    2,Bob,25
+    3,Charlie,35
+    """
+    dbt_test_helper.create_model("model1",
+                                 curr_csv=csv_data_curr,
+                                 curr_columns={"customer_id": "varchar", "name": "varchar", "age": "int"})
+    dbt_test_helper.create_model("model2",
+                                 curr_csv=csv_data_curr,
+                                 curr_columns={"customer_id": "varchar", "name": "varchar", "age": "int"},
+                                 depends_on=["model1"],
+                                 patch_func=python_node)
+    adapter: DbtAdapter = dbt_test_helper.context.adapter
+    assert not adapter.is_python_model('model1')
+    assert adapter.is_python_model('model2')
+
+    result = adapter.get_lineage()
+    assert result['nodes']['model2']['columns']['customer_id']['transformation_type'] == 'unknown'
+
+
+def test_source(dbt_test_helper):
+    csv_data_curr = """
+        customer_id,name,age
+        1,Alice,30
+        2,Bob,25
+        3,Charlie,35
+        """
+
+    dbt_test_helper.create_source(
+        "source1",
+        "table1",
+        curr_csv=csv_data_curr,
+        curr_columns={"customer_id": "varchar", "name": "varchar", "age": "int"})
+    adapter: DbtAdapter = dbt_test_helper.context.adapter
+    result = adapter.get_lineage()
+    assert result['nodes']['source1.table1']['columns']['customer_id']['transformation_type'] == 'source'
+
+
+def test_parse_error(dbt_test_helper):
+    dbt_test_helper.create_model("model1", curr_sql="select 1 as c", curr_columns={"c": "int"})
+    dbt_test_helper.create_model("model2", curr_sql='this is not a valid sql', curr_columns={"c": "int"})
+    adapter: DbtAdapter = dbt_test_helper.context.adapter
+    result = adapter.get_lineage()
+    assert result['nodes']['model2']['columns']['c']['transformation_type'] == 'unknown'
+
+
+def test_model_without_catalog(dbt_test_helper):
+    dbt_test_helper.create_model("model1", curr_sql="select 1 as c")
+    adapter: DbtAdapter = dbt_test_helper.context.adapter
+    result = adapter.get_lineage()
+    assert not hasattr(result['nodes']['model1'], 'columns')

tests/test_dbt.py

@@ -3,8 +3,6 @@ from unittest import TestCase
 from unittest.mock import patch, MagicMock
 
 from recce.adapter.dbt_adapter import load_manifest, load_catalog, DbtAdapter
-from recce.exceptions import RecceException
-from recce.util.cll import ColumnLevelDependencyColumn, ColumnLevelDependsOn
 
 current_dir = os.path.dirname(os.path.abspath(__file__))
 
@@ -40,176 +38,3 @@ class TestAdapterLineage(TestCase):
         lineage = dbt_adapter.get_lineage()
         assert lineage is not None
         assert len(lineage['nodes']['model.jaffle_shop.orders']['columns']) == 9
-
-
-class TestAdapterColumnLineage(TestCase):
-
-    def setUp(self) -> None:
-        manifest = load_manifest(path=os.path.join(current_dir, 'manifest.json'))
-        self.dbt_adapter = DbtAdapter(curr_manifest=manifest)
-        self.patcher_generate_sql = patch.object(self.dbt_adapter, 'generate_sql')
-        self.mock_generate_sql = self.patcher_generate_sql.start()
-
-        self.mock_adapter = MagicMock()
-        self.dbt_adapter.adapter = self.mock_adapter
-        self.mock_adapter.type.return_value = None
-
-        # mock 'is_python_model' per test case, default to False
-        self.patcher_is_python_model = patch.object(self.dbt_adapter, 'is_python_model', return_value=False)
-        self.mock_is_python_model = self.patcher_is_python_model.start()
-
-        # mock return value per test case
-        self.patcher_cll = patch('recce.adapter.dbt_adapter.cll')
-        self.mock_cll = self.patcher_cll.start()
-
-    def tearDown(self):
-        self.patcher_generate_sql.stop()
-        self.patcher_is_python_model.stop()
-        self.patcher_cll.stop()
-
-    def test_is_python_model(self):
-        self.assertFalse(self.dbt_adapter.is_python_model('model.jaffle_shop.orders'))
-
-    def test_seed(self):
-        nodes = {
-            'seed1': {
-                'id': 'seed1',
-                'name': 'seed1',
-                'resource_type': 'seed',
-                'raw_code': None,
-                'columns': {
-                    'a': {
-                        'name': 'a',
-                        'type': 'int'
-                    },
-                }
-            }
-        }
-        parents_map = {
-            'seed1': []
-        }
-
-        self.dbt_adapter.append_column_lineage(nodes, parents_map)
-        self.assertEqual(nodes['seed1']['columns']['a']['transformation_type'], 'source')
-        self.assertEqual(nodes['seed1']['columns']['a']['depends_on'], [])
-
-    def test_source(self):
-        nodes = {
-            'source1': {
-                'id': 'source1',
-                'name': 'source1',
-                'resource_type': 'source',
-                'raw_code': None,
-                'columns': {
-                    'a': {
-                        'name': 'a',
-                        'type': 'int'
-                    },
-                }
-            }
-        }
-        parents_map = {
-            'source1': []
-        }
-
-        self.dbt_adapter.append_column_lineage(nodes, parents_map)
-        self.assertEqual(nodes['source1']['columns']['a']['transformation_type'], 'source')
-        self.assertEqual(nodes['source1']['columns']['a']['depends_on'], [])
-
-    def test_python_model(self):
-        nodes = {
-            'py_model': {
-                'id': 'py_model',
-                'name': 'py_model',
-                'resource_type': 'model',
-                'raw_code': """
-                    def model(dbt, session):
-                        dbt.config(materialized = "table")
-                        df = dbt.ref("customers")
-                        return df
-                """,
-                'columns': {
-                    'a': {
-                        'name': 'a',
-                        'type': 'int'
-                    },
-                }
-            }
-        }
-        parents_map = {
-            'py_model': ['model.jaffle_shop.customers']
-        }
-
-        self.mock_is_python_model.return_value = True
-        self.dbt_adapter.append_column_lineage(nodes, parents_map)
-        self.assertEqual(nodes['py_model']['columns']['a']['transformation_type'], 'unknown')
-        self.assertEqual(nodes['py_model']['columns']['a']['depends_on'], [])
-
-    def test_model(self):
-        nodes = {
-            'model1': {
-                'id': 'model1',
-                'name': 'model1',
-                'resource_type': 'model',
-                'raw_code': 'select * from model2',
-                'columns': {
-                    'a': {
-                        'name': 'a',
-                        'type': 'int'
-                    },
-                }
-            }
-        }
-        parents_map = {
-            'model1': ['model2']
-        }
-
-        self.mock_cll.return_value = {'a': ColumnLevelDependencyColumn(
-            type='passthrough',
-            depends_on=[ColumnLevelDependsOn(node='model2', column='a')]
-        )}
-        self.dbt_adapter.append_column_lineage(nodes, parents_map)
-        self.assertEqual(nodes['model1']['columns']['a']['transformation_type'], 'passthrough')
-        self.assertEqual(nodes['model1']['columns']['a']['depends_on'][0].node, 'model2')
-        self.assertEqual(nodes['model1']['columns']['a']['depends_on'][0].column, 'a')
-
-    def test_model_parse_error(self):
-        nodes = {
-            'model1': {
-                'id': 'model1',
-                'name': 'model1',
-                'resource_type': 'model',
-                'raw_code': 'select * from model2',
-                'columns': {
-                    'a': {
-                        'name': 'a',
-                        'type': 'int'
-                    },
-                }
-            }
-        }
-        parents_map = {
-            'model1': ['model2']
-        }
-
-        self.mock_cll.side_effect = RecceException('Failed to parse SQL')
-        self.dbt_adapter.append_column_lineage(nodes, parents_map)
-        self.assertEqual(nodes['model1']['columns']['a']['transformation_type'], 'unknown')
-        self.assertEqual(nodes['model1']['columns']['a']['depends_on'], [])
-
-    def test_model_without_catalog(self):
-        # no 'columns' key in node
-        nodes = {
-            'model1': {
-                'id': 'model1',
-                'name': 'model1',
-                'resource_type': 'model',
-                'raw_code': 'select * from model2',
-            }
-        }
-        parents_map = {
-            'model1': ['model2']
-        }
-
-        self.dbt_adapter.append_column_lineage(nodes, parents_map)
-        self.assertIsNone(nodes['model1'].get('columns'))