recce-nightly 0.62.0.20250417__py3-none-any.whl → 1.30.0.20251221__py3-none-any.whl

tests/adapter/dbt_adapter/dbt_test_helper.py

@@ -1,283 +0,0 @@
-import os
-import textwrap
-import uuid
-from datetime import datetime
-from io import StringIO
-
-from dbt.contracts.graph.nodes import ModelNode, SnapshotNode, SeedNode, SourceDefinition
-from dbt.contracts.results import CatalogArtifact, ColumnMetadata, CatalogTable, TableMetadata
-
-from recce.adapter.dbt_adapter import DbtAdapter, as_manifest, load_manifest
-from recce.core import RecceContext
-
-
-class DbtTestHelper:
-
-    def __init__(self):
-        schema_prefix = "schema_" + uuid.uuid4().hex
-        self.base_schema = f"{schema_prefix}_base"
-        self.curr_schema = f"{schema_prefix}_curr"
-
-        current_dir = os.path.dirname(os.path.abspath(__file__))
-        project_dir = os.path.join(current_dir, 'test_proj')
-        profiles_dir = project_dir
-        manifest_path = os.path.join(project_dir, 'manifest.json')
-
-        dbt_adapter = DbtAdapter.load(
-            no_artifacts=True,
-            project_dir=project_dir,
-            profiles_dir=profiles_dir,
-        )
-
-        context = RecceContext()
-        context.adapter_type = 'dbt'
-        context.adapter = dbt_adapter
-        context.schema_prefix = schema_prefix
-        self.adapter = dbt_adapter
-        self.context = context
-        curr_writable_manifest = load_manifest(manifest_path)
-        base_writable_manifest = load_manifest(manifest_path)
-
-        self.curr_manifest = as_manifest(curr_writable_manifest)
-        self.base_manifest = as_manifest(base_writable_manifest)
-        now = datetime.now()
-        self.curr_catalog = CatalogArtifact.from_results(
-            generated_at=now,
-            nodes={},
-            sources={},
-            compile_results=None,
-            errors=None,
-        )
-        self.base_catalog = CatalogArtifact.from_results(
-            generated_at=now,
-            nodes={},
-            sources={},
-            compile_results=None,
-            errors=None,
-        )
-        self.context = context
-
-        self.adapter.execute(f"CREATE schema IF NOT EXISTS {self.base_schema}")
-        self.adapter.execute(f"CREATE schema IF NOT EXISTS {self.curr_schema}")
-        self.adapter.set_artifacts(
-            base_writable_manifest,
-            curr_writable_manifest,
-            self.curr_manifest,
-            self.base_manifest,
-            self.base_catalog,
-            self.curr_catalog,
-        )
-
-    def create_model(
-        self,
-        model_name,
-        base_csv=None,
-        curr_csv=None,
-        base_sql=None,
-        curr_sql=None,
-        depends_on=None,
-        disabled=False,
-        unique_id=None,
-        resource_type="model",
-        package_name="recce_test",
-        base_columns: dict[str, str] = None,
-        curr_columns: dict[str, str] = None,
-        patch_func=None,
-    ):
-        # unique_id = f"model.{package_name}.{model_name}"
-        unique_id = unique_id if unique_id else model_name
-        if depends_on is None:
-            depends_on = []
-
-        def _add_model_to_manifest(base):
-            if base:
-                schema = self.base_schema
-                manifest = self.base_manifest
-                catalog = self.base_catalog
-                csv = base_csv
-                sql = base_sql
-                columns = base_columns
-            else:
-                schema = self.curr_schema
-                manifest = self.curr_manifest
-                catalog = self.curr_catalog
-                csv = curr_csv
-                sql = curr_sql
-                columns = curr_columns
-
-            if csv:
-                dbt_adapter = self.adapter
-                csv = textwrap.dedent(csv)
-                with dbt_adapter.connection_named('create model'):
-                    import pandas as pd
-                    df = pd.read_csv(StringIO(csv))
-                    dbt_adapter.execute(f"CREATE TABLE {schema}.{model_name} AS SELECT * FROM df")
-            raw_code = sql if sql else csv
-
-            if columns:
-                index = 1
-                table = CatalogTable(
-                    TableMetadata(type="BASE TABLE", schema=schema, name=model_name), {}, {})
-                catalog.nodes[unique_id] = table
-                for column, column_type in columns.items():
-                    col_data = ColumnMetadata(type=column_type, index=index, name=column)
-                    catalog.nodes[unique_id].columns[column] = col_data
-                    index = index + 1
-
-            node_dict = {
-                "resource_type": resource_type,
-                "name": model_name,
-                "package_name": package_name,
-                "path": "",
-                "original_file_path": "",
-                "unique_id": unique_id,
-                "fqn": [
-                    package_name,
-                    model_name,
-                ],
-                "schema": schema,
-                "alias": model_name,
-                "checksum": {
-                    "name": "sha256",
-                    "checksum": hash(raw_code),
-                },
-                "raw_code": raw_code,
-                "config": {
-                    "materialized": "table",
-                    "tags": ["test_tag"],
-                },
-                "tags": ["test_tag"],
-                "depends_on": {
-                    "nodes": depends_on
-                },
-            }
-            if patch_func:
-                patch_func(node_dict)
-
-            if resource_type == "snapshot":
-                node = SnapshotNode.from_dict(node_dict)
-            elif resource_type == "seed":
-                node = SeedNode.from_dict(node_dict)
-            else:
-                node = ModelNode.from_dict(node_dict)
-
-            if disabled:
-                manifest.add_disabled_nofile(node)
-            else:
-                manifest.add_node_nofile(node)
-            return node
-
-        if base_csv or base_sql:
-            _add_model_to_manifest(True)
-
-        if curr_csv or curr_sql:
-            _add_model_to_manifest(False)
-
-        self.adapter.set_artifacts(
-            self.base_manifest.writable_manifest(),
-            self.curr_manifest.writable_manifest(),
-            self.curr_manifest,
-            self.base_manifest,
-            self.base_catalog,
-            self.curr_catalog)
-
-    def remove_model(self, model_name):
-        dbt_adapter = self.adapter
-        with dbt_adapter.connection_named('cleanup'):
-            dbt_adapter.execute(f"DROP TABLE IF EXISTS {self.base_schema}.{model_name}")
-            dbt_adapter.execute(f"DROP TABLE IF EXISTS {self.curr_schema}.{model_name} ")
-
-    def create_source(
-        self,
-        source_name,
-        table_name,
-        base_csv=None,
-        curr_csv=None,
-        unique_id=None,
-        package_name="recce_test",
-        base_columns: dict[str, str] = None,
-        curr_columns: dict[str, str] = None,
-        patch_func=None,
-    ):
-        # unique_id = f"model.{package_name}.{model_name}"
-        unique_id = unique_id if unique_id else f"{source_name}.{table_name}"
-
-        def _add_source_to_manifest(base):
-            if base:
-                schema = self.base_schema
-                manifest = self.base_manifest
-                catalog = self.base_catalog
-                csv = base_csv
-                columns = base_columns
-            else:
-                schema = self.curr_schema
-                manifest = self.curr_manifest
-                catalog = self.curr_catalog
-                csv = curr_csv
-                columns = curr_columns
-
-            if csv:
-                dbt_adapter = self.adapter
-                csv = textwrap.dedent(csv)
-                with dbt_adapter.connection_named('create source'):
-                    import pandas as pd
-                    df = pd.read_csv(StringIO(csv))
-                    dbt_adapter.execute(f"CREATE TABLE {schema}.{table_name} AS SELECT * FROM df")
-
-            if columns:
-                index = 1
-                table = CatalogTable(
-                    TableMetadata(type="BASE TABLE", schema=schema, name=table_name), {}, {})
-                catalog.sources[unique_id] = table
-                for column, column_type in columns.items():
-                    col_data = ColumnMetadata(type=column_type, index=index, name=column)
-                    catalog.sources[unique_id].columns[column] = col_data
-                    index = index + 1
-
-            node_dict = {
-                "resource_type": "source",
-                "source_name": source_name,
-                "name": table_name,
-                "identifier": table_name,
-                "source_description": "test source",
-                "loader": "",
-                "package_name": package_name,
-                "path": "",
-                "original_file_path": "",
-                "unique_id": unique_id,
-                "fqn": [
-                    package_name,
-                    source_name,
-                    table_name,
-                ],
-                "schema": schema,
-            }
-            if patch_func:
-                patch_func(node_dict)
-
-            source = SourceDefinition.from_dict(node_dict)
-            manifest.sources[unique_id] = source
-            return source
-
-        if base_csv:
-            _add_source_to_manifest(True)
-
-        if curr_csv:
-            _add_source_to_manifest(False)
-
-        self.adapter.set_artifacts(
-            self.base_manifest.writable_manifest(),
-            self.curr_manifest.writable_manifest(),
-            self.curr_manifest,
-            self.base_manifest,
-            self.base_catalog,
-            self.curr_catalog)
-
-    def cleanup(self):
-        dbt_adapter = self.adapter
-        with dbt_adapter.connection_named('cleanup'):
-            dbt_adapter.execute(f"DROP SCHEMA IF EXISTS {self.base_schema} CASCADE")
-            dbt_adapter.execute(f"DROP SCHEMA IF EXISTS {self.curr_schema} CASCADE")
-
-    def create_snapshot(self, sanpshot_name, base_csv, curr_csv, depends_on=[]):
-        self.create_model(sanpshot_name, base_csv, curr_csv, depends_on=depends_on, resource_type="snapshot")

tests/adapter/dbt_adapter/test_dbt_adapter.py

@@ -1,40 +0,0 @@
-from recce.adapter.dbt_adapter import DbtAdapter, dbt_supported_registry
-from recce.models import RunType
-
-
-def test_dbt_adapter_support_tasks(dbt_test_helper):
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-
-    # Test dbt package name
-    supported_dbt_packages = set([package.package_name for package in adapter.manifest.macros.values()])
-    assert 'audit_helper' in supported_dbt_packages
-    assert 'dbt_profiler' in supported_dbt_packages
-
-    # Test dbt task support
-    support_tasks = adapter.support_tasks()
-
-    for task_type in dbt_supported_registry:
-        task = task_type.value
-        assert task in support_tasks
-        assert support_tasks[task] is True
-
-
-def test_dbt_adapter_support_tasks_without_required_dbt_package(dbt_test_helper):
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-    # Mock the macros in manifest to simulate no required dbt package installed
-    adapter.manifest.macros = {}
-
-    support_tasks = adapter.support_tasks()
-
-    for task_type in dbt_supported_registry:
-        task = task_type.value
-        assert task in support_tasks
-        if task == RunType.PROFILE_DIFF.value:
-            assert support_tasks[task] is False
-        elif task == RunType.VALUE_DIFF.value:
-            assert support_tasks[task] is False
-        elif task == RunType.VALUE_DIFF_DETAIL.value:
-            assert support_tasks[task] is False
-
-    # Check the query_diff_with_primary_key is not supported
-    assert support_tasks["query_diff_with_primary_key"] is False

tests/adapter/dbt_adapter/test_dbt_cll.py

@@ -1,102 +0,0 @@
-from recce.adapter.dbt_adapter import DbtAdapter
-
-
-def test_cll_basic(dbt_test_helper):
-    dbt_test_helper.create_model("model1", curr_sql="select 1 as c", curr_columns={"c": "int"})
-    dbt_test_helper.create_model("model2", curr_sql='select c from {{ ref("model1") }}', curr_columns={"c": "int"},
-                                 depends_on=["model1"])
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-    result = adapter.get_cll_by_node_id("model1")
-    assert result['nodes']['model2']['columns']['c']['depends_on'][0].column == 'c'
-    assert result['nodes']['model2']['columns']['c']['depends_on'][0].node == 'model1'
-
-
-def test_cll_table_alisa(dbt_test_helper):
-    def patch_node(node):
-        node['alias'] = 'model1_alias'
-
-    dbt_test_helper.create_model("model1", curr_sql="select 1 as c", curr_columns={"c": "int"}, patch_func=patch_node)
-    dbt_test_helper.create_model("model2", curr_sql='select c from {{ ref("model1") }}', curr_columns={"c": "int"},
-                                 depends_on=["model1"])
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-    result = adapter.get_cll_by_node_id("model1")
-    assert result['nodes']['model2']['columns']['c']['depends_on'][0].column == 'c'
-    assert result['nodes']['model2']['columns']['c']['depends_on'][0].node == 'model1'
-
-
-def test_seed(dbt_test_helper):
-    csv_data_curr = """
-    customer_id,name,age
-    1,Alice,30
-    2,Bob,25
-    3,Charlie,35
-    """
-
-    dbt_test_helper.create_model("seed1",
-                                 curr_csv=csv_data_curr,
-                                 curr_columns={"customer_id": "varchar", "name": "varchar", "age": "int"},
-                                 resource_type="seed")
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-    result = adapter.get_cll_by_node_id("seed1")
-
-    assert result['nodes']['seed1']['columns']['customer_id']['transformation_type'] == 'source'
-    assert len(result['nodes']['seed1']['columns']['customer_id']['depends_on']) == 0
-
-
-def test_python_model(dbt_test_helper):
-    def python_node(node):
-        node['language'] = 'python'
-
-    csv_data_curr = """
-    customer_id,name,age
-    1,Alice,30
-    2,Bob,25
-    3,Charlie,35
-    """
-    dbt_test_helper.create_model("model1",
-                                 curr_csv=csv_data_curr,
-                                 curr_columns={"customer_id": "varchar", "name": "varchar", "age": "int"})
-    dbt_test_helper.create_model("model2",
-                                 curr_csv=csv_data_curr,
-                                 curr_columns={"customer_id": "varchar", "name": "varchar", "age": "int"},
-                                 depends_on=["model1"],
-                                 patch_func=python_node)
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-    assert not adapter.is_python_model('model1')
-    assert adapter.is_python_model('model2')
-
-    result = adapter.get_cll_by_node_id("model1")
-    assert result['nodes']['model2']['columns']['customer_id']['transformation_type'] == 'unknown'
-
-
-def test_source(dbt_test_helper):
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    dbt_test_helper.create_source(
-        "source1",
-        "table1",
-        curr_csv=csv_data_curr,
-        curr_columns={"customer_id": "varchar", "name": "varchar", "age": "int"})
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-    result = adapter.get_cll_by_node_id("source1.table1")
-    assert result['nodes']['source1.table1']['columns']['customer_id']['transformation_type'] == 'source'
-
-
-def test_parse_error(dbt_test_helper):
-    dbt_test_helper.create_model("model1", curr_sql="select 1 as c", curr_columns={"c": "int"})
-    dbt_test_helper.create_model("model2", curr_sql='this is not a valid sql', curr_columns={"c": "int"})
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-    result = adapter.get_cll_by_node_id("model2")
-    assert result['nodes']['model2']['columns']['c']['transformation_type'] == 'unknown'
-
-
-def test_model_without_catalog(dbt_test_helper):
-    dbt_test_helper.create_model("model1", curr_sql="select 1 as c")
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-    result = adapter.get_cll_by_node_id("model1")
-    assert not hasattr(result['nodes']['model1'], 'columns')

tests/adapter/dbt_adapter/test_selector.py

@@ -1,177 +0,0 @@
-from recce.adapter.dbt_adapter import DbtAdapter
-
-
-def test_select(dbt_test_helper):
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,35
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    dbt_test_helper.create_model("customers_1", csv_data_base, csv_data_curr)
-    dbt_test_helper.create_model("customers_2", csv_data_base, csv_data_base, depends_on=["customers_1"])
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-
-    # Test methods
-    node_ids = adapter.select_nodes('customers_1')
-    assert len(node_ids) == 1
-    node_ids = adapter.select_nodes('resource_type:model')
-    assert len(node_ids) == 2
-    node_ids = adapter.select_nodes('tag:test_tag')
-    assert len(node_ids) == 2
-    node_ids = adapter.select_nodes('tag:test_tag2')
-    assert len(node_ids) == 0
-    node_ids = adapter.select_nodes("config.materialized:incremental")
-    assert len(node_ids) == 0
-    node_ids = adapter.select_nodes("config.materialized:table")
-    assert len(node_ids) == 2
-
-    # Test state method
-    node_ids = adapter.select_nodes("state:modified")
-    assert len(node_ids) == 1
-
-    # Test set operation
-    node_ids = adapter.select_nodes("customers_1 customers_2")
-    assert len(node_ids) == 2
-    node_ids = adapter.select_nodes("customers_1,customers_2")
-    assert len(node_ids) == 0
-    node_ids = adapter.select_nodes("config.materialized:table,tag:test_tag")
-    assert len(node_ids) == 2
-    node_ids = adapter.select_nodes("config.materialized:table,tag:test_tag2")
-    assert len(node_ids) == 0
-    node_ids = adapter.select_nodes(exclude='customers_1')
-    assert len(node_ids) == 1
-    node_ids = adapter.select_nodes('customers_1', exclude='customers_2')
-    assert len(node_ids) == 1
-
-    # Test graph operation
-    node_ids = adapter.select_nodes("state:modified+")
-    assert len(node_ids) == 2
-    node_ids = adapter.select_nodes("+state:modified")
-    assert len(node_ids) == 1
-
-    # Test resource type: snapshot
-    dbt_test_helper.create_snapshot("snapshot_1", csv_data_base, csv_data_curr)
-    dbt_test_helper.create_model("use_snapshot", csv_data_base, csv_data_base, depends_on=["snapshot_1"])
-
-    node_ids = adapter.select_nodes('resource_type:snapshot')
-    assert len(node_ids) == 1
-
-    node_ids = adapter.select_nodes('resource_type:snapshot+')
-    assert len(node_ids) == 2
-
-    node_ids = adapter.select_nodes("state:modified,resource_type:snapshot")
-    assert len(node_ids) == 1
-
-
-def test_select_removed_by_graph(dbt_test_helper):
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,35
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    dbt_test_helper.create_model("customers_1", csv_data_base, csv_data_curr)
-    dbt_test_helper.create_model("customers_2", base_csv=csv_data_base, depends_on=["customers_1"])
-    dbt_test_helper.create_model("customers_3", curr_csv=csv_data_curr, depends_on=["customers_1"])
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-
-    # Test graph operation
-    node_ids = adapter.select_nodes("customers_1+")
-    assert len(node_ids) == 3
-    node_ids = adapter.select_nodes("+customers_2")
-    assert len(node_ids) == 2
-    node_ids = adapter.select_nodes("+customers_3")
-    assert len(node_ids) == 2
-
-
-def test_select_with_disabled(dbt_test_helper):
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,35
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    dbt_test_helper.create_model("customers_1", csv_data_base, csv_data_curr)
-    dbt_test_helper.create_model("customers_2", csv_data_base, csv_data_curr, disabled=True)
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-
-    # Test graph operation
-    node_ids = adapter.select_nodes("customers_1+")
-    assert len(node_ids) == 1
-
-
-def test_select_with_pacakage_mode_include_exclude(dbt_test_helper):
-    csv_data_curr = """
-            customer_id,name,age
-            1,Alice,30
-            2,Bob,25
-            3,Charlie,35
-            """
-
-    csv_data_base = """
-            customer_id,name,age
-            1,Alice,35
-            2,Bob,25
-            3,Charlie,35
-            """
-
-    """
-    The diagram of the models:
-    customers_1 ── customers_2 ── customers_3(*) ├── other_package.customers_4
-                                                 └── customers_5
-
-    Only customers_3 is changed
-    """
-    dbt_test_helper.create_model("customers_1", csv_data_base, csv_data_base)
-    dbt_test_helper.create_model("customers_2", csv_data_base, csv_data_base, depends_on=['customers_1'])
-    dbt_test_helper.create_model("customers_3", csv_data_base, csv_data_curr, depends_on=['customers_2'])
-    dbt_test_helper.create_model("customers_4", csv_data_base, csv_data_base, depends_on=['customers_3'],
-                                 package_name='other_package')
-    dbt_test_helper.create_model("customers_5", csv_data_base, csv_data_base, depends_on=['customers_3'])
-
-    adapter: DbtAdapter = dbt_test_helper.context.adapter
-
-    node_ids = adapter.select_nodes(packages=['other_package'])
-    assert len(node_ids) == 1
-
-    node_ids = adapter.select_nodes(view_mode='changed_models')
-    assert len(node_ids) == 4
-
-    node_ids = adapter.select_nodes(view_mode='changed_models', packages=['other_package'])
-    assert len(node_ids) == 1
-
-    node_ids = adapter.select_nodes(view_mode='changed_models', packages=['other_package'], exclude='customers_4')
-    assert len(node_ids) == 0
-    node_ids = adapter.select_nodes(view_mode='changed_models', packages=['other_package'], select='customers_1+')
-    assert len(node_ids) == 1
-    node_ids = adapter.select_nodes(view_mode='changed_models', select='+customers_5')
-    assert len(node_ids) == 3
-    node_ids = adapter.select_nodes(view_mode='all', select='+customers_5')
-    assert len(node_ids) == 4
-    node_ids = adapter.select_nodes(select='+customers_5')
-    assert len(node_ids) == 4

tests/tasks/conftest.py

@@ -1,4 +0,0 @@
-# Share the fixtures for the tasks tests
-
-# noinspection PyUnresolvedReferences
-from tests.adapter.dbt_adapter.conftest import dbt_test_helper