recce-nightly 0.62.0.20250417__py3-none-any.whl → 1.30.0.20251221__py3-none-any.whl

tests/tasks/test_histogram.py

@@ -1,137 +0,0 @@
-import pytest
-
-from recce.tasks.histogram import HistogramDiffTask, HistogramDiffCheckValidator, _is_histogram_supported
-
-
-def test_histogram(dbt_test_helper):
-    csv_data = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        4,Dolly,50
-        """
-
-    dbt_test_helper.create_model("customers", csv_data, csv_data)
-
-    params = {
-        "model": "customers",
-        "column_name": "age",
-        "column_type": "int"
-    }
-
-    task = HistogramDiffTask(params)
-    run_result = task.execute()
-
-    # {
-    #     'base': {'counts': [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], 'total': 4},
-    #     'current': {'counts': [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], 'total': 4},
-    #     'min': 25, 'max': 50,
-    #     'bin_edges': [25, 26, ..., 51],
-    #     'labels': ['25-26', ..., '51-52']
-    # }
-    assert run_result['current']['counts'][0] == 1
-    assert run_result['current']['counts'][-1] == 1
-    assert run_result['current']['total'] == 4
-    assert run_result['min'] == 25
-    assert run_result['max'] == 50
-    assert run_result['bin_edges'][0] == 25
-    assert run_result['bin_edges'][-1] == 51
-
-
-def test_histogram_emtpy(dbt_test_helper):
-    csv_data = """
-    customer_id,name,age
-    1,Alice,30
-    2,Bob,25
-    3,Charlie,35
-    4,Dolly,50
-    """
-
-    csv_data_zero = """
-    customer_id,name,age
-    """
-
-    dbt_test_helper.create_model("customers", csv_data_zero, csv_data_zero)
-    dbt_test_helper.create_model("customers2", csv_data, csv_data_zero)
-    dbt_test_helper.create_model("customers3", csv_data_zero, csv_data)
-
-    params = {
-        "model": "customers",
-        "column_name": "age",
-        "column_type": "int"
-    }
-
-    task = HistogramDiffTask(params)
-    run_result = task.execute()
-
-    assert len(run_result['current']['counts']) == 0
-    assert run_result['current']['total'] == 0
-    assert run_result['min'] is None
-    assert run_result['max'] is None
-    assert len(run_result['bin_edges']) == 0
-
-    params = {
-        "model": "customers2",
-        "column_name": "age",
-        "column_type": "int"
-    }
-
-    task = HistogramDiffTask(params)
-    run_result = task.execute()
-    assert run_result['base']['counts'][0] == 1
-    assert run_result['base']['counts'][-1] == 1
-    assert run_result['base']['total'] == 4
-    assert run_result['current']['counts'][0] == 0
-    assert run_result['current']['counts'][-1] == 0
-    assert run_result['current']['total'] == 0
-    assert run_result['min'] == 25
-    assert run_result['max'] == 50
-    assert run_result['bin_edges'][0] == 25
-    assert run_result['bin_edges'][-1] == 51
-
-    params = {
-        "model": "customers3",
-        "column_name": "age",
-        "column_type": "int"
-    }
-
-    task = HistogramDiffTask(params)
-    run_result = task.execute()
-    assert run_result['base']['counts'][0] == 0
-    assert run_result['base']['counts'][-1] == 0
-    assert run_result['base']['total'] == 0
-    assert run_result['current']['counts'][0] == 1
-    assert run_result['current']['counts'][-1] == 1
-    assert run_result['current']['total'] == 4
-    assert run_result['min'] == 25
-    assert run_result['max'] == 50
-    assert run_result['bin_edges'][0] == 25
-    assert run_result['bin_edges'][-1] == 51
-
-
-def test_validator():
-    def validate(params: dict = {}, view_options: dict = {}):
-        HistogramDiffCheckValidator().validate({
-            'name': 'test',
-            'type': 'histogram_diff',
-            'params': params,
-            'view_options': view_options,
-        })
-
-    validate({
-        "model": "customers",
-        "column_name": "age",
-        "column_type": "int",
-    })
-
-    with pytest.raises(ValueError):
-        validate({})
-
-
-def test_is_column_type_supported_by_histogram():
-    assert _is_histogram_supported("varchar") is False
-    assert _is_histogram_supported("varchar(16)") is False
-    assert _is_histogram_supported("varchar(256)") is False
-    assert _is_histogram_supported("bool") is False
-    assert _is_histogram_supported("int") is True

tests/tasks/test_lineage.py

@@ -1,42 +0,0 @@
-import pytest
-
-
-def test_validator():
-    from recce.tasks.lineage import LineageDiffCheckValidator
-    validator = LineageDiffCheckValidator()
-
-    def validate(params: dict):
-        validator.validate({
-            'name': 'test',
-            'type': 'schema_diff',
-            'params': params,
-        })
-
-    # Select all models
-    validate({})
-
-    # Select by selector
-    validate({
-        'select': 'customers',
-        'exclude': 'customers',
-        'packages': ['jaffle_shop'],
-        'view_mode': 'all',
-    })
-
-    # packages should be an array
-    with pytest.raises(ValueError):
-        validate({
-            'packages': 'jaffle_shop',
-        })
-
-    # view_mode should be 'all' or 'changed_models'
-    validate({
-        'view_mode': None,
-    })
-    validate({
-        'view_mode': 'all',
-    })
-    with pytest.raises(ValueError):
-        validate({
-            'view_mode': 'abc',
-        })

tests/tasks/test_preset_checks.py

@@ -1,50 +0,0 @@
-import pytest
-
-
-def test_default_validator():
-    from recce.tasks.core import CheckValidator
-
-    CheckValidator().validate({
-        "name": "test",
-        "type": "row_count_diff",
-        "params": {},
-    })
-
-    with pytest.raises(ValueError):
-        CheckValidator().validate({
-            "type": "row_count_diff",
-        })
-
-
-def test_query_diff_validator():
-    from recce.tasks.query import QueryDiffCheckValidator
-    QueryDiffCheckValidator().validate({
-        "name": "test",
-        "type": "query_diff",
-        "params": {
-            "sql_template": "select * from {{ model }}"
-        },
-    })
-    with pytest.raises(ValueError):
-        QueryDiffCheckValidator().validate({
-            "name": "test",
-            "type": "query_diff",
-        })
-
-
-def test_lineage_diff_validator():
-    from recce.tasks.lineage import LineageDiffCheckValidator
-    LineageDiffCheckValidator().validate({
-        "name": "test",
-        "type": "lineage_diff",
-        "params": {},
-    })
-    LineageDiffCheckValidator().validate({
-        "name": "test",
-        "type": "lineage_diff",
-        "view_options": {},
-    })
-    with pytest.raises(ValueError):
-        LineageDiffCheckValidator().validate({
-            "type": "lineage_diff",
-        })

tests/tasks/test_profile.py

@@ -1,73 +0,0 @@
-import pytest
-
-from recce.tasks import ProfileDiffTask, ProfileTask
-
-csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-csv_data_base = """
-    customer_id,name,age
-    1,Alice,35
-    2,Bob,25
-    3,Charlie,35
-    """
-
-
-def test_profile(dbt_test_helper):
-    dbt_test_helper.create_model("customers", None, csv_data_curr)
-    params = dict(model='customers')
-    task = ProfileTask(params)
-    run_result = task.execute()
-
-    assert len(run_result.current.data) == 3
-
-
-def test_profile_with_selected_columns(dbt_test_helper):
-    dbt_test_helper.create_model("customers", None, csv_data_curr)
-    params = dict(model='customers', columns=['name', 'age'])
-    task = ProfileTask(params)
-    run_result = task.execute()
-    assert len(run_result.current.data) == 2
-
-
-def test_profile_diff(dbt_test_helper):
-    dbt_test_helper.create_model("customers", csv_data_base, csv_data_curr)
-    params = dict(model='customers')
-    task = ProfileDiffTask(params)
-    run_result = task.execute()
-
-    assert len(run_result.current.data) == 3
-    assert len(run_result.base.data) == 3
-
-
-def test_profile_diff_with_selected_columns(dbt_test_helper):
-    dbt_test_helper.create_model("customers", csv_data_base, csv_data_curr)
-    params = dict(model='customers', columns=['name', 'age'])
-    task = ProfileDiffTask(params)
-    run_result = task.execute()
-    assert len(run_result.current.data) == 2
-    assert len(run_result.base.data) == 2
-
-
-def test_validator():
-    from recce.tasks.profile import ProfileCheckValidator
-
-    def validate(params: dict = {}, view_options: dict = {}):
-        ProfileCheckValidator().validate({
-            'name': 'test',
-            'type': 'profile_diff',
-            'params': params,
-            'view_options': view_options,
-        })
-
-    validate({
-        'model': 'customers',
-    })
-
-    with pytest.raises(ValueError):
-        validate({
-        })

tests/tasks/test_query.py

@@ -1,151 +0,0 @@
-import pytest
-
-from recce.tasks import QueryDiffTask, QueryTask
-
-
-def test_query_diff_in_client(dbt_test_helper):
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,35
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    dbt_test_helper.create_model("customers", csv_data_base, csv_data_curr)
-    params = dict(sql_template=f'select * from {{{{ ref("customers") }}}}')
-    task = QueryTask(params)
-    run_result = task.execute()
-    assert len(run_result.current.data) == 3
-
-
-def test_query_diff_in_client(dbt_test_helper):
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,35
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    dbt_test_helper.create_model("customers", csv_data_base, csv_data_curr)
-    params = dict(sql_template=f'select * from {{{{ ref("customers") }}}}')
-    task = QueryDiffTask(params)
-    run_result = task.execute()
-    assert len(run_result.base.data) == 3
-    assert len(run_result.current.data) == 3
-
-    params = dict(
-        base_sql_template=f'select * from {{{{ ref("customers") }}}} where customer_id <= 2',
-        sql_template=f'select * from {{{{ ref("customers") }}}}',
-    )
-    task = QueryDiffTask(params)
-    run_result = task.execute()
-    assert len(run_result.base.data) == 2
-    assert len(run_result.current.data) == 3
-
-
-def test_query_diff_in_warehouse(dbt_test_helper):
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,35
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    dbt_test_helper.create_model("customers", csv_data_base, csv_data_curr)
-    params = dict(sql_template=f'select * from {{{{ ref("customers") }}}}', primary_keys=['customer_id'])
-    task = QueryDiffTask(params)
-    run_result = task.execute()
-    assert len(run_result.diff.data) == 2
-
-    params = dict(
-        base_sql_template=f'select * from {{{{ ref("customers") }}}} where customer_id == 1',
-        sql_template=f'select * from {{{{ ref("customers") }}}}',
-        primary_keys=['customer_id'],
-    )
-    task = QueryDiffTask(params)
-    run_result = task.execute()
-    assert len(run_result.diff.data) == 4
-
-
-def test_validator():
-    from recce.tasks.query import QueryCheckValidator, QueryDiffCheckValidator
-
-    def validate(params: dict = {}, view_options: dict = {}):
-        QueryCheckValidator().validate({
-            'name': 'test',
-            'type': 'query',
-            'params': params,
-            'view_options': view_options,
-        })
-
-    def validate_diff(params: dict = {}, view_options: dict = {}):
-        QueryDiffCheckValidator().validate({
-            'name': 'test',
-            'type': 'query_diff',
-            'params': params,
-            'view_options': view_options,
-        })
-
-    # query
-    validate({
-        'sql_template': 'select * from abc'
-    })
-
-    # diff in client
-    validate_diff({
-        'sql_template': 'select * from abc'
-    })
-    validate_diff({
-        'sql_template': 'select * from abc',
-        'base_sql_template': 'select * from abc',
-    })
-
-    # diff in warehouse
-    validate_diff({
-        'primary_keys': ['customer_id'],
-        'sql_template': 'select * from abc',
-    })
-    validate_diff({
-        'sql_template': 'select * from abc',
-        'base_sql_template': 'select * from abc',
-        'primary_keys': ['customer_id']
-    })
-    validate_diff({
-        'sql_template': 'select * from abc',
-        'base_sql_template': 'select * from abc',
-        'primary_keys': ['customer_id']
-    })
-
-    # invalid
-    with pytest.raises(ValueError):
-        validate()
-
-    with pytest.raises(ValueError):
-        validate_diff()
-
-    with pytest.raises(ValueError):
-        validate_diff({'sql_template': 123, 'primary_keys': 'xyz'})
-
-    with pytest.raises(ValueError):
-        validate_diff({'sql_template': 's', 'primary_keys': 'xyz'})

tests/tasks/test_row_count.py

@@ -1,116 +0,0 @@
-import pytest
-
-from recce.tasks import RowCountDiffTask
-
-
-def test_row_count(dbt_test_helper):
-    csv_data_curr = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    csv_data_base = """
-        customer_id,name,age
-        1,Alice,35
-        2,Bob,25
-        """
-
-    dbt_test_helper.create_model("customers", csv_data_base, csv_data_curr, unique_id='model.customers')
-    task = RowCountDiffTask(dict(node_names=['customers']))
-    run_result = task.execute()
-    assert run_result['customers']['base'] == 2
-    assert run_result['customers']['curr'] == 3
-
-    task = RowCountDiffTask(dict(node_names=['customers_']))
-    run_result = task.execute()
-    assert run_result['customers_']['base'] is None
-    assert run_result['customers_']['curr'] is None
-
-    task = RowCountDiffTask(dict(node_ids=['model.customers']))
-    run_result = task.execute()
-    assert run_result['customers']['base'] == 2
-    assert run_result['customers']['curr'] == 3
-
-
-def test_row_count_with_selector(dbt_test_helper):
-    csv_data_1 = """
-        customer_id,name,age
-        1,Alice,30
-        2,Bob,25
-        3,Charlie,35
-        """
-
-    csv_data_2 = """
-        customer_id,name,age
-        1,Alice,35
-        2,Bob,25
-        """
-
-    dbt_test_helper.create_model("model_1", csv_data_1, csv_data_2, depends_on=[])
-    dbt_test_helper.create_model("model_2", csv_data_1, csv_data_1, depends_on=['model_1'],
-                                 package_name='other_package')
-    task = RowCountDiffTask(dict(select='model_1'))
-    run_result = task.execute()
-    assert len(run_result) == 1
-
-    task = RowCountDiffTask(dict(select='model_1+'))
-    run_result = task.execute()
-    assert len(run_result) == 2
-
-
-def test_validator():
-    from recce.tasks.rowcount import RowCountDiffCheckValidator
-
-    validator = RowCountDiffCheckValidator()
-
-    def validate(params: dict):
-        validator.validate({
-            'name': 'test',
-            'type': 'row_count_diff',
-            'params': params,
-        })
-
-    # Select all modesl
-    validate({})
-
-    # Select by node name
-    validate({
-        'node_names': ['abc'],
-    })
-    with pytest.raises(ValueError):
-        validate({
-            'node_names': 'abc',
-        })
-
-    # Select by node id
-    validate({
-        'node_ids': ['model.abc'],
-    })
-
-    # Select by selector
-    validate({
-        'select': 'customers',
-        'exclude': 'customers',
-        'packages': ['jaffle_shop'],
-        'view_mode': 'all',
-    })
-
-    # packages should be an array
-    with pytest.raises(ValueError):
-        validate({
-            'packages': 'jaffle_shop',
-        })
-
-    # view_mode should be 'all' or 'changed_models'
-    validate({
-        'view_mode': None,
-    })
-    validate({
-        'view_mode': 'all',
-    })
-    with pytest.raises(ValueError):
-        validate({
-            'view_mode': 'abc',
-        })

tests/tasks/test_schema.py

@@ -1,99 +0,0 @@
-import os
-from unittest import TestCase
-from unittest.mock import MagicMock
-
-import pytest
-
-from recce.adapter.dbt_adapter import load_manifest, load_catalog, DbtAdapter
-from recce.core import RecceContext, set_default_context
-from recce.run import schema_diff_should_be_approved
-
-
-def test_validator():
-    from recce.tasks.schema import SchemaDiffCheckValidator
-    validator = SchemaDiffCheckValidator()
-
-    def validate(params: dict):
-        validator.validate({
-            'name': 'test',
-            'type': 'schema_diff',
-            'params': params,
-        })
-
-    # Select all models
-    validate({})
-
-    # Select by node name
-    validate({
-        'node_id': 'abc',
-    })
-    validate({
-        'node_id': ['abc'],
-    })
-
-    # Select by selector
-    validate({
-        'select': 'customers',
-        'exclude': 'customers',
-        'packages': ['jaffle_shop'],
-        'view_mode': 'all',
-    })
-
-    # packages should be an array
-    with pytest.raises(ValueError):
-        validate({
-            'packages': 'jaffle_shop',
-        })
-
-    # view_mode should be 'all' or 'changed_models'
-    validate({
-        'view_mode': None,
-    })
-    validate({
-        'view_mode': 'all',
-    })
-    with pytest.raises(ValueError):
-        validate({
-            'view_mode': 'abc',
-        })
-
-
-test_root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-
-
-class TestSchemaDiffAutoApprove(TestCase):
-
-    def setUp(self):
-        self.default_context = MagicMock(spec=RecceContext)
-        manifest = load_manifest(path=os.path.join(test_root_path, 'manifest.json'))
-        catalog = load_catalog(path=os.path.join(test_root_path, 'catalog.json'))
-        dbt_adapter = DbtAdapter(curr_manifest=manifest, curr_catalog=catalog)
-        self.default_context.adapter = dbt_adapter
-
-        dbt_adapter.adapter = MagicMock()
-        dbt_adapter.adapter.type.return_value = None
-
-        dbt_adapter.select_nodes = MagicMock()
-        # Base and Current will be the same
-        self.default_context.get_lineage.return_value = dbt_adapter.get_lineage()
-        set_default_context(self.default_context)
-
-    def test_schema_diff_should_be_approved(self):
-        # Node_id is string
-        is_approved = schema_diff_should_be_approved({
-            'node_id': 'model.jaffle_shop.customers',
-        })
-        assert is_approved is True
-
-        # Node_id is list
-        is_approved = schema_diff_should_be_approved({
-            'node_id': ['model.jaffle_shop.customers'],
-        })
-        assert is_approved is True
-
-        # Select all models
-        self.default_context.adapter.select_nodes.return_value = ['model.jaffle_shop.customers']
-        is_approved = schema_diff_should_be_approved({
-            'select': 'customers',
-        })
-        assert is_approved is True