recce-nightly 1.3.0.20250507__py3-none-any.whl → 1.4.0.20250515__py3-none-any.whl

tests/tasks/test_preset_checks.py

@@ -4,47 +4,61 @@ import pytest
 def test_default_validator():
     from recce.tasks.core import CheckValidator
 
-    CheckValidator().validate({
-        "name": "test",
-        "type": "row_count_diff",
-        "params": {},
-    })
+    CheckValidator().validate(
+        {
+            "name": "test",
+            "type": "row_count_diff",
+            "params": {},
+        }
+    )
 
     with pytest.raises(ValueError):
-        CheckValidator().validate({
-            "type": "row_count_diff",
-        })
+        CheckValidator().validate(
+            {
+                "type": "row_count_diff",
+            }
+        )
 
 
 def test_query_diff_validator():
     from recce.tasks.query import QueryDiffCheckValidator
-    QueryDiffCheckValidator().validate({
-        "name": "test",
-        "type": "query_diff",
-        "params": {
-            "sql_template": "select * from {{ model }}"
-        },
-    })
-    with pytest.raises(ValueError):
-        QueryDiffCheckValidator().validate({
+
+    QueryDiffCheckValidator().validate(
+        {
             "name": "test",
             "type": "query_diff",
-        })
+            "params": {"sql_template": "select * from {{ model }}"},
+        }
+    )
+    with pytest.raises(ValueError):
+        QueryDiffCheckValidator().validate(
+            {
+                "name": "test",
+                "type": "query_diff",
+            }
+        )
 
 
 def test_lineage_diff_validator():
     from recce.tasks.lineage import LineageDiffCheckValidator
-    LineageDiffCheckValidator().validate({
-        "name": "test",
-        "type": "lineage_diff",
-        "params": {},
-    })
-    LineageDiffCheckValidator().validate({
-        "name": "test",
-        "type": "lineage_diff",
-        "view_options": {},
-    })
-    with pytest.raises(ValueError):
-        LineageDiffCheckValidator().validate({
+
+    LineageDiffCheckValidator().validate(
+        {
+            "name": "test",
             "type": "lineage_diff",
-        })
+            "params": {},
+        }
+    )
+    LineageDiffCheckValidator().validate(
+        {
+            "name": "test",
+            "type": "lineage_diff",
+            "view_options": {},
+        }
+    )
+    with pytest.raises(ValueError):
+        LineageDiffCheckValidator().validate(
+            {
+                "type": "lineage_diff",
+            }
+        )

tests/tasks/test_profile.py

@@ -1,6 +1,10 @@
 import pytest
+from agate import data_types
+from jinja2 import Template
+from sqlglot import parse_one
 
 from recce.tasks import ProfileDiffTask, ProfileTask
+from recce.tasks.profile import PROFILE_COLUMN_JINJA_TEMPLATE
 
 csv_data_curr = """
         customer_id,name,age
@@ -19,7 +23,7 @@ csv_data_base = """
 
 def test_profile(dbt_test_helper):
     dbt_test_helper.create_model("customers", None, csv_data_curr)
-    params = dict(model='customers')
+    params = dict(model="customers")
     task = ProfileTask(params)
     run_result = task.execute()
 
@@ -28,7 +32,7 @@ def test_profile(dbt_test_helper):
 
 def test_profile_with_selected_columns(dbt_test_helper):
     dbt_test_helper.create_model("customers", None, csv_data_curr)
-    params = dict(model='customers', columns=['name', 'age'])
+    params = dict(model="customers", columns=["name", "age"])
     task = ProfileTask(params)
     run_result = task.execute()
     assert len(run_result.current.data) == 2
@@ -36,7 +40,7 @@ def test_profile_with_selected_columns(dbt_test_helper):
 
 def test_profile_diff(dbt_test_helper):
     dbt_test_helper.create_model("customers", csv_data_base, csv_data_curr)
-    params = dict(model='customers')
+    params = dict(model="customers")
     task = ProfileDiffTask(params)
     run_result = task.execute()
 
@@ -46,7 +50,7 @@ def test_profile_diff(dbt_test_helper):
 
 def test_profile_diff_with_selected_columns(dbt_test_helper):
     dbt_test_helper.create_model("customers", csv_data_base, csv_data_curr)
-    params = dict(model='customers', columns=['name', 'age'])
+    params = dict(model="customers", columns=["name", "age"])
     task = ProfileDiffTask(params)
     run_result = task.execute()
     assert len(run_result.current.data) == 2
@@ -57,17 +61,338 @@ def test_validator():
     from recce.tasks.profile import ProfileCheckValidator
 
     def validate(params: dict = {}, view_options: dict = {}):
-        ProfileCheckValidator().validate({
-            'name': 'test',
-            'type': 'profile_diff',
-            'params': params,
-            'view_options': view_options,
-        })
+        ProfileCheckValidator().validate(
+            {
+                "name": "test",
+                "type": "profile_diff",
+                "params": params,
+                "view_options": view_options,
+            }
+        )
 
-    validate({
-        'model': 'customers',
-    })
+    validate(
+        {
+            "model": "customers",
+        }
+    )
 
     with pytest.raises(ValueError):
-        validate({
-        })
+        validate({})
+
+
+def test_profile_column_jinja_template():
+
+    class DummyAdapter:
+        def __init__(self, database_type="duckdb"):
+            self.database_type = database_type
+
+        def quote(self, col):
+            quote_marks = {
+                "athena": '"',
+                "bigquery": "`",
+                "databricks": "`",
+                "duckdb": '"',
+                "postgres": '"',
+                "redshift": '"',
+                "snowflake": '"',
+                "sqlite": '"',
+                "sqlserver": '"',
+                "trino": '"',
+            }
+            quote_mark = quote_marks.get(self.database_type, '"')
+            return f"{quote_mark}{col}{quote_mark}"
+
+    class DummyDbt:
+        def __init__(self, database_type="Postgres"):
+            self.database_type = database_type
+
+        def type_bigint(self):
+            bigint_types = {
+                "athena": "BIGINT",
+                "bigquery": "INT64",
+                "databricks": "BIGINT",
+                "duckdb": "BIGINT",
+                "postgres": "BIGINT",
+                "redshift": "BIGINT",
+                "snowflake": "NUMBER",
+                "sqlite": "INTEGER",
+                "sqlserver": "BIGINT",
+                "trino": "BIGINT",
+            }
+            return bigint_types.get(self.database_type, "bigint")
+
+        def type_numeric(self):
+            numeric_types = {
+                "athena": "DECIMAL",
+                "bigquery": "NUMERIC",
+                "databricks": "DECIMAL",
+                "duckdb": "DECIMAL",
+                "postgres": "NUMERIC",
+                "redshift": "DECIMAL",
+                "snowflake": "NUMBER",
+                "sqlite": "NUMERIC",
+                "sqlserver": "DECIMAL",
+                "trino": "DECIMAL",
+            }
+            return numeric_types.get(self.database_type, "numeric")
+
+        def type_string(self):
+            string_types = {
+                "athena": "VARCHAR",
+                "bigquery": "STRING",
+                "databricks": "STRING",
+                "duckdb": "VARCHAR",
+                "postgres": "TEXT",
+                "redshift": "VARCHAR",
+                "snowflake": "VARCHAR",
+                "sqlite": "TEXT",
+                "sqlserver": "VARCHAR",
+                "trino": "VARCHAR",
+            }
+            return string_types.get(self.database_type, "varchar")
+
+    test_db_column_types = {
+        "athena": [
+            "boolean",
+            "int",
+            "double",
+            "decimal",
+            "char",
+            "string",
+            "binary",
+            "date",
+            "array",
+            "struct",
+            "map",
+        ],
+        "bigquery": [
+            "array",
+            "bool",
+            "bytes",
+            "date",
+            "datetime",
+            "interval",
+            "json",
+            "int64",
+            "bignumeric",
+            "range",
+            "string",
+            "struct",
+            "time",
+            "timestamp",
+        ],
+        "databricks": [
+            "bigint",
+            "binary",
+            "boolean",
+            "date",
+            "decimal",
+            "double",
+            "float",
+            "int",
+            "interval",
+            "void",
+            "string",
+            "timestamp",
+            "array",
+            "map",
+            "struct",
+            "variant",
+            "object",
+        ],
+        "duckdb": [
+            "boolean",
+            "integer",
+            "bigint",
+            "real",
+            "double",
+            "decimal",
+            "varchar",
+            "date",
+            "time",
+            "timestamp",
+            "blob",
+            "array",
+            "list",
+            "struct",
+            "map",
+            "union",
+            "uuid",
+            "json",
+        ],
+        "postgres": [
+            "bigint",
+            "bigserial",
+            "bit",
+            "boolean",
+            "box",
+            "bytea",
+            "character",
+            "cidr",
+            "circle",
+            "date",
+            "double precision",
+            "inet",
+            "integer",
+            "interval",
+            "json",
+            "jsonb",
+            "line",
+            "lseg",
+            "macaddr",
+            "money",
+            "numeric",
+            "path",
+            "pg_lsn",
+            "point",
+            "polygon",
+            "real",
+            "smallint",
+            "smallserial",
+            "serial",
+            "text",
+            "time",
+            "timestamp",
+            "tsquery",
+            "tsvector",
+            "txid_snapshot",
+            "uuid",
+            "xml",
+            "array",
+        ],
+        "redshift": [
+            "smallint",
+            "integer",
+            "bigint",
+            "decimal",
+            "real",
+            "double precision",
+            "boolean",
+            "char",
+            "varchar",
+            "date",
+            "timestamp",
+            "timestamptz",
+            "super",
+            "time",
+            "timetz",
+            "varbyte",
+            "geometry",
+            "geography",
+            "hllsketch",
+        ],
+        "snowflake": [
+            "number",
+            "decimal",
+            "numeric",
+            "int",
+            "integer",
+            "bigint",
+            "smallint",
+            "tinyint",
+            "byteint",
+            "float",
+            "float4",
+            "float8",
+            "double",
+            "double precision",
+            "real",
+            "varchar",
+            "char",
+            "character",
+            "string",
+            "text",
+            "binary",
+            "varbinary",
+            "boolean",
+            "date",
+            "datetime",
+            "time",
+            "timestamp",
+            "timestamp_ltz",
+            "timestamp_ntz",
+            "timestamp_tz",
+            "variant",
+            "object",
+            "array",
+            "geography",
+            "geometry",
+        ],
+        "sqlite": ["integer", "real", "text", "blob", "numeric", "boolean", "date", "datetime", "json"],
+        "trino": [
+            "boolean",
+            "tinyint",
+            "smallint",
+            "integer",
+            "bigint",
+            "real",
+            "double",
+            "decimal",
+            "varchar",
+            "char",
+            "varbinary",
+            "json",
+            "date",
+            "time",
+            "timestamp",
+            "timestamp with time zone",
+            "interval year to month",
+            "interval day to second",
+            "array",
+            "map",
+            "row",
+            "ipaddress",
+            "uuid",
+            "hyperloglog",
+            "p4hyperloglog",
+            "qdigest",
+        ],
+        "sqlserver": [
+            "bigint",
+            "int",
+            "smallint",
+            "tinyint",
+            "bit",
+            "decimal",
+            "numeric",
+            "money",
+            "smallmoney",
+            "float",
+            "real",
+            "date",
+            "datetime",
+            "datetime2",
+            "smalldatetime",
+            "time",
+            "char",
+            "varchar",
+            "text",
+            "nchar",
+            "nvarchar",
+            "ntext",
+            "binary",
+            "varbinary",
+            "image",
+            "timestamp",
+            "uniqueidentifier",
+            "xml",
+            "cursor",
+            "table",
+            "sql_variant",
+        ],
+    }
+
+    for db_type, column_types in test_db_column_types.items():
+        for column_type in column_types:
+            context = {
+                "column_type": column_type,
+                "column_name": "profile_column",
+                "db_type": db_type,
+                "relation": "test_table",
+                "adapter": DummyAdapter(),
+                "dbt": DummyDbt(),
+            }
+
+            sql = Template(PROFILE_COLUMN_JINJA_TEMPLATE).render(context)
+            dialect = db_type if db_type != "sqlserver" else "tsql"
+            parse_one(sql, read=dialect)

tests/tasks/test_query.py

@@ -73,7 +73,7 @@ def test_query_diff_in_warehouse(dbt_test_helper):
         """
 
     dbt_test_helper.create_model("customers", csv_data_base, csv_data_curr)
-    params = dict(sql_template=f'select * from {{{{ ref("customers") }}}}', primary_keys=['customer_id'])
+    params = dict(sql_template=f'select * from {{{{ ref("customers") }}}}', primary_keys=["customer_id"])
     task = QueryDiffTask(params)
     run_result = task.execute()
     assert len(run_result.diff.data) == 2
@@ -81,7 +81,7 @@ def test_query_diff_in_warehouse(dbt_test_helper):
     params = dict(
         base_sql_template=f'select * from {{{{ ref("customers") }}}} where customer_id == 1',
         sql_template=f'select * from {{{{ ref("customers") }}}}',
-        primary_keys=['customer_id'],
+        primary_keys=["customer_id"],
     )
     task = QueryDiffTask(params)
     run_result = task.execute()
@@ -92,50 +92,50 @@ def test_validator():
     from recce.tasks.query import QueryCheckValidator, QueryDiffCheckValidator
 
     def validate(params: dict = {}, view_options: dict = {}):
-        QueryCheckValidator().validate({
-            'name': 'test',
-            'type': 'query',
-            'params': params,
-            'view_options': view_options,
-        })
+        QueryCheckValidator().validate(
+            {
+                "name": "test",
+                "type": "query",
+                "params": params,
+                "view_options": view_options,
+            }
+        )
 
     def validate_diff(params: dict = {}, view_options: dict = {}):
-        QueryDiffCheckValidator().validate({
-            'name': 'test',
-            'type': 'query_diff',
-            'params': params,
-            'view_options': view_options,
-        })
+        QueryDiffCheckValidator().validate(
+            {
+                "name": "test",
+                "type": "query_diff",
+                "params": params,
+                "view_options": view_options,
+            }
+        )
 
     # query
-    validate({
-        'sql_template': 'select * from abc'
-    })
+    validate({"sql_template": "select * from abc"})
 
     # diff in client
-    validate_diff({
-        'sql_template': 'select * from abc'
-    })
-    validate_diff({
-        'sql_template': 'select * from abc',
-        'base_sql_template': 'select * from abc',
-    })
+    validate_diff({"sql_template": "select * from abc"})
+    validate_diff(
+        {
+            "sql_template": "select * from abc",
+            "base_sql_template": "select * from abc",
+        }
+    )
 
     # diff in warehouse
-    validate_diff({
-        'primary_keys': ['customer_id'],
-        'sql_template': 'select * from abc',
-    })
-    validate_diff({
-        'sql_template': 'select * from abc',
-        'base_sql_template': 'select * from abc',
-        'primary_keys': ['customer_id']
-    })
-    validate_diff({
-        'sql_template': 'select * from abc',
-        'base_sql_template': 'select * from abc',
-        'primary_keys': ['customer_id']
-    })
+    validate_diff(
+        {
+            "primary_keys": ["customer_id"],
+            "sql_template": "select * from abc",
+        }
+    )
+    validate_diff(
+        {"sql_template": "select * from abc", "base_sql_template": "select * from abc", "primary_keys": ["customer_id"]}
+    )
+    validate_diff(
+        {"sql_template": "select * from abc", "base_sql_template": "select * from abc", "primary_keys": ["customer_id"]}
+    )
 
     # invalid
     with pytest.raises(ValueError):
@@ -145,7 +145,7 @@ def test_validator():
         validate_diff()
 
     with pytest.raises(ValueError):
-        validate_diff({'sql_template': 123, 'primary_keys': 'xyz'})
+        validate_diff({"sql_template": 123, "primary_keys": "xyz"})
 
     with pytest.raises(ValueError):
-        validate_diff({'sql_template': 's', 'primary_keys': 'xyz'})
+        validate_diff({"sql_template": "s", "primary_keys": "xyz"})