recce-nightly 1.2.0.20250506__py3-none-any.whl → 1.26.0.20251124__py3-none-any.whl

tests/adapter/dbt_adapter/dbt_test_helper.py

@@ -4,8 +4,18 @@ import uuid
 from datetime import datetime
 from io import StringIO
 
-from dbt.contracts.graph.nodes import ModelNode, SnapshotNode, SeedNode, SourceDefinition
-from dbt.contracts.results import CatalogArtifact, ColumnMetadata, CatalogTable, TableMetadata
+from dbt.contracts.graph.nodes import (
+    ModelNode,
+    SeedNode,
+    SnapshotNode,
+    SourceDefinition,
+)
+from dbt.contracts.results import (
+    CatalogArtifact,
+    CatalogTable,
+    ColumnMetadata,
+    TableMetadata,
+)
 
 from recce.adapter.dbt_adapter import DbtAdapter, as_manifest, load_manifest
 from recce.core import RecceContext
@@ -19,9 +29,9 @@ class DbtTestHelper:
         self.curr_schema = f"{schema_prefix}_curr"
 
         current_dir = os.path.dirname(os.path.abspath(__file__))
-        project_dir = os.path.join(current_dir, 'test_proj')
+        project_dir = os.path.join(current_dir, "test_proj")
         profiles_dir = project_dir
-        manifest_path = os.path.join(project_dir, 'manifest.json')
+        manifest_path = os.path.join(project_dir, "manifest.json")
 
         dbt_adapter = DbtAdapter.load(
             no_artifacts=True,
@@ -30,7 +40,7 @@ class DbtTestHelper:
         )
 
         context = RecceContext()
-        context.adapter_type = 'dbt'
+        context.adapter_type = "dbt"
         context.adapter = dbt_adapter
         context.schema_prefix = schema_prefix
         self.adapter = dbt_adapter
@@ -108,16 +118,19 @@ class DbtTestHelper:
             if csv:
                 dbt_adapter = self.adapter
                 csv = textwrap.dedent(csv)
-                with dbt_adapter.connection_named('create model'):
+                with dbt_adapter.connection_named("create model"):
                     import pandas as pd
-                    df = pd.read_csv(StringIO(csv))
+
+                    df = pd.read_csv(StringIO(csv))  # noqa: F841
+                    # DuckDB disables python_scan_all_frames by default after 1.1,
+                    # ref: https://github.com/duckdb/duckdb/pull/13896
+                    dbt_adapter.execute("SET python_scan_all_frames=true")
                     dbt_adapter.execute(f"CREATE TABLE {schema}.{model_name} AS SELECT * FROM df")
             raw_code = sql if sql else csv
 
             if columns:
                 index = 1
-                table = CatalogTable(
-                    TableMetadata(type="BASE TABLE", schema=schema, name=model_name), {}, {})
+                table = CatalogTable(TableMetadata(type="BASE TABLE", schema=schema, name=model_name), {}, {})
                 catalog.nodes[unique_id] = table
                 for column, column_type in columns.items():
                     col_data = ColumnMetadata(type=column_type, index=index, name=column)
@@ -147,9 +160,7 @@ class DbtTestHelper:
                     "tags": ["test_tag"],
                 },
                 "tags": ["test_tag"],
-                "depends_on": {
-                    "nodes": depends_on
-                },
+                "depends_on": {"nodes": depends_on},
             }
             if patch_func:
                 patch_func(node_dict)
@@ -179,11 +190,12 @@ class DbtTestHelper:
             self.curr_manifest,
             self.base_manifest,
             self.base_catalog,
-            self.curr_catalog)
+            self.curr_catalog,
+        )
 
     def remove_model(self, model_name):
         dbt_adapter = self.adapter
-        with dbt_adapter.connection_named('cleanup'):
+        with dbt_adapter.connection_named("cleanup"):
             dbt_adapter.execute(f"DROP TABLE IF EXISTS {self.base_schema}.{model_name}")
             dbt_adapter.execute(f"DROP TABLE IF EXISTS {self.curr_schema}.{model_name} ")
 
@@ -199,8 +211,7 @@ class DbtTestHelper:
         curr_columns: dict[str, str] = None,
         patch_func=None,
     ):
-        # unique_id = f"model.{package_name}.{model_name}"
-        unique_id = unique_id if unique_id else f"{source_name}.{table_name}"
+        unique_id = unique_id if unique_id else f"source.{package_name}.{source_name}.{table_name}"
 
         def _add_source_to_manifest(base):
             if base:
@@ -219,15 +230,18 @@ class DbtTestHelper:
             if csv:
                 dbt_adapter = self.adapter
                 csv = textwrap.dedent(csv)
-                with dbt_adapter.connection_named('create source'):
+                with dbt_adapter.connection_named("create source"):
                     import pandas as pd
-                    df = pd.read_csv(StringIO(csv))
+
+                    df = pd.read_csv(StringIO(csv))  # noqa: F841
+                    # DuckDB disables python_scan_all_frames by default after 1.1,
+                    # ref: https://github.com/duckdb/duckdb/pull/13896
+                    dbt_adapter.execute("SET python_scan_all_frames=true")
                     dbt_adapter.execute(f"CREATE TABLE {schema}.{table_name} AS SELECT * FROM df")
 
             if columns:
                 index = 1
-                table = CatalogTable(
-                    TableMetadata(type="BASE TABLE", schema=schema, name=table_name), {}, {})
+                table = CatalogTable(TableMetadata(type="BASE TABLE", schema=schema, name=table_name), {}, {})
                 catalog.sources[unique_id] = table
                 for column, column_type in columns.items():
                     col_data = ColumnMetadata(type=column_type, index=index, name=column)
@@ -271,11 +285,12 @@ class DbtTestHelper:
             self.curr_manifest,
             self.base_manifest,
             self.base_catalog,
-            self.curr_catalog)
+            self.curr_catalog,
+        )
 
     def cleanup(self):
         dbt_adapter = self.adapter
-        with dbt_adapter.connection_named('cleanup'):
+        with dbt_adapter.connection_named("cleanup"):
             dbt_adapter.execute(f"DROP SCHEMA IF EXISTS {self.base_schema} CASCADE")
             dbt_adapter.execute(f"DROP SCHEMA IF EXISTS {self.curr_schema} CASCADE")
 

tests/adapter/dbt_adapter/test_dbt_adapter.py

@@ -1,15 +1,9 @@
 from recce.adapter.dbt_adapter import DbtAdapter, dbt_supported_registry
-from recce.models import RunType
 
 
 def test_dbt_adapter_support_tasks(dbt_test_helper):
     adapter: DbtAdapter = dbt_test_helper.context.adapter
 
-    # Test dbt package name
-    supported_dbt_packages = set([package.package_name for package in adapter.manifest.macros.values()])
-    assert 'audit_helper' in supported_dbt_packages
-    assert 'dbt_profiler' in supported_dbt_packages
-
     # Test dbt task support
     support_tasks = adapter.support_tasks()
 
@@ -29,12 +23,3 @@ def test_dbt_adapter_support_tasks_without_required_dbt_package(dbt_test_helper)
     for task_type in dbt_supported_registry:
         task = task_type.value
         assert task in support_tasks
-        if task == RunType.PROFILE_DIFF.value:
-            assert support_tasks[task] is False
-        elif task == RunType.VALUE_DIFF.value:
-            assert support_tasks[task] is False
-        elif task == RunType.VALUE_DIFF_DETAIL.value:
-            assert support_tasks[task] is False
-
-    # Check the query_diff_with_primary_key is not supported
-    assert support_tasks["query_diff_with_primary_key"] is False