recce-nightly 1.2.0.20250506__py3-none-any.whl → 1.26.0.20251124__py3-none-any.whl

recce/tasks/core.py

@@ -1,11 +1,11 @@
 from abc import ABC, abstractmethod
-from typing import List, Union, Optional, Literal
+from typing import List, Literal, Optional, Union
 
 from pydantic import BaseModel
 
 from recce.core import default_context
 from recce.exceptions import RecceCancelException
-from recce.models import Run, Check
+from recce.models import Check, Run
 from recce.util.pydantic_model import pydantic_model_dump
 
 
@@ -63,6 +63,7 @@ class TaskResultDiffer(ABC):
     @staticmethod
     def diff(base, current):
         from deepdiff import DeepDiff
+
         diff = DeepDiff(base, current, ignore_order=True)
         return diff if diff else None
 
@@ -76,15 +77,12 @@ class TaskResultDiffer(ABC):
         select: Optional[str] = None,
         exclude: Optional[str] = None,
         packages: Optional[list[str]] = None,
-        view_mode: Optional[Literal['all', 'changed_models']] = None,
+        view_mode: Optional[Literal["all", "changed_models"]] = None,
     ) -> List[str]:
         nodes = default_context().adapter.select_nodes(
-            select=select,
-            exclude=exclude,
-            packages=packages,
-            view_mode=view_mode
+            select=select, exclude=exclude, packages=packages, view_mode=view_mode
         )
-        return [node for node in nodes if not node.startswith('test.')]
+        return [node for node in nodes if not node.startswith("test.")]
 
     @abstractmethod
     def _check_result_changed_fn(self, result):
@@ -100,10 +98,10 @@ class TaskResultDiffer(ABC):
         Should be implemented by subclass.
         """
         params = self.run.params
-        if params.get('model'):
-            return [TaskResultDiffer.get_node_id_by_name(params.get('model'))]
-        elif params.get('node_names'):
-            names = params.get('node_names', [])
+        if params.get("model"):
+            return [TaskResultDiffer.get_node_id_by_name(params.get("model"))]
+        elif params.get("node_names"):
+            names = params.get("node_names", [])
             return [TaskResultDiffer.get_node_id_by_name(name) for name in names]
         else:
             # No related node ids in the params
@@ -125,7 +123,7 @@ class CheckValidator:
         try:
             check = Check(**check)
         except Exception as e:
-            raise ValueError(f'Invalid check format. {str(e)}')
+            raise ValueError(f"Invalid check format. {str(e)}")
 
         self.validate_check(check)
 

recce/tasks/dataframe.py

@@ -10,20 +10,43 @@ from pydantic import BaseModel, Field
 
 
 class DataFrameColumnType(Enum):
-    NUMBER = 'number'
-    INTEGER = 'integer'
-    TEXT = 'text'
-    BOOLEAN = 'boolean'
-    DATE = 'date'
-    DATETIME = 'datetime'
-    TIMEDELTA = 'timedelta'
-    UNKNOWN = 'unknown'
+    NUMBER = "number"
+    INTEGER = "integer"
+    TEXT = "text"
+    BOOLEAN = "boolean"
+    DATE = "date"
+    DATETIME = "datetime"
+    TIMEDELTA = "timedelta"
+    UNKNOWN = "unknown"
+
+    @classmethod
+    def from_string(cls, type_str: str) -> "DataFrameColumnType":
+        """Convert string to DataFrameColumnType enum.
+
+        Args:
+            type_str: String representation of the type (e.g., "integer", "text")
+
+        Returns:
+            DataFrameColumnType enum value
+        """
+        type_str = type_str.lower().strip()
+        try:
+            return cls(type_str)
+        except ValueError:
+            return cls.UNKNOWN
 
 
 class DataFrameColumn(BaseModel):
+    key: t.Optional[str] = None
     name: str
     type: DataFrameColumnType
 
+    def __init__(self, **data):
+        """Initialize DataFrameColumn, auto-setting key=name if key is missing."""
+        if "key" not in data or data["key"] is None:
+            data["key"] = data.get("name")
+        super().__init__(**data)
+
 
 class DataFrame(BaseModel):
     columns: t.List[DataFrameColumn]
@@ -32,19 +55,21 @@ class DataFrame(BaseModel):
     more: t.Optional[bool] = Field(None, description="Whether there are more rows to fetch")
 
     @staticmethod
-    def from_agate(table: 'agate.Table', limit: t.Optional[int] = None, more: t.Optional[bool] = None):
+    def from_agate(table: "agate.Table", limit: t.Optional[int] = None, more: t.Optional[bool] = None):
         from recce.adapter.dbt_adapter import dbt_version
-        if dbt_version < 'v1.8':
+
+        if dbt_version < "v1.8":
             import dbt.clients.agate_helper as agate_helper
         else:
             import dbt_common.clients.agate_helper as agate_helper
 
         import agate
+
         columns = []
 
         for col_name, col_type in zip(table.column_names, table.column_types):
 
-            has_integer = hasattr(agate_helper, 'Integer')
+            has_integer = hasattr(agate_helper, "Integer")
 
             if isinstance(col_type, agate.Number):
                 col_type = DataFrameColumnType.NUMBER
@@ -62,7 +87,7 @@ class DataFrame(BaseModel):
                 col_type = DataFrameColumnType.INTEGER
             else:
                 col_type = DataFrameColumnType.UNKNOWN
-            columns.append(DataFrameColumn(name=col_name, type=col_type))
+            columns.append(DataFrameColumn(key=col_name, name=col_name, type=col_type))
 
         def _row_values(row):
             # If the value is Decimal, check if it's finite. If not, convert it to float(xxx) (GitHub issue #476)
@@ -78,23 +103,23 @@ class DataFrame(BaseModel):
         return df
 
     @staticmethod
-    def from_pandas(pandas_df: 'pandas.DataFrame', limit: t.Optional[int] = None, more: t.Optional[bool] = None):
+    def from_pandas(pandas_df: "pandas.DataFrame", limit: t.Optional[int] = None, more: t.Optional[bool] = None):
         columns = []
         for column in pandas_df.columns:
             dtype = pandas_df[column].dtype
-            if dtype == 'int64':
+            if dtype == "int64":
                 col_type = DataFrameColumnType.INTEGER
-            elif dtype == 'float64':
+            elif dtype == "float64":
                 col_type = DataFrameColumnType.NUMBER
-            elif dtype == 'object':
+            elif dtype == "object":
                 col_type = DataFrameColumnType.TEXT
-            elif dtype == 'bool':
+            elif dtype == "bool":
                 col_type = DataFrameColumnType.BOOLEAN
             else:
                 col_type = DataFrameColumnType.UNKNOWN
             columns.append(DataFrameColumn(name=column, type=col_type))
 
-        s = pandas_df.to_json(orient='values')
+        s = pandas_df.to_json(orient="values")
         data = json.loads(s)
 
         df = DataFrame(
@@ -104,3 +129,42 @@ class DataFrame(BaseModel):
             more=more,
         )
         return df
+
+    @staticmethod
+    def from_data(
+        columns: t.Dict[str, str],
+        data: t.List[tuple],
+        limit: t.Optional[int] = None,
+        more: t.Optional[bool] = None,
+    ):
+        """Create a DataFrame from columns and data directly.
+
+        Args:
+            columns: Dict defining the schema where keys are column names and values are type strings.
+                     Type strings can be: "number", "integer", "text", "boolean", "date", "datetime", "timedelta"
+            data: List of rows (each row is a list/tuple/sequence of values)
+            limit: Optional limit on the number of rows returned
+            more: Optional flag indicating whether there are more rows to fetch
+
+        Returns:
+            DataFrame instance
+
+        Examples:
+            # Using simple dict format
+            columns = {"idx": "integer", "name": "text", "impacted": "boolean"}
+            data = [[0, "model_a", True], [1, "model_b", False]]
+            df = DataFrame.from_data(columns, data)
+        """
+        # Convert dict columns to DataFrameColumn objects
+        processed_columns = []
+        for key, type_str in columns.items():
+            col_type = DataFrameColumnType.from_string(type_str)
+            processed_columns.append(DataFrameColumn(key=key, name=key, type=col_type))
+
+        df = DataFrame(
+            columns=processed_columns,
+            data=data,
+            limit=limit,
+            more=more,
+        )
+        return df

recce/tasks/histogram.py

@@ -9,34 +9,66 @@ from pydantic import BaseModel
 from recce.core import default_context
 from recce.models import Check
 from recce.tasks import Task
-from recce.tasks.core import TaskResultDiffer, CheckValidator
+from recce.tasks.core import CheckValidator, TaskResultDiffer
 from recce.tasks.query import QueryMixin
 
 sql_datetime_types = [
-    "DATE", "DATETIME", "TIMESTAMP", "TIME",
+    "DATE",
+    "DATETIME",
+    "TIMESTAMP",
+    "TIME",
     "YEAR",  # Specific to MySQL/MariaDB
-    "DATETIME2", "SMALLDATETIME", "DATETIMEOFFSET",  # Specific to SQL Server
+    "DATETIME2",
+    "SMALLDATETIME",
+    "DATETIMEOFFSET",  # Specific to SQL Server
     "INTERVAL",  # Common in PostgreSQL and Oracle
-    "TIMESTAMPTZ", "TIMETZ",  # Specific to PostgreSQL
-    "TIMESTAMP WITH TIME ZONE", "TIMESTAMP WITH LOCAL TIME ZONE",  # Oracle
-    "TIMESTAMP_LTZ", "TIMESTAMP_NTZ", "TIMESTAMP_TZ",  # Specific to Snowflake
+    "TIMESTAMPTZ",
+    "TIMETZ",  # Specific to PostgreSQL
+    "TIMESTAMP WITH TIME ZONE",
+    "TIMESTAMP WITH LOCAL TIME ZONE",  # Oracle
+    "TIMESTAMP_LTZ",
+    "TIMESTAMP_NTZ",
+    "TIMESTAMP_TZ",  # Specific to Snowflake
 ]
 
 sql_integer_types = [
-    "TINYINT", "SMALLINT", "MEDIUMINT", "INT", "INTEGER", "BIGINT",  # Common across most databases
-    "INT2", "INT4", "INT8",  # PostgreSQL specific aliases
+    "TINYINT",
+    "SMALLINT",
+    "MEDIUMINT",
+    "INT",
+    "INTEGER",
+    "BIGINT",  # Common across most databases
+    "INT2",
+    "INT4",
+    "INT8",  # PostgreSQL specific aliases
     "UNSIGNED BIG INT",  # SQLite specific
     "NUMBER",  # Oracle, can be used as an integer with precision and scale
     "NUMERIC",  # Generally available in many SQL databases, used with precision and scale
-    "SMALLSERIAL", "SERIAL", "BIGSERIAL",  # PostgreSQL auto-increment types
-    "IDENTITY", "SMALLIDENTITY", "BIGIDENTITY",  # SQL Server specific auto-increment types
+    "SMALLSERIAL",
+    "SERIAL",
+    "BIGSERIAL",  # PostgreSQL auto-increment types
+    "IDENTITY",
+    "SMALLIDENTITY",
+    "BIGIDENTITY",  # SQL Server specific auto-increment types
     "BYTEINT",  # Specific to Snowflake, for storing very small integers
 ]
 
 sql_not_supported_types = [
-    "CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT",
-    "NCHAR", "NVARCHAR", "VARCHAR2", "NVARCHAR2", "CLOB", "NCLOB",
-    "VARCHAR(MAX)", "XML", "JSON",
+    "CHAR",
+    "VARCHAR",
+    "TINYTEXT",
+    "TEXT",
+    "MEDIUMTEXT",
+    "LONGTEXT",
+    "NCHAR",
+    "NVARCHAR",
+    "VARCHAR2",
+    "NVARCHAR2",
+    "CLOB",
+    "NCLOB",
+    "VARCHAR(MAX)",
+    "XML",
+    "JSON",
     "BOOLEAN",  # PostgreSQL, SQLite, and others with native boolean support
     "TINYINT(1)",  # MySQL/MariaDB uses TINYINT(1) to represent boolean values
     "BIT",  # SQL Server and others use BIT to represent boolean values, where 1 is true and 0 is false
@@ -185,7 +217,7 @@ def query_numeric_histogram(task, node, column, column_type, min_value, max_valu
                 else:
                     counts[num_bins - 1] += count
         base_result = {
-            'counts': counts,
+            "counts": counts,
         }
     if curr is not None:
         counts = [0] * num_bins
@@ -199,7 +231,7 @@ def query_numeric_histogram(task, node, column, column_type, min_value, max_valu
                 else:
                     counts[num_bins - 1] += count
         curr_result = {
-            'counts': counts,
+            "counts": counts,
         }
     return base_result, curr_result, bin_edges, labels
 
@@ -209,7 +241,7 @@ def query_datetime_histogram(task, node, column, min_value, max_value):
     print(max_value, min_value, days_delta)
     # _type = None
     if days_delta > 365 * 4:
-        _type = 'yearly'
+        _type = "yearly"
         dmin = date(min_value.year, 1, 1)
         if max_value.year < 3000:
             dmax = date(max_value.year, 1, 1) + relativedelta(years=+1)
@@ -237,7 +269,7 @@ def query_datetime_histogram(task, node, column, min_value, max_value):
         else:
             dmax = date(3000, 1, 1)
         period = relativedelta(dmax, dmin)
-        num_buckets = (period.years * 12 + period.months)
+        num_buckets = period.years * 12 + period.months
         bin_edges = [dmin + relativedelta(months=i) for i in range(num_buckets + 1)]
         sql = f"""
         SELECT
@@ -285,18 +317,18 @@ def query_datetime_histogram(task, node, column, min_value, max_value):
 
     base_counts = [0] * num_buckets
     print(_type)
-    for (d, v) in base.rows:
+    for d, v in base.rows:
         i = bin_edges.index(d.date()) if isinstance(d, datetime) else bin_edges.index(d)
         base_counts[i] = v
     curr_counts = [0] * num_buckets
-    for (d, v) in curr.rows:
+    for d, v in curr.rows:
         i = bin_edges.index(d.date()) if isinstance(d, datetime) else bin_edges.index(d)
         curr_counts[i] = v
     base_result = {
-        'counts': base_counts,
+        "counts": base_counts,
     }
     curr_result = {
-        'counts': curr_counts,
+        "counts": curr_counts,
     }
 
     return base_result, curr_result, bin_edges
@@ -310,6 +342,7 @@ class HistogramDiffTask(Task, QueryMixin):
 
     def execute(self):
         from recce.adapter.dbt_adapter import DbtAdapter
+
         result = {}
 
         dbt_adapter: DbtAdapter = default_context().adapter
@@ -353,29 +386,31 @@ class HistogramDiffTask(Task, QueryMixin):
             labels = None
             if min_value is None or max_value is None:
                 base_result = {
-                    'counts': [],
+                    "counts": [],
                 }
                 current_result = {
-                    'counts': [],
+                    "counts": [],
                 }
                 bin_edges = []
                 labels = []
             elif column_type.upper() in sql_datetime_types:
                 base_result, current_result, bin_edges = query_datetime_histogram(
-                    self, node, column, min_value, max_value)
+                    self, node, column, min_value, max_value
+                )
             else:
                 base_result, current_result, bin_edges, labels = query_numeric_histogram(
-                    self, node, column, column_type, min_value, max_value, num_bins)
+                    self, node, column, column_type, min_value, max_value, num_bins
+                )
             if base_result:
-                base_result['total'] = base_total
+                base_result["total"] = base_total
             if current_result:
-                current_result['total'] = curr_total
-            result['base'] = base_result
-            result['current'] = current_result
-            result['min'] = min_value
-            result['max'] = max_value
-            result['bin_edges'] = bin_edges
-            result['labels'] = labels
+                current_result["total"] = curr_total
+            result["base"] = base_result
+            result["current"] = current_result
+            result["min"] = min_value
+            result["max"] = max_value
+            result["bin_edges"] = bin_edges
+            result["labels"] = labels
         return result
 
     def cancel(self):
@@ -386,7 +421,7 @@ class HistogramDiffTask(Task, QueryMixin):
 
 class HistogramDiffTaskResultDiffer(TaskResultDiffer):
     def _check_result_changed_fn(self, result):
-        return TaskResultDiffer.diff(result['base'], result['current'])
+        return TaskResultDiffer.diff(result["base"], result["current"])
 
 
 class HistogramDiffCheckValidator(CheckValidator):

recce/tasks/lineage.py

@@ -1,4 +1,4 @@
-from typing import Optional, Literal
+from typing import Literal, Optional
 
 from pydantic import BaseModel
 
@@ -10,7 +10,7 @@ class LineageDiffParams(BaseModel):
     select: Optional[str] = None
     exclude: Optional[str] = None
     packages: Optional[list[str]] = None
-    view_mode: Optional[Literal['all', 'changed_models']] = None
+    view_mode: Optional[Literal["all", "changed_models"]] = None
 
 
 class LineageDiffCheckValidator(CheckValidator):