recce-nightly 1.10.0.20250625__py3-none-any.whl → 1.30.0.20251221__py3-none-any.whl

recce/tasks/utils.py

@@ -0,0 +1,147 @@
+"""Utility functions for task operations."""
+
+from typing import List, Optional
+
+from recce.tasks.dataframe import DataFrame
+
+
+def strip_identifier_quotes(identifier: str) -> str:
+    """
+    Strip SQL identifier quotes from a column name.
+
+    Different databases use different quoting styles:
+    - Double quotes: "column" (PostgreSQL, Snowflake, etc.)
+    - Backticks: `column` (MySQL, BigQuery)
+    - Square brackets: [column] (SQL Server)
+
+    Args:
+        identifier: Column name that may be quoted
+
+    Returns:
+        Column name with quotes stripped
+
+    Examples:
+        >>> strip_identifier_quotes('"myColumn"')
+        'myColumn'
+        >>> strip_identifier_quotes('`my_column`')
+        'my_column'
+        >>> strip_identifier_quotes('[Column Name]')
+        'Column Name'
+        >>> strip_identifier_quotes('regular_column')
+        'regular_column'
+    """
+    if not identifier or len(identifier) < 2:
+        return identifier
+
+    # Check for double quotes
+    if identifier.startswith('"') and identifier.endswith('"'):
+        return identifier[1:-1]
+
+    # Check for backticks
+    if identifier.startswith("`") and identifier.endswith("`"):
+        return identifier[1:-1]
+
+    # Check for square brackets
+    if identifier.startswith("[") and identifier.endswith("]"):
+        return identifier[1:-1]
+
+    return identifier
+
+
+def normalize_keys_to_columns(
+    keys: Optional[List[str]],
+    column_keys: List[str],
+) -> Optional[List[str]]:
+    """
+    Normalize user-provided keys to match actual column keys from the warehouse.
+
+    Different warehouses return column names in different cases:
+    - Snowflake: UPPERCASE (unless quoted)
+    - PostgreSQL/Redshift: lowercase (unless quoted)
+    - BigQuery: preserves original case
+
+    This function first attempts an exact match (for quoted columns that preserve
+    case), then falls back to case-insensitive matching to align user input
+    with the actual column keys returned by the warehouse.
+
+    Args:
+        keys: User-provided keys (e.g., primary_keys from params)
+        column_keys: Actual column keys from the query result
+
+    Returns:
+        List of keys normalized to match column_keys casing,
+        or None if keys is None.
+        If a key doesn't match any column, it's preserved as-is.
+
+    Examples:
+        >>> normalize_keys_to_columns(["payment_id"], ["PAYMENT_ID", "ORDER_ID"])
+        ["PAYMENT_ID"]
+
+        >>> normalize_keys_to_columns(["ID", "NAME"], ["id", "name", "value"])
+        ["id", "name"]
+
+        >>> normalize_keys_to_columns(["preCommitID"], ["preCommitID", "order_id"])
+        ["preCommitID"]  # Exact match preserved for quoted columns
+
+        >>> normalize_keys_to_columns(['"customerID"'], ["customerID", "amount"])
+        ["customerID"]  # Quotes stripped, then matched
+
+        >>> normalize_keys_to_columns(['`my_column`'], ["MY_COLUMN"])
+        ["MY_COLUMN"]  # Backticks stripped, then case-insensitive match
+    """
+    if keys is None:
+        return None
+
+    # Strip quotes from all keys first - quotes are for SQL execution,
+    # but the frontend should receive unquoted column names
+    unquoted_keys = [strip_identifier_quotes(key) for key in keys]
+
+    if not column_keys:
+        return unquoted_keys
+
+    # Build both exact and case-insensitive lookup maps
+    exact_key_set = set(column_keys)
+    case_insensitive_map = {col.lower(): col for col in column_keys}
+
+    normalized = []
+    for key in unquoted_keys:
+        if key in exact_key_set:
+            # Exact match - use as-is (handles quoted columns that preserved case)
+            normalized.append(key)
+        else:
+            # Case-insensitive fallback
+            actual_key = case_insensitive_map.get(key.lower())
+            normalized.append(actual_key if actual_key is not None else key)
+
+    return normalized
+
+
+def normalize_boolean_flag_columns(df: "DataFrame") -> "DataFrame":
+    """
+    Normalize boolean flag columns (in_a, in_b) to lowercase for cross-warehouse consistency.
+
+    Different warehouses return column names in different cases:
+    - Snowflake: IN_A, IN_B (UPPERCASE)
+    - PostgreSQL/Redshift: in_a, in_b (lowercase)
+    - BigQuery: preserves original case
+
+    This function ensures these columns are always lowercase in the DataFrame
+    sent to the frontend, enabling exact string matching.
+
+    Args:
+        df: DataFrame that may contain IN_A/IN_B columns
+
+    Returns:
+        DataFrame with in_a/in_b columns normalized to lowercase
+    """
+    from .dataframe import DataFrame, DataFrameColumn
+
+    normalized_columns = []
+    for col in df.columns:
+        key_upper = col.key.upper() if col.key else ""
+        if key_upper in ("IN_A", "IN_B"):
+            normalized_columns.append(DataFrameColumn(key=col.key.lower(), name=col.name.lower(), type=col.type))
+        else:
+            normalized_columns.append(col)
+
+    return DataFrame(columns=normalized_columns, data=df.data, limit=df.limit, more=df.more)

recce/tasks/valuediff.py

@@ -7,6 +7,7 @@ from ..exceptions import RecceException
 from ..models import Check
 from .core import CheckValidator, Task, TaskResultDiffer
 from .dataframe import DataFrame
+from .utils import normalize_boolean_flag_columns, normalize_keys_to_columns
 
 
 class ValueDiffParams(BaseModel):
@@ -91,6 +92,17 @@ class ValueDiffTask(Task, ValueDiffMixin):
         model: str,
         columns: List[str] = None,
     ):
+        """
+        Query value diff between base and current relations.
+        Compares column values between base and current relations using the primary key.
+        Mutates `self.params.primary_key` to normalize primary key names to match actual column names.
+
+        :param dbt_adapter: The dbt adapter instance.
+        :param primary_key: Single column name or list of column names for composite key.
+        :param model: The model name to compare.
+        :param columns: Optional list of columns to compare. If None, uses common columns.
+        :return: ValueDiffResult with summary and per-column match data, or None if invalid.
+        """
         import agate
 
         column_groups = {}
@@ -159,7 +171,7 @@ class ValueDiffTask(Task, ValueDiffMixin):
                 match_status,
                 count(*) as count_records
             from joined
-            group by column_name, match_status
+            group by 1, 2
         )
 
         select
@@ -250,6 +262,16 @@ class ValueDiffTask(Task, ValueDiffMixin):
         column_types = [agate.Text(), agate.Number(), agate.Number()]
         table = agate.Table(row, column_names=column_names, column_types=column_types)
 
+        # Normalize primary_key to match actual column keys
+        # For ValueDiff, 'columns' refers to the model's column list (from metadata), not a DataFrame result.
+        composite = isinstance(primary_key, list)
+        if composite:
+            self.params.primary_key = normalize_keys_to_columns(primary_key, columns)  # columns list from the model
+        else:
+            normalized = normalize_keys_to_columns([primary_key], columns)
+            if normalized:
+                self.params.primary_key = normalized[0]
+
         return ValueDiffResult(
             summary=ValueDiffResult.Summary(total=total, added=added, removed=removed),
             data=DataFrame.from_agate(table),
@@ -353,61 +375,53 @@ class ValueDiffDetailTask(Task, ValueDiffMixin):
                 columns.insert(0, primary_key)
 
         sql_template = r"""
-        with a_query as (
-            select {{ columns | join(',\n') }} from {{ base_relation }}
-        ),
-
-        b_query as (
-            select {{ columns | join(',\n') }} from {{ curr_relation }}
-        ),
-
-        a_intersect_b as (
-            select * from a_query
-            {{ dbt.intersect() }}
-            select * from b_query
-        ),
-
-        a_except_b as (
-            select * from a_query
-            {{ dbt.except() }}
-            select * from b_query
-        ),
-
-        b_except_a as (
-            select * from b_query
-            {{ dbt.except() }}
-            select * from a_query
-        ),
-
-        all_records as (
-            select
-                *,
-                true as in_a,
-                true as in_b
-            from a_intersect_b
-
-            union all
-
-            select
-                *,
-                true as in_a,
-                false as in_b
-            from a_except_b
-
-            union all
-
-            select
-                *,
-                false as in_a,
-                true as in_b
-            from b_except_a
-        )
-
-        select * from all_records
-        where not (in_a and in_b)
-        order by {{ primary_keys | join(',\n') }}, in_a desc, in_b desc
-        limit {{ limit }}
-        """
+                       with a_query as (select {{ columns | join (',\n') }}
+                       from {{ base_relation }}
+                           ), b_query as (
+                       select {{ columns | join (',\n') }}
+                       from {{ curr_relation }}
+                           ), a_intersect_b as (
+                       select *
+                       from a_query
+                           {{ dbt.intersect() }}
+                       select *
+                       from b_query
+                           ), a_except_b as (
+                       select *
+                       from a_query
+                           {{ dbt.except() }}
+                       select *
+                       from b_query
+                           ), b_except_a as (
+                       select *
+                       from b_query
+                           {{ dbt.except() }}
+                       select *
+                       from a_query
+                           ), all_records as (
+                       select
+                           *, true as in_a, true as in_b
+                       from a_intersect_b
+
+                       union all
+
+                       select
+                           *, true as in_a, false as in_b
+                       from a_except_b
+
+                       union all
+
+                       select
+                           *, false as in_a, true as in_b
+                       from b_except_a
+                           )
+
+                       select *
+                       from all_records
+                       where not (in_a and in_b)
+                       order by {{ primary_keys | join (',\n') }}, in_a desc, in_b desc
+                           limit {{ limit }}
+                       """
 
         sql = dbt_adapter.generate_sql(
             sql_template,
@@ -423,7 +437,21 @@ class ValueDiffDetailTask(Task, ValueDiffMixin):
         _, table = dbt_adapter.execute(sql, fetch=True)
         self.check_cancel()
 
-        return DataFrame.from_agate(table)
+        result_df = DataFrame.from_agate(table)
+        # Normalize in_a/in_b columns to lowercase for cross-warehouse consistency
+        result_df = normalize_boolean_flag_columns(result_df)
+
+        # Normalize primary_key to match actual column keys from result
+        column_keys = [col.key for col in result_df.columns]
+        composite = isinstance(primary_key, list)
+        if composite:
+            self.params.primary_key = normalize_keys_to_columns(primary_key, column_keys)
+        else:
+            normalized = normalize_keys_to_columns([primary_key], column_keys)
+            if normalized:
+                self.params.primary_key = normalized[0]
+
+        return result_df
 
     def execute(self):
         from recce.adapter.dbt_adapter import DbtAdapter

recce/util/api_token.py

@@ -17,7 +17,10 @@ def show_invalid_api_token_message():
     Show the message when the API token is invalid.
     """
     console.print("[[red]Error[/red]] Invalid Recce Cloud API token.")
-    console.print(f"Please check your API token from {RECCE_CLOUD_BASE_URL}/settings#tokens")
+    console.print("Please associate with your Recce Cloud account by the following command 'recce connect-to-cloud'.")
+    console.print(
+        "For more information, please visit: https://docs.reccehq.com/recce-cloud/share-recce-session-securely/#configure-recce-cloud-association-manually"
+    )
 
 
 def prepare_api_token(
@@ -30,7 +33,13 @@ def prepare_api_token(
     # Verify the API token for Recce Cloud Share Link
     api_token = get_recce_api_token()
     new_api_token = kwargs.get("api_token")
-    if api_token != new_api_token and new_api_token is not None:
+    if new_api_token is not None and new_api_token.startswith("rct-"):
+        # Task Token
+        valid = RecceCloud(new_api_token).verify_token()
+        if not valid:
+            raise RecceConfigException("Invalid Recce Cloud Task token")
+        api_token = new_api_token
+    elif api_token != new_api_token and new_api_token is not None:
         # Handle the API token provided by option `--api-token`
         valid = RecceCloud(new_api_token).verify_token()
         if not valid:

recce/util/breaking.py

@@ -78,11 +78,20 @@ def _diff_select_scope(old_scope: Scope, new_scope: Scope, scope_changes_map: di
             if change.category == "breaking":
                 change_category = "breaking"
 
+    # check if the upstream scopes sources table are the same
+    if len(old_scope.sources) != len(new_scope.sources):
+        change_category = "breaking"
+    else:
+        old_source_tables = [s.name for s in old_scope.sources.values() if isinstance(s, exp.Table)]
+        new_source_tables = [s.name for s in new_scope.sources.values() if isinstance(s, exp.Table)]
+        if sorted(old_source_tables) != sorted(new_source_tables):
+            change_category = "breaking"
+
     # check if non-select expressions are the same
     old_select = old_scope.expression  # type: exp.Select
     new_select = new_scope.expression  # type: exp.Select
     for arg_key in old_select.args.keys() | new_select.args.keys():
-        if arg_key in ["expressions", "with", "from"]:
+        if arg_key in ["expressions", "with", "from", "with_", "from_"]:
             continue
 
         if old_select.args.get(arg_key) != new_select.args.get(arg_key):

recce/util/cll.py

@@ -10,7 +10,6 @@ from sqlglot.optimizer.qualify import qualify
 
 from recce.exceptions import RecceException
 from recce.models.types import CllColumn, CllColumnDep
-from recce.util import SingletonMeta
 
 CllResult = Tuple[
     List[CllColumnDep],  # Model to column dependencies
@@ -19,7 +18,7 @@ CllResult = Tuple[
 
 
 @dataclass
-class CLLPerformanceTracking(metaclass=SingletonMeta):
+class CLLPerformanceTracking:
     lineage_start = None
     lineage_elapsed = None
     column_lineage_start = None

recce/util/cloud/__init__.py

@@ -0,0 +1,15 @@
+"""
+Recce Cloud API client modules.
+
+This package provides modular access to Recce Cloud API endpoints.
+"""
+
+from recce.util.cloud.base import CloudBase
+from recce.util.cloud.check_events import CheckEventsCloud
+from recce.util.cloud.checks import ChecksCloud
+
+__all__ = [
+    "CloudBase",
+    "CheckEventsCloud",
+    "ChecksCloud",
+]

recce/util/cloud/base.py

@@ -0,0 +1,115 @@
+"""
+Base class for Recce Cloud API clients.
+
+This module provides the common functionality shared across all cloud API clients.
+"""
+
+import os
+from typing import Dict, Optional
+
+import requests
+
+from recce.util.recce_cloud import (
+    DOCKER_INTERNAL_URL_PREFIX,
+    LOCALHOST_URL_PREFIX,
+    RECCE_CLOUD_API_HOST,
+    RecceCloudException,
+)
+
+
+class CloudBase:
+    """
+    Base class for Recce Cloud API operations.
+
+    Provides common functionality for making authenticated requests to the Recce Cloud API,
+    including request handling, error management, and Docker environment URL conversion.
+
+    Attributes:
+        token: Authentication token (API token or GitHub token)
+        token_type: Type of token ("api_token" or "github_token")
+        base_url: Base URL for API v1 endpoints
+        base_url_v2: Base URL for API v2 endpoints
+    """
+
+    def __init__(self, token: str):
+        """
+        Initialize the CloudBase client.
+
+        Args:
+            token: Authentication token for Recce Cloud API
+
+        Raises:
+            ValueError: If token is None
+        """
+        if token is None:
+            raise ValueError("Token cannot be None.")
+
+        self.token = token
+        self.token_type = "github_token" if token.startswith(("ghp_", "gho_", "ghu_", "ghs_", "ghr_")) else "api_token"
+        self.base_url = f"{RECCE_CLOUD_API_HOST}/api/v1"
+        self.base_url_v2 = f"{RECCE_CLOUD_API_HOST}/api/v2"
+
+    def _request(self, method: str, url: str, headers: Optional[Dict] = None, **kwargs):
+        """
+        Make an authenticated HTTP request to Recce Cloud API.
+
+        Args:
+            method: HTTP method (GET, POST, PATCH, DELETE, etc.)
+            url: Full URL for the request
+            headers: Optional additional headers
+            **kwargs: Additional arguments passed to requests.request
+
+        Returns:
+            Response object from requests library
+        """
+        headers = {
+            **(headers or {}),
+            "Authorization": f"Bearer {self.token}",
+        }
+        url = self._replace_localhost_with_docker_internal(url)
+        return requests.request(method, url, headers=headers, **kwargs)
+
+    @staticmethod
+    def _replace_localhost_with_docker_internal(url: str) -> Optional[str]:
+        """
+        Convert localhost URLs to docker internal URLs if running in Docker.
+
+        This is useful for local development when Recce is running inside a Docker container
+        and needs to access localhost services on the host machine.
+
+        Args:
+            url: URL that might contain localhost
+
+        Returns:
+            URL with localhost replaced by host.docker.internal if in Docker, otherwise original URL
+        """
+        if url is None:
+            return None
+
+        if (
+            os.environ.get("RECCE_SHARE_INSTANCE_ENV") == "docker"
+            or os.environ.get("RECCE_TASK_INSTANCE_ENV") == "docker"
+            or os.environ.get("RECCE_INSTANCE_ENV") == "docker"
+        ):
+            if url.startswith(LOCALHOST_URL_PREFIX):
+                return url.replace(LOCALHOST_URL_PREFIX, DOCKER_INTERNAL_URL_PREFIX)
+
+        return url
+
+    def _raise_for_status(self, response, message: str):
+        """
+        Raise RecceCloudException if the response status is not successful.
+
+        Args:
+            response: Response object from requests
+            message: Error message to include in the exception
+
+        Raises:
+            RecceCloudException: If response status code is not 2xx
+        """
+        if not response.ok:
+            raise RecceCloudException(
+                message=message,
+                reason=response.text,
+                status_code=response.status_code,
+            )