recce-nightly 1.10.0.20250625__py3-none-any.whl → 1.30.0.20251221__py3-none-any.whl

recce/state/local.py

@@ -0,0 +1,56 @@
+import logging
+import os
+from typing import Optional, Tuple, Union
+
+from .state import RecceState
+from .state_loader import RecceStateLoader
+
+logger = logging.getLogger("uvicorn")
+
+
+class FileStateLoader(RecceStateLoader):
+    def __init__(
+        self,
+        review_mode: bool = False,
+        state_file: Optional[str] = None,
+        initial_state: Optional[RecceState] = None,
+    ):
+        super().__init__(review_mode=review_mode, state_file=state_file, initial_state=initial_state)
+
+    def verify(self) -> bool:
+        if self.review_mode is True and self.state_file is None:
+            self.error_message = "Recce can not launch without a state file."
+            self.hint_message = "Please provide a state file in the command argument."
+            return False
+        return True
+
+    def _load_state(self) -> Tuple[RecceState, str]:
+        state = RecceState.from_file(self.state_file) if self.state_file else None
+        state_tag = None
+        return state, state_tag
+
+    def _export_state(self, state: RecceState = None) -> Tuple[Union[str, None], str]:
+        """
+        Store the state to a file. Store happens when terminating the server or run instance.
+        """
+
+        if self.state_file is None:
+            return "No state file is provided. Skip storing the state.", None
+
+        logger.info(f"Store recce state to '{self.state_file}'")
+        message = self._export_state_to_file(self.state_file)
+        tag = None
+
+        return message, tag
+
+    def purge(self) -> bool:
+        if self.state_file is not None:
+            try:
+                os.remove(self.state_file)
+                return True
+            except Exception as e:
+                self.error_message = f"Failed to remove the state file: {e}"
+                return False
+        else:
+            self.error_message = "No state file is provided. Skip removing the state file."
+            return False

recce/state/state.py

@@ -0,0 +1,119 @@
+"""Define the type to serialize/de-serialize the state of the recce instance."""
+
+import json
+import logging
+from datetime import datetime
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+from recce import get_version
+from recce.exceptions import RecceException
+from recce.git import current_branch
+from recce.models.types import Check, Run
+from recce.pull_request import PullRequestInfo
+from recce.util.io import SupportedFileTypes, file_io_factory
+from recce.util.pydantic_model import pydantic_model_dump, pydantic_model_json_dump
+
+logger = logging.getLogger("uvicorn")
+
+
+class GitRepoInfo(BaseModel):
+    branch: Optional[str] = None
+
+    @staticmethod
+    def from_current_repository() -> Optional["GitRepoInfo"]:
+        branch = current_branch()
+        if branch is None:
+            return None
+
+        return GitRepoInfo(branch=branch)
+
+    def to_dict(self):
+        return pydantic_model_dump(self)
+
+
+class RecceStateMetadata(BaseModel):
+    schema_version: str = "v0"
+    recce_version: str = Field(default_factory=lambda: get_version())
+    generated_at: str = Field(default_factory=lambda: datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"))
+
+
+class ArtifactsRoot(BaseModel):
+    """
+    Root of the artifacts.
+
+    base: artifacts of the base env. key is file name, value is dict
+    current: artifacts of the current env. key is file name, value is dict
+    """
+
+    base: Dict[str, Optional[dict]] = {}
+    current: Dict[str, Optional[dict]] = {}
+
+
+class RecceState(BaseModel):
+    metadata: Optional[RecceStateMetadata] = None
+    runs: Optional[List[Run]] = Field(default_factory=list)
+    checks: Optional[List[Check]] = Field(default_factory=list)
+    artifacts: ArtifactsRoot = ArtifactsRoot(base={}, current={})
+    git: Optional[GitRepoInfo] = None
+    pull_request: Optional[PullRequestInfo] = None
+
+    @staticmethod
+    def from_json(json_content: str):
+        dict_data = json.loads(json_content)
+        state = RecceState(**dict_data)
+        metadata = state.metadata
+
+        if metadata:
+            if metadata.schema_version is None:
+                pass
+            if metadata.schema_version == "v0":
+                pass
+            else:
+                raise RecceException(f"Unsupported state file version: {metadata.schema_version}")
+        return state
+
+    @staticmethod
+    def from_file(file_path: str, file_type: SupportedFileTypes = SupportedFileTypes.FILE):
+        """
+        Load the state from a recce state file.
+        """
+        from pathlib import Path
+
+        logger.debug(f"Load state file from: '{file_path}'")
+        if not Path(file_path).is_file():
+            return None
+
+        io = file_io_factory(file_type)
+        json_content = io.read(file_path)
+        return RecceState.from_json(json_content)
+
+    def to_json(self):
+        return pydantic_model_json_dump(self)
+
+    def to_file(self, file_path: str, file_type: SupportedFileTypes = SupportedFileTypes.FILE):
+
+        json_data = self.to_json()
+        io = file_io_factory(file_type)
+
+        io.write(file_path, json_data)
+        return f"The state file is stored at '{file_path}'"
+
+    def _merge_run(self, run: Run):
+        for r in self.runs:
+            if r.run_id == run.run_id:
+                break
+        else:
+            self.runs.append(run)
+
+    def _merge_check(self, check: Check):
+        for c in self.checks:
+            if c.check_id == check.check_id:
+                c.merge(check)
+                break
+        else:
+            self.checks.append(check)
+
+    def _merge_artifacts(self, artifacts: ArtifactsRoot):
+        self.artifacts.merge(artifacts)

recce/state/state_loader.py

@@ -0,0 +1,174 @@
+import logging
+import threading
+import time
+from abc import ABC, abstractmethod
+from typing import Dict, Literal, Optional, Tuple, Union, final
+
+from recce.exceptions import RecceException
+from recce.pull_request import fetch_pr_metadata
+
+from ..util.io import SupportedFileTypes, file_io_factory
+from .const import RECCE_API_TOKEN_MISSING
+from .state import RecceState
+
+logger = logging.getLogger("uvicorn")
+
+
+class RecceStateLoader(ABC):
+    def __init__(
+        self,
+        review_mode: bool = False,
+        cloud_mode: bool = False,
+        state_file: Optional[str] = None,
+        cloud_options: Optional[Dict[str, str]] = None,
+        initial_state: Optional[RecceState] = None,
+    ):
+        self.review_mode = review_mode
+        self.cloud_mode = cloud_mode
+        self.state_file = state_file
+        self.cloud_options = cloud_options or {}
+        self.error_message = None
+        self.hint_message = None
+        self.state: RecceState | None = initial_state
+        self.state_lock = threading.Lock()
+        self.state_etag = None
+        self.pr_info = None
+        self.catalog: Literal["github", "preview", "session"] = "github"
+        self.share_id = None
+        self.session_id = None
+
+        if self.cloud_mode:
+            if self.cloud_options.get("github_token"):
+                self.catalog = "github"
+                self.pr_info = fetch_pr_metadata(
+                    cloud=self.cloud_mode, github_token=self.cloud_options.get("github_token")
+                )
+                if self.pr_info.id is None:
+                    raise RecceException("Cannot get the pull request information from GitHub.")
+            elif self.cloud_options.get("api_token"):
+                if self.cloud_options.get("session_id"):
+                    self.catalog = "session"
+                    self.session_id = self.cloud_options.get("session_id")
+                else:
+                    self.catalog = "preview"
+                    self.share_id = self.cloud_options.get("share_id")
+            else:
+                raise RecceException(RECCE_API_TOKEN_MISSING.error_message)
+
+    @property
+    def token(self):
+        return self.cloud_options.get("github_token") or self.cloud_options.get("api_token")
+
+    @abstractmethod
+    def verify(self) -> bool:
+        """
+        Verify the state loader configuration.
+        Returns:
+            bool: True if the configuration is valid, False otherwise.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    @property
+    def error_and_hint(self) -> (Union[str, None], Union[str, None]):
+        return self.error_message, self.hint_message
+
+    def update(self, state: RecceState):
+        self.state = state
+
+    @final
+    def load(self, refresh=False) -> RecceState:
+        if self.state is not None and refresh is False:
+            return self.state
+        self.state_lock.acquire()
+        try:
+            self.state, self.state_etag = self._load_state()
+        finally:
+            self.state_lock.release()
+        return self.state
+
+    @abstractmethod
+    def _load_state(self) -> Tuple[RecceState, str]:
+        """
+        Load the state from the specified source (file or cloud).
+        Returns:
+            RecceState: The loaded state object.
+            str: The etag of the state file (if applicable).
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    def save_as(self, state_file: str, state: RecceState = None):
+        if self.cloud_mode:
+            raise Exception("Cannot save the state to Recce Cloud.")
+
+        self.state_file = state_file
+        self.export(state)
+
+    @final
+    def export(self, state: RecceState = None) -> Union[str, None]:
+        if state is not None:
+            self.update(state)
+
+        start_time = time.time()
+        self.state_lock.acquire()
+        try:
+            message, state_etag = self._export_state()
+            self.state_etag = state_etag
+            end_time = time.time()
+            elapsed_time = end_time - start_time
+        finally:
+            self.state_lock.release()
+        logger.info(f"Store state completed in {elapsed_time:.2f} seconds")
+        return message
+
+    @abstractmethod
+    def _export_state(self) -> Tuple[Union[str, None], str]:
+        """
+        Export the current Recce state to a file or cloud storage.
+        Returns:
+            str: A message indicating the result of the export operation.
+            str: The etag of the exported state file (if applicable).
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    def _export_state_to_file(self, file_path: str, file_type: SupportedFileTypes = SupportedFileTypes.FILE) -> str:
+        """
+        Store the state to a file. Store happens when terminating the server or run instance.
+        """
+
+        json_data = self.state.to_json()
+        io = file_io_factory(file_type)
+
+        io.write(file_path, json_data)
+        return f"The state file is stored at '{file_path}'"
+
+    def refresh(self):
+        new_state = self.load(refresh=True)
+        return new_state
+
+    def check_conflict(self) -> bool:
+        return False
+
+    def info(self) -> dict:
+        if self.state is None:
+            self.error_message = "No state is loaded."
+            return None
+
+        state_info = {
+            "mode": "cloud" if self.cloud_mode else "local",
+            "source": None,
+        }
+        if self.cloud_mode:
+            state_info["source"] = "Recce Cloud"
+            state_info["pull_request"] = self.pr_info
+        else:
+            state_info["source"] = self.state_file
+        return state_info
+
+    @abstractmethod
+    def purge(self) -> bool:
+        """
+        Purge the state file or cloud storage.
+        Returns:
+            bool: True if the purge was successful, False otherwise.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")

recce/summary.py

@@ -271,11 +271,24 @@ class LineageGraph:
 def _build_lineage_graph(base, current) -> LineageGraph:
     graph = LineageGraph()
 
+    # Get the current package name to filter nodes (from the current manifest metadata)
+    package_name = None
+    manifest_metadata = current.get("manifest_metadata")
+    if manifest_metadata and hasattr(manifest_metadata, "project_name"):
+        # The default package name is the project name
+        package_name = manifest_metadata.project_name
+
     # Init Graph nodes with base & current nodes
     for node_id, node_data in base.get("nodes", {}).items():
+        # Skip nodes that are not from the current package
+        if package_name and node_data.get("package_name") != package_name:
+            continue
         graph.create_node(node_id, node_data, "base")
 
     for node_id, node_data in current.get("nodes", {}).items():
+        # Skip nodes that are not from the current package
+        if package_name and node_data.get("package_name") != package_name:
+            continue
         if node_id not in graph.nodes:
             node = Node(node_id, node_data, "current")
             graph.nodes[node_id] = node
@@ -286,9 +299,15 @@ def _build_lineage_graph(base, current) -> LineageGraph:
     # Build edges
     for child_id, parents in base.get("parent_map", {}).items():
         for parent_id in parents:
+            if child_id not in graph.nodes or parent_id not in graph.nodes:
+                continue
+
             graph.create_edge(parent_id, child_id, "base")
     for child_id, parents in current.get("parent_map", {}).items():
         for parent_id in parents:
+            if child_id not in graph.nodes or parent_id not in graph.nodes:
+                continue
+
             graph.create_edge(parent_id, child_id, "current")
 
     return graph
@@ -301,7 +320,8 @@ def _build_node_schema(lineage, node_id):
 def _get_node_row_count_diff(node_id, node_name):
     row_count_runs = RunDAO().list(type_filter=RunType.ROW_COUNT_DIFF)
     for run in row_count_runs:
-        if node_id in run.params.get("node_ids", []):
+        node_ids = (run.params or {}).get("node_ids") or []
+        if node_id in node_ids:
             result = run.result.get(node_name, {})
             diff = TaskResultDiffer.diff(result.get("base"), result.get("curr"))
             return diff, result
@@ -491,9 +511,11 @@ No changed module was detected.
         if check_content:
             content += check_content
 
-        if ctx.state_loader.cloud_mode:
+        if ctx.state_loader.cloud_mode and ctx.state_loader.pr_info is not None:
             pr_info = ctx.state_loader.pr_info
-            content += f"\nSee PR page: {RECCE_CLOUD_HOST}/{pr_info.repository}/pulls/{pr_info.id}\n"
+            if pr_info.repository is not None and pr_info.id is not None:
+                # the classic route will be deprecated soon
+                content += f"\nSee PR page: {RECCE_CLOUD_HOST}/classic/{pr_info.repository}/pulls/{pr_info.id}\n"
 
         return content
 

recce/tasks/dataframe.py

@@ -19,11 +19,34 @@ class DataFrameColumnType(Enum):
     TIMEDELTA = "timedelta"
     UNKNOWN = "unknown"
 
+    @classmethod
+    def from_string(cls, type_str: str) -> "DataFrameColumnType":
+        """Convert string to DataFrameColumnType enum.
+
+        Args:
+            type_str: String representation of the type (e.g., "integer", "text")
+
+        Returns:
+            DataFrameColumnType enum value
+        """
+        type_str = type_str.lower().strip()
+        try:
+            return cls(type_str)
+        except ValueError:
+            return cls.UNKNOWN
+
 
 class DataFrameColumn(BaseModel):
+    key: t.Optional[str] = None
     name: str
     type: DataFrameColumnType
 
+    def __init__(self, **data):
+        """Initialize DataFrameColumn, auto-setting key=name if key is missing."""
+        if "key" not in data or data["key"] is None:
+            data["key"] = data.get("name")
+        super().__init__(**data)
+
 
 class DataFrame(BaseModel):
     columns: t.List[DataFrameColumn]
@@ -64,7 +87,7 @@ class DataFrame(BaseModel):
                 col_type = DataFrameColumnType.INTEGER
             else:
                 col_type = DataFrameColumnType.UNKNOWN
-            columns.append(DataFrameColumn(name=col_name, type=col_type))
+            columns.append(DataFrameColumn(key=col_name, name=col_name, type=col_type))
 
         def _row_values(row):
             # If the value is Decimal, check if it's finite. If not, convert it to float(xxx) (GitHub issue #476)
@@ -106,3 +129,42 @@ class DataFrame(BaseModel):
             more=more,
         )
         return df
+
+    @staticmethod
+    def from_data(
+        columns: t.Dict[str, str],
+        data: t.List[tuple],
+        limit: t.Optional[int] = None,
+        more: t.Optional[bool] = None,
+    ):
+        """Create a DataFrame from columns and data directly.
+
+        Args:
+            columns: Dict defining the schema where keys are column names and values are type strings.
+                     Type strings can be: "number", "integer", "text", "boolean", "date", "datetime", "timedelta"
+            data: List of rows (each row is a list/tuple/sequence of values)
+            limit: Optional limit on the number of rows returned
+            more: Optional flag indicating whether there are more rows to fetch
+
+        Returns:
+            DataFrame instance
+
+        Examples:
+            # Using simple dict format
+            columns = {"idx": "integer", "name": "text", "impacted": "boolean"}
+            data = [[0, "model_a", True], [1, "model_b", False]]
+            df = DataFrame.from_data(columns, data)
+        """
+        # Convert dict columns to DataFrameColumn objects
+        processed_columns = []
+        for key, type_str in columns.items():
+            col_type = DataFrameColumnType.from_string(type_str)
+            processed_columns.append(DataFrameColumn(key=key, name=key, type=col_type))
+
+        df = DataFrame(
+            columns=processed_columns,
+            data=data,
+            limit=limit,
+            more=more,
+        )
+        return df

recce/tasks/query.py

@@ -8,6 +8,7 @@ from ..exceptions import RecceException
 from ..models import Check
 from .core import CheckValidator, Task, TaskResultDiffer
 from .dataframe import DataFrame
+from .utils import normalize_boolean_flag_columns, normalize_keys_to_columns
 from .valuediff import ValueDiffMixin
 
 QUERY_LIMIT = 2000
@@ -147,6 +148,10 @@ class QueryDiffTask(Task, QueryMixin, ValueDiffMixin):
         base_sql_template: Optional[str] = None,
         preview_change: bool = False,
     ):
+        """
+        Execute diff queries on base and current environments without join.
+        Note: Mutates self.params.primary_keys to normalize values with actual column keys.
+        """
         limit = QUERY_LIMIT
 
         self.connection = dbt_adapter.get_thread_connection()
@@ -159,9 +164,17 @@ class QueryDiffTask(Task, QueryMixin, ValueDiffMixin):
         current, current_more = self.execute_sql_with_limit(sql_template, base=False, limit=limit)
         self.check_cancel()
 
+        base_df = DataFrame.from_agate(base, limit=limit, more=base_more)
+        current_df = DataFrame.from_agate(current, limit=limit, more=current_more)
+
+        # Normalize primary_keys if present (for non-join diff, use current columns as reference)
+        if self.params.primary_keys:
+            column_keys = [col.key for col in current_df.columns]
+            self.params.primary_keys = normalize_keys_to_columns(self.params.primary_keys, column_keys)
+
         return QueryDiffResult(
-            base=DataFrame.from_agate(base, limit=limit, more=base_more),
-            current=DataFrame.from_agate(current, limit=limit, more=current_more),
+            base=base_df,
+            current=current_df,
         )
 
     def _query_diff_join(
@@ -172,6 +185,22 @@ class QueryDiffTask(Task, QueryMixin, ValueDiffMixin):
         base_sql_template: Optional[str] = None,
         preview_change: bool = False,
     ):
+        """
+        Execute diff queries on base and current environments using SQL join operations.
+        This method performs a set-based diff using INTERSECT and EXCEPT operations
+        to identify rows that differ between base and current query results.
+
+        Note: Mutates self.params.primary_keys to normalize values with actual column keys.
+
+        :param dbt_adapter: The dbt adapter instance for executing SQL
+        :param sql_template: SQL template to execute on the current environment
+        :param primary_keys: List of column names to use as primary keys for ordering
+        :param base_sql_template: Optional SQL template for the base environment.
+            If None, sql_template is used for both environments.
+        :param preview_change: If True, run base_sql_template against current environment
+            instead of base environment
+        :return: QueryDiffResult containing the diff DataFrame with in_a/in_b flags
+        """
 
         query_template = r"""
         with a_query as (
@@ -251,7 +280,15 @@ class QueryDiffTask(Task, QueryMixin, ValueDiffMixin):
         _, table = dbt_adapter.execute(sql, fetch=True)
         self.check_cancel()
 
-        return QueryDiffResult(diff=DataFrame.from_agate(table))
+        diff_df = DataFrame.from_agate(table)
+        # Normalize in_a/in_b columns to lowercase for cross-warehouse consistency
+        diff_df = normalize_boolean_flag_columns(diff_df)
+
+        # Normalize primary_keys to match actual column keys from warehouse
+        column_keys = [col.key for col in diff_df.columns]
+        self.params.primary_keys = normalize_keys_to_columns(primary_keys, column_keys)
+
+        return QueryDiffResult(diff=diff_df)
 
     @staticmethod
     def _select_single_model(model_name):

recce/tasks/rowcount.py

@@ -263,7 +263,10 @@ class RowCountDiffResultDiffer(TaskResultDiffer):
 
     def _get_changed_nodes(self) -> Union[List[str], None]:
         if self.changes:
-            return self.changes.affected_root_keys.items
+            # Both affected_root_keys of deepdiff v7 (OrderedSet) and v8 (SetOrdered) are iterable
+            # Convert to list directly
+            return list(self.changes.affected_root_keys)
+        return None
 
 
 class RowCountDiffCheckValidator(CheckValidator):

recce/tasks/schema.py

@@ -45,7 +45,10 @@ class SchemaDiffResultDiffer:
 
     def _get_changed_nodes(self) -> Union[List[str], None]:
         if self.changes:
-            return self.changes.affected_root_keys.items
+            # Both affected_root_keys of deepdiff v7 (OrderedSet) and v8 (SetOrdered) are iterable
+            # Convert to list directly
+            return list(self.changes.affected_root_keys)
+        return None
 
 
 class SchemaDiffParams(BaseModel):