recce-nightly 1.9.0.20250623__py3-none-any.whl → 1.25.0.20251112a2066__py3-none-any.whl

recce/state/local.py

@@ -0,0 +1,56 @@
+import logging
+import os
+from typing import Optional, Tuple, Union
+
+from .state import RecceState
+from .state_loader import RecceStateLoader
+
+logger = logging.getLogger("uvicorn")
+
+
+class FileStateLoader(RecceStateLoader):
+    def __init__(
+        self,
+        review_mode: bool = False,
+        state_file: Optional[str] = None,
+        initial_state: Optional[RecceState] = None,
+    ):
+        super().__init__(review_mode=review_mode, state_file=state_file, initial_state=initial_state)
+
+    def verify(self) -> bool:
+        if self.review_mode is True and self.state_file is None:
+            self.error_message = "Recce can not launch without a state file."
+            self.hint_message = "Please provide a state file in the command argument."
+            return False
+        return True
+
+    def _load_state(self) -> Tuple[RecceState, str]:
+        state = RecceState.from_file(self.state_file) if self.state_file else None
+        state_tag = None
+        return state, state_tag
+
+    def _export_state(self, state: RecceState = None) -> Tuple[Union[str, None], str]:
+        """
+        Store the state to a file. Store happens when terminating the server or run instance.
+        """
+
+        if self.state_file is None:
+            return "No state file is provided. Skip storing the state.", None
+
+        logger.info(f"Store recce state to '{self.state_file}'")
+        message = self._export_state_to_file(self.state_file)
+        tag = None
+
+        return message, tag
+
+    def purge(self) -> bool:
+        if self.state_file is not None:
+            try:
+                os.remove(self.state_file)
+                return True
+            except Exception as e:
+                self.error_message = f"Failed to remove the state file: {e}"
+                return False
+        else:
+            self.error_message = "No state file is provided. Skip removing the state file."
+            return False

recce/state/state.py

@@ -0,0 +1,119 @@
+"""Define the type to serialize/de-serialize the state of the recce instance."""
+
+import json
+import logging
+from datetime import datetime
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+from recce import get_version
+from recce.exceptions import RecceException
+from recce.git import current_branch
+from recce.models.types import Check, Run
+from recce.pull_request import PullRequestInfo
+from recce.util.io import SupportedFileTypes, file_io_factory
+from recce.util.pydantic_model import pydantic_model_dump, pydantic_model_json_dump
+
+logger = logging.getLogger("uvicorn")
+
+
+class GitRepoInfo(BaseModel):
+    branch: Optional[str] = None
+
+    @staticmethod
+    def from_current_repository() -> Optional["GitRepoInfo"]:
+        branch = current_branch()
+        if branch is None:
+            return None
+
+        return GitRepoInfo(branch=branch)
+
+    def to_dict(self):
+        return pydantic_model_dump(self)
+
+
+class RecceStateMetadata(BaseModel):
+    schema_version: str = "v0"
+    recce_version: str = Field(default_factory=lambda: get_version())
+    generated_at: str = Field(default_factory=lambda: datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"))
+
+
+class ArtifactsRoot(BaseModel):
+    """
+    Root of the artifacts.
+
+    base: artifacts of the base env. key is file name, value is dict
+    current: artifacts of the current env. key is file name, value is dict
+    """
+
+    base: Dict[str, Optional[dict]] = {}
+    current: Dict[str, Optional[dict]] = {}
+
+
+class RecceState(BaseModel):
+    metadata: Optional[RecceStateMetadata] = None
+    runs: Optional[List[Run]] = Field(default_factory=list)
+    checks: Optional[List[Check]] = Field(default_factory=list)
+    artifacts: ArtifactsRoot = ArtifactsRoot(base={}, current={})
+    git: Optional[GitRepoInfo] = None
+    pull_request: Optional[PullRequestInfo] = None
+
+    @staticmethod
+    def from_json(json_content: str):
+        dict_data = json.loads(json_content)
+        state = RecceState(**dict_data)
+        metadata = state.metadata
+
+        if metadata:
+            if metadata.schema_version is None:
+                pass
+            if metadata.schema_version == "v0":
+                pass
+            else:
+                raise RecceException(f"Unsupported state file version: {metadata.schema_version}")
+        return state
+
+    @staticmethod
+    def from_file(file_path: str, file_type: SupportedFileTypes = SupportedFileTypes.FILE):
+        """
+        Load the state from a recce state file.
+        """
+        from pathlib import Path
+
+        logger.debug(f"Load state file from: '{file_path}'")
+        if not Path(file_path).is_file():
+            return None
+
+        io = file_io_factory(file_type)
+        json_content = io.read(file_path)
+        return RecceState.from_json(json_content)
+
+    def to_json(self):
+        return pydantic_model_json_dump(self)
+
+    def to_file(self, file_path: str, file_type: SupportedFileTypes = SupportedFileTypes.FILE):
+
+        json_data = self.to_json()
+        io = file_io_factory(file_type)
+
+        io.write(file_path, json_data)
+        return f"The state file is stored at '{file_path}'"
+
+    def _merge_run(self, run: Run):
+        for r in self.runs:
+            if r.run_id == run.run_id:
+                break
+        else:
+            self.runs.append(run)
+
+    def _merge_check(self, check: Check):
+        for c in self.checks:
+            if c.check_id == check.check_id:
+                c.merge(check)
+                break
+        else:
+            self.checks.append(check)
+
+    def _merge_artifacts(self, artifacts: ArtifactsRoot):
+        self.artifacts.merge(artifacts)

recce/state/state_loader.py

@@ -0,0 +1,174 @@
+import logging
+import threading
+import time
+from abc import ABC, abstractmethod
+from typing import Dict, Literal, Optional, Tuple, Union, final
+
+from recce.exceptions import RecceException
+from recce.pull_request import fetch_pr_metadata
+
+from ..util.io import SupportedFileTypes, file_io_factory
+from .const import RECCE_API_TOKEN_MISSING
+from .state import RecceState
+
+logger = logging.getLogger("uvicorn")
+
+
+class RecceStateLoader(ABC):
+    def __init__(
+        self,
+        review_mode: bool = False,
+        cloud_mode: bool = False,
+        state_file: Optional[str] = None,
+        cloud_options: Optional[Dict[str, str]] = None,
+        initial_state: Optional[RecceState] = None,
+    ):
+        self.review_mode = review_mode
+        self.cloud_mode = cloud_mode
+        self.state_file = state_file
+        self.cloud_options = cloud_options or {}
+        self.error_message = None
+        self.hint_message = None
+        self.state: RecceState | None = initial_state
+        self.state_lock = threading.Lock()
+        self.state_etag = None
+        self.pr_info = None
+        self.catalog: Literal["github", "preview", "snapshot"] = "github"
+        self.share_id = None
+        self.session_id = None
+
+        if self.cloud_mode:
+            if self.cloud_options.get("github_token"):
+                self.catalog = "github"
+                self.pr_info = fetch_pr_metadata(
+                    cloud=self.cloud_mode, github_token=self.cloud_options.get("github_token")
+                )
+                if self.pr_info.id is None:
+                    raise RecceException("Cannot get the pull request information from GitHub.")
+            elif self.cloud_options.get("api_token"):
+                if self.cloud_options.get("session_id"):
+                    self.catalog = "session"
+                    self.session_id = self.cloud_options.get("session_id")
+                else:
+                    self.catalog = "preview"
+                    self.share_id = self.cloud_options.get("share_id")
+            else:
+                raise RecceException(RECCE_API_TOKEN_MISSING.error_message)
+
+    @property
+    def token(self):
+        return self.cloud_options.get("github_token") or self.cloud_options.get("api_token")
+
+    @abstractmethod
+    def verify(self) -> bool:
+        """
+        Verify the state loader configuration.
+        Returns:
+            bool: True if the configuration is valid, False otherwise.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    @property
+    def error_and_hint(self) -> (Union[str, None], Union[str, None]):
+        return self.error_message, self.hint_message
+
+    def update(self, state: RecceState):
+        self.state = state
+
+    @final
+    def load(self, refresh=False) -> RecceState:
+        if self.state is not None and refresh is False:
+            return self.state
+        self.state_lock.acquire()
+        try:
+            self.state, self.state_etag = self._load_state()
+        finally:
+            self.state_lock.release()
+        return self.state
+
+    @abstractmethod
+    def _load_state(self) -> Tuple[RecceState, str]:
+        """
+        Load the state from the specified source (file or cloud).
+        Returns:
+            RecceState: The loaded state object.
+            str: The etag of the state file (if applicable).
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    def save_as(self, state_file: str, state: RecceState = None):
+        if self.cloud_mode:
+            raise Exception("Cannot save the state to Recce Cloud.")
+
+        self.state_file = state_file
+        self.export(state)
+
+    @final
+    def export(self, state: RecceState = None) -> Union[str, None]:
+        if state is not None:
+            self.update(state)
+
+        start_time = time.time()
+        self.state_lock.acquire()
+        try:
+            message, state_etag = self._export_state()
+            self.state_etag = state_etag
+            end_time = time.time()
+            elapsed_time = end_time - start_time
+        finally:
+            self.state_lock.release()
+        logger.info(f"Store state completed in {elapsed_time:.2f} seconds")
+        return message
+
+    @abstractmethod
+    def _export_state(self) -> Tuple[Union[str, None], str]:
+        """
+        Export the current Recce state to a file or cloud storage.
+        Returns:
+            str: A message indicating the result of the export operation.
+            str: The etag of the exported state file (if applicable).
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    def _export_state_to_file(self, file_path: str, file_type: SupportedFileTypes = SupportedFileTypes.FILE) -> str:
+        """
+        Store the state to a file. Store happens when terminating the server or run instance.
+        """
+
+        json_data = self.state.to_json()
+        io = file_io_factory(file_type)
+
+        io.write(file_path, json_data)
+        return f"The state file is stored at '{file_path}'"
+
+    def refresh(self):
+        new_state = self.load(refresh=True)
+        return new_state
+
+    def check_conflict(self) -> bool:
+        return False
+
+    def info(self) -> dict:
+        if self.state is None:
+            self.error_message = "No state is loaded."
+            return None
+
+        state_info = {
+            "mode": "cloud" if self.cloud_mode else "local",
+            "source": None,
+        }
+        if self.cloud_mode:
+            state_info["source"] = "Recce Cloud"
+            state_info["pull_request"] = self.pr_info
+        else:
+            state_info["source"] = self.state_file
+        return state_info
+
+    @abstractmethod
+    def purge(self) -> bool:
+        """
+        Purge the state file or cloud storage.
+        Returns:
+            bool: True if the purge was successful, False otherwise.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")

recce/summary.py

@@ -493,7 +493,8 @@ No changed module was detected.
 
         if ctx.state_loader.cloud_mode:
             pr_info = ctx.state_loader.pr_info
-            content += f"\nSee PR page: {RECCE_CLOUD_HOST}/{pr_info.repository}/pulls/{pr_info.id}\n"
+            # the classic route will be deprecated soon
+            content += f"\nSee PR page: {RECCE_CLOUD_HOST}/classic/{pr_info.repository}/pulls/{pr_info.id}\n"
 
         return content
 

recce/tasks/dataframe.py

@@ -2,6 +2,7 @@ import json
 import typing as t
 from decimal import Decimal
 from enum import Enum
+from typing import Sequence
 
 if t.TYPE_CHECKING:
     import agate
@@ -19,15 +20,32 @@ class DataFrameColumnType(Enum):
     TIMEDELTA = "timedelta"
     UNKNOWN = "unknown"
 
+    @classmethod
+    def from_string(cls, type_str: str) -> "DataFrameColumnType":
+        """Convert string to DataFrameColumnType enum.
+
+        Args:
+            type_str: String representation of the type (e.g., "integer", "text")
+
+        Returns:
+            DataFrameColumnType enum value
+        """
+        type_str = type_str.lower().strip()
+        try:
+            return cls(type_str)
+        except ValueError:
+            return cls.UNKNOWN
+
 
 class DataFrameColumn(BaseModel):
+    key: str
     name: str
     type: DataFrameColumnType
 
 
 class DataFrame(BaseModel):
     columns: t.List[DataFrameColumn]
-    data: t.List[tuple]
+    data: t.List[Sequence]
     limit: t.Optional[int] = Field(None, description="Limit the number of rows returned")
     more: t.Optional[bool] = Field(None, description="Whether there are more rows to fetch")
 
@@ -64,7 +82,7 @@ class DataFrame(BaseModel):
                 col_type = DataFrameColumnType.INTEGER
             else:
                 col_type = DataFrameColumnType.UNKNOWN
-            columns.append(DataFrameColumn(name=col_name, type=col_type))
+            columns.append(DataFrameColumn(key=col_name, name=col_name, type=col_type))
 
         def _row_values(row):
             # If the value is Decimal, check if it's finite. If not, convert it to float(xxx) (GitHub issue #476)
@@ -106,3 +124,42 @@ class DataFrame(BaseModel):
             more=more,
         )
         return df
+
+    @staticmethod
+    def from_data(
+        columns: t.Dict[str, str],
+        data: t.List[Sequence],
+        limit: t.Optional[int] = None,
+        more: t.Optional[bool] = None,
+    ):
+        """Create a DataFrame from columns and data directly.
+
+        Args:
+            columns: Dict defining the schema where keys are column names and values are type strings.
+                     Type strings can be: "number", "integer", "text", "boolean", "date", "datetime", "timedelta"
+            data: List of rows (each row is a list/tuple/sequence of values)
+            limit: Optional limit on the number of rows returned
+            more: Optional flag indicating whether there are more rows to fetch
+
+        Returns:
+            DataFrame instance
+
+        Examples:
+            # Using simple dict format
+            columns = {"idx": "integer", "name": "text", "impacted": "boolean"}
+            data = [[0, "model_a", True], [1, "model_b", False]]
+            df = DataFrame.from_data(columns, data)
+        """
+        # Convert dict columns to DataFrameColumn objects
+        processed_columns = []
+        for key, type_str in columns.items():
+            col_type = DataFrameColumnType.from_string(type_str)
+            processed_columns.append(DataFrameColumn(key=key, name=key, type=col_type))
+
+        df = DataFrame(
+            columns=processed_columns,
+            data=data,
+            limit=limit,
+            more=more,
+        )
+        return df

recce/tasks/rowcount.py

@@ -263,7 +263,10 @@ class RowCountDiffResultDiffer(TaskResultDiffer):
 
     def _get_changed_nodes(self) -> Union[List[str], None]:
         if self.changes:
-            return self.changes.affected_root_keys.items
+            # Both affected_root_keys of deepdiff v7 (OrderedSet) and v8 (SetOrdered) are iterable
+            # Convert to list directly
+            return list(self.changes.affected_root_keys)
+        return None
 
 
 class RowCountDiffCheckValidator(CheckValidator):

recce/tasks/schema.py

@@ -45,7 +45,10 @@ class SchemaDiffResultDiffer:
 
     def _get_changed_nodes(self) -> Union[List[str], None]:
         if self.changes:
-            return self.changes.affected_root_keys.items
+            # Both affected_root_keys of deepdiff v7 (OrderedSet) and v8 (SetOrdered) are iterable
+            # Convert to list directly
+            return list(self.changes.affected_root_keys)
+        return None
 
 
 class SchemaDiffParams(BaseModel):

recce/tasks/valuediff.py

@@ -159,7 +159,7 @@ class ValueDiffTask(Task, ValueDiffMixin):
                 match_status,
                 count(*) as count_records
             from joined
-            group by column_name, match_status
+            group by 1, 2
         )
 
         select

recce/util/api_token.py

@@ -17,7 +17,10 @@ def show_invalid_api_token_message():
     Show the message when the API token is invalid.
     """
     console.print("[[red]Error[/red]] Invalid Recce Cloud API token.")
-    console.print(f"Please check your API token from {RECCE_CLOUD_BASE_URL}/settings#tokens")
+    console.print("Please associate with your Recce Cloud account by the following command 'recce connect-to-cloud'.")
+    console.print(
+        "For more information, please visit: https://docs.reccehq.com/recce-cloud/share-recce-session-securely/#configure-recce-cloud-association-manually"
+    )
 
 
 def prepare_api_token(
@@ -30,7 +33,13 @@ def prepare_api_token(
     # Verify the API token for Recce Cloud Share Link
     api_token = get_recce_api_token()
     new_api_token = kwargs.get("api_token")
-    if api_token != new_api_token and new_api_token is not None:
+    if new_api_token is not None and new_api_token.startswith("rct-"):
+        # Task Token
+        valid = RecceCloud(new_api_token).verify_token()
+        if not valid:
+            raise RecceConfigException("Invalid Recce Cloud Task token")
+        api_token = new_api_token
+    elif api_token != new_api_token and new_api_token is not None:
         # Handle the API token provided by option `--api-token`
         valid = RecceCloud(new_api_token).verify_token()
         if not valid:

recce/util/breaking.py

@@ -78,6 +78,15 @@ def _diff_select_scope(old_scope: Scope, new_scope: Scope, scope_changes_map: di
             if change.category == "breaking":
                 change_category = "breaking"
 
+    # check if the upstream scopes sources table are the same
+    if len(old_scope.sources) != len(new_scope.sources):
+        change_category = "breaking"
+    else:
+        old_source_tables = [s.name for s in old_scope.sources.values() if isinstance(s, exp.Table)]
+        new_source_tables = [s.name for s in new_scope.sources.values() if isinstance(s, exp.Table)]
+        if sorted(old_source_tables) != sorted(new_source_tables):
+            change_category = "breaking"
+
     # check if non-select expressions are the same
     old_select = old_scope.expression  # type: exp.Select
     new_select = new_scope.expression  # type: exp.Select

recce/util/cll.py

@@ -10,7 +10,6 @@ from sqlglot.optimizer.qualify import qualify
 
 from recce.exceptions import RecceException
 from recce.models.types import CllColumn, CllColumnDep
-from recce.util import SingletonMeta
 
 CllResult = Tuple[
     List[CllColumnDep],  # Model to column dependencies
@@ -19,7 +18,7 @@ CllResult = Tuple[
 
 
 @dataclass
-class CLLPerformanceTracking(metaclass=SingletonMeta):
+class CLLPerformanceTracking:
     lineage_start = None
     lineage_elapsed = None
     column_lineage_start = None

recce/util/io.py

@@ -37,12 +37,12 @@ class AbstractFileIO(metaclass=ABCMeta):
 class FileIO(AbstractFileIO, ABC):
     @staticmethod
     def write(path: str, data: str, **kwargs):
-        with open(path, "w") as f:
+        with open(path, "w", encoding="utf-8") as f:
             f.write(data)
 
     @staticmethod
     def read(path: str, **kwargs) -> str:
-        with open(path, "r") as f:
+        with open(path, "r", encoding="utf-8") as f:
             return f.read()
 
 

recce/util/lineage.py

@@ -7,19 +7,17 @@ def find_upstream(node_ids: Iterable, parent_map):
     visited = set()
     upstream = set()
 
-    def dfs(current):
+    stack = list(node_ids)
+    while stack:
+        current = stack.pop()
         if current in visited:
-            return
+            continue
         visited.add(current)
-
         parents = parent_map.get(current, [])
         for parent in parents:
-            upstream.add(parent)
-            dfs(parent)
-
-    for node_id in node_ids:
-        dfs(node_id)
-
+            if parent not in upstream:
+                upstream.add(parent)
+                stack.append(parent)
     return upstream
 
 
@@ -27,19 +25,17 @@ def find_downstream(node_ids: Iterable, child_map):
     visited = set()
     downstream = set()
 
-    def dfs(current):
+    stack = list(node_ids)
+    while stack:
+        current = stack.pop()
         if current in visited:
-            return
+            continue
         visited.add(current)
-
         children = child_map.get(current, [])
         for child in children:
-            downstream.add(child)
-            dfs(child)
-
-    for node_id in node_ids:
-        dfs(node_id)
-
+            if child not in downstream:
+                downstream.add(child)
+                stack.append(child)
     return downstream
 
 
@@ -80,3 +76,8 @@ def filter_dependency_maps(
             c_map[node_id] = {c for c in children if c in relevant_ids}
 
     return p_map, c_map
+
+
+def build_column_key(node_id: str, column_name: str) -> str:
+    """Build a unique column key from node name and column name."""
+    return f"{node_id}_{column_name}"