recce-nightly 0.62.0.20250417__py3-none-any.whl → 1.30.0.20251221__py3-none-any.whl

recce/models/types.py

@@ -1,26 +1,28 @@
 import uuid
 from datetime import datetime, timezone
 from enum import Enum
-from typing import Optional, Literal
+from typing import Dict, List, Literal, Optional, Set
 
-from pydantic import BaseModel, UUID4, Field
+from pydantic import UUID4, BaseModel, Field
+
+from recce.util.pydantic_model import pydantic_model_dump
 
 
 class RunType(Enum):
-    SIMPLE = 'simple'
+    SIMPLE = "simple"
     QUERY = "query"
-    QUERY_BASE = 'query_base'
-    QUERY_DIFF = 'query_diff'
-    VALUE_DIFF = 'value_diff'
-    VALUE_DIFF_DETAIL = 'value_diff_detail'
-    SCHEMA_DIFF = 'schema_diff'
-    PROFILE = 'profile'
-    PROFILE_DIFF = 'profile_diff'
-    ROW_COUNT = 'row_count'
-    ROW_COUNT_DIFF = 'row_count_diff'
-    LINEAGE_DIFF = 'lineage_diff'
-    TOP_K_DIFF = 'top_k_diff'
-    HISTOGRAM_DIFF = 'histogram_diff'
+    QUERY_BASE = "query_base"
+    QUERY_DIFF = "query_diff"
+    VALUE_DIFF = "value_diff"
+    VALUE_DIFF_DETAIL = "value_diff_detail"
+    SCHEMA_DIFF = "schema_diff"
+    PROFILE = "profile"
+    PROFILE_DIFF = "profile_diff"
+    ROW_COUNT = "row_count"
+    ROW_COUNT_DIFF = "row_count_diff"
+    LINEAGE_DIFF = "lineage_diff"
+    TOP_K_DIFF = "top_k_diff"
+    HISTOGRAM_DIFF = "histogram_diff"
 
     def __str__(self):
         return self.value
@@ -32,12 +34,10 @@ class RunProgress(BaseModel):
 
 
 class RunStatus(Enum):
-    FINISHED = 'finished'
-    FAILED = 'failed'
-    CANCELLED = 'cancelled'
-    RUNNING = 'running'
-    # This is a special status only in v0.36.0. Replaced by FINISHED. To be removed in the future.
-    SUCCESSFUL = 'successful'
+    FINISHED = "finished"
+    FAILED = "failed"
+    CANCELLED = "cancelled"
+    RUNNING = "running"
 
 
 class Run(BaseModel):
@@ -52,6 +52,39 @@ class Run(BaseModel):
     run_id: UUID4 = Field(default_factory=uuid.uuid4)
     run_at: str = Field(default_factory=lambda: datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"))
 
+    def __init__(self, **data):
+        type = data.get("type")
+
+        if "result" in data and data["result"] is not None:
+            result = data.get("result")
+
+            if type in [RunType.QUERY.value, RunType.QUERY_BASE.value]:
+                from recce.tasks.query import QueryResult
+
+                data["result"] = pydantic_model_dump(QueryResult(**result))
+            elif type == RunType.QUERY_DIFF.value:
+                from recce.tasks.query import QueryDiffResult
+
+                data["result"] = pydantic_model_dump(QueryDiffResult(**result))
+            elif type == RunType.PROFILE.value:
+                from recce.tasks.profile import ProfileResult
+
+                data["result"] = pydantic_model_dump(ProfileResult(**result))
+            elif type == RunType.PROFILE_DIFF.value:
+                from recce.tasks.profile import ProfileDiffResult
+
+                data["result"] = pydantic_model_dump(ProfileDiffResult(**result))
+            elif type == RunType.VALUE_DIFF.value:
+                from recce.tasks.valuediff import ValueDiffResult
+
+                data["result"] = pydantic_model_dump(ValueDiffResult(**result))
+            elif type == RunType.VALUE_DIFF_DETAIL.value:
+                from recce.tasks.valuediff import ValueDiffDetailResult
+
+                data["result"] = pydantic_model_dump(ValueDiffDetailResult(**result))
+
+        super().__init__(**data)
+
 
 class Check(BaseModel):
     name: str
@@ -60,8 +93,11 @@ class Check(BaseModel):
     params: Optional[dict] = {}
     view_options: Optional[dict] = {}
     check_id: UUID4 = Field(default_factory=uuid.uuid4)
+    session_id: Optional[UUID4] = Field(default=None)
     is_checked: bool = False
     is_preset: bool = False
+    created_by: Optional[str] = None
+    updated_by: Optional[str] = None
     created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc).replace(microsecond=0))
     updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc).replace(microsecond=0))
 
@@ -84,15 +120,15 @@ class Check(BaseModel):
 
 
 ChangeStatus = Literal[
-    'added',
-    'removed',
-    'modified',
+    "added",
+    "removed",
+    "modified",
 ]
 ChangeCategory = Literal[
-    'breaking',
-    'non_breaking',
-    'partial_breaking',
-    'unknown',
+    "breaking",
+    "non_breaking",
+    "partial_breaking",
+    "unknown",
 ]
 
 
@@ -110,3 +146,73 @@ class LineageDiff(BaseModel):
     base: dict
     current: dict
     diff: dict[str, NodeDiff]
+
+
+# Column Level Linage
+class CllColumnDep(BaseModel):
+    node: str
+    column: str
+
+
+class CllColumn(BaseModel):
+    id: Optional[str] = None
+    table_id: Optional[str] = None
+    name: Optional[str] = None
+
+    # data type
+    type: Optional[str] = None
+
+    # transformation type
+    transformation_type: Literal["source", "passthrough", "renamed", "derived", "unknown"] = "unknown"
+
+    # change analysis
+    change_status: Optional[ChangeStatus] = None
+
+    # column-to-column dependencies
+    depends_on: List[CllColumnDep] = Field(default_factory=list)
+
+
+class CllNode(BaseModel):
+    id: str
+    name: str
+    package_name: str
+    resource_type: str
+    raw_code: Optional[str] = None
+    source_name: Optional[str] = None
+
+    # change analysis
+    change_status: Optional[ChangeStatus] = None
+    change_category: Optional[ChangeCategory] = None
+
+    # Column to column dependencies
+    columns: Dict[str, CllColumn] = Field(default_factory=dict)
+
+    # If the node is impacted. Only used if option 'change_analysis' is set
+    impacted: Optional[bool] = None
+
+    @classmethod
+    def build_cll_node(cls, manifest, resource_key, node_id) -> Optional["CllNode"]:
+        resources = getattr(manifest, resource_key)
+        if node_id not in resources:
+            return None
+        n = resources[node_id]
+        if resource_key == "nodes" and n.resource_type not in ["model", "seed", "snapshot"]:
+            return None
+        cll_node = CllNode(
+            id=n.unique_id,
+            name=n.name,
+            package_name=n.package_name,
+            resource_type=n.resource_type,
+        )
+        if resource_key == "sources":
+            cll_node.source_name = n.source_name
+        elif resource_key == "nodes":
+            cll_node.raw_code = n.raw_code
+        return cll_node
+
+
+class CllData(BaseModel):
+    nodes: Dict[str, CllNode] = Field(default_factory=dict)
+    columns: Dict[str, CllColumn] = Field(default_factory=dict)
+    parent_map: Dict[str, Set[str]] = Field(default_factory=dict)
+    child_map: Dict[str, Set[str]] = Field(default_factory=dict)

recce/pull_request.py

@@ -28,14 +28,14 @@ def fetch_pr_metadata(**kwargs):
     # fetch from github action event path
     metadata = fetch_pr_metadata_from_event_path()
     if metadata is not None:
-        pr_info.id = metadata.get('github_pr_id')
-        pr_info.url = metadata.get('github_pr_url')
-        pr_info.title = metadata.get('github_pr_title')
-        pr_info.repository = metadata.get('github_repository')
+        pr_info.id = metadata.get("github_pr_id")
+        pr_info.url = metadata.get("github_pr_url")
+        pr_info.title = metadata.get("github_pr_title")
+        pr_info.repository = metadata.get("github_repository")
     else:
         repo = hosting_repo()
-        pr, message = recce_pr_information(github_token=kwargs.get('github_token'))
-        if kwargs.get('cloud') and message:
+        pr, message = recce_pr_information(github_token=kwargs.get("github_token"))
+        if kwargs.get("cloud") and message:
             print(message)
 
         if pr:
@@ -47,14 +47,14 @@ def fetch_pr_metadata(**kwargs):
             pr_info.branch = pr.head.ref
 
     # fetch from cli arguments
-    if pr_info.url is None and 'github_pull_request_url' in kwargs:
-        pr_info.url = kwargs.get('github_pull_request_url')
+    if pr_info.url is None and "github_pull_request_url" in kwargs:
+        pr_info.url = kwargs.get("github_pull_request_url")
 
     if pr_info.branch is None:
-        pr_info.branch = kwargs.get('git_current_branch')
+        pr_info.branch = kwargs.get("git_current_branch")
 
     if pr_info.base_branch is None:
-        pr_info.base_branch = kwargs.get('git_base_branch')
+        pr_info.base_branch = kwargs.get("git_base_branch")
 
     # fetch from env
     if pr_info.url is None:
@@ -65,17 +65,17 @@ def fetch_pr_metadata(**kwargs):
 
 def fetch_pr_metadata_from_event_path() -> Optional[dict]:
     """
-        If recce run is running in a GitHub Action, this function will return the pull request metadata.
+    If recce run is running in a GitHub Action, this function will return the pull request metadata.
 
-        Example:
-        {
-            "github_pr_id": 1,
-            "github_pr_url": "https://github.com/xyz/abc/pull/1,
-            "github_pr_title": "Update README.md",
-            "github_repository": "xyz/abc"
-        }
+    Example:
+    {
+        "github_pr_id": 1,
+        "github_pr_url": "https://github.com/xyz/abc/pull/1,
+        "github_pr_title": "Update README.md",
+        "github_repository": "xyz/abc"
+    }
 
-        :return: dict
+    :return: dict
     """
 
     # get the event json from the path in GITHUB_EVENT_PATH
@@ -83,7 +83,7 @@ def fetch_pr_metadata_from_event_path() -> Optional[dict]:
     github_repository = os.getenv("GITHUB_REPOSITORY")
     if event_path:
         try:
-            with open(event_path, "r") as event_file:
+            with open(event_path, "r", encoding="utf-8") as event_file:
                 event_data = json.load(event_file)
 
             pr_id = event_data["number"]
@@ -92,10 +92,12 @@ def fetch_pr_metadata_from_event_path() -> Optional[dict]:
                 pr_url = pull_request_data["_links"]["html"]["href"]
                 pr_api = pull_request_data["_links"]["self"]["href"]
                 pr_title = _fetch_pr_title(pr_api)
-                return dict(github_pr_id=pr_id,
-                            github_pr_url=pr_url,
-                            github_pr_title=pr_title,
-                            github_repository=github_repository)
+                return dict(
+                    github_pr_id=pr_id,
+                    github_pr_url=pr_url,
+                    github_pr_title=pr_title,
+                    github_repository=github_repository,
+                )
             else:
                 print("Not a pull request event, skip.")
         except Exception as e:
@@ -115,7 +117,7 @@ def _fetch_pr_title(endpoint) -> Optional[str]:
 
         if response.status_code == 200:
             pull_request_data = response.json()
-            return pull_request_data.get('title')
+            return pull_request_data.get("title")
     except Exception as e:
         print("Cannot fetch PR title: ", e)