recce-nightly 1.2.0.20250506__py3-none-any.whl → 1.26.0.20251124__py3-none-any.whl

recce/run.py

@@ -2,14 +2,18 @@ import os
 import sys
 import time
 from datetime import datetime, timezone
-from typing import List
+from typing import Dict, List, Tuple
 
 from deepdiff import DeepDiff
 from rich import box
 from rich.console import Console
 from rich.table import Table
 
-from recce.apis.check_func import create_check_from_run, create_check_without_run, purge_preset_checks
+from recce.apis.check_func import (
+    create_check_from_run,
+    create_check_without_run,
+    purge_preset_checks,
+)
 from recce.apis.run_func import submit_run
 from recce.config import RecceConfig
 from recce.core import default_context
@@ -19,13 +23,13 @@ from recce.summary import generate_markdown_summary
 
 def check_github_ci_env(**kwargs):
     """Check if the environment is GitHub CI"""
-    if 'GITHUB_ACTIONS' not in os.environ:
+    if "GITHUB_ACTIONS" not in os.environ:
         return False, None
 
     # Get the PR number
-    github_server_url = os.environ.get('GITHUB_SERVER_URL', 'https://github.com')
-    github_repository = os.environ.get('GITHUB_REPOSITORY', '')
-    github_ref_name = os.environ.get('GITHUB_REF_NAME', '')
+    github_server_url = os.environ.get("GITHUB_SERVER_URL", "https://github.com")
+    github_repository = os.environ.get("GITHUB_REPOSITORY", "")
+    github_ref_name = os.environ.get("GITHUB_REF_NAME", "")
     github_pull_request_url = f"{github_server_url}/{github_repository}/pull/{github_ref_name}"
 
     return True, github_pull_request_url
@@ -33,21 +37,22 @@ def check_github_ci_env(**kwargs):
 
 def load_preset_checks(checks: list):
     console = Console()
-    table = Table(title='Recce Preset Checks', box=box.HORIZONTALS, title_style='bold dark_orange3')
-    table.add_column('Name')
-    table.add_column('Type')
-    table.add_column('Description')
+    table = Table(title="Recce Preset Checks", box=box.HORIZONTALS, title_style="bold dark_orange3")
+    table.add_column("Name")
+    table.add_column("Type")
+    table.add_column("Description")
     for check in checks:
         is_check = False
-        name = check.get('name')
-        description = check.get('description', '')
-        check_type = check.get('type')
-        check_params = check.get('params', {})
-        check_options = check.get('view_options', {})
-
-        create_check_without_run(name, description, check_type, check_params, check_options, is_preset=True,
-                                 is_checked=is_check)
-        table.add_row(name, check_type.replace('_', ' ').title(), description.strip())
+        name = check.get("name")
+        description = check.get("description", "")
+        check_type = check.get("type")
+        check_params = check.get("params", {})
+        check_options = check.get("view_options", {})
+
+        create_check_without_run(
+            name, description, check_type, check_params, check_options, is_preset=True, is_checked=is_check
+        )
+        table.add_row(name, check_type.replace("_", " ").title(), description.strip())
     console.print(table)
 
 
@@ -55,28 +60,28 @@ def schema_diff_should_be_approved(check_params: dict) -> bool:
     try:
         context = default_context()
 
-        if 'node_id' in check_params:
+        if "node_id" in check_params:
             # If the node_id is provided, then use it
-            if isinstance(check_params['node_id'], str):
-                selected_node_ids = [check_params['node_id']]
+            if isinstance(check_params["node_id"], str):
+                selected_node_ids = [check_params["node_id"]]
             else:
-                selected_node_ids = check_params.get('node_id', [])
+                selected_node_ids = check_params.get("node_id", [])
         else:
             # Otherwise, select the nodes based on the select/exclude/packages/view_mode
             selected_node_ids = context.adapter.select_nodes(
-                select=check_params.get('select'),
-                exclude=check_params.get('exclude'),
-                packages=check_params.get('packages'),
-                view_mode=check_params.get('view_mode'),
+                select=check_params.get("select"),
+                exclude=check_params.get("exclude"),
+                packages=check_params.get("packages"),
+                view_mode=check_params.get("view_mode"),
             )
 
-        selected_node_ids = [node for node in selected_node_ids if not node.startswith('test.')]
+        selected_node_ids = [node for node in selected_node_ids if not node.startswith("test.")]
 
         def _get_selected_node_columns_from_lineage(lineage, node_ids: list[str]):
             nodes = {}
-            for node_id, node in lineage.get('nodes', {}).items():
+            for node_id, node in lineage.get("nodes", {}).items():
                 if node_id in node_ids:
-                    nodes[node_id] = node.get('columns', {})
+                    nodes[node_id] = node.get("columns", {})
             return nodes
 
         base_nodes = _get_selected_node_columns_from_lineage(context.get_lineage(base=True), selected_node_ids)
@@ -100,25 +105,25 @@ def run_should_be_approved(run):
             return False
         # If the row count are exactly the same, then the check should be approved
         for column, row_count_result in run.result.items():
-            if row_count_result['base'] != row_count_result['curr']:
+            if row_count_result["base"] != row_count_result["curr"]:
                 return False
         return True
     return False
 
 
-async def execute_preset_checks(preset_checks: list) -> (int, List[dict]):
+async def execute_preset_checks(preset_checks: List, is_skip_query: bool) -> Tuple[int, List[Dict]]:
     """
     Execute the preset checks
     """
     console = Console()
     rc = 0
     failed_checks = []
-    table = Table(title='Recce Preset Checks', box=box.HORIZONTALS, title_style='bold dark_orange3')
-    table.add_column('Status')
-    table.add_column('Name')
-    table.add_column('Type')
-    table.add_column('Execution Time')
-    table.add_column('Failed Reason')
+    table = Table(title="Recce Preset Checks", box=box.HORIZONTALS, title_style="bold dark_orange3")
+    table.add_column("Status")
+    table.add_column("Name")
+    table.add_column("Type")
+    table.add_column("Execution Time")
+    table.add_column("Failed Reason")
 
     # Purge the existing preset checks before running the new ones
     purge_preset_checks()
@@ -126,11 +131,11 @@ async def execute_preset_checks(preset_checks: list) -> (int, List[dict]):
     # Execute the preset checks
     for check in preset_checks:
         run = None
-        check_name = check.get('name')
-        check_type = check.get('type')
-        check_description = check.get('description', '')
-        check_params = check.get('params') if check.get('params') else {}
-        check_options = check.get('view_options', {})
+        check_name = check.get("name")
+        check_type = check.get("type")
+        check_description = check.get("description", "")
+        check_params = check.get("params") if check.get("params") else {}
+        check_options = check.get("view_options", {})
 
         try:
             # verify the check
@@ -138,59 +143,82 @@ async def execute_preset_checks(preset_checks: list) -> (int, List[dict]):
                 raise ValueError(f"Invalid check type: {check_type}")
 
             start = time.time()
-            if check_type in ['schema_diff', 'lineage_diff']:
-                is_check = schema_diff_should_be_approved(check_params) if check_type == 'schema_diff' else False
-                create_check_without_run(check_name, check_description, check_type, check_params, check_options,
-                                         is_preset=True, is_checked=is_check)
+            if check_type in ["schema_diff", "lineage_diff"]:
+                is_check = schema_diff_should_be_approved(check_params) if check_type == "schema_diff" else False
+                create_check_without_run(
+                    check_name,
+                    check_description,
+                    check_type,
+                    check_params,
+                    check_options,
+                    is_preset=True,
+                    is_checked=is_check,
+                )
             else:
-                run, future = submit_run(check_type, params=check_params)
-                await future
-                is_check = run_should_be_approved(run)
-                create_check_from_run(run.run_id, check_name, check_description, check_options, is_preset=True,
-                                      is_checked=is_check)
+                if not is_skip_query:
+                    run, future = submit_run(check_type, params=check_params)
+                    await future
+                    is_check = run_should_be_approved(run)
+                    create_check_from_run(
+                        run.run_id, check_name, check_description, check_options, is_preset=True, is_checked=is_check
+                    )
+                else:
+                    create_check_without_run(
+                        check_name, check_description, check_type, check_params, check_options, is_preset=True
+                    )
+                    continue
 
             end = time.time()
-            table.add_row('[[green]Success[/green]]', check_name, check_type.replace('_', ' ').title(),
-                          f'{end - start:.2f} seconds', 'N/A')
+            table.add_row(
+                "[[green]Success[/green]]",
+                check_name,
+                check_type.replace("_", " ").title(),
+                f"{end - start:.2f} seconds",
+                "N/A",
+            )
         except Exception as e:
             rc = 1
             if run is None:
-                table.add_row('[[red]Error[/red]]', check_name, check_type.replace('_', ' ').title(), 'N/A', str(e))
-                failed_checks.append({
-                    'check_name': check_name,
-                    'check_type': check_type,
-                    'check_description': check_description,
-                    'failed_type': 'error',
-                    'failed_reason': str(e)
-                })
+                table.add_row("[[red]Error[/red]]", check_name, check_type.replace("_", " ").title(), "N/A", str(e))
+                failed_checks.append(
+                    {
+                        "check_name": check_name,
+                        "check_type": check_type,
+                        "check_description": check_description,
+                        "failed_type": "error",
+                        "failed_reason": str(e),
+                    }
+                )
             else:
                 create_check_from_run(run.run_id, check_name, check_description, check_options, is_preset=True)
-                table.add_row('[[red]Failed[/red]]', check_name, check_type.replace('_', ' ').title(), 'N/A', run.error)
-                failed_checks.append({
-                    'check_name': check_name,
-                    'check_type': check_type,
-                    'check_description': 'N/A',
-                    'failed_type': 'failed',
-                    'failed_reason': run.error
-                })
+                table.add_row("[[red]Failed[/red]]", check_name, check_type.replace("_", " ").title(), "N/A", run.error)
+                failed_checks.append(
+                    {
+                        "check_name": check_name,
+                        "check_type": check_type,
+                        "check_description": "N/A",
+                        "failed_type": "failed",
+                        "failed_reason": run.error,
+                    }
+                )
 
     console.print(table)
     return rc, failed_checks
 
 
-async def execute_state_checks(checks: list) -> (int, List[dict]):
+async def execute_state_checks(checks: List, is_skip_query: bool) -> Tuple[int, List[Dict]]:
     """
     Execute the checks from loaded state
     """
     console = Console()
     rc = 0
     failed_checks = []
-    table = Table(title='Recce Checks', box=box.HORIZONTALS, title_style='bold dark_orange3')
-    table.add_column('Status')
-    table.add_column('Name')
-    table.add_column('Type')
-    table.add_column('Execution Time')
-    table.add_column('Failed Reason')
+    table = Table(title="Recce Checks", box=box.HORIZONTALS, title_style="bold dark_orange3")
+    table.add_column("Status")
+    table.add_column("Name")
+    table.add_column("Type")
+    table.add_column("Execution Time")
+    table.add_column("Failed Reason")
 
     # Execute loaded checks
     for check in checks:
@@ -210,33 +238,42 @@ async def execute_state_checks(checks: list) -> (int, List[dict]):
                 raise ValueError(f"Invalid check type: {check_type}")
 
             start = time.time()
-            if check_type not in ['schema_diff']:
+            if check_type not in ["schema_diff", "lineage_diff"] and not is_skip_query:
                 run, future = submit_run(check_type, params=check_params, check_id=check_id)
                 await future
 
             end = time.time()
-            table.add_row('[[green]Success[/green]]', check_name, check_type.replace('_', ' ').title(),
-                          f'{end - start:.2f} seconds', 'N/A')
+            table.add_row(
+                "[[green]Success[/green]]",
+                check_name,
+                check_type.replace("_", " ").title(),
+                f"{end - start:.2f} seconds",
+                "N/A",
+            )
         except Exception as e:
             rc = 1
             if run is None:
-                table.add_row('[[red]Error[/red]]', check_name, check_type.replace('_', ' ').title(), 'N/A', str(e))
-                failed_checks.append({
-                    'check_name': check_name,
-                    'check_type': check_type,
-                    'check_description': check_description,
-                    'failed_type': 'error',
-                    'failed_reason': str(e)
-                })
+                table.add_row("[[red]Error[/red]]", check_name, check_type.replace("_", " ").title(), "N/A", str(e))
+                failed_checks.append(
+                    {
+                        "check_name": check_name,
+                        "check_type": check_type,
+                        "check_description": check_description,
+                        "failed_type": "error",
+                        "failed_reason": str(e),
+                    }
+                )
             else:
-                table.add_row('[[red]Failed[/red]]', check_name, check_type.replace('_', ' ').title(), 'N/A', run.error)
-                failed_checks.append({
-                    'check_name': check_name,
-                    'check_type': check_type,
-                    'check_description': 'N/A',
-                    'failed_type': 'failed',
-                    'failed_reason': run.error
-                })
+                table.add_row("[[red]Failed[/red]]", check_name, check_type.replace("_", " ").title(), "N/A", run.error)
+                failed_checks.append(
+                    {
+                        "check_name": check_name,
+                        "check_type": check_type,
+                        "check_description": "N/A",
+                        "failed_type": "failed",
+                        "failed_reason": run.error,
+                    }
+                )
 
     console.print(table)
     return rc, failed_checks
@@ -244,24 +281,27 @@ async def execute_state_checks(checks: list) -> (int, List[dict]):
 
 def process_failed_checks(failed_checks: List[dict], error_log=None):
     from py_markdown_table.markdown_table import markdown_table
+
     failed_check_table = []
     for check in failed_checks:
-        name = check.get('check_name')
-        check_type = check.get('check_type')
-        failed_type = check.get('failed_type')
-        failed_reason = check.get('failed_reason')
-        failed_check_table.append({
-            'Name': name,
-            'Type': check_type,
-            'Kind of Failed': failed_type,
-            'Failed Reason': failed_reason.replace('\n', ' ')
-        })
-
-    content = '# Recce Runc Failed Checks\n'
-    content += markdown_table(failed_check_table).set_params(quote=False, row_sep='markdown').get_markdown()
+        name = check.get("check_name")
+        check_type = check.get("check_type")
+        failed_type = check.get("failed_type")
+        failed_reason = check.get("failed_reason")
+        failed_check_table.append(
+            {
+                "Name": name,
+                "Type": check_type,
+                "Kind of Failed": failed_type,
+                "Failed Reason": failed_reason.replace("\n", " "),
+            }
+        )
+
+    content = "# Recce Runc Failed Checks\n"
+    content += markdown_table(failed_check_table).set_params(quote=False, row_sep="markdown").get_markdown()
 
     if error_log:
-        with open(error_log, 'w') as f:
+        with open(error_log, "w", encoding="utf-8") as f:
             f.write(content)
         print(f"The failed checks are stored at '{error_log}'")
     else:
@@ -271,49 +311,53 @@ def process_failed_checks(failed_checks: List[dict], error_log=None):
 async def cli_run(output_state_file: str, **kwargs):
     """The main function of 'recce run' command. It will execute the default runs and store the state."""
     console = Console()
-    error_log = kwargs.get('error_log')
-    if kwargs.get('sqlmesh', False):
+    error_log = kwargs.get("error_log")
+    if kwargs.get("sqlmesh", False):
         console.print("[[red]Error[/red]] SQLMesh adapter is not supported.")
         sys.exit(1)
 
     from recce.core import load_context
+
     ctx = load_context(**kwargs)
 
-    is_skip_query = kwargs.get('skip_query', False)
+    is_skip_query = kwargs.get("skip_query", False)
+    is_skip_check = kwargs.get("skip_check", False)
 
     # Prepare the artifact by collecting the lineage
     console.rule("DBT Artifacts")
     from recce.adapter.dbt_adapter import DbtAdapter
+
     dbt_adaptor: DbtAdapter = ctx.adapter
     dbt_adaptor.print_lineage_info()
 
     # Execute the preset checks
     rc = 0
     if ctx.state_loader.state is None:
-        preset_checks = RecceConfig().get('checks')
-        if is_skip_query or preset_checks is None or len(preset_checks) == 0:
+        preset_checks = RecceConfig().get("checks")
+        if is_skip_check or preset_checks is None or len(preset_checks) == 0:
             # Skip the preset checks
             pass
         else:
             console.rule("Preset checks")
-            _, failed_checks = await execute_preset_checks(preset_checks)
+            _, failed_checks = await execute_preset_checks(preset_checks, is_skip_query)
             if failed_checks:
                 console.print("[[yellow]Warning[/yellow]] Preset checks failed. Please see the failed reason.")
                 process_failed_checks(failed_checks, error_log)
     else:
         state_checks = ctx.state_loader.state.checks
-        if is_skip_query or state_checks is None or len(state_checks) == 0:
+        if is_skip_check or state_checks is None or len(state_checks) == 0:
             # Skip the checks in the state
             pass
         else:
             console.rule("Checks")
-            _, failed_checks = await execute_state_checks(state_checks)
+            _, failed_checks = await execute_state_checks(state_checks, is_skip_query)
             if failed_checks:
                 console.print("[[yellow]Warning[/yellow]] Checks failed. Please see the failed reason.")
                 process_failed_checks(failed_checks, error_log)
 
     from recce.event import log_load_state
-    log_load_state(command='run')
+
+    log_load_state(command="run")
 
     # Export the state
     console.rule("Export state")
@@ -321,12 +365,12 @@ async def cli_run(output_state_file: str, **kwargs):
     msg = ctx.state_loader.export(ctx.export_state())
     console.print(msg)
 
-    summary_path = kwargs.get('summary')
+    summary_path = kwargs.get("summary")
     if summary_path:
         dirs = os.path.dirname(summary_path)
         if dirs:
             os.makedirs(dirs, exist_ok=True)
-        with open(summary_path, 'w') as f:
+        with open(summary_path, "w", encoding="utf-8") as f:
             f.write(generate_markdown_summary(ctx))
         console.print(f"The summary is stored at '{summary_path}'")