recce-nightly 1.2.0.20250506__py3-none-any.whl → 1.26.0.20251124__py3-none-any.whl

recce/adapter/dbt_adapter/dbt_version.py

@@ -2,6 +2,7 @@ class DbtVersion:
 
     def __init__(self):
         from dbt import version as dbt_version
+
         dbt_version = self.parse(dbt_version.__version__)
         if dbt_version.is_prerelease:
             dbt_version = self.parse(dbt_version.base_version)
@@ -10,10 +11,12 @@ class DbtVersion:
     @staticmethod
     def parse(version: str):
         from packaging import version as v
+
         return v.parse(version)
 
     def as_version(self, other):
         from packaging.version import Version
+
         if isinstance(other, Version):
             return other
         if isinstance(other, str):

recce/adapter/sqlmesh_adapter.py

@@ -1,9 +1,9 @@
 import typing as t
 from dataclasses import dataclass
-from typing import Optional, Dict, Type
+from typing import Dict, Optional, Type
 
 import pandas as pd
-from sqlglot import parse_one, Expression, select
+from sqlglot import Expression, parse_one, select
 from sqlglot.optimizer import traverse_scope
 from sqlmesh.core.context import Context as SqlmeshContext
 from sqlmesh.core.environment import Environment
@@ -11,7 +11,7 @@ from sqlmesh.core.state_sync import StateReader
 
 from recce.adapter.base import BaseAdapter
 from recce.models import RunType
-from recce.tasks import Task, QueryTask, QueryDiffTask, RowCountDiffTask
+from recce.tasks import QueryDiffTask, QueryTask, RowCountDiffTask, Task
 
 sqlmesh_supported_registry: Dict[RunType, Type[Task]] = {
     RunType.QUERY: QueryTask,
@@ -37,7 +37,7 @@ class SqlmeshAdapter(BaseAdapter):
 
         for snapshot in state_reader.get_snapshots(env.snapshots, hydrate_seeds=True).values():
 
-            if snapshot.node_type.lower() != 'model':
+            if snapshot.node_type.lower() != "model":
                 continue
 
             model = snapshot.model
@@ -45,16 +45,16 @@ class SqlmeshAdapter(BaseAdapter):
                 continue
 
             node = {
-                'unique_id': model.name,
-                'name': model.name,
-                'resource_type': snapshot.node_type.lower(),
-                'checksum': {'checksum': snapshot.fingerprint.data_hash},
+                "unique_id": model.name,
+                "name": model.name,
+                "resource_type": snapshot.node_type.lower(),
+                "checksum": {"checksum": snapshot.fingerprint.data_hash},
             }
 
             columns = {}
             for column, type in model.columns_to_types.items():
-                columns[column] = {'name': column, 'type': str(type)}
-            node['columns'] = columns
+                columns[column] = {"name": column, "type": str(type)}
+            node["columns"] = columns
 
             nodes[snapshot.name] = node
             parents = [snapshotId.name for snapshotId in snapshot.parents]
@@ -81,15 +81,15 @@ class SqlmeshAdapter(BaseAdapter):
 
     @classmethod
     def load(cls, **kwargs):
-        sqlmesh_envs = kwargs.get('sqlmesh_envs')
+        sqlmesh_envs = kwargs.get("sqlmesh_envs")
         if sqlmesh_envs is None:
-            raise Exception('\'--sqlmesh-envs SOURCE:TARGET\' is required')
+            raise Exception("'--sqlmesh-envs SOURCE:TARGET' is required")
 
-        envs = sqlmesh_envs.split(':')
+        envs = sqlmesh_envs.split(":")
         if len(envs) != 2:
             raise Exception('sqlmesh_envs must be in the format of "SOURCE:TARGET"')
 
-        sqlmesh_config = kwargs.get('sqlmesh_config', None)
+        sqlmesh_config = kwargs.get("sqlmesh_config", None)
         context = SqlmeshContext(config=sqlmesh_config)
         base_env = context.state_reader.get_environment(envs[0])
         curr_env = context.state_reader.get_environment(envs[1])
@@ -100,18 +100,14 @@ class SqlmeshAdapter(BaseAdapter):
 
         return cls(context=context, base_env=base_env, curr_env=curr_env)
 
-    def replace_virtual_tables(
-        self,
-        sql: t.Union[Expression, str],
-        base: bool = None
-    ) -> Expression:
-        '''
+    def replace_virtual_tables(self, sql: t.Union[Expression, str], base: bool = None) -> Expression:
+        """
         Replace virtual tables based on the env name.
 
         Args:
             sql: SQL expression to replace virtual tables
             base: True: replace virtual tables with base env, False: replace virtual tables with current env, None: no replacement
-        '''
+        """
         if isinstance(sql, str):
             expression = parse_one(sql, dialect=self.context.default_dialect)
         else:
@@ -119,30 +115,23 @@ class SqlmeshAdapter(BaseAdapter):
 
         if base is not None:
             env = self.base_env if base else self.curr_env
-            if env.name != 'prod':
+            if env.name != "prod":
                 model_names = [model.name for model in self.context.models.values()]
                 for scope in traverse_scope(expression):
                     for table in scope.tables:
-                        if f'{table.db}.{table.name}' in model_names:
-                            table.args['db'] = f"{table.args['db']}__{env.name}"
+                        if f"{table.db}.{table.name}" in model_names:
+                            table.args["db"] = f"{table.args['db']}__{env.name}"
 
         return expression
 
     def fetchdf_with_limit(
-        self,
-        sql: t.Union[Expression, str],
-        base: Optional[bool] = None,
-        limit: Optional[int] = None
+        self, sql: t.Union[Expression, str], base: Optional[bool] = None, limit: Optional[int] = None
     ) -> (pd.DataFrame, bool):
         expression = self.replace_virtual_tables(sql, base=base)
         if limit:
-            expression = select(
-                '*'
-            ).from_(
-                '__QUERY'
-            ).with_(
-                '__QUERY', as_=expression
-            ).limit(limit + 1 if limit else None)
+            expression = (
+                select("*").from_("__QUERY").with_("__QUERY", as_=expression).limit(limit + 1 if limit else None)
+            )
         df = self.context.fetchdf(expression)
         if limit and len(df) > limit:
             df = df.head(limit)

recce/apis/check_api.py

@@ -2,23 +2,27 @@ from datetime import datetime, timezone
 from typing import Optional
 from uuid import UUID
 
-from fastapi import APIRouter, HTTPException, BackgroundTasks
+from fastapi import APIRouter, BackgroundTasks, HTTPException
 from pydantic import BaseModel
 
-from recce.apis.check_func import create_check_without_run, create_check_from_run, export_persistent_state
+from recce.apis.check_func import (
+    create_check_from_run,
+    create_check_without_run,
+    export_persistent_state,
+)
 from recce.apis.run_func import submit_run
 from recce.event import log_api_event
 from recce.exceptions import RecceException
-from recce.models import RunType, RunDAO, Check, CheckDAO, Run
+from recce.models import Check, CheckDAO, Run, RunDAO, RunType
 
-check_router = APIRouter(tags=['check'])
+check_router = APIRouter(tags=["check"])
 
 
 class CreateCheckIn(BaseModel):
     name: Optional[str] = None
-    description: str = ''
+    description: str = ""
     run_id: Optional[str] = None
-    type: Optional[RunType] = None,
+    type: Optional[RunType] = (None,)
     params: Optional[dict] = None
     view_options: Optional[dict] = None
     track_props: Optional[dict] = None
@@ -39,16 +43,17 @@ class CheckOut(BaseModel):
     def from_check(cls, check: Check):
         check_related_runs = RunDAO().list_by_check_id(check.check_id)
         last_run = check_related_runs[-1] if len(check_related_runs) > 0 else None
-        return CheckOut(check_id=check.check_id,
-                        name=check.name,
-                        description=check.description,
-                        type=check.type.value,
-                        params=check.params,
-                        view_options=check.view_options,
-                        is_checked=check.is_checked,
-                        is_preset=check.is_preset,
-                        last_run=last_run,
-                        )
+        return CheckOut(
+            check_id=check.check_id,
+            name=check.name,
+            description=check.description,
+            type=check.type.value,
+            params=check.params,
+            view_options=check.view_options,
+            is_checked=check.is_checked,
+            is_preset=check.is_preset,
+            last_run=last_run,
+        )
 
 
 @check_router.post("/checks", status_code=201, response_model=CheckOut)
@@ -59,7 +64,7 @@ async def create_check(check_in: CreateCheckIn, background_tasks: BackgroundTask
                 check_in.run_id,
                 check_name=check_in.name,
                 check_description=check_in.description,
-                check_view_options=check_in.view_options
+                check_view_options=check_in.view_options,
             )
         else:
             check = create_check_without_run(
@@ -67,12 +72,15 @@ async def create_check(check_in: CreateCheckIn, background_tasks: BackgroundTask
                 check_description=check_in.description,
                 check_type=check_in.type,
                 params=check_in.params,
-                check_view_options=check_in.view_options
+                check_view_options=check_in.view_options,
             )
-        log_api_event('create_check', dict(
-            type=str(check.type),
-            track_props=check_in.track_props,
-        ))
+        log_api_event(
+            "create_check",
+            dict(
+                type=str(check.type),
+                track_props=check_in.track_props,
+            ),
+        )
     except NameError as e:
         raise HTTPException(status_code=404, detail=str(e))
     except ValueError as e:
@@ -93,10 +101,13 @@ async def run_check_handler(check_id: UUID, input: RunCheckIn):
         raise HTTPException(status_code=404, detail=f"Check ID '{check_id}' not found")
 
     try:
-        log_api_event('rerun_check', dict(
-            type=str(check.type),
-            rerun=True,
-        ))
+        log_api_event(
+            "rerun_check",
+            dict(
+                type=str(check.type),
+                rerun=True,
+            ),
+        )
         run, future = submit_run(check.type, check.params, check_id=check_id)
     except RecceException as e:
         raise HTTPException(status_code=400, detail=str(e))
@@ -122,7 +133,7 @@ async def list_checks_handler():
 async def get_check_handler(check_id: UUID):
     check = CheckDAO().find_check_by_id(check_id)
     if check is None:
-        raise HTTPException(status_code=404, detail='Not Found')
+        raise HTTPException(status_code=404, detail="Not Found")
 
     runs = RunDAO().list_by_check_id(check_id)
     last_run = runs[-1] if len(runs) > 0 else None
@@ -144,7 +155,7 @@ class PatchCheckIn(BaseModel):
 async def update_check_handler(check_id: UUID, patch: PatchCheckIn, background_tasks: BackgroundTasks):
     check = CheckDAO().find_check_by_id(check_id)
     if check is None:
-        raise HTTPException(status_code=404, detail='Not Found')
+        raise HTTPException(status_code=404, detail="Not Found")
 
     if patch.name is not None:
         check.name = patch.name

recce/apis/check_func.py

@@ -5,13 +5,13 @@ from fastapi import HTTPException
 
 from recce.apis.run_func import generate_run_name
 from recce.core import default_context
-from recce.models import RunDAO, RunType, Check, CheckDAO
+from recce.models import Check, CheckDAO, RunDAO, RunType
 
 
 def validate_schema_diff_check(params):
-    node_id = params.get('node_id')
+    node_id = params.get("node_id")
     if node_id is None:
-        raise HTTPException(status_code=400, detail='node_id is required for schema diff')
+        raise HTTPException(status_code=400, detail="node_id is required for schema diff")
     node_name = default_context().get_node_name_by_id(node_id)
     if node_name is None:
         raise HTTPException(status_code=400, detail=f"node_id '{node_id}' not found in dbt manifest")
@@ -45,8 +45,8 @@ def _get_ref_model(sql_template: str) -> Optional[str]:
 def _generate_check_name(check_type, params, view_options):
     now = datetime.utcnow().strftime("%d %b %Y")
     if check_type == RunType.SCHEMA_DIFF:
-        if params.get('node_id'):
-            nodeIds = params.get('node_id') if isinstance(params.get('node_id'), list) else [params.get('node_id')]
+        if params.get("node_id"):
+            nodeIds = params.get("node_id") if isinstance(params.get("node_id"), list) else [params.get("node_id")]
             if len(nodeIds) == 1:
                 node_name = get_node_name_by_id(nodeIds[0])
                 return f"schema diff of {node_name}".capitalize()
@@ -54,7 +54,7 @@ def _generate_check_name(check_type, params, view_options):
                 return f"schema diff of {len(nodeIds)} nodes".capitalize()
         return f"{'schema diff'.capitalize()} - {now}"
     elif check_type == RunType.LINEAGE_DIFF:
-        nodes = view_options.get('node_ids') if view_options else params.get('node_ids')
+        nodes = view_options.get("node_ids") if view_options else params.get("node_ids")
         if nodes is not None:
             return f"lineage diff of {len(nodes)} nodes".capitalize()
         return f"{'lineage diff'.capitalize()} - {now}"
@@ -62,10 +62,11 @@ def _generate_check_name(check_type, params, view_options):
         return f"{'check'.capitalize()} - {now}"
 
 
-def create_check_from_run(run_id, check_name=None, check_description='', check_view_options=None, is_preset=False,
-                          is_checked=False):
+def create_check_from_run(
+    run_id, check_name=None, check_description="", check_view_options=None, is_preset=False, is_checked=False
+):
     if run_id is None:
-        raise ValueError('run_id is required')
+        raise ValueError("run_id is required")
 
     run = RunDAO().find_run_by_id(run_id)
     if run is None:
@@ -76,29 +77,34 @@ def create_check_from_run(run_id, check_name=None, check_description='', check_v
 
     _validate_check(run_type, run_params)
     name = check_name if check_name is not None else generate_run_name(run)
-    check = Check(name=name,
-                  description=check_description,
-                  type=run_type,
-                  params=run_params,
-                  view_options=check_view_options,
-                  is_preset=is_preset,
-                  is_checked=is_checked)
+    check = Check(
+        name=name,
+        description=check_description,
+        type=run_type,
+        params=run_params,
+        view_options=check_view_options,
+        is_preset=is_preset,
+        is_checked=is_checked,
+    )
     CheckDAO().create(check)
     run.check_id = check.check_id
 
     return check
 
 
-def create_check_without_run(check_name, check_description, check_type, params, check_view_options, is_preset=False,
-                             is_checked=False):
+def create_check_without_run(
+    check_name, check_description, check_type, params, check_view_options, is_preset=False, is_checked=False
+):
     name = check_name if check_name is not None else _generate_check_name(check_type, params, check_view_options)
-    check = Check(name=name,
-                  description=check_description,
-                  type=check_type,
-                  params=params,
-                  view_options=check_view_options,
-                  is_preset=is_preset,
-                  is_checked=is_checked)
+    check = Check(
+        name=name,
+        description=check_description,
+        type=check_type,
+        params=params,
+        view_options=check_view_options,
+        is_preset=is_preset,
+        is_checked=is_checked,
+    )
     CheckDAO().create(check)
     return check
 
@@ -119,6 +125,6 @@ def export_persistent_state():
     if state_loader is not None:
         is_conflict = state_loader.check_conflict()
         if is_conflict:
-            ctx.sync_state('merge')
+            ctx.sync_state("merge")
         else:
-            ctx.sync_state('overwrite')
+            ctx.sync_state("overwrite")

recce/apis/run_api.py

@@ -1,16 +1,16 @@
 import asyncio
-from typing import Optional, List
+from typing import List, Optional
 from uuid import UUID
 
 from fastapi import APIRouter, HTTPException, Query
 from pydantic import BaseModel
 
-from recce.apis.run_func import submit_run, cancel_run, materialize_run_results
+from recce.apis.run_func import cancel_run, materialize_run_results, submit_run
 from recce.event import log_api_event
 from recce.exceptions import RecceException
 from recce.models import RunDAO
 
-run_router = APIRouter(tags=['run'])
+run_router = APIRouter(tags=["run"])
 
 
 class CreateRunIn(BaseModel):
@@ -23,10 +23,13 @@ class CreateRunIn(BaseModel):
 
 @run_router.post("/runs", status_code=201)
 async def create_run_handler(input: CreateRunIn):
-    log_api_event('create_run', dict(
-        type=input.type,
-        track_props=input.track_props,
-    ))
+    log_api_event(
+        "create_run",
+        dict(
+            type=input.type,
+            track_props=input.track_props,
+        ),
+    )
     try:
         run, future = submit_run(input.type, input.params)
     except RecceException as e:
@@ -51,7 +54,7 @@ async def cancel_run_handler(run_id: UUID):
 async def wait_run_handler(run_id: UUID, timeout: int = Query(None, description="Maximum number of seconds to wait")):
     run = RunDAO().find_run_by_id(run_id)
     if run is None:
-        raise HTTPException(status_code=404, detail='Not Found')
+        raise HTTPException(status_code=404, detail="Not Found")
 
     start_time = asyncio.get_event_loop().time()
     while run.result is None and run.error is None:
@@ -65,18 +68,21 @@ async def wait_run_handler(run_id: UUID, timeout: int = Query(None, description=
 async def list_run_handler():
     runs = RunDAO().list() or []
 
-    result = [{
-        'run_id': run.run_id,
-        'run_at': run.run_at,
-        'name': run.name,
-        'type': run.type,
-        'params': run.params,
-        'status': run.status,
-        'check_id': run.check_id,
-    } for run in runs]
+    result = [
+        {
+            "run_id": run.run_id,
+            "run_at": run.run_at,
+            "name": run.name,
+            "type": run.type,
+            "params": run.params,
+            "status": run.status,
+            "check_id": run.check_id,
+        }
+        for run in runs
+    ]
 
     # sort by run_at
-    result = sorted(result, key=lambda x: x['run_at'], reverse=True)
+    result = sorted(result, key=lambda x: x["run_at"], reverse=True)
 
     return result
 
@@ -101,7 +107,7 @@ async def search_runs_handler(search: SearchRunsIn):
         result.append(run)
 
     if search.limit:
-        return result[-search.limit:]
+        return result[-search.limit :]
 
     return result
 

recce/apis/run_func.py

@@ -4,11 +4,11 @@ from typing import List, Optional
 
 from recce.core import default_context
 from recce.exceptions import RecceException
-from recce.models import RunType, Run, RunDAO
+from recce.models import Run, RunDAO, RunType
 from recce.models.types import RunStatus
 
 running_tasks = {}
-logger = logging.getLogger('uvicorn')
+logger = logging.getLogger("uvicorn")
 
 
 def _get_ref_model(sql_template: str) -> Optional[str]:
@@ -33,26 +33,26 @@ def generate_run_name(run):
     now = dateutil.parser.parse(run.run_at)
 
     if run_type == RunType.QUERY:
-        ref = _get_ref_model(params.get('sql_template'))
+        ref = _get_ref_model(params.get("sql_template"))
         if ref:
             return f"query of {ref}".capitalize()
         return f"{'query'.capitalize()} - {now}"
     elif run_type == RunType.QUERY_DIFF:
-        ref = _get_ref_model(params.get('sql_template'))
+        ref = _get_ref_model(params.get("sql_template"))
         if ref:
             return f"query diff of {ref}".capitalize()
         return f"{'query diff'.capitalize()} - {now}"
     elif run_type == RunType.VALUE_DIFF:
-        model = params.get('model')
+        model = params.get("model")
         return f"value diff of {model}".capitalize()
     elif run_type == RunType.VALUE_DIFF_DETAIL:
-        model = params.get('model')
+        model = params.get("model")
         return f"value diff detail of {model}".capitalize()
     elif run_type == RunType.PROFILE_DIFF:
-        model = params.get('model')
+        model = params.get("model")
         return f"profile diff of {model}".capitalize()
     elif run_type == RunType.ROW_COUNT_DIFF:
-        nodes = params.get('node_names')
+        nodes = params.get("node_names")
         if nodes:
             if len(nodes) == 1:
                 node = nodes[0]
@@ -62,23 +62,27 @@ def generate_run_name(run):
         else:
             return "row count of multiple nodes".capitalize()
     elif run_type == RunType.TOP_K_DIFF:
-        model = params.get('model')
-        column = params.get('column_name')
+        model = params.get("model")
+        column = params.get("column_name")
         return f"top-k diff of {model}.{column} ".capitalize()
     elif run_type == RunType.HISTOGRAM_DIFF:
-        model = params.get('model')
-        column = params.get('column_name')
+        model = params.get("model")
+        column = params.get("column_name")
         return f"histogram diff of {model}.{column} ".capitalize()
     else:
         return f"{'run'.capitalize()} - {now}"
 
 
 def create_task(run_type: RunType, params: dict):
-    if default_context().adapter_type == 'sqlmesh':
-        from recce.adapter.sqlmesh_adapter import sqlmesh_supported_registry as sqlmesh_registry
+    if default_context().adapter_type == "sqlmesh":
+        from recce.adapter.sqlmesh_adapter import (
+            sqlmesh_supported_registry as sqlmesh_registry,
+        )
+
         registry = sqlmesh_registry
     else:
         from recce.adapter.dbt_adapter import dbt_supported_registry as dbt_registry
+
         registry = dbt_registry
 
     taskClz = registry.get(run_type)
@@ -101,6 +105,7 @@ def submit_run(type, params, check_id=None):
     context = default_context()
     if context.review_mode is True:
         from recce.adapter.dbt_adapter import DbtAdapter
+
         dbt_adaptor: DbtAdapter = context.adapter
         if dbt_adaptor.adapter is None:
             raise RecceException("Recce Server is not launched under DBT project folder.")
@@ -113,7 +118,7 @@ def submit_run(type, params, check_id=None):
     running_tasks[run.run_id] = task
 
     def progress_listener(message=None, percentage=None):
-        run.progress = {'message': message, 'percentage': percentage}
+        run.progress = {"message": message, "percentage": percentage}
 
     task.progress_listener = progress_listener
 
@@ -124,7 +129,7 @@ def submit_run(type, params, check_id=None):
             run.result = result
             run.status = RunStatus.FINISHED
         if error is not None:
-            failed_reason = str(error) if str(error) != 'None' else repr(error)
+            failed_reason = str(error) if str(error) != "None" else repr(error)
             run.error = failed_reason
             if run.status != RunStatus.CANCELLED:
                 run.status = RunStatus.FAILED
@@ -140,9 +145,10 @@ def submit_run(type, params, check_id=None):
             if isinstance(e, RecceException) and e.is_raise is False:
                 return None
             import sentry_sdk
+
             sentry_sdk.capture_exception(e)
-            failed_reason = str(e) if str(e) != 'None' else repr(e)
-            failed_reason = failed_reason.replace('. ', ".\n")
+            failed_reason = str(e) if str(e) != "None" else repr(e)
+            failed_reason = failed_reason.replace(". ", ".\n")
             logger.error(f"Failed to execute {run_type} task: {failed_reason}")
             return None
 
@@ -164,7 +170,7 @@ def cancel_run(run_id):
 
 
 def materialize_run_results(runs: List[Run], nodes: List[str] = None):
-    '''
+    """
     Materialize the run results for nodes. It walks through all runs and get the last results for primary run types.
 
     The result format
@@ -180,11 +186,11 @@ def materialize_run_results(runs: List[Run], nodes: List[str] = None):
           },
        },
     }
-    '''
+    """
 
     context = default_context()
     if context:
-        mame_to_unique_id = context.build_name_to_unique_id_index(excluded_types={'semantic_model', 'metric'})
+        mame_to_unique_id = context.build_name_to_unique_id_index(excluded_types={"semantic_model", "metric"})
     else:
         mame_to_unique_id = {}
 
@@ -205,7 +211,7 @@ def materialize_run_results(runs: List[Run], nodes: List[str] = None):
                     node_result = result[key] = {}
                 else:
                     node_result = result.get(key)
-                node_result['row_count_diff'] = {'run_id': run.run_id, 'result': node_run_result}
+                node_result["row_count_diff"] = {"run_id": run.run_id, "result": node_run_result}
         elif run.type == RunType.ROW_COUNT:
             for model_name, node_run_result in run.result.items():
                 key = mame_to_unique_id.get(model_name, model_name)
@@ -218,5 +224,5 @@ def materialize_run_results(runs: List[Run], nodes: List[str] = None):
                     node_result = result[key] = {}
                 else:
                     node_result = result.get(key)
-                node_result['row_count'] = {'run_id': run.run_id, 'result': node_run_result}
+                node_result["row_count"] = {"run_id": run.run_id, "result": node_run_result}
     return result