recce-nightly 1.2.0.20250506__py3-none-any.whl → 1.26.0.20251124__py3-none-any.whl

recce/config.py

@@ -5,11 +5,11 @@ from recce import yaml
 from recce.exceptions import RecceConfigException
 from recce.util import SingletonMeta
 
-RECCE_CONFIG_FILE = 'recce.yml'
-RECCE_PRESET_CHECK_COMMENT = '''Preset Checks
+RECCE_CONFIG_FILE = "recce.yml"
+RECCE_PRESET_CHECK_COMMENT = """Preset Checks
 Please see https://docs.datarecce.io/features/preset-checks/
-'''
-RECCE_ERROR_LOG_FILE = 'recce_error.log'
+"""
+RECCE_ERROR_LOG_FILE = "recce_error.log"
 console = Console()
 
 
@@ -21,83 +21,92 @@ class RecceConfig(metaclass=SingletonMeta):
 
     def load(self):
         try:
-            with open(self.config_file, 'r') as f:
+            with open(self.config_file, "r", encoding="utf-8") as f:
                 config = yaml.safe_load(f)
                 self.config = config if config else {}
             self._verify_preset_checks()
         except FileNotFoundError:
-            console.print(f'Recce config file not found. Generating default config file at \'{self.config_file}\'')
+            console.print(f"[[orange3]NOTICE[/orange3]] Generate default Recce config file at '{self.config_file}'")
             self.config = self.generate_template()
             self.save()
 
     def _verify_preset_checks(self):
         from recce.tasks.core import CheckValidator
 
-        if not self.config.get('checks'):
+        if not self.config.get("checks"):
             return
 
-        for check in self.config['checks']:
+        for check in self.config["checks"]:
             try:
-                check_type = check.get('type')
+                check_type = check.get("type")
                 if check_type is None:
                     raise ValueError(f'Check type is required for check "{check}"')
-                if check_type == 'lineage_diff':
+                if check_type == "lineage_diff":
                     from recce.tasks.lineage import LineageDiffCheckValidator
+
                     validator = LineageDiffCheckValidator()
-                elif check_type == 'schema_diff':
+                elif check_type == "schema_diff":
                     from recce.tasks.schema import SchemaDiffCheckValidator
+
                     validator = SchemaDiffCheckValidator()
-                elif check_type == 'row_count_diff':
+                elif check_type == "row_count_diff":
                     from recce.tasks.rowcount import RowCountDiffCheckValidator
+
                     validator = RowCountDiffCheckValidator()
-                elif check_type == 'query':
+                elif check_type == "query":
                     from recce.tasks.query import QueryCheckValidator
+
                     validator = QueryCheckValidator()
-                elif check_type == 'query_diff':
+                elif check_type == "query_diff":
                     from recce.tasks.query import QueryDiffCheckValidator
+
                     validator = QueryDiffCheckValidator()
-                elif check_type == 'value_diff' or check_type == 'value_diff_detail':
+                elif check_type == "value_diff" or check_type == "value_diff_detail":
                     from recce.tasks.valuediff import ValueDiffCheckValidator
+
                     validator = ValueDiffCheckValidator()
-                elif check_type == 'profile_diff':
+                elif check_type == "profile_diff":
                     from recce.tasks.profile import ProfileCheckValidator
+
                     validator = ProfileCheckValidator()
-                elif check_type == 'top_k_diff':
+                elif check_type == "top_k_diff":
                     from recce.tasks.top_k import TopKDiffCheckValidator
+
                     validator = TopKDiffCheckValidator()
-                elif check_type == 'histogram_diff':
+                elif check_type == "histogram_diff":
                     from recce.tasks.histogram import HistogramDiffCheckValidator
+
                     validator = HistogramDiffCheckValidator()
                 else:
                     validator = CheckValidator()
                 validator.validate(check)
             except Exception as e:
                 import json
+
                 raise RecceConfigException(
-                    f"Load preset checks failed from '{self.config_file}'\n{json.dumps(check, indent=2)}",
-                    cause=e)
+                    f"Load preset checks failed from '{self.config_file}'\n{json.dumps(check, indent=2)}", cause=e
+                )
 
     def generate_template(self):
-        data = yaml.CommentedMap(
-            checks=yaml.CommentedSeq())
-        data.yaml_set_comment_before_after_key('checks', before=RECCE_PRESET_CHECK_COMMENT)
+        data = yaml.CommentedMap(checks=yaml.CommentedSeq())
+        data.yaml_set_comment_before_after_key("checks", before=RECCE_PRESET_CHECK_COMMENT)
         # Define default preset checks
         default_checks = [
             yaml.CommentedMap(
-                name='Row count diff',
-                description='Check the row count diff for all table models.',
-                type='row_count_diff',
-                params={'select': 'state:modified,config.materialized:table'},
+                name="Row count diff",
+                description="Check the row count diff for all table models.",
+                type="row_count_diff",
+                params={"select": "state:modified,config.materialized:table"},
             ),
             yaml.CommentedMap(
-                name='Schema diff',
-                description='Check the schema diff for all nodes.',
-                type='schema_diff',
-            )
+                name="Schema diff",
+                description="Check the schema diff for all nodes.",
+                type="schema_diff",
+            ),
         ]
 
         for check in default_checks:
-            data['checks'].append(check)
+            data["checks"].append(check)
 
         return data
 
@@ -108,7 +117,7 @@ class RecceConfig(metaclass=SingletonMeta):
         self.config[key] = value
 
     def save(self):
-        with open(RECCE_CONFIG_FILE, 'w') as f:
+        with open(RECCE_CONFIG_FILE, "w", encoding="utf-8") as f:
             yaml.dump(self.config, f)
 
     def __str__(self):

recce/connect_to_cloud.py

@@ -0,0 +1,138 @@
+import base64
+import os.path
+import random
+import threading
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from pathlib import Path
+from typing import Tuple
+from urllib.parse import parse_qs, urlparse
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import padding, rsa
+from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey, RSAPublicKey
+from rich.console import Console
+
+from recce.event import update_recce_api_token
+from recce.exceptions import RecceConfigException
+from recce.util.onboarding_state import update_onboarding_state
+from recce.util.recce_cloud import RECCE_CLOUD_BASE_URL, RecceCloud
+
+console = Console()
+
+static_folder_path = Path(__file__).parent / "data"
+_server_lock = threading.Lock()
+_connection_url = None
+
+
+def decrypt_code(private_key: RSAPrivateKey, code: str) -> str:
+    ciphertext = base64.b64decode(code)
+    plaintext = private_key.decrypt(
+        ciphertext,
+        padding.OAEP(
+            mgf=padding.MGF1(algorithm=hashes.SHA1()),  # Node.js uses SHA1 by default
+            algorithm=hashes.SHA1(),
+            label=None,
+        ),
+    )
+    return plaintext.decode("utf-8")
+
+
+def handle_callback_request(query_string: str, private_key: RSAPrivateKey):
+    query_params = parse_qs(query_string)
+    code = query_params.get("code", [None])[0]
+    if not code:
+        raise RecceConfigException("Missing `code` in query")
+
+    api_token = decrypt_code(private_key, code)
+    if not RecceCloud(api_token).verify_token():
+        raise RecceConfigException("Invalid Recce Cloud API token")
+
+    update_recce_api_token(api_token)
+    update_onboarding_state(api_token, False)
+
+    return api_token  # for testability/debugging
+
+
+def make_callback_handler(private_key: RSAPrivateKey):
+    class OneTimeHTTPRequestHandler(BaseHTTPRequestHandler):
+        def do_GET(self):
+            try:
+                with open(os.path.join(static_folder_path, "auth_callback.html"), "r", encoding="utf-8") as f:
+                    callback_html_content = f.read()
+
+                # Parse query parameters
+                parsed_url = urlparse(self.path)
+
+                handle_callback_request(parsed_url.query, private_key)
+
+                # Construct HTML content
+                self.send_response(200)
+                self.send_header("Content-Type", "text/html")
+                self.send_header("Content-Length", str(len(callback_html_content.encode())))
+                self.end_headers()
+                self.wfile.write(callback_html_content.encode())
+
+            except Exception:
+                console.print_exception()
+                self.send_response(500)
+                self.end_headers()
+                self.wfile.write(b"<h1>Internal Server Error</h1>")
+            finally:
+                # Shut down the server after handling the first request
+                # Shutdown in a new thread to avoid deadlock
+                self.server.server_close()
+                threading.Thread(target=self.server.shutdown, daemon=True).start()
+
+        def log_message(self, format, *args):
+            # Suppress default logging
+            return
+
+    return OneTimeHTTPRequestHandler
+
+
+def is_callback_server_running():
+    return _server_lock.locked()
+
+
+def get_connection_url():
+    return _connection_url
+
+
+def run_one_time_http_server(private_key: RSAPrivateKey, port=8080):
+    handler = make_callback_handler(private_key)
+    server = HTTPServer(("localhost", port), handler)
+    server.serve_forever()
+
+
+def prepare_connection_url(public_key: RSAPublicKey):
+    public_key_pem_bytes = public_key.public_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PublicFormat.SubjectPublicKeyInfo,
+    )
+    public_key_pem_str = base64.b64encode(public_key_pem_bytes).decode("utf-8")
+    callback_port = random.randint(10000, 15000)
+    connect_url = f"{RECCE_CLOUD_BASE_URL}/connect?_key={public_key_pem_str}&_port={callback_port}"
+    return connect_url, callback_port
+
+
+def generate_key_pair() -> Tuple[RSAPrivateKey, RSAPublicKey]:
+    key_size = 2048  # Should be at least 2048
+
+    private_key = rsa.generate_private_key(
+        public_exponent=65537, key_size=key_size, backend=default_backend()  # Do not change
+    )
+
+    public_key = private_key.public_key()
+    return private_key, public_key
+
+
+def connect_to_cloud_background_task(private_key: RSAPrivateKey, callback_port, connection_url):
+    if is_callback_server_running():
+        return
+
+    with _server_lock:
+        global _connection_url
+        _connection_url = connection_url
+        run_one_time_http_server(private_key, callback_port)
+        _connection_url = None

recce/core.py

@@ -3,15 +3,21 @@ import json
 import logging
 import os
 from dataclasses import dataclass, field
-from typing import Callable, Dict, Optional, List, Tuple, Set
+from typing import Callable, Dict, List, Optional, Set, Tuple
 
 from recce.adapter.base import BaseAdapter
 from recce.models import Check, Run
 from recce.models.types import LineageDiff
-from recce.state import RecceState, RecceStateMetadata, GitRepoInfo, PullRequestInfo, RecceStateLoader
+from recce.state import (
+    GitRepoInfo,
+    PullRequestInfo,
+    RecceState,
+    RecceStateLoader,
+    RecceStateMetadata,
+)
 from recce.util.recce_cloud import set_recce_cloud_onboarding_state
 
-logger = logging.getLogger('uvicorn')
+logger = logging.getLogger("uvicorn")
 
 
 @dataclass
@@ -25,8 +31,8 @@ class RecceContext:
 
     @classmethod
     def load(cls, **kwargs):
-        state_loader: RecceStateLoader = kwargs.get('state_loader')
-        is_review_mode = kwargs.get('review', False)
+        state_loader: RecceStateLoader = kwargs.get("state_loader")
+        is_review_mode = kwargs.get("review", False)
 
         context = cls(
             review_mode=is_review_mode,
@@ -34,14 +40,16 @@ class RecceContext:
         )
 
         # Initiate the adapter
-        if kwargs.get('sqlmesh', False):
-            logger.warning('SQLMesh adapter is still in EXPERIMENTAL mode.')
+        if kwargs.get("sqlmesh", False):
+            logger.warning("SQLMesh adapter is still in EXPERIMENTAL mode.")
             from recce.adapter.sqlmesh_adapter import SqlmeshAdapter
-            context.adapter_type = 'sqlmesh'
+
+            context.adapter_type = "sqlmesh"
             context.adapter = SqlmeshAdapter.load(**kwargs)
         else:
             from recce.adapter.dbt_adapter import DbtAdapter
-            context.adapter_type = 'dbt'
+
+            context.adapter_type = "dbt"
             context.adapter = DbtAdapter.load(**kwargs)
 
         # Import state
@@ -52,7 +60,7 @@ class RecceContext:
 
             if is_review_mode:
                 if not state:
-                    raise Exception('The state file is required for review mode')
+                    raise Exception("The state file is required for review mode")
 
         return context
 
@@ -76,14 +84,14 @@ class RecceContext:
         curr = self.get_lineage(base=False)
         base = self.get_lineage(base=True)
 
-        for unique_id, node in curr['nodes'].items():
-            if excluded_types and node.get('resource_type') in excluded_types:
+        for unique_id, node in curr["nodes"].items():
+            if excluded_types and node.get("resource_type") in excluded_types:
                 continue
-            name_to_unique_id[node['name']] = unique_id
-        for unique_id, node in base['nodes'].items():
-            if excluded_types and node.get('resource_type') in excluded_types:
+            name_to_unique_id[node["name"]] = unique_id
+        for unique_id, node in base["nodes"].items():
+            if excluded_types and node.get("resource_type") in excluded_types:
                 continue
-            name_to_unique_id[node['name']] = unique_id
+            name_to_unique_id[node["name"]] = unique_id
         return name_to_unique_id
 
     def start_monitor_artifacts(self, callback: Callable = None):
@@ -118,7 +126,7 @@ class RecceContext:
             state.git = self.state_loader.state.git
             state.pull_request = self.state_loader.state.pull_request
         else:
-            git = GitRepoInfo.from_current_repositroy()
+            git = GitRepoInfo.from_current_repository()
             if git:
                 state.git = git
             if self.state_loader.pr_info:
@@ -137,10 +145,10 @@ class RecceContext:
         state.runs = self.runs
         state.checks = self.checks
         state.artifacts = self.adapter.export_artifacts()
-        git = GitRepoInfo.from_current_repositroy()
+        git = GitRepoInfo.from_current_repository()
         if git:
             state.git = git
-        pr = PullRequestInfo(url=os.getenv('RECCE_PR_URL'))
+        pr = PullRequestInfo(url=os.getenv("RECCE_PR_URL"))
         state.pull_request = pr
 
         return state
@@ -152,38 +160,37 @@ class RecceContext:
         :param method: merge, revert, overwrite
 
         """
-        if method == 'merge':
+        if method == "merge":
             self.state_loader.refresh()
             self.import_state(self.state_loader.state, merge=True)
             state = self.export_state()
             self.state_loader.export(state)
-        elif method == 'revert':
+        elif method == "revert":
             self.state_loader.refresh()
             self.import_state(self.state_loader.state, merge=False)
-        elif method == 'overwrite':
+        elif method == "overwrite":
             state = self.export_state()
             self.state_loader.export(state)
         else:
-            raise Exception(f'Unsupported method: {method}')
+            raise Exception(f"Unsupported method: {method}")
 
     def _merge_checks(self, import_checks: list[Check]):
         checks = list(self.checks)
         imports = 0
 
         def _calculate_checksum(c: Check):
-            payload = json.dumps({
-                'type': str(c.type),
-                'params': c.params,
-                'view_options': c.view_options,
-            }, sort_keys=True)
+            payload = json.dumps(
+                {
+                    "type": str(c.type),
+                    "params": c.params,
+                    "view_options": c.view_options,
+                },
+                sort_keys=True,
+            )
             return hashlib.sha256(payload.encode()).hexdigest()
 
-        checksum_map = {
-            _calculate_checksum(c): c for c in self.checks if c.is_preset
-        }
-        check_map = {
-            c.check_id: c for c in self.checks
-        }
+        checksum_map = {_calculate_checksum(c): c for c in self.checks if c.is_preset}
+        check_map = {c.check_id: c for c in self.checks}
 
         # merge checks
         for imported in import_checks:
@@ -219,12 +226,12 @@ class RecceContext:
         return imports
 
     def import_state(self, import_state: RecceState, merge: bool = True):
-        '''
+        """
         Import the state from another RecceState object.
 
         :param import_state: the state to import
         :param merge: whether to merge the state or replace the current state
-        '''
+        """
         import_runs = 0
         import_checks = 0
         if merge:
@@ -261,24 +268,25 @@ class RecceContext:
     def mark_onboarding_completed(self):
         if self.state_loader.cloud_mode:
             try:
-                token = self.state_loader.cloud_options.get('token')
-                set_recce_cloud_onboarding_state(token, 'completed')
+                token = self.state_loader.cloud_options.get("github_token")
+                set_recce_cloud_onboarding_state(token, "completed")
             except Exception as e:
-                logger.debug(f'Failed to mark onboarding completed in Recce Cloud. Reason: {str(e)}')
+                logger.debug(f"Failed to mark onboarding completed in Recce Cloud. Reason: {str(e)}")
         else:
             # Skip the onboarding state for non-cloud mode
             pass
 
     @staticmethod
     def verify_required_artifacts(**kwargs) -> Tuple[bool, Optional[str]]:
-        if kwargs.get('sqlmesh', False):
+        if kwargs.get("sqlmesh", False):
             pass
         else:
             from recce.adapter.dbt_adapter import DbtAdapter
+
             try:
                 DbtAdapter.load(**kwargs)
             except FileNotFoundError as e:
-                return False, f"Cannot load the manifest: '{e.filename}'"
+                return False, f"Cannot load the manifest: '{e.filename}'. Type 'recce debug'."
 
         return True, None
 
@@ -289,18 +297,18 @@ class RecceContext:
         """
         The state loader mode is used for telemetry purpose.
         """
-        if os.environ.get('DEMO', False):
-            return 'demo'
+        if os.environ.get("DEMO", False):
+            return "demo"
 
         if not self.state_loader:
-            return 'none'
+            return "none"
 
         if self.state_loader.cloud_mode:
-            return 'cloud'
+            return "cloud"
         elif self.state_loader.state_file:
-            return 'file'
+            return "file"
         else:
-            return 'none'
+            return "none"
 
 
 recce_context: Optional[RecceContext] = None