recce-nightly 1.15.0.20250806__py3-none-any.whl → 1.26.0.20251124__py3-none-any.whl

recce/util/recce_cloud.py

@@ -42,6 +42,7 @@ class RecceCloud:
         self.token = token
         self.token_type = "github_token" if token.startswith(("ghp_", "gho_", "ghu_", "ghs_", "ghr_")) else "api_token"
         self.base_url = f"{RECCE_CLOUD_API_HOST}/api/v1"
+        self.base_url_v2 = f"{RECCE_CLOUD_API_HOST}/api/v2"
 
     def _request(self, method, url, headers: Dict = None, **kwargs):
         headers = {
@@ -81,6 +82,19 @@ class RecceCloud:
         response = self._fetch_presigned_url(method, repository, artifact_name, metadata, pr_id, branch)
         return response.get("presigned_url")
 
+    def _replace_localhost_with_docker_internal(self, url: str) -> str:
+        if url is None:
+            return None
+        if (
+            os.environ.get("RECCE_SHARE_INSTANCE_ENV") == "docker"
+            or os.environ.get("RECCE_TASK_INSTANCE_ENV") == "docker"
+            or os.environ.get("RECCE_INSTANCE_ENV") == "docker"
+        ):
+            # For local development, convert the presigned URL from localhost to host.docker.internal
+            if url.startswith(LOCALHOST_URL_PREFIX):
+                return url.replace(LOCALHOST_URL_PREFIX, DOCKER_INTERNAL_URL_PREFIX)
+        return url
+
     def get_presigned_url_by_share_id(
         self,
         method: PresignedUrlMethod,
@@ -89,10 +103,13 @@ class RecceCloud:
     ) -> str:
         response = self._fetch_presigned_url_by_share_id(method, share_id, metadata=metadata)
         presigned_url = response.get("presigned_url")
-        # Check if the CLI is running in Docker Recce Share Instance
-        if os.environ.get("RECCE_SHARE_INSTANCE_ENV") == "docker" and presigned_url.startswith(LOCALHOST_URL_PREFIX):
-            # For local development, convert the presigned URL from localhost to host.docker.internal
-            presigned_url = presigned_url.replace(LOCALHOST_URL_PREFIX, DOCKER_INTERNAL_URL_PREFIX)
+        if not presigned_url:
+            raise RecceCloudException(
+                message="Failed to get presigned URL from Recce Cloud.",
+                reason="No presigned URL returned from the server.",
+                status_code=404,
+            )
+        presigned_url = self._replace_localhost_with_docker_internal(presigned_url)
         return presigned_url
 
     def get_download_presigned_url_by_github_repo_with_tags(
@@ -167,12 +184,22 @@ class RecceCloud:
             )
         return response.json()
 
-    def purge_artifacts(self, pr_info: PullRequestInfo):
-        api_url = f"{self.base_url}/{pr_info.repository}/pulls/{pr_info.id}/artifacts"
+    def purge_artifacts(self, repository: str, pr_id: int = None, branch: str = None):
+        if pr_id is not None:
+            api_url = f"{self.base_url}/{repository}/pulls/{pr_id}/artifacts"
+            error_message = "Failed to purge artifacts from Recce Cloud."
+        elif branch is not None:
+            api_url = f"{self.base_url}/{repository}/commits/{branch}/artifacts"
+            error_message = "Failed to delete artifacts from Recce Cloud."
+        else:
+            raise ValueError(
+                "Please either run this command from within a pull request context "
+                "or specify a branch using the --branch option."
+            )
         response = self._request("DELETE", api_url)
         if response.status_code != 204:
             raise RecceCloudException(
-                message="Failed to purge artifacts from Recce Cloud.",
+                message=error_message,
                 reason=response.text,
                 status_code=response.status_code,
             )
@@ -232,8 +259,161 @@ class RecceCloud:
             logger.warning(f"Failed to set Onboarding State in Recce Cloud. Reason: {str(e)}")
         return
 
+    def get_session(self, session_id: str):
+        api_url = f"{self.base_url_v2}/sessions/{session_id}"
+        response = self._request("GET", api_url)
+        if response.status_code == 403:
+            return {"status": "error", "message": response.json().get("detail")}
+        if response.status_code != 200:
+            raise RecceCloudException(
+                message="Failed to get session from Recce Cloud.",
+                reason=response.text,
+                status_code=response.status_code,
+            )
+        data = response.json()
+        if data["success"] is not True:
+            raise RecceCloudException(
+                message="Failed to get session from Recce Cloud.",
+                reason=data.get("message", "Unknown error"),
+                status_code=response.status_code,
+            )
+        return data["session"]
+
+    def update_session(self, org_id: str, project_id: str, session_id: str, adapter_type: str):
+        api_url = f"{self.base_url_v2}/organizations/{org_id}/projects/{project_id}/sessions/{session_id}"
+        data = {"adapter_type": adapter_type}
+        response = self._request("PATCH", api_url, json=data)
+        if response.status_code == 403:
+            return {"status": "error", "message": response.json().get("detail")}
+        if response.status_code != 200:
+            raise RecceCloudException(
+                message="Failed to update session in Recce Cloud.",
+                reason=response.text,
+                status_code=response.status_code,
+            )
+        return response.json()
+
+    def get_download_urls_by_session_id(self, org_id: str, project_id: str, session_id: str) -> dict[str, str]:
+        api_url = f"{self.base_url_v2}/organizations/{org_id}/projects/{project_id}/sessions/{session_id}/download-url"
+        response = self._request("GET", api_url)
+        if response.status_code != 200:
+            raise RecceCloudException(
+                message="Failed to download session from Recce Cloud.",
+                reason=response.text,
+                status_code=response.status_code,
+            )
+        data = response.json()
+        if data["presigned_urls"] is None:
+            raise RecceCloudException(
+                message="No presigned URLs returned from the server.",
+                reason="",
+                status_code=404,
+            )
+
+        presigned_urls = data["presigned_urls"]
+        for key, url in presigned_urls.items():
+            presigned_urls[key] = self._replace_localhost_with_docker_internal(url)
+        return presigned_urls
+
+    def get_base_session_download_urls(self, org_id: str, project_id: str) -> dict[str, str]:
+        """Get download URLs for the base session of a project."""
+        api_url = f"{self.base_url_v2}/organizations/{org_id}/projects/{project_id}/base-session/download-url"
+        response = self._request("GET", api_url)
+        if response.status_code != 200:
+            raise RecceCloudException(
+                message="Failed to download base session from Recce Cloud.",
+                reason=response.text,
+                status_code=response.status_code,
+            )
+        data = response.json()
+        if data["presigned_urls"] is None:
+            raise RecceCloudException(
+                message="No presigned URLs returned from the server.",
+                reason="",
+                status_code=404,
+            )
+
+        presigned_urls = data["presigned_urls"]
+        for key, url in presigned_urls.items():
+            presigned_urls[key] = self._replace_localhost_with_docker_internal(url)
+        return presigned_urls
+
+    def get_upload_urls_by_session_id(self, org_id: str, project_id: str, session_id: str) -> dict[str, str]:
+        api_url = f"{self.base_url_v2}/organizations/{org_id}/projects/{project_id}/sessions/{session_id}/upload-url"
+        response = self._request("GET", api_url)
+        if response.status_code != 200:
+            raise RecceCloudException(
+                message="Failed to get upload URLs for session from Recce Cloud.",
+                reason=response.text,
+                status_code=response.status_code,
+            )
+        data = response.json()
+        if data["presigned_urls"] is None:
+            raise RecceCloudException(
+                message="No presigned URLs returned from the server.",
+                reason="",
+                status_code=404,
+            )
+
+        presigned_urls = data["presigned_urls"]
+        for key, url in presigned_urls.items():
+            presigned_urls[key] = self._replace_localhost_with_docker_internal(url)
+        return presigned_urls
+
+    def post_recce_state_uploaded_by_session_id(self, org_id: str, project_id: str, session_id: str):
+        api_url = f"{self.base_url_v2}/organizations/{org_id}/projects/{project_id}/sessions/{session_id}/recce-state-uploaded"
+        response = self._request("POST", api_url)
+        if response.status_code != 204:
+            raise RecceCloudException(
+                message="Failed to notify state uploaded for session in Recce Cloud.",
+                reason=response.text,
+                status_code=response.status_code,
+            )
+
+    def list_organizations(self) -> list:
+        """List all organizations the user has access to."""
+        api_url = f"{self.base_url_v2}/organizations"
+        response = self._request("GET", api_url)
+        if response.status_code != 200:
+            raise RecceCloudException(
+                message="Failed to list organizations from Recce Cloud.",
+                reason=response.text,
+                status_code=response.status_code,
+            )
+        data = response.json()
+        return data.get("organizations", [])
+
+    def list_projects(self, org_id: str) -> list:
+        """List all projects in an organization."""
+        api_url = f"{self.base_url_v2}/organizations/{org_id}/projects"
+        response = self._request("GET", api_url)
+        if response.status_code != 200:
+            raise RecceCloudException(
+                message="Failed to list projects from Recce Cloud.",
+                reason=response.text,
+                status_code=response.status_code,
+            )
+        data = response.json()
+        return data.get("projects", [])
+
+    def list_sessions(self, org_id: str, project_id: str) -> list:
+        """List all sessions in a project."""
+        api_url = f"{self.base_url_v2}/organizations/{org_id}/projects/{project_id}/sessions"
+        response = self._request("GET", api_url)
+        if response.status_code != 200:
+            raise RecceCloudException(
+                message="Failed to list sessions from Recce Cloud.",
+                reason=response.text,
+                status_code=response.status_code,
+            )
+        data = response.json()
+        return data.get("sessions", [])
+
 
 def get_recce_cloud_onboarding_state(token: str) -> str:
+    if token and token.startswith("rct-"):
+        return "undefined"
+
     try:
         recce_cloud = RecceCloud(token)
         user_info = recce_cloud.get_user_info()

recce/yaml/__init__.py

@@ -34,7 +34,7 @@ def dump(data, stream: Any = None, *, transform: Any = None) -> Any:
 
 def safe_load_yaml(file_path):
     try:
-        with open(file_path, "r") as f:
+        with open(file_path, "r", encoding="utf-8") as f:
             payload = safe_load(f)
     except yaml.YAMLError as e:
         print(e)
@@ -45,7 +45,7 @@ def safe_load_yaml(file_path):
 
 
 def round_trip_load_yaml(file_path):
-    with open(file_path, "r") as f:
+    with open(file_path, "r", encoding="utf-8") as f:
         try:
             payload = load(f)
         except yaml.YAMLError as e:

recce_cloud/__init__.py

@@ -0,0 +1,24 @@
+"""Recce Cloud - Lightweight CLI for Recce Cloud operations."""
+
+import os
+
+
+def get_version():
+    """Get version from VERSION file."""
+    # Try recce_cloud/VERSION first (for standalone package)
+    version_file = os.path.join(os.path.dirname(__file__), "VERSION")
+    if os.path.exists(version_file):
+        with open(version_file) as fh:
+            return fh.read().strip()
+
+    # Fallback to ../recce/VERSION (for development)
+    version_file = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "recce", "VERSION"))
+    if os.path.exists(version_file):
+        with open(version_file) as fh:
+            return fh.read().strip()
+
+    # Last resort
+    return "unknown"
+
+
+__version__ = get_version()

recce_cloud/api/__init__.py

@@ -0,0 +1,17 @@
+"""Recce Cloud API client module."""
+
+from recce_cloud.api.base import BaseRecceCloudClient
+from recce_cloud.api.client import RecceCloudClient
+from recce_cloud.api.exceptions import RecceCloudException
+from recce_cloud.api.factory import create_platform_client
+from recce_cloud.api.github import GitHubRecceCloudClient
+from recce_cloud.api.gitlab import GitLabRecceCloudClient
+
+__all__ = [
+    "BaseRecceCloudClient",
+    "RecceCloudClient",
+    "RecceCloudException",
+    "create_platform_client",
+    "GitHubRecceCloudClient",
+    "GitLabRecceCloudClient",
+]

recce_cloud/api/base.py

@@ -0,0 +1,111 @@
+"""
+Base API client for Recce Cloud.
+"""
+
+import os
+from abc import ABC, abstractmethod
+from typing import Dict, Optional
+
+import requests
+
+from recce_cloud.api.exceptions import RecceCloudException
+
+
+class BaseRecceCloudClient(ABC):
+    """Abstract base class for platform-specific Recce Cloud API clients."""
+
+    def __init__(self, token: str, api_host: Optional[str] = None):
+        """
+        Initialize the API client.
+
+        Args:
+            token: Authentication token (GITHUB_TOKEN, CI_JOB_TOKEN, or RECCE_API_TOKEN)
+            api_host: Recce Cloud API host (defaults to RECCE_CLOUD_API_HOST or https://cloud.datarecce.io)
+        """
+        self.token = token
+        self.api_host = api_host or os.getenv("RECCE_CLOUD_API_HOST", "https://cloud.datarecce.io")
+
+    def _make_request(self, method: str, url: str, **kwargs) -> Dict:
+        """
+        Make an HTTP request to Recce Cloud API.
+
+        Args:
+            method: HTTP method (GET, POST, PUT, etc.)
+            url: Full URL for the request
+            **kwargs: Additional arguments passed to requests
+
+        Returns:
+            Response JSON as dictionary
+
+        Raises:
+            RecceCloudException: If the request fails
+        """
+        headers = kwargs.pop("headers", {})
+        headers.update(
+            {
+                "Authorization": f"Bearer {self.token}",
+                "Content-Type": "application/json",
+            }
+        )
+
+        try:
+            response = requests.request(method, url, headers=headers, **kwargs)
+            response.raise_for_status()
+
+            # Handle empty responses (e.g., 204 No Content)
+            if response.status_code == 204 or not response.content:
+                return {}
+
+            return response.json()
+        except requests.exceptions.HTTPError as e:
+            reason = str(e)
+            if e.response is not None:
+                try:
+                    error_detail = e.response.json()
+                    reason = error_detail.get("message", str(e))
+                except Exception:
+                    reason = e.response.text or str(e)
+            raise RecceCloudException(reason=reason, status_code=e.response.status_code if e.response else None)
+        except requests.exceptions.RequestException as e:
+            raise RecceCloudException(reason=str(e))
+
+    @abstractmethod
+    def touch_recce_session(
+        self,
+        branch: str,
+        adapter_type: str,
+        cr_number: Optional[int] = None,
+        commit_sha: Optional[str] = None,
+        session_type: Optional[str] = None,
+    ) -> Dict:
+        """
+        Create or touch a Recce session.
+
+        Args:
+            branch: Branch name
+            adapter_type: DBT adapter type (e.g., 'postgres', 'snowflake', 'bigquery')
+            cr_number: Change request number (PR/MR number) for CR sessions
+            commit_sha: Commit SHA (GitLab requires this)
+            session_type: Session type ("cr", "prod", "dev") - determines if cr_number is used
+
+        Returns:
+            Dictionary containing:
+                - session_id: Session ID
+                - manifest_upload_url: Presigned URL for manifest.json upload
+                - catalog_upload_url: Presigned URL for catalog.json upload
+        """
+        pass
+
+    @abstractmethod
+    def upload_completed(self, session_id: str, commit_sha: Optional[str] = None) -> Dict:
+        """
+        Notify Recce Cloud that upload is complete.
+
+        Args:
+            session_id: Session ID from touch_recce_session
+            commit_sha: Commit SHA (GitLab requires this)
+
+        Returns:
+            Empty dictionary or acknowledgement
+        """
+        pass

recce_cloud/api/client.py

@@ -0,0 +1,150 @@
+"""
+Recce Cloud API client for lightweight operations.
+
+Simplified version of recce.util.recce_cloud.RecceCloud with only
+the methods needed for upload-session functionality.
+"""
+
+import os
+
+import requests
+
+from recce_cloud.api.exceptions import RecceCloudException
+
+RECCE_CLOUD_API_HOST = os.environ.get("RECCE_CLOUD_API_HOST", "https://cloud.datarecce.io")
+
+DOCKER_INTERNAL_URL_PREFIX = "http://host.docker.internal"
+LOCALHOST_URL_PREFIX = "http://localhost"
+
+
+class RecceCloudClient:
+    """
+    Lightweight Recce Cloud API client.
+
+    Supports authentication with Recce Cloud API token (starts with "rct-").
+    """
+
+    def __init__(self, token: str):
+        if token is None:
+            raise ValueError("Token cannot be None.")
+        self.token = token
+        self.base_url_v2 = f"{RECCE_CLOUD_API_HOST}/api/v2"
+
+    def _request(self, method: str, url: str, headers: dict = None, **kwargs):
+        """Make authenticated HTTP request to Recce Cloud API."""
+        headers = {
+            **(headers or {}),
+            "Authorization": f"Bearer {self.token}",
+        }
+        return requests.request(method, url, headers=headers, **kwargs)
+
+    def _replace_localhost_with_docker_internal(self, url: str) -> str:
+        """Convert localhost URLs to docker internal URLs if running in Docker."""
+        if url is None:
+            return None
+        if (
+            os.environ.get("RECCE_SHARE_INSTANCE_ENV") == "docker"
+            or os.environ.get("RECCE_TASK_INSTANCE_ENV") == "docker"
+            or os.environ.get("RECCE_INSTANCE_ENV") == "docker"
+        ):
+            # For local development, convert the presigned URL from localhost to host.docker.internal
+            if url.startswith(LOCALHOST_URL_PREFIX):
+                return url.replace(LOCALHOST_URL_PREFIX, DOCKER_INTERNAL_URL_PREFIX)
+        return url
+
+    def get_session(self, session_id: str) -> dict:
+        """
+        Get session information from Recce Cloud.
+
+        Args:
+            session_id: The session ID to retrieve
+
+        Returns:
+            dict containing session information with keys:
+                - org_id: Organization ID
+                - project_id: Project ID
+                - ... other session fields
+
+        Raises:
+            RecceCloudException: If the request fails
+        """
+        api_url = f"{self.base_url_v2}/sessions/{session_id}"
+        response = self._request("GET", api_url)
+        if response.status_code == 403:
+            return {"status": "error", "message": response.json().get("detail")}
+        if response.status_code != 200:
+            raise RecceCloudException(
+                reason=response.text,
+                status_code=response.status_code,
+            )
+        data = response.json()
+        if data["success"] is not True:
+            raise RecceCloudException(
+                reason=data.get("message", "Unknown error"),
+                status_code=response.status_code,
+            )
+        return data["session"]
+
+    def get_upload_urls_by_session_id(self, org_id: str, project_id: str, session_id: str) -> dict:
+        """
+        Get presigned S3 upload URLs for a session.
+
+        Args:
+            org_id: Organization ID
+            project_id: Project ID
+            session_id: Session ID
+
+        Returns:
+            dict with keys:
+                - manifest_url: Presigned URL for uploading manifest.json
+                - catalog_url: Presigned URL for uploading catalog.json
+
+        Raises:
+            RecceCloudException: If the request fails
+        """
+        api_url = f"{self.base_url_v2}/organizations/{org_id}/projects/{project_id}/sessions/{session_id}/upload-url"
+        response = self._request("GET", api_url)
+        if response.status_code != 200:
+            raise RecceCloudException(
+                reason=response.text,
+                status_code=response.status_code,
+            )
+        data = response.json()
+        if data["presigned_urls"] is None:
+            raise RecceCloudException(
+                reason="No presigned URLs returned from the server.",
+                status_code=404,
+            )
+
+        presigned_urls = data["presigned_urls"]
+        for key, url in presigned_urls.items():
+            presigned_urls[key] = self._replace_localhost_with_docker_internal(url)
+        return presigned_urls
+
+    def update_session(self, org_id: str, project_id: str, session_id: str, adapter_type: str) -> dict:
+        """
+        Update session metadata with adapter type.
+
+        Args:
+            org_id: Organization ID
+            project_id: Project ID
+            session_id: Session ID
+            adapter_type: dbt adapter type (e.g., "postgres", "snowflake", "bigquery")
+
+        Returns:
+            dict containing updated session information
+
+        Raises:
+            RecceCloudException: If the request fails
+        """
+        api_url = f"{self.base_url_v2}/organizations/{org_id}/projects/{project_id}/sessions/{session_id}"
+        data = {"adapter_type": adapter_type}
+        response = self._request("PATCH", api_url, json=data)
+        if response.status_code == 403:
+            return {"status": "error", "message": response.json().get("detail")}
+        if response.status_code != 200:
+            raise RecceCloudException(
+                reason=response.text,
+                status_code=response.status_code,
+            )
+        return response.json()

recce_cloud/api/exceptions.py

@@ -0,0 +1,26 @@
+"""
+Exceptions for Recce Cloud API.
+"""
+
+import json
+
+
+class RecceCloudException(Exception):
+    """Exception raised when Recce Cloud API returns an error."""
+
+    def __init__(self, reason: str, status_code: int = None):
+        """
+        Initialize exception.
+
+        Args:
+            reason: Error reason/message
+            status_code: HTTP status code (optional)
+        """
+        try:
+            reason = json.loads(reason).get("detail", reason)
+        except (json.JSONDecodeError, AttributeError):
+            pass
+
+        super().__init__(reason)
+        self.reason = reason
+        self.status_code = status_code

recce_cloud/api/factory.py

@@ -0,0 +1,63 @@
+"""
+Factory for creating platform-specific API clients.
+"""
+
+import os
+from typing import Optional
+
+from recce_cloud.api.github import GitHubRecceCloudClient
+from recce_cloud.api.gitlab import GitLabRecceCloudClient
+from recce_cloud.ci_providers import CIDetector
+from recce_cloud.ci_providers.base import CIInfo
+
+
+def create_platform_client(
+    token: str,
+    ci_info: Optional[CIInfo] = None,
+    api_host: Optional[str] = None,
+):
+    """
+    Create a platform-specific Recce Cloud API client based on CI environment.
+
+    Args:
+        token: Authentication token (GITHUB_TOKEN, CI_JOB_TOKEN, or RECCE_API_TOKEN)
+        ci_info: CI information (auto-detected if not provided)
+        api_host: Recce Cloud API host (optional)
+
+    Returns:
+        GitHubRecceCloudClient or GitLabRecceCloudClient
+
+    Raises:
+        ValueError: If platform is not supported or required information is missing
+    """
+    # Auto-detect CI info if not provided
+    if ci_info is None:
+        ci_info = CIDetector.detect()
+
+    if ci_info.platform == "github-actions":
+        repository = ci_info.repository or os.getenv("GITHUB_REPOSITORY")
+        if not repository:
+            raise ValueError("GitHub repository information is required but not detected")
+
+        return GitHubRecceCloudClient(token=token, repository=repository, api_host=api_host)
+
+    elif ci_info.platform == "gitlab-ci":
+        project_path = ci_info.repository or os.getenv("CI_PROJECT_PATH")
+        repository_url = os.getenv("CI_PROJECT_URL")
+
+        if not project_path:
+            raise ValueError("GitLab project path is required but not detected")
+        if not repository_url:
+            raise ValueError("GitLab project URL is required but not detected")
+
+        return GitLabRecceCloudClient(
+            token=token,
+            project_path=project_path,
+            repository_url=repository_url,
+            api_host=api_host,
+        )
+
+    else:
+        raise ValueError(
+            f"Unsupported platform: {ci_info.platform}. " "Only GitHub Actions and GitLab CI are supported."
+        )