recce-nightly 1.2.0.20250506__py3-none-any.whl → 1.26.0.20251124__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of recce-nightly might be problematic. Click here for more details.
- recce/VERSION +1 -1
- recce/__init__.py +27 -22
- recce/adapter/base.py +11 -14
- recce/adapter/dbt_adapter/__init__.py +810 -480
- recce/adapter/dbt_adapter/dbt_version.py +3 -0
- recce/adapter/sqlmesh_adapter.py +24 -35
- recce/apis/check_api.py +39 -28
- recce/apis/check_func.py +33 -27
- recce/apis/run_api.py +25 -19
- recce/apis/run_func.py +29 -23
- recce/artifact.py +119 -51
- recce/cli.py +1299 -323
- recce/config.py +42 -33
- recce/connect_to_cloud.py +138 -0
- recce/core.py +55 -47
- recce/data/404.html +1 -1
- recce/data/__next.__PAGE__.txt +10 -0
- recce/data/__next._full.txt +23 -0
- recce/data/__next._head.txt +8 -0
- recce/data/__next._index.txt +8 -0
- recce/data/__next._tree.txt +5 -0
- recce/data/_next/static/52aV_JrNUZU6dMFgvTQEO/_buildManifest.js +11 -0
- recce/data/_next/static/52aV_JrNUZU6dMFgvTQEO/_clientMiddlewareManifest.json +1 -0
- recce/data/_next/static/chunks/02b996c7f6a29a06.js +4 -0
- recce/data/_next/static/chunks/19c10d219a6a21ff.js +1 -0
- recce/data/_next/static/chunks/2df9ec28a061971d.js +11 -0
- recce/data/_next/static/chunks/3098c987393bda15.js +1 -0
- recce/data/_next/static/chunks/393dc43e483f717a.css +2 -0
- recce/data/_next/static/chunks/399e8d91a7e45073.js +2 -0
- recce/data/_next/static/chunks/4d0186f631230245.js +1 -0
- recce/data/_next/static/chunks/5794ba9e10a9c060.js +11 -0
- recce/data/_next/static/chunks/715761c929a3f28b.js +110 -0
- recce/data/_next/static/chunks/71f88fcc615bf282.js +1 -0
- recce/data/_next/static/chunks/80d2a95eaf1201ea.js +1 -0
- recce/data/_next/static/chunks/9979c6109bbbee35.js +1 -0
- recce/data/_next/static/chunks/99d638224186c118.js +1 -0
- recce/data/_next/static/chunks/d003eb36240e92f3.js +1 -0
- recce/data/_next/static/chunks/d3167cdfec4fc351.js +1 -0
- recce/data/_next/static/chunks/e124bccf574a3361.css +1 -0
- recce/data/_next/static/chunks/f40141db1bdb46f0.css +6 -0
- recce/data/_next/static/chunks/fcc53a88741a52f9.js +1 -0
- recce/data/_next/static/chunks/turbopack-b1920d28cfb1f28d.js +3 -0
- recce/data/_next/static/media/favicon.a8d38d84.ico +0 -0
- recce/data/_next/static/media/montserrat-cyrillic-800-normal.d80d830d.woff2 +0 -0
- recce/data/_next/static/media/montserrat-cyrillic-800-normal.f9d58125.woff +0 -0
- recce/data/_next/static/media/montserrat-cyrillic-ext-800-normal.076c2a93.woff2 +0 -0
- recce/data/_next/static/media/montserrat-cyrillic-ext-800-normal.a4fa76b5.woff +0 -0
- recce/data/_next/static/media/montserrat-latin-800-normal.cde454cc.woff2 +0 -0
- recce/data/_next/static/media/montserrat-latin-800-normal.d5761935.woff +0 -0
- recce/data/_next/static/media/montserrat-latin-ext-800-normal.40ec0659.woff2 +0 -0
- recce/data/_next/static/media/montserrat-latin-ext-800-normal.b671449b.woff +0 -0
- recce/data/_next/static/media/montserrat-vietnamese-800-normal.9f7b8541.woff +0 -0
- recce/data/_next/static/media/montserrat-vietnamese-800-normal.f9eb854e.woff2 +0 -0
- recce/data/_next/static/media/reload-image.7aa931c7.svg +4 -0
- recce/data/_not-found/__next._full.txt +17 -0
- recce/data/_not-found/__next._head.txt +8 -0
- recce/data/_not-found/__next._index.txt +8 -0
- recce/data/_not-found/__next._not-found.__PAGE__.txt +5 -0
- recce/data/_not-found/__next._not-found.txt +4 -0
- recce/data/_not-found/__next._tree.txt +3 -0
- recce/data/_not-found.html +1 -0
- recce/data/_not-found.txt +17 -0
- recce/data/auth_callback.html +68 -0
- recce/data/imgs/reload-image.svg +4 -0
- recce/data/index.html +1 -27
- recce/data/index.txt +23 -7
- recce/diff.py +6 -12
- recce/event/__init__.py +86 -74
- recce/event/collector.py +33 -22
- recce/event/track.py +49 -27
- recce/exceptions.py +1 -1
- recce/git.py +7 -7
- recce/github.py +57 -53
- recce/mcp_server.py +716 -0
- recce/models/__init__.py +4 -1
- recce/models/check.py +6 -7
- recce/models/run.py +1 -0
- recce/models/types.py +131 -28
- recce/pull_request.py +27 -25
- recce/run.py +165 -121
- recce/server.py +303 -111
- recce/state/__init__.py +31 -0
- recce/state/cloud.py +632 -0
- recce/state/const.py +26 -0
- recce/state/local.py +56 -0
- recce/state/state.py +119 -0
- recce/state/state_loader.py +174 -0
- recce/summary.py +188 -143
- recce/tasks/__init__.py +19 -3
- recce/tasks/core.py +11 -13
- recce/tasks/dataframe.py +82 -18
- recce/tasks/histogram.py +69 -34
- recce/tasks/lineage.py +2 -2
- recce/tasks/profile.py +152 -86
- recce/tasks/query.py +139 -87
- recce/tasks/rowcount.py +37 -31
- recce/tasks/schema.py +18 -15
- recce/tasks/top_k.py +35 -35
- recce/tasks/valuediff.py +216 -152
- recce/util/__init__.py +3 -0
- recce/util/api_token.py +80 -0
- recce/util/breaking.py +87 -85
- recce/util/cll.py +274 -219
- recce/util/io.py +22 -17
- recce/util/lineage.py +65 -16
- recce/util/logger.py +1 -1
- recce/util/onboarding_state.py +45 -0
- recce/util/perf_tracking.py +85 -0
- recce/util/recce_cloud.py +322 -72
- recce/util/singleton.py +4 -4
- recce/yaml/__init__.py +7 -10
- recce_cloud/__init__.py +24 -0
- recce_cloud/api/__init__.py +17 -0
- recce_cloud/api/base.py +111 -0
- recce_cloud/api/client.py +150 -0
- recce_cloud/api/exceptions.py +26 -0
- recce_cloud/api/factory.py +63 -0
- recce_cloud/api/github.py +76 -0
- recce_cloud/api/gitlab.py +82 -0
- recce_cloud/artifact.py +57 -0
- recce_cloud/ci_providers/__init__.py +9 -0
- recce_cloud/ci_providers/base.py +82 -0
- recce_cloud/ci_providers/detector.py +147 -0
- recce_cloud/ci_providers/github_actions.py +136 -0
- recce_cloud/ci_providers/gitlab_ci.py +130 -0
- recce_cloud/cli.py +245 -0
- recce_cloud/upload.py +214 -0
- {recce_nightly-1.2.0.20250506.dist-info → recce_nightly-1.26.0.20251124.dist-info}/METADATA +68 -37
- recce_nightly-1.26.0.20251124.dist-info/RECORD +180 -0
- {recce_nightly-1.2.0.20250506.dist-info → recce_nightly-1.26.0.20251124.dist-info}/WHEEL +1 -1
- {recce_nightly-1.2.0.20250506.dist-info → recce_nightly-1.26.0.20251124.dist-info}/top_level.txt +1 -0
- tests/adapter/dbt_adapter/conftest.py +9 -5
- tests/adapter/dbt_adapter/dbt_test_helper.py +37 -22
- tests/adapter/dbt_adapter/test_dbt_adapter.py +0 -15
- tests/adapter/dbt_adapter/test_dbt_cll.py +656 -41
- tests/adapter/dbt_adapter/test_selector.py +22 -21
- tests/recce_cloud/__init__.py +0 -0
- tests/recce_cloud/test_ci_providers.py +351 -0
- tests/recce_cloud/test_cli.py +372 -0
- tests/recce_cloud/test_client.py +273 -0
- tests/recce_cloud/test_platform_clients.py +333 -0
- tests/tasks/conftest.py +1 -1
- tests/tasks/test_histogram.py +58 -66
- tests/tasks/test_lineage.py +36 -23
- tests/tasks/test_preset_checks.py +45 -31
- tests/tasks/test_profile.py +339 -15
- tests/tasks/test_query.py +46 -46
- tests/tasks/test_row_count.py +65 -46
- tests/tasks/test_schema.py +65 -42
- tests/tasks/test_top_k.py +22 -18
- tests/tasks/test_valuediff.py +43 -32
- tests/test_cli.py +174 -60
- tests/test_cli_mcp_optional.py +45 -0
- tests/test_cloud_listing_cli.py +324 -0
- tests/test_config.py +7 -9
- tests/test_connect_to_cloud.py +82 -0
- tests/test_core.py +151 -4
- tests/test_dbt.py +7 -7
- tests/test_mcp_server.py +332 -0
- tests/test_pull_request.py +1 -1
- tests/test_server.py +25 -19
- tests/test_summary.py +29 -17
- recce/data/_next/static/Kcbs3GEIyH2LxgLYat0es/_buildManifest.js +0 -1
- recce/data/_next/static/chunks/1f229bf6-d9fe92e56db8d93b.js +0 -1
- recce/data/_next/static/chunks/29e3cc0d-8c150e37dff9631b.js +0 -1
- recce/data/_next/static/chunks/368-7587b306577df275.js +0 -65
- recce/data/_next/static/chunks/36e1c10d-bb0210cbd6573a8d.js +0 -1
- recce/data/_next/static/chunks/3998a672-eaad84bdd88cc73e.js +0 -1
- recce/data/_next/static/chunks/3a92ee20-3b5d922d4157af5e.js +0 -1
- recce/data/_next/static/chunks/450c323b-1bb5db526e54435a.js +0 -1
- recce/data/_next/static/chunks/47d8844f-79a1b53c66a7d7ec.js +0 -1
- recce/data/_next/static/chunks/6dc81886-c94b9b91bc2c3caf.js +0 -1
- recce/data/_next/static/chunks/6ef81909-694dc38134099299.js +0 -1
- recce/data/_next/static/chunks/700-3b65fc3666820d00.js +0 -2
- recce/data/_next/static/chunks/7a8a3e83-d7fa409d97b38b2b.js +0 -1
- recce/data/_next/static/chunks/7f27ae6c-413f6b869a04183a.js +0 -1
- recce/data/_next/static/chunks/8d700b6a-f0b1f6b9e0d97ce2.js +0 -1
- recce/data/_next/static/chunks/9746af58-d74bef4d03eea6ab.js +0 -1
- recce/data/_next/static/chunks/a30376cd-7d806e1602f2dc3a.js +0 -1
- recce/data/_next/static/chunks/app/_not-found/page-8a886fa0855c3105.js +0 -1
- recce/data/_next/static/chunks/app/layout-9102e22cb73f74d6.js +0 -1
- recce/data/_next/static/chunks/app/page-cee661090afbd6aa.js +0 -1
- recce/data/_next/static/chunks/b63b1b3f-7395c74e11a14e95.js +0 -1
- recce/data/_next/static/chunks/c132bf7d-8102037f9ccf372a.js +0 -1
- recce/data/_next/static/chunks/c1ceaa8b-a1e442154d23515e.js +0 -1
- recce/data/_next/static/chunks/cd9f8d63-cf0d5a7b0f7a92e8.js +0 -54
- recce/data/_next/static/chunks/ce84277d-f42c2c58049cea2d.js +0 -1
- recce/data/_next/static/chunks/e24bf851-0f8cbc99656833e7.js +0 -1
- recce/data/_next/static/chunks/fee69bc6-f17d36c080742e74.js +0 -1
- recce/data/_next/static/chunks/framework-ded83d71b51ce901.js +0 -1
- recce/data/_next/static/chunks/main-a0859f1f36d0aa6c.js +0 -1
- recce/data/_next/static/chunks/main-app-0225a2255968e566.js +0 -1
- recce/data/_next/static/chunks/pages/_app-d5672bf3d8b6371b.js +0 -1
- recce/data/_next/static/chunks/pages/_error-ed75be3f25588548.js +0 -1
- recce/data/_next/static/chunks/webpack-567d72f0bc0820d5.js +0 -1
- recce/data/_next/static/css/c9ecb46a4b21c126.css +0 -14
- recce/data/_next/static/media/montserrat-cyrillic-800-normal.22628180.woff2 +0 -0
- recce/data/_next/static/media/montserrat-cyrillic-800-normal.31d693bb.woff +0 -0
- recce/data/_next/static/media/montserrat-cyrillic-ext-800-normal.7e2c1e62.woff +0 -0
- recce/data/_next/static/media/montserrat-cyrillic-ext-800-normal.94a63aea.woff2 +0 -0
- recce/data/_next/static/media/montserrat-latin-800-normal.6f8fa298.woff2 +0 -0
- recce/data/_next/static/media/montserrat-latin-800-normal.97e20d5e.woff +0 -0
- recce/data/_next/static/media/montserrat-latin-ext-800-normal.013b84f9.woff2 +0 -0
- recce/data/_next/static/media/montserrat-latin-ext-800-normal.aff52ab0.woff +0 -0
- recce/data/_next/static/media/montserrat-vietnamese-800-normal.5f21869b.woff +0 -0
- recce/data/_next/static/media/montserrat-vietnamese-800-normal.c0035377.woff2 +0 -0
- recce/state.py +0 -753
- recce_nightly-1.2.0.20250506.dist-info/RECORD +0 -142
- tests/test_state.py +0 -123
- /recce/data/_next/static/{Kcbs3GEIyH2LxgLYat0es → 52aV_JrNUZU6dMFgvTQEO}/_ssgManifest.js +0 -0
- /recce/data/_next/static/chunks/{polyfills-42372ed130431b0a.js → a6dad97d9634a72d.js} +0 -0
- {recce_nightly-1.2.0.20250506.dist-info → recce_nightly-1.26.0.20251124.dist-info}/entry_points.txt +0 -0
- {recce_nightly-1.2.0.20250506.dist-info → recce_nightly-1.26.0.20251124.dist-info}/licenses/LICENSE +0 -0
recce/cli.py
CHANGED
|
@@ -5,17 +5,39 @@ from typing import List
|
|
|
5
5
|
|
|
6
6
|
import click
|
|
7
7
|
import uvicorn
|
|
8
|
+
from click import Abort
|
|
8
9
|
|
|
9
10
|
from recce import event
|
|
10
|
-
from recce.artifact import
|
|
11
|
-
|
|
12
|
-
|
|
11
|
+
from recce.artifact import (
|
|
12
|
+
delete_dbt_artifacts,
|
|
13
|
+
download_dbt_artifacts,
|
|
14
|
+
upload_artifacts_to_session,
|
|
15
|
+
upload_dbt_artifacts,
|
|
16
|
+
)
|
|
17
|
+
from recce.config import RECCE_CONFIG_FILE, RECCE_ERROR_LOG_FILE, RecceConfig
|
|
18
|
+
from recce.connect_to_cloud import (
|
|
19
|
+
generate_key_pair,
|
|
20
|
+
prepare_connection_url,
|
|
21
|
+
run_one_time_http_server,
|
|
22
|
+
)
|
|
23
|
+
from recce.exceptions import RecceConfigException
|
|
13
24
|
from recce.git import current_branch, current_default_branch
|
|
14
|
-
from recce.run import
|
|
15
|
-
from recce.
|
|
25
|
+
from recce.run import check_github_ci_env, cli_run
|
|
26
|
+
from recce.server import RecceServerMode
|
|
27
|
+
from recce.state import (
|
|
28
|
+
CloudStateLoader,
|
|
29
|
+
FileStateLoader,
|
|
30
|
+
RecceCloudStateManager,
|
|
31
|
+
RecceShareStateManager,
|
|
32
|
+
)
|
|
16
33
|
from recce.summary import generate_markdown_summary
|
|
34
|
+
from recce.util.api_token import prepare_api_token, show_invalid_api_token_message
|
|
17
35
|
from recce.util.logger import CustomFormatter
|
|
18
|
-
from recce.util.
|
|
36
|
+
from recce.util.onboarding_state import update_onboarding_state
|
|
37
|
+
from recce.util.recce_cloud import (
|
|
38
|
+
RecceCloudException,
|
|
39
|
+
)
|
|
40
|
+
|
|
19
41
|
from .core import RecceContext, set_default_context
|
|
20
42
|
from .event.track import TrackCommand
|
|
21
43
|
|
|
@@ -24,11 +46,17 @@ event.init()
|
|
|
24
46
|
|
|
25
47
|
def create_state_loader(review_mode, cloud_mode, state_file, cloud_options):
|
|
26
48
|
from rich.console import Console
|
|
49
|
+
|
|
27
50
|
console = Console()
|
|
28
51
|
|
|
29
52
|
try:
|
|
30
|
-
|
|
31
|
-
|
|
53
|
+
state_loader = (
|
|
54
|
+
CloudStateLoader(review_mode=review_mode, cloud_options=cloud_options)
|
|
55
|
+
if cloud_mode
|
|
56
|
+
else FileStateLoader(review_mode=review_mode, state_file=state_file)
|
|
57
|
+
)
|
|
58
|
+
state_loader.load()
|
|
59
|
+
return state_loader
|
|
32
60
|
except RecceCloudException as e:
|
|
33
61
|
console.print("[[red]Error[/red]] Failed to load recce state file")
|
|
34
62
|
console.print(f"Reason: {e.reason}")
|
|
@@ -39,13 +67,91 @@ def create_state_loader(review_mode, cloud_mode, state_file, cloud_options):
|
|
|
39
67
|
exit(1)
|
|
40
68
|
|
|
41
69
|
|
|
70
|
+
def patch_derived_args(args):
|
|
71
|
+
"""
|
|
72
|
+
Patch derived args based on other args.
|
|
73
|
+
"""
|
|
74
|
+
if args.get("session_id") or args.get("share_url"):
|
|
75
|
+
args["cloud"] = True
|
|
76
|
+
args["review"] = True
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def create_state_loader_by_args(state_file=None, **kwargs):
|
|
80
|
+
"""
|
|
81
|
+
Create a state loader based on CLI arguments.
|
|
82
|
+
|
|
83
|
+
This function handles the cloud options logic that is shared between
|
|
84
|
+
server and mcp-server commands.
|
|
85
|
+
|
|
86
|
+
Args:
|
|
87
|
+
state_file: Optional path to state file
|
|
88
|
+
**kwargs: CLI arguments including api_token, cloud, review, session_id, share_url, etc.
|
|
89
|
+
|
|
90
|
+
Returns:
|
|
91
|
+
state_loader: The created state loader instance
|
|
92
|
+
"""
|
|
93
|
+
from rich.console import Console
|
|
94
|
+
|
|
95
|
+
console = Console()
|
|
96
|
+
|
|
97
|
+
api_token = kwargs.get("api_token")
|
|
98
|
+
is_review = kwargs.get("review", False)
|
|
99
|
+
is_cloud = kwargs.get("cloud", False)
|
|
100
|
+
cloud_options = None
|
|
101
|
+
|
|
102
|
+
# Handle share_url and session_id
|
|
103
|
+
share_url = kwargs.get("share_url")
|
|
104
|
+
session_id = kwargs.get("session_id")
|
|
105
|
+
|
|
106
|
+
if share_url:
|
|
107
|
+
share_id = share_url.split("/")[-1]
|
|
108
|
+
if not share_id:
|
|
109
|
+
console.print("[[red]Error[/red]] Invalid share URL format.")
|
|
110
|
+
exit(1)
|
|
111
|
+
|
|
112
|
+
if is_cloud:
|
|
113
|
+
# Cloud mode
|
|
114
|
+
if share_url:
|
|
115
|
+
cloud_options = {
|
|
116
|
+
"host": kwargs.get("state_file_host"),
|
|
117
|
+
"api_token": api_token,
|
|
118
|
+
"share_id": share_id,
|
|
119
|
+
}
|
|
120
|
+
elif session_id:
|
|
121
|
+
cloud_options = {
|
|
122
|
+
"host": kwargs.get("state_file_host"),
|
|
123
|
+
"api_token": api_token,
|
|
124
|
+
"session_id": session_id,
|
|
125
|
+
}
|
|
126
|
+
else:
|
|
127
|
+
cloud_options = {
|
|
128
|
+
"host": kwargs.get("state_file_host"),
|
|
129
|
+
"github_token": kwargs.get("cloud_token"),
|
|
130
|
+
"password": kwargs.get("password"),
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
# Create state loader
|
|
134
|
+
state_loader = create_state_loader(is_review, is_cloud, state_file, cloud_options)
|
|
135
|
+
|
|
136
|
+
return state_loader
|
|
137
|
+
|
|
138
|
+
|
|
42
139
|
def handle_debug_flag(**kwargs):
|
|
43
|
-
if kwargs.get(
|
|
140
|
+
if kwargs.get("debug"):
|
|
44
141
|
import logging
|
|
142
|
+
|
|
45
143
|
ch = logging.StreamHandler()
|
|
46
144
|
ch.setFormatter(CustomFormatter())
|
|
47
145
|
logging.basicConfig(handlers=[ch], level=logging.DEBUG)
|
|
48
146
|
|
|
147
|
+
# Explicitly set uvicorn logger to DEBUG level
|
|
148
|
+
uvicorn_logger = logging.getLogger("uvicorn")
|
|
149
|
+
uvicorn_logger.setLevel(logging.DEBUG)
|
|
150
|
+
|
|
151
|
+
# Set all child loggers to DEBUG as well
|
|
152
|
+
for handler in uvicorn_logger.handlers:
|
|
153
|
+
handler.setLevel(logging.DEBUG)
|
|
154
|
+
|
|
49
155
|
|
|
50
156
|
def add_options(options):
|
|
51
157
|
def _add_options(func):
|
|
@@ -57,43 +163,109 @@ def add_options(options):
|
|
|
57
163
|
|
|
58
164
|
|
|
59
165
|
dbt_related_options = [
|
|
60
|
-
click.option(
|
|
61
|
-
click.option(
|
|
62
|
-
click.option(
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
166
|
+
click.option("--target", "-t", help="Which target to load for the given profile.", type=click.STRING),
|
|
167
|
+
click.option("--profile", help="Which existing profile to load.", type=click.STRING),
|
|
168
|
+
click.option(
|
|
169
|
+
"--project-dir",
|
|
170
|
+
help="Which directory to look in for the dbt_project.yml file.",
|
|
171
|
+
type=click.Path(),
|
|
172
|
+
envvar="DBT_PROJECT_DIR",
|
|
173
|
+
),
|
|
174
|
+
click.option(
|
|
175
|
+
"--profiles-dir",
|
|
176
|
+
help="Which directory to look in for the profiles.yml file.",
|
|
177
|
+
type=click.Path(),
|
|
178
|
+
envvar="DBT_PROFILES_DIR",
|
|
179
|
+
),
|
|
66
180
|
]
|
|
67
181
|
|
|
68
182
|
sqlmesh_related_options = [
|
|
69
|
-
click.option(
|
|
70
|
-
click.option(
|
|
71
|
-
click.option(
|
|
183
|
+
click.option("--sqlmesh", is_flag=True, help="Use SQLMesh ", hidden=True),
|
|
184
|
+
click.option("--sqlmesh-envs", is_flag=False, help="SQLMesh envs to compare. SOURCE:TARGET", hidden=True),
|
|
185
|
+
click.option("--sqlmesh-config", is_flag=False, help="SQLMesh config name to use", hidden=True),
|
|
72
186
|
]
|
|
73
187
|
|
|
74
188
|
recce_options = [
|
|
75
|
-
click.option(
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
189
|
+
click.option(
|
|
190
|
+
"--config",
|
|
191
|
+
help="Path to the recce config file.",
|
|
192
|
+
type=click.Path(),
|
|
193
|
+
default=RECCE_CONFIG_FILE,
|
|
194
|
+
show_default=True,
|
|
195
|
+
),
|
|
196
|
+
click.option(
|
|
197
|
+
"--error-log", help="Path to the error log file.", type=click.Path(), default=RECCE_ERROR_LOG_FILE, hidden=True
|
|
198
|
+
),
|
|
199
|
+
click.option("--debug", is_flag=True, help="Enable debug mode.", hidden=True),
|
|
80
200
|
]
|
|
81
201
|
|
|
82
202
|
recce_cloud_options = [
|
|
83
|
-
click.option(
|
|
84
|
-
click.option(
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
203
|
+
click.option("--cloud", is_flag=True, help="Fetch the state file from cloud."),
|
|
204
|
+
click.option(
|
|
205
|
+
"--cloud-token", help="The GitHub token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN"
|
|
206
|
+
),
|
|
207
|
+
click.option(
|
|
208
|
+
"--state-file-host",
|
|
209
|
+
help="The host to fetch the state file from.",
|
|
210
|
+
type=click.STRING,
|
|
211
|
+
envvar="RECCE_STATE_FILE_HOST",
|
|
212
|
+
default="",
|
|
213
|
+
hidden=True,
|
|
214
|
+
),
|
|
215
|
+
click.option(
|
|
216
|
+
"--password",
|
|
217
|
+
"-p",
|
|
218
|
+
help="The password to encrypt the state file in cloud.",
|
|
219
|
+
type=click.STRING,
|
|
220
|
+
envvar="RECCE_STATE_PASSWORD",
|
|
221
|
+
),
|
|
222
|
+
]
|
|
223
|
+
|
|
224
|
+
recce_cloud_auth_options = [
|
|
225
|
+
click.option(
|
|
226
|
+
"--api-token",
|
|
227
|
+
help="The personal token generated by Recce Cloud.",
|
|
228
|
+
type=click.STRING,
|
|
229
|
+
envvar="RECCE_API_TOKEN",
|
|
230
|
+
)
|
|
90
231
|
]
|
|
91
232
|
|
|
92
233
|
recce_dbt_artifact_dir_options = [
|
|
93
|
-
click.option(
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
234
|
+
click.option(
|
|
235
|
+
"--target-path",
|
|
236
|
+
help="dbt artifacts directory for your development branch.",
|
|
237
|
+
type=click.STRING,
|
|
238
|
+
default="target",
|
|
239
|
+
),
|
|
240
|
+
click.option(
|
|
241
|
+
"--target-base-path",
|
|
242
|
+
help="dbt artifacts directory to be used as the base for the comparison.",
|
|
243
|
+
type=click.STRING,
|
|
244
|
+
default="target-base",
|
|
245
|
+
),
|
|
246
|
+
]
|
|
247
|
+
|
|
248
|
+
recce_hidden_options = [
|
|
249
|
+
click.option(
|
|
250
|
+
"--mode",
|
|
251
|
+
envvar="RECCE_SERVER_MODE",
|
|
252
|
+
type=click.Choice(RecceServerMode.available_members(), case_sensitive=False),
|
|
253
|
+
hidden=True,
|
|
254
|
+
),
|
|
255
|
+
click.option(
|
|
256
|
+
"--share-url",
|
|
257
|
+
help="The share URL triggers this instance.",
|
|
258
|
+
type=click.STRING,
|
|
259
|
+
envvar="RECCE_SHARE_URL",
|
|
260
|
+
hidden=True,
|
|
261
|
+
),
|
|
262
|
+
click.option(
|
|
263
|
+
"--session-id",
|
|
264
|
+
help="The session ID triggers this instance.",
|
|
265
|
+
type=click.STRING,
|
|
266
|
+
envvar=["RECCE_SESSION_ID", "RECCE_SNAPSHOT_ID"], # Backward compatibility with RECCE_SNAPSHOT_ID
|
|
267
|
+
hidden=True,
|
|
268
|
+
),
|
|
97
269
|
]
|
|
98
270
|
|
|
99
271
|
|
|
@@ -105,8 +277,9 @@ def _execute_sql(context, sql_template, base=False):
|
|
|
105
277
|
exit(1)
|
|
106
278
|
|
|
107
279
|
from recce.adapter.dbt_adapter import DbtAdapter
|
|
280
|
+
|
|
108
281
|
dbt_adapter: DbtAdapter = context.adapter
|
|
109
|
-
with dbt_adapter.connection_named(
|
|
282
|
+
with dbt_adapter.connection_named("recce"):
|
|
110
283
|
sql = dbt_adapter.generate_sql(sql_template, base)
|
|
111
284
|
response, result = dbt_adapter.execute(sql, fetch=True, auto_begin=True)
|
|
112
285
|
table = result
|
|
@@ -119,13 +292,15 @@ def _execute_sql(context, sql_template, base=False):
|
|
|
119
292
|
def cli(ctx, **kwargs):
|
|
120
293
|
"""Recce: Data validation toolkit for comprehensive PR review"""
|
|
121
294
|
from rich.console import Console
|
|
295
|
+
|
|
122
296
|
from recce import __is_recce_outdated__, __latest_version__
|
|
297
|
+
|
|
123
298
|
if __is_recce_outdated__ is True:
|
|
124
|
-
error_console = Console(stderr=True, style=
|
|
299
|
+
error_console = Console(stderr=True, style="bold")
|
|
125
300
|
error_console.print(
|
|
126
301
|
f"[[yellow]Update Available[/yellow]] A new version of Recce {__latest_version__} is available.",
|
|
127
302
|
)
|
|
128
|
-
error_console.print("Please update using the command: 'pip install --upgrade recce'.", end=
|
|
303
|
+
error_console.print("Please update using the command: 'pip install --upgrade recce'.", end="\n\n")
|
|
129
304
|
|
|
130
305
|
|
|
131
306
|
@cli.command(cls=TrackCommand)
|
|
@@ -134,12 +309,110 @@ def version():
|
|
|
134
309
|
Show version information
|
|
135
310
|
"""
|
|
136
311
|
from recce import __version__
|
|
312
|
+
|
|
137
313
|
print(__version__)
|
|
138
314
|
|
|
139
315
|
|
|
316
|
+
@cli.command(cls=TrackCommand)
|
|
317
|
+
@add_options(dbt_related_options)
|
|
318
|
+
@add_options(recce_dbt_artifact_dir_options)
|
|
319
|
+
def debug(**kwargs):
|
|
320
|
+
"""
|
|
321
|
+
Diagnose and verify Recce setup for the development and the base environments
|
|
322
|
+
"""
|
|
323
|
+
|
|
324
|
+
from rich.console import Console
|
|
325
|
+
|
|
326
|
+
from recce.adapter.dbt_adapter import DbtAdapter
|
|
327
|
+
from recce.core import load_context
|
|
328
|
+
|
|
329
|
+
console = Console()
|
|
330
|
+
|
|
331
|
+
def check_artifacts(env_name, target_path):
|
|
332
|
+
console.rule(f"{env_name} Environment", style="orange3")
|
|
333
|
+
if not target_path.is_dir():
|
|
334
|
+
console.print(f"[[red]MISS[/red]] Directory not found: {target_path}")
|
|
335
|
+
return [False, False, False]
|
|
336
|
+
|
|
337
|
+
console.print(f"[[green]OK[/green]] Directory exists: {target_path}")
|
|
338
|
+
|
|
339
|
+
manifest_path = target_path / "manifest.json"
|
|
340
|
+
manifest_is_ready = manifest_path.is_file()
|
|
341
|
+
if manifest_is_ready:
|
|
342
|
+
console.print(f"[[green]OK[/green]] Manifest JSON file exists : {manifest_path}")
|
|
343
|
+
else:
|
|
344
|
+
console.print(f"[[red]MISS[/red]] Manifest JSON file not found: {manifest_path}")
|
|
345
|
+
|
|
346
|
+
catalog_path = target_path / "catalog.json"
|
|
347
|
+
catalog_is_ready = catalog_path.is_file()
|
|
348
|
+
if catalog_is_ready:
|
|
349
|
+
console.print(f"[[green]OK[/green]] Catalog JSON file exists: {catalog_path}")
|
|
350
|
+
else:
|
|
351
|
+
console.print(f"[[red]MISS[/red]] Catalog JSON file not found: {catalog_path}")
|
|
352
|
+
|
|
353
|
+
return [True, manifest_is_ready, catalog_is_ready]
|
|
354
|
+
|
|
355
|
+
project_dir_path = Path(kwargs.get("project_dir") or "./")
|
|
356
|
+
target_path = project_dir_path.joinpath(Path(kwargs.get("target_path", "target")))
|
|
357
|
+
target_base_path = project_dir_path.joinpath(Path(kwargs.get("target_base_path", "target-base")))
|
|
358
|
+
|
|
359
|
+
curr_is_ready = check_artifacts("Development", target_path)
|
|
360
|
+
base_is_ready = check_artifacts("Base", target_base_path)
|
|
361
|
+
|
|
362
|
+
console.rule("Warehouse Connection", style="orange3")
|
|
363
|
+
conn_is_ready = True
|
|
364
|
+
try:
|
|
365
|
+
context_kwargs = {**kwargs, "target_base_path": kwargs.get("target_path")}
|
|
366
|
+
ctx = load_context(**context_kwargs)
|
|
367
|
+
dbt_adapter: DbtAdapter = ctx.adapter
|
|
368
|
+
sql = dbt_adapter.generate_sql("select 1", False)
|
|
369
|
+
dbt_adapter.execute(sql, fetch=True, auto_begin=True)
|
|
370
|
+
console.print("[[green]OK[/green]] Connection test")
|
|
371
|
+
except Exception:
|
|
372
|
+
conn_is_ready = False
|
|
373
|
+
console.print("[[red]FAIL[/red]] Connection test")
|
|
374
|
+
|
|
375
|
+
console.rule("Result", style="orange3")
|
|
376
|
+
if all(curr_is_ready) and all(base_is_ready) and conn_is_ready:
|
|
377
|
+
console.print("[[green]OK[/green]] Ready to launch! Type 'recce server'.")
|
|
378
|
+
elif all(curr_is_ready) and conn_is_ready:
|
|
379
|
+
console.print("[[orange3]OK[/orange3]] Ready to launch with [i]limited features[/i]. Type 'recce server'.")
|
|
380
|
+
|
|
381
|
+
if not curr_is_ready[0]:
|
|
382
|
+
console.print(
|
|
383
|
+
"[[orange3]TIP[/orange3]] Run dbt or overwrite the default directory of the development environment with '--target-path'."
|
|
384
|
+
)
|
|
385
|
+
else:
|
|
386
|
+
if not curr_is_ready[1]:
|
|
387
|
+
console.print(
|
|
388
|
+
"[[orange3]TIP[/orange3]] 'dbt run' to generate the manifest JSON file for the development environment."
|
|
389
|
+
)
|
|
390
|
+
if not curr_is_ready[2]:
|
|
391
|
+
console.print(
|
|
392
|
+
"[[orange3]TIP[/orange3]] 'dbt docs generate' to generate the catalog JSON file for the development environment."
|
|
393
|
+
)
|
|
394
|
+
|
|
395
|
+
if not base_is_ready[0]:
|
|
396
|
+
console.print(
|
|
397
|
+
"[[orange3]TIP[/orange3]] Run dbt with '--target-path target-base' or overwrite the default directory of the base environment with '--target-base-path'."
|
|
398
|
+
)
|
|
399
|
+
else:
|
|
400
|
+
if not base_is_ready[1]:
|
|
401
|
+
console.print(
|
|
402
|
+
"[[orange3]TIP[/orange3]] 'dbt run --target-path target-base' to generate the manifest JSON file for the base environment."
|
|
403
|
+
)
|
|
404
|
+
if not base_is_ready[2]:
|
|
405
|
+
console.print(
|
|
406
|
+
"[[orange3]TIP[/orange3]] 'dbt docs generate --target-path target-base' to generate the catalog JSON file for the base environment."
|
|
407
|
+
)
|
|
408
|
+
|
|
409
|
+
if not conn_is_ready:
|
|
410
|
+
console.print("[[orange3]TIP[/orange3]] Run 'dbt debug' to check the connection.")
|
|
411
|
+
|
|
412
|
+
|
|
140
413
|
@cli.command(hidden=True, cls=TrackCommand)
|
|
141
|
-
@click.option(
|
|
142
|
-
@click.option(
|
|
414
|
+
@click.option("--sql", help="Sql template to query", required=True)
|
|
415
|
+
@click.option("--base", is_flag=True, help="Run the query on the base environment")
|
|
143
416
|
@add_options(dbt_related_options)
|
|
144
417
|
def query(sql, base: bool = False, **kwargs):
|
|
145
418
|
"""
|
|
@@ -155,20 +428,25 @@ def query(sql, base: bool = False, **kwargs):
|
|
|
155
428
|
"""
|
|
156
429
|
context = RecceContext.load(**kwargs)
|
|
157
430
|
result = _execute_sql(context, sql, base=base)
|
|
158
|
-
print(result.to_string(na_rep=
|
|
431
|
+
print(result.to_string(na_rep="-", index=False))
|
|
159
432
|
|
|
160
433
|
|
|
161
434
|
def _split_comma_separated(ctx, param, value):
|
|
162
|
-
return value.split(
|
|
435
|
+
return value.split(",") if value else None
|
|
163
436
|
|
|
164
437
|
|
|
165
438
|
@cli.command(hidden=True, cls=TrackCommand)
|
|
166
|
-
@click.option(
|
|
167
|
-
@click.option(
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
439
|
+
@click.option("--sql", help="Sql template to query.", required=True)
|
|
440
|
+
@click.option(
|
|
441
|
+
"--primary-keys",
|
|
442
|
+
type=click.STRING,
|
|
443
|
+
help="Comma-separated list of primary key columns.",
|
|
444
|
+
callback=_split_comma_separated,
|
|
445
|
+
)
|
|
446
|
+
@click.option("--keep-shape", is_flag=True, help="Keep unchanged columns. Otherwise, unchanged columns are hidden.")
|
|
447
|
+
@click.option(
|
|
448
|
+
"--keep-equal", is_flag=True, help='Keep values that are equal. Otherwise, equal values are shown as "-".'
|
|
449
|
+
)
|
|
172
450
|
@add_options(dbt_related_options)
|
|
173
451
|
def diff(sql, primary_keys: List[str] = None, keep_shape: bool = False, keep_equal: bool = False, **kwargs):
|
|
174
452
|
"""
|
|
@@ -189,27 +467,34 @@ def diff(sql, primary_keys: List[str] = None, keep_shape: bool = False, keep_equ
|
|
|
189
467
|
after.set_index(primary_keys, inplace=True)
|
|
190
468
|
|
|
191
469
|
before_aligned, after_aligned = before.align(after)
|
|
192
|
-
diff = before_aligned.compare(
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
print(diff.to_string(na_rep='-') if not diff.empty else 'no changes')
|
|
470
|
+
diff = before_aligned.compare(
|
|
471
|
+
after_aligned, result_names=("base", "current"), keep_equal=keep_equal, keep_shape=keep_shape
|
|
472
|
+
)
|
|
473
|
+
print(diff.to_string(na_rep="-") if not diff.empty else "no changes")
|
|
197
474
|
|
|
198
475
|
|
|
199
476
|
@cli.command(cls=TrackCommand)
|
|
200
|
-
@click.argument(
|
|
201
|
-
@click.option(
|
|
202
|
-
@click.option(
|
|
203
|
-
@click.option(
|
|
204
|
-
@click.option(
|
|
205
|
-
|
|
206
|
-
|
|
477
|
+
@click.argument("state_file", required=False)
|
|
478
|
+
@click.option("--host", default="localhost", show_default=True, help="The host to bind to.")
|
|
479
|
+
@click.option("--port", default=8000, show_default=True, help="The port to bind to.", type=int)
|
|
480
|
+
@click.option("--lifetime", default=0, show_default=True, help="The lifetime of the server in seconds.", type=int)
|
|
481
|
+
@click.option(
|
|
482
|
+
"--idle-timeout",
|
|
483
|
+
default=0,
|
|
484
|
+
show_default=True,
|
|
485
|
+
help="The idle timeout in seconds. If 0, idle timeout is disabled. Maximum value is capped by lifetime.",
|
|
486
|
+
type=int,
|
|
487
|
+
)
|
|
488
|
+
@click.option("--review", is_flag=True, help="Open the state file in the review mode.")
|
|
489
|
+
@click.option("--single-env", is_flag=True, help="Launch in single environment mode directly.")
|
|
207
490
|
@add_options(dbt_related_options)
|
|
208
491
|
@add_options(sqlmesh_related_options)
|
|
209
492
|
@add_options(recce_options)
|
|
210
493
|
@add_options(recce_dbt_artifact_dir_options)
|
|
211
494
|
@add_options(recce_cloud_options)
|
|
212
|
-
|
|
495
|
+
@add_options(recce_cloud_auth_options)
|
|
496
|
+
@add_options(recce_hidden_options)
|
|
497
|
+
def server(host, port, lifetime, idle_timeout=0, state_file=None, **kwargs):
|
|
213
498
|
"""
|
|
214
499
|
Launch the recce server
|
|
215
500
|
|
|
@@ -230,59 +515,72 @@ def server(host, port, lifetime, state_file=None, **kwargs):
|
|
|
230
515
|
recce server --review recce_state.json
|
|
231
516
|
|
|
232
517
|
\b
|
|
233
|
-
# Launch the server
|
|
518
|
+
# Launch the server using the state from the PR of your current branch. (Requires GitHub token)
|
|
519
|
+
export GITHUB_TOKEN=<your-github-token>
|
|
234
520
|
recce server --cloud
|
|
235
521
|
recce server --review --cloud
|
|
236
522
|
|
|
237
523
|
"""
|
|
238
524
|
|
|
239
|
-
from .server import app, AppState
|
|
240
525
|
from rich.console import Console
|
|
526
|
+
from rich.prompt import Confirm
|
|
241
527
|
|
|
242
|
-
|
|
528
|
+
from .server import AppState, app
|
|
529
|
+
|
|
530
|
+
RecceConfig(config_file=kwargs.get("config"))
|
|
243
531
|
|
|
244
532
|
handle_debug_flag(**kwargs)
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
533
|
+
patch_derived_args(kwargs)
|
|
534
|
+
|
|
535
|
+
server_mode = kwargs.get("mode") if kwargs.get("mode") else RecceServerMode.server
|
|
536
|
+
is_review = kwargs.get("review", False)
|
|
537
|
+
is_cloud = kwargs.get("cloud", False)
|
|
249
538
|
flag = {
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
539
|
+
"single_env_onboarding": False,
|
|
540
|
+
"show_relaunch_hint": False,
|
|
541
|
+
"preview": False,
|
|
542
|
+
"read_only": False,
|
|
253
543
|
}
|
|
254
|
-
|
|
255
|
-
cloud_options = {
|
|
256
|
-
'host': kwargs.get('state_file_host'),
|
|
257
|
-
'token': kwargs.get('cloud_token'),
|
|
258
|
-
'password': kwargs.get('password'),
|
|
259
|
-
}
|
|
260
|
-
cloud_onboarding_state = get_recce_cloud_onboarding_state(kwargs.get('cloud_token'))
|
|
261
|
-
flag['show_onboarding_guide'] = False if cloud_onboarding_state == 'completed' else True
|
|
262
|
-
|
|
263
|
-
auth_options = {}
|
|
264
|
-
api_token = kwargs.get('api_token') if kwargs.get('api_token') else get_recce_api_token()
|
|
265
|
-
auth_options['api_token'] = api_token
|
|
266
|
-
|
|
267
|
-
# Check Single Environment Onboarding Mode if the review mode is False
|
|
268
|
-
if not os.path.isdir(kwargs.get('target_base_path')) and is_review is False:
|
|
269
|
-
# Mark as single env onboarding mode if user provides the target-path only
|
|
270
|
-
flag['single_env_onboarding'] = True
|
|
271
|
-
flag['show_relaunch_hint'] = True
|
|
272
|
-
target_path = kwargs.get('target_path')
|
|
273
|
-
target_base_path = kwargs.get('target_base_path')
|
|
274
|
-
# Use the target path as the base path
|
|
275
|
-
kwargs['target_base_path'] = target_path
|
|
544
|
+
console = Console()
|
|
276
545
|
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
546
|
+
# Prepare API token
|
|
547
|
+
try:
|
|
548
|
+
api_token = prepare_api_token(**kwargs)
|
|
549
|
+
kwargs["api_token"] = api_token
|
|
550
|
+
except RecceConfigException:
|
|
551
|
+
show_invalid_api_token_message()
|
|
552
|
+
exit(1)
|
|
553
|
+
auth_options = {
|
|
554
|
+
"api_token": api_token,
|
|
555
|
+
}
|
|
284
556
|
|
|
285
|
-
|
|
557
|
+
# Check Single Environment Onboarding Mode if not in cloud mode and not in review mode
|
|
558
|
+
if not is_cloud and not is_review:
|
|
559
|
+
project_dir_path = Path(kwargs.get("project_dir") or "./")
|
|
560
|
+
target_base_path = project_dir_path.joinpath(Path(kwargs.get("target_base_path", "target-base")))
|
|
561
|
+
if not target_base_path.is_dir():
|
|
562
|
+
# Mark as single env onboarding mode if user provides the target-path only
|
|
563
|
+
flag["single_env_onboarding"] = True
|
|
564
|
+
flag["show_relaunch_hint"] = True
|
|
565
|
+
# Use the target path as the base path
|
|
566
|
+
kwargs["target_base_path"] = kwargs.get("target_path")
|
|
567
|
+
|
|
568
|
+
# Server mode:
|
|
569
|
+
#
|
|
570
|
+
# It's used to determine the features disabled in the Web UI. Only used in the cloud-managed recce instances.
|
|
571
|
+
#
|
|
572
|
+
# Read-Only: No run query, no checklist
|
|
573
|
+
# Preview (Metadata-Only): No run query
|
|
574
|
+
if server_mode == RecceServerMode.preview:
|
|
575
|
+
flag["preview"] = True
|
|
576
|
+
elif server_mode == RecceServerMode.read_only:
|
|
577
|
+
flag["read_only"] = True
|
|
578
|
+
|
|
579
|
+
# Onboarding State logic update here
|
|
580
|
+
update_onboarding_state(api_token, flag.get("single_env_onboarding"))
|
|
581
|
+
|
|
582
|
+
# Create state loader using shared function
|
|
583
|
+
state_loader = create_state_loader_by_args(state_file, **kwargs)
|
|
286
584
|
|
|
287
585
|
if not state_loader.verify():
|
|
288
586
|
error, hint = state_loader.error_and_hint
|
|
@@ -290,38 +588,104 @@ def server(host, port, lifetime, state_file=None, **kwargs):
|
|
|
290
588
|
console.print(f"{hint}")
|
|
291
589
|
exit(1)
|
|
292
590
|
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
591
|
+
try:
|
|
592
|
+
result, message = RecceContext.verify_required_artifacts(**kwargs)
|
|
593
|
+
except Exception as e:
|
|
594
|
+
result = False
|
|
595
|
+
error_type = type(e).__name__
|
|
596
|
+
error_message = str(e)
|
|
597
|
+
message = f"{error_type}: {error_message}"
|
|
299
598
|
if not result:
|
|
599
|
+
console.rule("Notice", style="orange3")
|
|
300
600
|
console.print(f"[[red]Error[/red]] {message}")
|
|
301
601
|
exit(1)
|
|
302
602
|
|
|
303
|
-
|
|
304
|
-
|
|
603
|
+
if state_loader.review_mode:
|
|
604
|
+
console.rule("Recce Server : Review Mode")
|
|
605
|
+
elif flag.get("single_env_onboarding"):
|
|
606
|
+
# Show warning message
|
|
607
|
+
console.rule("Notice", style="orange3")
|
|
608
|
+
console.print(
|
|
609
|
+
"Recce will launch with limited features (no environment comparison).\n"
|
|
610
|
+
"\n"
|
|
611
|
+
"For full functionality, set up a base environment first.\n"
|
|
612
|
+
"Setup help: 'recce debug' or https://docs.datarecce.io/configure-diff/\n"
|
|
613
|
+
)
|
|
614
|
+
|
|
615
|
+
single_env_flag = kwargs.get("single_env", False)
|
|
616
|
+
if not single_env_flag:
|
|
617
|
+
lanch_in_single_env = Confirm.ask("Continue to launch Recce?")
|
|
618
|
+
if not lanch_in_single_env:
|
|
619
|
+
exit(0)
|
|
620
|
+
|
|
621
|
+
console.rule("Recce Server : Limited Features")
|
|
622
|
+
else:
|
|
623
|
+
console.rule("Recce Server")
|
|
624
|
+
|
|
625
|
+
# Validate idle_timeout: cap at lifetime if it exceeds lifetime
|
|
626
|
+
if idle_timeout > 0:
|
|
627
|
+
# If lifetime is set (> 0) and idle_timeout exceeds it, cap to lifetime
|
|
628
|
+
if lifetime > 0 and idle_timeout > lifetime:
|
|
629
|
+
effective_idle_timeout = lifetime
|
|
630
|
+
console.print(
|
|
631
|
+
f"[[yellow]Warning[/yellow]] idle_timeout ({idle_timeout}s) exceeds lifetime ({lifetime}s). "
|
|
632
|
+
f"Capping idle_timeout to {effective_idle_timeout}s."
|
|
633
|
+
)
|
|
634
|
+
else:
|
|
635
|
+
# Use idle_timeout as-is (either lifetime is 0, or idle_timeout <= lifetime)
|
|
636
|
+
effective_idle_timeout = idle_timeout
|
|
637
|
+
else:
|
|
638
|
+
# idle_timeout is 0 or negative, disable idle timeout
|
|
639
|
+
effective_idle_timeout = 0
|
|
640
|
+
|
|
641
|
+
state = AppState(
|
|
642
|
+
command=server_mode,
|
|
643
|
+
state_loader=state_loader,
|
|
644
|
+
kwargs=kwargs,
|
|
645
|
+
flag=flag,
|
|
646
|
+
auth_options=auth_options,
|
|
647
|
+
lifetime=lifetime,
|
|
648
|
+
idle_timeout=effective_idle_timeout,
|
|
649
|
+
share_url=kwargs.get("share_url"),
|
|
650
|
+
organization_name=os.environ.get("RECCE_SESSION_ORGANIZATION_NAME"),
|
|
651
|
+
web_url=os.environ.get("RECCE_CLOUD_WEB_URL"),
|
|
652
|
+
)
|
|
305
653
|
app.state = state
|
|
306
654
|
|
|
307
|
-
|
|
655
|
+
if server_mode == RecceServerMode.read_only:
|
|
656
|
+
set_default_context(RecceContext.load(**kwargs, state_loader=state_loader))
|
|
657
|
+
|
|
658
|
+
uvicorn.run(app, host=host, port=port, lifespan="on")
|
|
308
659
|
|
|
309
660
|
|
|
310
|
-
DEFAULT_RECCE_STATE_FILE =
|
|
661
|
+
DEFAULT_RECCE_STATE_FILE = "recce_state.json"
|
|
311
662
|
|
|
312
663
|
|
|
313
664
|
@cli.command(cls=TrackCommand)
|
|
314
|
-
@click.option(
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
@click.option(
|
|
324
|
-
|
|
665
|
+
@click.option(
|
|
666
|
+
"-o",
|
|
667
|
+
"--output",
|
|
668
|
+
help="Path of the output state file.",
|
|
669
|
+
type=click.Path(),
|
|
670
|
+
default=DEFAULT_RECCE_STATE_FILE,
|
|
671
|
+
show_default=True,
|
|
672
|
+
)
|
|
673
|
+
@click.option("--state-file", help="Path of the import state file.", type=click.Path())
|
|
674
|
+
@click.option("--summary", help="Path of the summary markdown file.", type=click.Path())
|
|
675
|
+
@click.option("--skip-query", is_flag=True, help="Skip running the queries for the checks.")
|
|
676
|
+
@click.option("--skip-check", is_flag=True, help="Skip running the checks.")
|
|
677
|
+
@click.option(
|
|
678
|
+
"--git-current-branch",
|
|
679
|
+
help="The git branch of the current environment.",
|
|
680
|
+
type=click.STRING,
|
|
681
|
+
envvar="GITHUB_HEAD_REF",
|
|
682
|
+
)
|
|
683
|
+
@click.option(
|
|
684
|
+
"--git-base-branch", help="The git branch of the base environment.", type=click.STRING, envvar="GITHUB_BASE_REF"
|
|
685
|
+
)
|
|
686
|
+
@click.option(
|
|
687
|
+
"--github-pull-request-url", help="The github pull request url to use for the lineage.", type=click.STRING
|
|
688
|
+
)
|
|
325
689
|
@add_options(dbt_related_options)
|
|
326
690
|
@add_options(sqlmesh_related_options)
|
|
327
691
|
@add_options(recce_options)
|
|
@@ -347,25 +711,31 @@ def run(output, **kwargs):
|
|
|
347
711
|
|
|
348
712
|
"""
|
|
349
713
|
from rich.console import Console
|
|
714
|
+
|
|
350
715
|
handle_debug_flag(**kwargs)
|
|
351
716
|
console = Console()
|
|
352
717
|
is_github_action, pr_url = check_github_ci_env(**kwargs)
|
|
353
718
|
if is_github_action is True and pr_url is not None:
|
|
354
|
-
kwargs[
|
|
719
|
+
kwargs["github_pull_request_url"] = pr_url
|
|
355
720
|
|
|
356
721
|
# Initialize Recce Config
|
|
357
|
-
RecceConfig(config_file=kwargs.get(
|
|
358
|
-
|
|
359
|
-
cloud_mode = kwargs.get(
|
|
360
|
-
state_file = kwargs.get(
|
|
361
|
-
cloud_options =
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
722
|
+
RecceConfig(config_file=kwargs.get("config"))
|
|
723
|
+
|
|
724
|
+
cloud_mode = kwargs.get("cloud", False)
|
|
725
|
+
state_file = kwargs.get("state_file")
|
|
726
|
+
cloud_options = (
|
|
727
|
+
{
|
|
728
|
+
"host": kwargs.get("state_file_host"),
|
|
729
|
+
"github_token": kwargs.get("cloud_token"),
|
|
730
|
+
"password": kwargs.get("password"),
|
|
731
|
+
}
|
|
732
|
+
if cloud_mode
|
|
733
|
+
else None
|
|
734
|
+
)
|
|
366
735
|
|
|
367
|
-
state_loader = create_state_loader(
|
|
368
|
-
|
|
736
|
+
state_loader = create_state_loader(
|
|
737
|
+
review_mode=False, cloud_mode=cloud_mode, state_file=state_file, cloud_options=cloud_options
|
|
738
|
+
)
|
|
369
739
|
|
|
370
740
|
if not state_loader.verify():
|
|
371
741
|
error, hint = state_loader.error_and_hint
|
|
@@ -380,7 +750,7 @@ def run(output, **kwargs):
|
|
|
380
750
|
|
|
381
751
|
# Verify the output state file path
|
|
382
752
|
try:
|
|
383
|
-
if os.path.isdir(output) or output.endswith(
|
|
753
|
+
if os.path.isdir(output) or output.endswith("/"):
|
|
384
754
|
|
|
385
755
|
output_dir = Path(output)
|
|
386
756
|
# Create the directory if not exists
|
|
@@ -388,7 +758,8 @@ def run(output, **kwargs):
|
|
|
388
758
|
output = os.path.join(output, DEFAULT_RECCE_STATE_FILE)
|
|
389
759
|
console.print(
|
|
390
760
|
f"[[yellow]Warning[/yellow]] The path '{output_dir}' is a directory. "
|
|
391
|
-
f"The state file will be saved as '{output}'."
|
|
761
|
+
f"The state file will be saved as '{output}'."
|
|
762
|
+
)
|
|
392
763
|
else:
|
|
393
764
|
# Create the parent directory if not exists
|
|
394
765
|
output_dir = Path(output).parent
|
|
@@ -402,30 +773,43 @@ def run(output, **kwargs):
|
|
|
402
773
|
|
|
403
774
|
|
|
404
775
|
@cli.command(cls=TrackCommand)
|
|
405
|
-
@click.argument(
|
|
406
|
-
@click.option(
|
|
407
|
-
|
|
408
|
-
|
|
776
|
+
@click.argument("state_file", required=False)
|
|
777
|
+
@click.option(
|
|
778
|
+
"--format",
|
|
779
|
+
"-f",
|
|
780
|
+
help="Output format. Currently only markdown is supported.",
|
|
781
|
+
type=click.Choice(["markdown", "mermaid", "check"], case_sensitive=False),
|
|
782
|
+
default="markdown",
|
|
783
|
+
show_default=True,
|
|
784
|
+
hidden=True,
|
|
785
|
+
)
|
|
409
786
|
@add_options(dbt_related_options)
|
|
410
787
|
@add_options(recce_options)
|
|
411
788
|
@add_options(recce_cloud_options)
|
|
412
789
|
def summary(state_file, **kwargs):
|
|
413
790
|
"""
|
|
414
|
-
|
|
791
|
+
Generate a summary of the recce state file
|
|
415
792
|
"""
|
|
416
793
|
from rich.console import Console
|
|
794
|
+
|
|
417
795
|
from .core import load_context
|
|
796
|
+
|
|
418
797
|
handle_debug_flag(**kwargs)
|
|
419
798
|
console = Console()
|
|
420
|
-
cloud_mode = kwargs.get(
|
|
421
|
-
cloud_options =
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
799
|
+
cloud_mode = kwargs.get("cloud", False)
|
|
800
|
+
cloud_options = (
|
|
801
|
+
{
|
|
802
|
+
"host": kwargs.get("state_file_host"),
|
|
803
|
+
"github_token": kwargs.get("cloud_token"),
|
|
804
|
+
"password": kwargs.get("password"),
|
|
805
|
+
}
|
|
806
|
+
if cloud_mode
|
|
807
|
+
else None
|
|
808
|
+
)
|
|
426
809
|
|
|
427
|
-
state_loader = create_state_loader(
|
|
428
|
-
|
|
810
|
+
state_loader = create_state_loader(
|
|
811
|
+
review_mode=True, cloud_mode=cloud_mode, state_file=state_file, cloud_options=cloud_options
|
|
812
|
+
)
|
|
429
813
|
|
|
430
814
|
if not state_loader.verify():
|
|
431
815
|
error, hint = state_loader.error_and_hint
|
|
@@ -440,56 +824,93 @@ def summary(state_file, **kwargs):
|
|
|
440
824
|
console.print(f"{e}")
|
|
441
825
|
exit(1)
|
|
442
826
|
|
|
443
|
-
output = generate_markdown_summary(ctx, summary_format=kwargs.get(
|
|
827
|
+
output = generate_markdown_summary(ctx, summary_format=kwargs.get("format"))
|
|
444
828
|
print(output)
|
|
445
829
|
|
|
446
830
|
|
|
447
|
-
@cli.
|
|
831
|
+
@cli.command(cls=TrackCommand)
|
|
832
|
+
def connect_to_cloud():
|
|
833
|
+
"""
|
|
834
|
+
Connect OSS to Cloud
|
|
835
|
+
"""
|
|
836
|
+
import webbrowser
|
|
837
|
+
|
|
838
|
+
from rich.console import Console
|
|
839
|
+
|
|
840
|
+
console = Console()
|
|
841
|
+
|
|
842
|
+
# Prepare RSA keys for connecting to cloud
|
|
843
|
+
private_key, public_key = generate_key_pair()
|
|
844
|
+
|
|
845
|
+
connect_url, callback_port = prepare_connection_url(public_key)
|
|
846
|
+
console.rule("Connecting to Recce Cloud")
|
|
847
|
+
console.print("Attempting to automatically open the Recce Cloud authorization page in your default browser.")
|
|
848
|
+
console.print("If the browser does not open, please open the following URL:")
|
|
849
|
+
console.print(connect_url)
|
|
850
|
+
webbrowser.open(connect_url)
|
|
851
|
+
|
|
852
|
+
# Launch a callback HTTP server for fetching the api-token
|
|
853
|
+
run_one_time_http_server(private_key, port=callback_port)
|
|
854
|
+
|
|
855
|
+
|
|
856
|
+
@cli.group("cloud", short_help="Manage Recce Cloud state file.")
|
|
448
857
|
def cloud(**kwargs):
|
|
449
858
|
# Manage Recce Cloud.
|
|
450
859
|
pass
|
|
451
860
|
|
|
452
861
|
|
|
453
862
|
@cloud.command(cls=TrackCommand)
|
|
454
|
-
@click.option(
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
863
|
+
@click.option("--cloud-token", help="The GitHub token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN")
|
|
864
|
+
@click.option(
|
|
865
|
+
"--state-file-host",
|
|
866
|
+
help="The host to fetch the state file from.",
|
|
867
|
+
type=click.STRING,
|
|
868
|
+
envvar="RECCE_STATE_FILE_HOST",
|
|
869
|
+
default="",
|
|
870
|
+
hidden=True,
|
|
871
|
+
)
|
|
872
|
+
@click.option(
|
|
873
|
+
"--password",
|
|
874
|
+
"-p",
|
|
875
|
+
help="The password to encrypt the state file in cloud.",
|
|
876
|
+
type=click.STRING,
|
|
877
|
+
envvar="RECCE_STATE_PASSWORD",
|
|
878
|
+
)
|
|
879
|
+
@click.option("--force", "-f", help="Bypasses the confirmation prompt. Purge the state file directly.", is_flag=True)
|
|
461
880
|
@add_options(recce_options)
|
|
462
881
|
def purge(**kwargs):
|
|
463
882
|
"""
|
|
464
|
-
|
|
883
|
+
Purge the state file from cloud
|
|
465
884
|
"""
|
|
466
885
|
from rich.console import Console
|
|
886
|
+
|
|
467
887
|
handle_debug_flag(**kwargs)
|
|
468
888
|
console = Console()
|
|
469
889
|
state_loader = None
|
|
470
890
|
cloud_options = {
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
891
|
+
"host": kwargs.get("state_file_host"),
|
|
892
|
+
"github_token": kwargs.get("cloud_token"),
|
|
893
|
+
"password": kwargs.get("password"),
|
|
474
894
|
}
|
|
475
|
-
force_to_purge = kwargs.get(
|
|
895
|
+
force_to_purge = kwargs.get("force", False)
|
|
476
896
|
|
|
477
897
|
try:
|
|
478
|
-
console.rule(
|
|
479
|
-
state_loader =
|
|
480
|
-
|
|
898
|
+
console.rule("Check Recce State from Cloud")
|
|
899
|
+
state_loader = create_state_loader(
|
|
900
|
+
review_mode=False, cloud_mode=True, state_file=None, cloud_options=cloud_options
|
|
901
|
+
)
|
|
481
902
|
except Exception:
|
|
482
903
|
console.print("[[yellow]Skip[/yellow]] Cannot access existing state file from cloud. Purge it directly.")
|
|
483
904
|
|
|
484
905
|
if state_loader is None:
|
|
485
906
|
try:
|
|
486
|
-
if force_to_purge is True or click.confirm(
|
|
907
|
+
if force_to_purge is True or click.confirm("\nDo you want to purge the state file?"):
|
|
487
908
|
rc, err_msg = RecceCloudStateManager(cloud_options).purge_cloud_state()
|
|
488
909
|
if rc is True:
|
|
489
|
-
console.rule(
|
|
910
|
+
console.rule("Purged Successfully")
|
|
490
911
|
else:
|
|
491
|
-
console.rule(
|
|
492
|
-
console.print(f
|
|
912
|
+
console.rule("Failed to Purge", style="red")
|
|
913
|
+
console.print(f"Reason: {err_msg}")
|
|
493
914
|
|
|
494
915
|
except click.exceptions.Abort:
|
|
495
916
|
pass
|
|
@@ -500,21 +921,21 @@ def purge(**kwargs):
|
|
|
500
921
|
console.print("[[yellow]Skip[/yellow]] No state file found in cloud.")
|
|
501
922
|
return 0
|
|
502
923
|
|
|
503
|
-
pr_info = info.get(
|
|
504
|
-
console.print(
|
|
505
|
-
console.print(
|
|
506
|
-
console.print(f
|
|
507
|
-
console.print(f
|
|
924
|
+
pr_info = info.get("pull_request")
|
|
925
|
+
console.print("[green]State File hosted by[/green]", info.get("source"))
|
|
926
|
+
console.print("[green]GitHub Repository[/green]", info.get("pull_request").repository)
|
|
927
|
+
console.print(f"[green]GitHub Pull Request[/green]\n{pr_info.title} #{pr_info.id}")
|
|
928
|
+
console.print(f"Branch merged into [blue]{pr_info.base_branch}[/blue] from [blue]{pr_info.branch}[/blue]")
|
|
508
929
|
console.print(pr_info.url)
|
|
509
930
|
|
|
510
931
|
try:
|
|
511
|
-
if force_to_purge is True or click.confirm(
|
|
932
|
+
if force_to_purge is True or click.confirm("\nDo you want to purge the state file?"):
|
|
512
933
|
response = state_loader.purge()
|
|
513
934
|
if response is True:
|
|
514
|
-
console.rule(
|
|
935
|
+
console.rule("Purged Successfully")
|
|
515
936
|
else:
|
|
516
|
-
console.rule(
|
|
517
|
-
console.print(f
|
|
937
|
+
console.rule("Failed to Purge", style="red")
|
|
938
|
+
console.print(f"Reason: {state_loader.error_message}")
|
|
518
939
|
except click.exceptions.Abort:
|
|
519
940
|
pass
|
|
520
941
|
|
|
@@ -522,32 +943,43 @@ def purge(**kwargs):
|
|
|
522
943
|
|
|
523
944
|
|
|
524
945
|
@cloud.command(cls=TrackCommand)
|
|
525
|
-
@click.argument(
|
|
526
|
-
@click.option(
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
946
|
+
@click.argument("state_file", type=click.Path(exists=True))
|
|
947
|
+
@click.option("--cloud-token", help="The GitHub token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN")
|
|
948
|
+
@click.option(
|
|
949
|
+
"--state-file-host",
|
|
950
|
+
help="The host to fetch the state file from.",
|
|
951
|
+
type=click.STRING,
|
|
952
|
+
envvar="RECCE_STATE_FILE_HOST",
|
|
953
|
+
default="",
|
|
954
|
+
hidden=True,
|
|
955
|
+
)
|
|
956
|
+
@click.option(
|
|
957
|
+
"--password",
|
|
958
|
+
"-p",
|
|
959
|
+
help="The password to encrypt the state file in cloud.",
|
|
960
|
+
type=click.STRING,
|
|
961
|
+
envvar="RECCE_STATE_PASSWORD",
|
|
962
|
+
)
|
|
532
963
|
@add_options(recce_options)
|
|
533
964
|
def upload(state_file, **kwargs):
|
|
534
965
|
"""
|
|
535
|
-
|
|
966
|
+
Upload the state file to cloud
|
|
536
967
|
"""
|
|
537
968
|
from rich.console import Console
|
|
538
969
|
|
|
539
970
|
handle_debug_flag(**kwargs)
|
|
540
971
|
cloud_options = {
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
972
|
+
"host": kwargs.get("state_file_host"),
|
|
973
|
+
"github_token": kwargs.get("cloud_token"),
|
|
974
|
+
"password": kwargs.get("password"),
|
|
544
975
|
}
|
|
545
976
|
|
|
546
977
|
console = Console()
|
|
547
978
|
|
|
548
979
|
# load local state
|
|
549
|
-
state_loader = create_state_loader(
|
|
550
|
-
|
|
980
|
+
state_loader = create_state_loader(
|
|
981
|
+
review_mode=False, cloud_mode=False, state_file=state_file, cloud_options=cloud_options
|
|
982
|
+
)
|
|
551
983
|
|
|
552
984
|
if not state_loader.verify():
|
|
553
985
|
error, hint = state_loader.error_and_hint
|
|
@@ -565,34 +997,50 @@ def upload(state_file, **kwargs):
|
|
|
565
997
|
|
|
566
998
|
cloud_state_file_exists = state_manager.check_cloud_state_exists()
|
|
567
999
|
|
|
568
|
-
if cloud_state_file_exists and not click.confirm(
|
|
1000
|
+
if cloud_state_file_exists and not click.confirm("\nDo you want to overwrite the existing state file?"):
|
|
569
1001
|
return 0
|
|
570
1002
|
|
|
571
1003
|
console.print(state_manager.upload_state_to_cloud(state_loader.state))
|
|
572
1004
|
|
|
573
1005
|
|
|
574
1006
|
@cloud.command(cls=TrackCommand)
|
|
575
|
-
@click.option(
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
1007
|
+
@click.option(
|
|
1008
|
+
"-o",
|
|
1009
|
+
"--output",
|
|
1010
|
+
help="Path of the downloaded state file.",
|
|
1011
|
+
type=click.STRING,
|
|
1012
|
+
default=DEFAULT_RECCE_STATE_FILE,
|
|
1013
|
+
show_default=True,
|
|
1014
|
+
)
|
|
1015
|
+
@click.option("--cloud-token", help="The GitHub token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN")
|
|
1016
|
+
@click.option(
|
|
1017
|
+
"--state-file-host",
|
|
1018
|
+
help="The host to fetch the state file from.",
|
|
1019
|
+
type=click.STRING,
|
|
1020
|
+
envvar="RECCE_STATE_FILE_HOST",
|
|
1021
|
+
default="",
|
|
1022
|
+
hidden=True,
|
|
1023
|
+
)
|
|
1024
|
+
@click.option(
|
|
1025
|
+
"--password",
|
|
1026
|
+
"-p",
|
|
1027
|
+
help="The password to encrypt the state file in cloud.",
|
|
1028
|
+
type=click.STRING,
|
|
1029
|
+
envvar="RECCE_STATE_PASSWORD",
|
|
1030
|
+
)
|
|
583
1031
|
@add_options(recce_options)
|
|
584
1032
|
def download(**kwargs):
|
|
585
1033
|
"""
|
|
586
|
-
|
|
1034
|
+
Download the state file to cloud
|
|
587
1035
|
"""
|
|
588
1036
|
from rich.console import Console
|
|
589
1037
|
|
|
590
1038
|
handle_debug_flag(**kwargs)
|
|
591
|
-
filepath = kwargs.get(
|
|
1039
|
+
filepath = kwargs.get("output")
|
|
592
1040
|
cloud_options = {
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
1041
|
+
"host": kwargs.get("state_file_host"),
|
|
1042
|
+
"github_token": kwargs.get("cloud_token"),
|
|
1043
|
+
"password": kwargs.get("password"),
|
|
596
1044
|
}
|
|
597
1045
|
|
|
598
1046
|
console = Console()
|
|
@@ -608,7 +1056,7 @@ def download(**kwargs):
|
|
|
608
1056
|
cloud_state_file_exists = state_manager.check_cloud_state_exists()
|
|
609
1057
|
|
|
610
1058
|
if not cloud_state_file_exists:
|
|
611
|
-
console.print(
|
|
1059
|
+
console.print("[yellow]Skip[/yellow] No state file found in cloud.")
|
|
612
1060
|
return 0
|
|
613
1061
|
|
|
614
1062
|
state_manager.download_state_from_cloud(filepath)
|
|
@@ -616,41 +1064,60 @@ def download(**kwargs):
|
|
|
616
1064
|
|
|
617
1065
|
|
|
618
1066
|
@cloud.command(cls=TrackCommand)
|
|
619
|
-
@click.option(
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
1067
|
+
@click.option("--cloud-token", help="The GitHub token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN")
|
|
1068
|
+
@click.option(
|
|
1069
|
+
"--branch",
|
|
1070
|
+
"-b",
|
|
1071
|
+
help="The branch of the provided artifacts.",
|
|
1072
|
+
type=click.STRING,
|
|
1073
|
+
envvar="GITHUB_HEAD_REF",
|
|
1074
|
+
default=current_branch(),
|
|
1075
|
+
show_default=True,
|
|
1076
|
+
)
|
|
1077
|
+
@click.option(
|
|
1078
|
+
"--target-path",
|
|
1079
|
+
help="dbt artifacts directory for your artifacts.",
|
|
1080
|
+
type=click.STRING,
|
|
1081
|
+
default="target",
|
|
1082
|
+
show_default=True,
|
|
1083
|
+
)
|
|
1084
|
+
@click.option(
|
|
1085
|
+
"--password",
|
|
1086
|
+
"-p",
|
|
1087
|
+
help="The password to encrypt the dbt artifacts in cloud.",
|
|
1088
|
+
type=click.STRING,
|
|
1089
|
+
envvar="RECCE_STATE_PASSWORD",
|
|
1090
|
+
required=True,
|
|
1091
|
+
)
|
|
627
1092
|
@add_options(recce_options)
|
|
628
1093
|
def upload_artifacts(**kwargs):
|
|
629
1094
|
"""
|
|
630
|
-
|
|
1095
|
+
Upload the dbt artifacts to cloud
|
|
631
1096
|
|
|
632
|
-
|
|
633
|
-
|
|
1097
|
+
Upload the dbt artifacts (metadata.json, catalog.json) to Recce Cloud for the given branch.
|
|
1098
|
+
The password is used to encrypt the dbt artifacts in the cloud. You will need the password to download the dbt artifacts.
|
|
634
1099
|
|
|
635
|
-
|
|
636
|
-
|
|
1100
|
+
By default, the artifacts are uploaded to the current branch. You can specify the branch using the --branch option.
|
|
1101
|
+
The target path is set to 'target' by default. You can specify the target path using the --target-path option.
|
|
637
1102
|
"""
|
|
638
1103
|
from rich.console import Console
|
|
1104
|
+
|
|
639
1105
|
console = Console()
|
|
640
|
-
cloud_token = kwargs.get(
|
|
641
|
-
password = kwargs.get(
|
|
642
|
-
target_path = kwargs.get(
|
|
643
|
-
branch = kwargs.get(
|
|
1106
|
+
cloud_token = kwargs.get("cloud_token")
|
|
1107
|
+
password = kwargs.get("password")
|
|
1108
|
+
target_path = kwargs.get("target_path")
|
|
1109
|
+
branch = kwargs.get("branch")
|
|
644
1110
|
|
|
645
1111
|
try:
|
|
646
|
-
rc = upload_dbt_artifacts(
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
console.rule(
|
|
1112
|
+
rc = upload_dbt_artifacts(
|
|
1113
|
+
target_path, branch=branch, token=cloud_token, password=password, debug=kwargs.get("debug", False)
|
|
1114
|
+
)
|
|
1115
|
+
console.rule("Uploaded Successfully")
|
|
650
1116
|
console.print(
|
|
651
|
-
f'Uploaded dbt artifacts to Recce Cloud for branch "{branch}" from "{os.path.abspath(target_path)}"'
|
|
1117
|
+
f'Uploaded dbt artifacts to Recce Cloud for branch "{branch}" from "{os.path.abspath(target_path)}"'
|
|
1118
|
+
)
|
|
652
1119
|
except Exception as e:
|
|
653
|
-
console.rule(
|
|
1120
|
+
console.rule("Failed to Upload", style="red")
|
|
654
1121
|
console.print("[[red]Error[/red]] Failed to upload the dbt artifacts to cloud.")
|
|
655
1122
|
console.print(f"Reason: {e}")
|
|
656
1123
|
rc = 1
|
|
@@ -659,22 +1126,32 @@ def upload_artifacts(**kwargs):
|
|
|
659
1126
|
|
|
660
1127
|
def _download_artifacts(branch, cloud_token, console, kwargs, password, target_path):
|
|
661
1128
|
try:
|
|
662
|
-
rc = download_dbt_artifacts(
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
1129
|
+
rc = download_dbt_artifacts(
|
|
1130
|
+
target_path,
|
|
1131
|
+
branch=branch,
|
|
1132
|
+
token=cloud_token,
|
|
1133
|
+
password=password,
|
|
1134
|
+
force=kwargs.get("force", False),
|
|
1135
|
+
debug=kwargs.get("debug", False),
|
|
1136
|
+
)
|
|
1137
|
+
console.rule("Downloaded Successfully")
|
|
666
1138
|
console.print(
|
|
667
|
-
f'Downloaded dbt artifacts from Recce Cloud for branch "{branch}" to "{os.path.abspath(target_path)}"'
|
|
1139
|
+
f'Downloaded dbt artifacts from Recce Cloud for branch "{branch}" to "{os.path.abspath(target_path)}"'
|
|
1140
|
+
)
|
|
668
1141
|
except Exception as e:
|
|
669
|
-
console.rule(
|
|
1142
|
+
console.rule("Failed to Download", style="red")
|
|
670
1143
|
console.print("[[red]Error[/red]] Failed to download the dbt artifacts from cloud.")
|
|
671
1144
|
reason = str(e)
|
|
672
1145
|
|
|
673
|
-
if
|
|
1146
|
+
if (
|
|
1147
|
+
"Requests specifying Server Side Encryption with Customer provided keys must provide the correct secret key"
|
|
1148
|
+
in reason
|
|
1149
|
+
):
|
|
674
1150
|
console.print("Reason: Decryption failed due to incorrect password.")
|
|
675
1151
|
console.print(
|
|
676
|
-
"Please provide the correct password to decrypt the dbt artifacts. Or re-upload the dbt artifacts with a new password."
|
|
677
|
-
|
|
1152
|
+
"Please provide the correct password to decrypt the dbt artifacts. Or re-upload the dbt artifacts with a new password."
|
|
1153
|
+
)
|
|
1154
|
+
elif "The specified key does not exist" in reason:
|
|
678
1155
|
console.print("Reason: The dbt artifacts is not found in the cloud.")
|
|
679
1156
|
console.print("Please upload the dbt artifacts to the cloud before downloading it.")
|
|
680
1157
|
else:
|
|
@@ -684,89 +1161,415 @@ def _download_artifacts(branch, cloud_token, console, kwargs, password, target_p
|
|
|
684
1161
|
|
|
685
1162
|
|
|
686
1163
|
@cloud.command(cls=TrackCommand)
|
|
687
|
-
@click.option(
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
1164
|
+
@click.option("--cloud-token", help="The GitHub token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN")
|
|
1165
|
+
@click.option(
|
|
1166
|
+
"--branch",
|
|
1167
|
+
"-b",
|
|
1168
|
+
help="The branch of the selected artifacts.",
|
|
1169
|
+
type=click.STRING,
|
|
1170
|
+
envvar="GITHUB_BASE_REF",
|
|
1171
|
+
default=current_branch(),
|
|
1172
|
+
show_default=True,
|
|
1173
|
+
)
|
|
1174
|
+
@click.option(
|
|
1175
|
+
"--target-path",
|
|
1176
|
+
help="The dbt artifacts directory for your artifacts.",
|
|
1177
|
+
type=click.STRING,
|
|
1178
|
+
default="target",
|
|
1179
|
+
show_default=True,
|
|
1180
|
+
)
|
|
1181
|
+
@click.option(
|
|
1182
|
+
"--password",
|
|
1183
|
+
"-p",
|
|
1184
|
+
help="The password to decrypt the dbt artifacts in cloud.",
|
|
1185
|
+
type=click.STRING,
|
|
1186
|
+
envvar="RECCE_STATE_PASSWORD",
|
|
1187
|
+
required=True,
|
|
1188
|
+
)
|
|
1189
|
+
@click.option("--force", "-f", help="Bypasses the confirmation prompt. Download the artifacts directly.", is_flag=True)
|
|
697
1190
|
@add_options(recce_options)
|
|
698
1191
|
def download_artifacts(**kwargs):
|
|
699
1192
|
"""
|
|
700
|
-
|
|
1193
|
+
Download the dbt artifacts from cloud
|
|
701
1194
|
|
|
702
|
-
|
|
703
|
-
|
|
1195
|
+
Download the dbt artifacts (metadata.json, catalog.json) from Recce Cloud for the given branch.
|
|
1196
|
+
The password is used to decrypt the dbt artifacts in the cloud.
|
|
704
1197
|
|
|
705
|
-
|
|
706
|
-
|
|
1198
|
+
By default, the artifacts are downloaded from the current branch. You can specify the branch using the --branch option.
|
|
1199
|
+
The target path is set to 'target' by default. You can specify the target path using the --target-path option.
|
|
707
1200
|
"""
|
|
708
1201
|
from rich.console import Console
|
|
1202
|
+
|
|
709
1203
|
console = Console()
|
|
710
|
-
cloud_token = kwargs.get(
|
|
711
|
-
password = kwargs.get(
|
|
712
|
-
target_path = kwargs.get(
|
|
713
|
-
branch = kwargs.get(
|
|
1204
|
+
cloud_token = kwargs.get("cloud_token")
|
|
1205
|
+
password = kwargs.get("password")
|
|
1206
|
+
target_path = kwargs.get("target_path")
|
|
1207
|
+
branch = kwargs.get("branch")
|
|
714
1208
|
return _download_artifacts(branch, cloud_token, console, kwargs, password, target_path)
|
|
715
1209
|
|
|
716
1210
|
|
|
717
1211
|
@cloud.command(cls=TrackCommand)
|
|
718
|
-
@click.option(
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
1212
|
+
@click.option("--cloud-token", help="The GitHub token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN")
|
|
1213
|
+
@click.option(
|
|
1214
|
+
"--branch",
|
|
1215
|
+
"-b",
|
|
1216
|
+
help="The branch of the selected artifacts.",
|
|
1217
|
+
type=click.STRING,
|
|
1218
|
+
envvar="GITHUB_BASE_REF",
|
|
1219
|
+
default=current_default_branch(),
|
|
1220
|
+
show_default=True,
|
|
1221
|
+
)
|
|
1222
|
+
@click.option(
|
|
1223
|
+
"--target-path",
|
|
1224
|
+
help="The dbt artifacts directory for your artifacts.",
|
|
1225
|
+
type=click.STRING,
|
|
1226
|
+
default="target-base",
|
|
1227
|
+
show_default=True,
|
|
1228
|
+
)
|
|
1229
|
+
@click.option(
|
|
1230
|
+
"--password",
|
|
1231
|
+
"-p",
|
|
1232
|
+
help="The password to decrypt the dbt artifacts in cloud.",
|
|
1233
|
+
type=click.STRING,
|
|
1234
|
+
envvar="RECCE_STATE_PASSWORD",
|
|
1235
|
+
required=True,
|
|
1236
|
+
)
|
|
1237
|
+
@click.option("--force", "-f", help="Bypasses the confirmation prompt. Download the artifacts directly.", is_flag=True)
|
|
728
1238
|
@add_options(recce_options)
|
|
729
1239
|
def download_base_artifacts(**kwargs):
|
|
730
1240
|
"""
|
|
731
|
-
|
|
1241
|
+
Download the base dbt artifacts from cloud
|
|
732
1242
|
|
|
733
|
-
|
|
734
|
-
|
|
1243
|
+
Download the base dbt artifacts (metadata.json, catalog.json) from Recce Cloud.
|
|
1244
|
+
This is useful when you start to set up the base dbt artifacts for the first time.
|
|
735
1245
|
|
|
736
|
-
|
|
1246
|
+
Please make sure you have uploaded the dbt artifacts before downloading them.
|
|
737
1247
|
"""
|
|
738
1248
|
from rich.console import Console
|
|
1249
|
+
|
|
739
1250
|
console = Console()
|
|
740
|
-
cloud_token = kwargs.get(
|
|
741
|
-
password = kwargs.get(
|
|
742
|
-
target_path = kwargs.get(
|
|
743
|
-
branch = kwargs.get(
|
|
1251
|
+
cloud_token = kwargs.get("cloud_token")
|
|
1252
|
+
password = kwargs.get("password")
|
|
1253
|
+
target_path = kwargs.get("target_path")
|
|
1254
|
+
branch = kwargs.get("branch")
|
|
1255
|
+
# If recce can't infer default branch from "GITHUB_BASE_REF" and current_default_branch()
|
|
1256
|
+
if branch is None:
|
|
1257
|
+
console.print(
|
|
1258
|
+
"[[red]Error[/red]] Please provide your base branch name with '--branch' to download the base " "artifacts."
|
|
1259
|
+
)
|
|
1260
|
+
exit(1)
|
|
1261
|
+
|
|
744
1262
|
return _download_artifacts(branch, cloud_token, console, kwargs, password, target_path)
|
|
745
1263
|
|
|
746
1264
|
|
|
747
|
-
@
|
|
1265
|
+
@cloud.command(cls=TrackCommand)
|
|
1266
|
+
@click.option("--cloud-token", help="The GitHub token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN")
|
|
1267
|
+
@click.option(
|
|
1268
|
+
"--branch",
|
|
1269
|
+
"-b",
|
|
1270
|
+
help="The branch to delete artifacts from.",
|
|
1271
|
+
type=click.STRING,
|
|
1272
|
+
envvar="GITHUB_HEAD_REF",
|
|
1273
|
+
default=current_branch(),
|
|
1274
|
+
show_default=True,
|
|
1275
|
+
)
|
|
1276
|
+
@click.option("--force", "-f", help="Bypasses the confirmation prompt. Delete the artifacts directly.", is_flag=True)
|
|
1277
|
+
@add_options(recce_options)
|
|
1278
|
+
def delete_artifacts(**kwargs):
|
|
1279
|
+
"""
|
|
1280
|
+
Delete the dbt artifacts from cloud
|
|
1281
|
+
|
|
1282
|
+
Delete the dbt artifacts (metadata.json, catalog.json) from Recce Cloud for the given branch.
|
|
1283
|
+
This will permanently remove the artifacts from the cloud storage.
|
|
1284
|
+
|
|
1285
|
+
By default, the artifacts are deleted from the current branch. You can specify the branch using the --branch option.
|
|
1286
|
+
"""
|
|
1287
|
+
from rich.console import Console
|
|
1288
|
+
|
|
1289
|
+
console = Console()
|
|
1290
|
+
cloud_token = kwargs.get("cloud_token")
|
|
1291
|
+
branch = kwargs.get("branch")
|
|
1292
|
+
force = kwargs.get("force", False)
|
|
1293
|
+
|
|
1294
|
+
if not force:
|
|
1295
|
+
if not click.confirm(f'Do you want to delete artifacts from branch "{branch}"?'):
|
|
1296
|
+
console.print("Deletion cancelled.")
|
|
1297
|
+
return 0
|
|
1298
|
+
|
|
1299
|
+
try:
|
|
1300
|
+
delete_dbt_artifacts(branch=branch, token=cloud_token, debug=kwargs.get("debug", False))
|
|
1301
|
+
console.print(f"[[green]Success[/green]] Artifacts deleted from branch: {branch}")
|
|
1302
|
+
return 0
|
|
1303
|
+
except click.exceptions.Abort:
|
|
1304
|
+
pass
|
|
1305
|
+
except RecceCloudException as e:
|
|
1306
|
+
console.print("[[red]Error[/red]] Failed to delete the dbt artifacts from cloud.")
|
|
1307
|
+
console.print(f"Reason: {e.reason}")
|
|
1308
|
+
exit(1)
|
|
1309
|
+
except Exception as e:
|
|
1310
|
+
console.print("[[red]Error[/red]] Failed to delete the dbt artifacts from cloud.")
|
|
1311
|
+
console.print(f"Reason: {e}")
|
|
1312
|
+
exit(1)
|
|
1313
|
+
|
|
1314
|
+
|
|
1315
|
+
@cloud.command(cls=TrackCommand, name="list-organizations")
|
|
1316
|
+
@click.option("--api-token", help="The Recce Cloud API token.", type=click.STRING, envvar="RECCE_API_TOKEN")
|
|
1317
|
+
@add_options(recce_options)
|
|
1318
|
+
def list_organizations(**kwargs):
|
|
1319
|
+
"""
|
|
1320
|
+
List organizations from Recce Cloud
|
|
1321
|
+
|
|
1322
|
+
Lists all organizations that the authenticated user has access to.
|
|
1323
|
+
"""
|
|
1324
|
+
from rich.console import Console
|
|
1325
|
+
from rich.table import Table
|
|
1326
|
+
|
|
1327
|
+
console = Console()
|
|
1328
|
+
handle_debug_flag(**kwargs)
|
|
1329
|
+
|
|
1330
|
+
try:
|
|
1331
|
+
api_token = prepare_api_token(**kwargs)
|
|
1332
|
+
except RecceConfigException:
|
|
1333
|
+
show_invalid_api_token_message()
|
|
1334
|
+
exit(1)
|
|
1335
|
+
|
|
1336
|
+
try:
|
|
1337
|
+
from recce.util.recce_cloud import RecceCloud
|
|
1338
|
+
|
|
1339
|
+
cloud = RecceCloud(api_token)
|
|
1340
|
+
organizations = cloud.list_organizations()
|
|
1341
|
+
|
|
1342
|
+
if not organizations:
|
|
1343
|
+
console.print("No organizations found.")
|
|
1344
|
+
return
|
|
1345
|
+
|
|
1346
|
+
table = Table(title="Organizations")
|
|
1347
|
+
table.add_column("ID", style="cyan")
|
|
1348
|
+
table.add_column("Name", style="green")
|
|
1349
|
+
table.add_column("Display Name", style="yellow")
|
|
1350
|
+
|
|
1351
|
+
for org in organizations:
|
|
1352
|
+
table.add_row(str(org.get("id", "")), org.get("name", ""), org.get("display_name", ""))
|
|
1353
|
+
|
|
1354
|
+
console.print(table)
|
|
1355
|
+
|
|
1356
|
+
except RecceCloudException as e:
|
|
1357
|
+
console.print(f"[[red]Error[/red]] {e}")
|
|
1358
|
+
exit(1)
|
|
1359
|
+
except Exception as e:
|
|
1360
|
+
console.print(f"[[red]Error[/red]] {e}")
|
|
1361
|
+
exit(1)
|
|
1362
|
+
|
|
1363
|
+
|
|
1364
|
+
@cloud.command(cls=TrackCommand, name="list-projects")
|
|
1365
|
+
@click.option(
|
|
1366
|
+
"--organization",
|
|
1367
|
+
"-o",
|
|
1368
|
+
help="Organization ID (can also be set via RECCE_ORGANIZATION_ID environment variable)",
|
|
1369
|
+
type=click.STRING,
|
|
1370
|
+
envvar="RECCE_ORGANIZATION_ID",
|
|
1371
|
+
)
|
|
1372
|
+
@click.option("--api-token", help="The Recce Cloud API token.", type=click.STRING, envvar="RECCE_API_TOKEN")
|
|
1373
|
+
@add_options(recce_options)
|
|
1374
|
+
def list_projects(**kwargs):
|
|
1375
|
+
"""
|
|
1376
|
+
List projects from Recce Cloud
|
|
1377
|
+
|
|
1378
|
+
Lists all projects in the specified organization that the authenticated user has access to.
|
|
1379
|
+
|
|
1380
|
+
Examples:
|
|
1381
|
+
|
|
1382
|
+
# Using environment variable
|
|
1383
|
+
export RECCE_ORGANIZATION_ID=8
|
|
1384
|
+
recce cloud list-projects
|
|
1385
|
+
|
|
1386
|
+
# Using command line argument
|
|
1387
|
+
recce cloud list-projects --organization 8
|
|
1388
|
+
|
|
1389
|
+
# Override environment variable
|
|
1390
|
+
export RECCE_ORGANIZATION_ID=8
|
|
1391
|
+
recce cloud list-projects --organization 10
|
|
1392
|
+
"""
|
|
1393
|
+
from rich.console import Console
|
|
1394
|
+
from rich.table import Table
|
|
1395
|
+
|
|
1396
|
+
console = Console()
|
|
1397
|
+
handle_debug_flag(**kwargs)
|
|
1398
|
+
|
|
1399
|
+
try:
|
|
1400
|
+
api_token = prepare_api_token(**kwargs)
|
|
1401
|
+
except RecceConfigException:
|
|
1402
|
+
show_invalid_api_token_message()
|
|
1403
|
+
exit(1)
|
|
1404
|
+
|
|
1405
|
+
organization = kwargs.get("organization")
|
|
1406
|
+
if not organization:
|
|
1407
|
+
console.print("[[red]Error[/red]] Organization ID is required. Please provide it via:")
|
|
1408
|
+
console.print(" --organization <id> or set RECCE_ORGANIZATION_ID environment variable")
|
|
1409
|
+
exit(1)
|
|
1410
|
+
|
|
1411
|
+
try:
|
|
1412
|
+
from recce.util.recce_cloud import RecceCloud
|
|
1413
|
+
|
|
1414
|
+
cloud = RecceCloud(api_token)
|
|
1415
|
+
projects = cloud.list_projects(organization)
|
|
1416
|
+
|
|
1417
|
+
if not projects:
|
|
1418
|
+
console.print(f"No projects found in organization {organization}.")
|
|
1419
|
+
return
|
|
1420
|
+
|
|
1421
|
+
table = Table(title=f"Projects in Organization {organization}")
|
|
1422
|
+
table.add_column("ID", style="cyan")
|
|
1423
|
+
table.add_column("Name", style="green")
|
|
1424
|
+
table.add_column("Display Name", style="yellow")
|
|
1425
|
+
|
|
1426
|
+
for project in projects:
|
|
1427
|
+
table.add_row(str(project.get("id", "")), project.get("name", ""), project.get("display_name", ""))
|
|
1428
|
+
|
|
1429
|
+
console.print(table)
|
|
1430
|
+
|
|
1431
|
+
except RecceCloudException as e:
|
|
1432
|
+
console.print(f"[[red]Error[/red]] {e}")
|
|
1433
|
+
exit(1)
|
|
1434
|
+
except Exception as e:
|
|
1435
|
+
console.print(f"[[red]Error[/red]] {e}")
|
|
1436
|
+
exit(1)
|
|
1437
|
+
|
|
1438
|
+
|
|
1439
|
+
@cloud.command(cls=TrackCommand, name="list-sessions")
|
|
1440
|
+
@click.option(
|
|
1441
|
+
"--organization",
|
|
1442
|
+
"-o",
|
|
1443
|
+
help="Organization ID (can also be set via RECCE_ORGANIZATION_ID environment variable)",
|
|
1444
|
+
type=click.STRING,
|
|
1445
|
+
envvar="RECCE_ORGANIZATION_ID",
|
|
1446
|
+
)
|
|
1447
|
+
@click.option(
|
|
1448
|
+
"--project",
|
|
1449
|
+
"-p",
|
|
1450
|
+
help="Project ID (can also be set via RECCE_PROJECT_ID environment variable)",
|
|
1451
|
+
type=click.STRING,
|
|
1452
|
+
envvar="RECCE_PROJECT_ID",
|
|
1453
|
+
)
|
|
1454
|
+
@click.option("--api-token", help="The Recce Cloud API token.", type=click.STRING, envvar="RECCE_API_TOKEN")
|
|
1455
|
+
@add_options(recce_options)
|
|
1456
|
+
def list_sessions(**kwargs):
|
|
1457
|
+
"""
|
|
1458
|
+
List sessions from Recce Cloud
|
|
1459
|
+
|
|
1460
|
+
Lists all sessions in the specified project that the authenticated user has access to.
|
|
1461
|
+
|
|
1462
|
+
Examples:
|
|
1463
|
+
|
|
1464
|
+
# Using environment variables
|
|
1465
|
+
export RECCE_ORGANIZATION_ID=8
|
|
1466
|
+
export RECCE_PROJECT_ID=7
|
|
1467
|
+
recce cloud list-sessions
|
|
1468
|
+
|
|
1469
|
+
# Using command line arguments
|
|
1470
|
+
recce cloud list-sessions --organization 8 --project 7
|
|
1471
|
+
|
|
1472
|
+
# Mixed usage (env + CLI override)
|
|
1473
|
+
export RECCE_ORGANIZATION_ID=8
|
|
1474
|
+
recce cloud list-sessions --project 7
|
|
1475
|
+
|
|
1476
|
+
# Override environment variables
|
|
1477
|
+
export RECCE_ORGANIZATION_ID=8
|
|
1478
|
+
export RECCE_PROJECT_ID=7
|
|
1479
|
+
recce cloud list-sessions --organization 10 --project 9
|
|
1480
|
+
"""
|
|
1481
|
+
from rich.console import Console
|
|
1482
|
+
from rich.table import Table
|
|
1483
|
+
|
|
1484
|
+
console = Console()
|
|
1485
|
+
handle_debug_flag(**kwargs)
|
|
1486
|
+
|
|
1487
|
+
try:
|
|
1488
|
+
api_token = prepare_api_token(**kwargs)
|
|
1489
|
+
except RecceConfigException:
|
|
1490
|
+
show_invalid_api_token_message()
|
|
1491
|
+
exit(1)
|
|
1492
|
+
|
|
1493
|
+
organization = kwargs.get("organization")
|
|
1494
|
+
project = kwargs.get("project")
|
|
1495
|
+
|
|
1496
|
+
# Validate required parameters
|
|
1497
|
+
if not organization:
|
|
1498
|
+
console.print("[[red]Error[/red]] Organization ID is required. Please provide it via:")
|
|
1499
|
+
console.print(" --organization <id> or set RECCE_ORGANIZATION_ID environment variable")
|
|
1500
|
+
exit(1)
|
|
1501
|
+
|
|
1502
|
+
if not project:
|
|
1503
|
+
console.print("[[red]Error[/red]] Project ID is required. Please provide it via:")
|
|
1504
|
+
console.print(" --project <id> or set RECCE_PROJECT_ID environment variable")
|
|
1505
|
+
exit(1)
|
|
1506
|
+
|
|
1507
|
+
try:
|
|
1508
|
+
from recce.util.recce_cloud import RecceCloud
|
|
1509
|
+
|
|
1510
|
+
cloud = RecceCloud(api_token)
|
|
1511
|
+
sessions = cloud.list_sessions(organization, project)
|
|
1512
|
+
|
|
1513
|
+
if not sessions:
|
|
1514
|
+
console.print(f"No sessions found in project {project}.")
|
|
1515
|
+
return
|
|
1516
|
+
|
|
1517
|
+
table = Table(title=f"Sessions in Project {project}")
|
|
1518
|
+
table.add_column("ID", style="cyan")
|
|
1519
|
+
table.add_column("Name", style="green")
|
|
1520
|
+
table.add_column("Is Base", style="yellow")
|
|
1521
|
+
|
|
1522
|
+
for session in sessions:
|
|
1523
|
+
is_base = "✓" if session.get("is_base", False) else ""
|
|
1524
|
+
table.add_row(session.get("id", ""), session.get("name", ""), is_base)
|
|
1525
|
+
|
|
1526
|
+
console.print(table)
|
|
1527
|
+
|
|
1528
|
+
except RecceCloudException as e:
|
|
1529
|
+
console.print(f"[[red]Error[/red]] {e}")
|
|
1530
|
+
exit(1)
|
|
1531
|
+
except Exception as e:
|
|
1532
|
+
console.print(f"[[red]Error[/red]] {e}")
|
|
1533
|
+
exit(1)
|
|
1534
|
+
|
|
1535
|
+
|
|
1536
|
+
@cli.group("github", short_help="GitHub related commands", hidden=True)
|
|
748
1537
|
def github(**kwargs):
|
|
749
1538
|
pass
|
|
750
1539
|
|
|
751
1540
|
|
|
752
|
-
@github.command(
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
1541
|
+
@github.command(
|
|
1542
|
+
cls=TrackCommand, short_help="Download the artifacts from the GitHub repository based on the current Pull Request."
|
|
1543
|
+
)
|
|
1544
|
+
@click.option(
|
|
1545
|
+
"--github-token",
|
|
1546
|
+
help="The github token to use for accessing GitHub repo.",
|
|
1547
|
+
type=click.STRING,
|
|
1548
|
+
envvar="GITHUB_TOKEN",
|
|
1549
|
+
)
|
|
1550
|
+
@click.option(
|
|
1551
|
+
"--github-repo",
|
|
1552
|
+
help="The github repo to use for accessing GitHub repo.",
|
|
1553
|
+
type=click.STRING,
|
|
1554
|
+
envvar="GITHUB_REPOSITORY",
|
|
1555
|
+
)
|
|
758
1556
|
def artifact(**kwargs):
|
|
759
1557
|
from recce.github import recce_ci_artifact
|
|
1558
|
+
|
|
760
1559
|
return recce_ci_artifact(**kwargs)
|
|
761
1560
|
|
|
762
1561
|
|
|
763
1562
|
@cli.command(cls=TrackCommand)
|
|
764
|
-
@click.argument(
|
|
765
|
-
@click.option(
|
|
766
|
-
|
|
1563
|
+
@click.argument("state_file", type=click.Path(exists=True))
|
|
1564
|
+
@click.option(
|
|
1565
|
+
"--api-token",
|
|
1566
|
+
help="The personal token generated by Recce Cloud.",
|
|
1567
|
+
type=click.STRING,
|
|
1568
|
+
envvar="RECCE_API_TOKEN",
|
|
1569
|
+
)
|
|
767
1570
|
def share(state_file, **kwargs):
|
|
768
1571
|
"""
|
|
769
|
-
|
|
1572
|
+
Share the state file
|
|
770
1573
|
"""
|
|
771
1574
|
from rich.console import Console
|
|
772
1575
|
|
|
@@ -775,19 +1578,21 @@ def share(state_file, **kwargs):
|
|
|
775
1578
|
cloud_options = None
|
|
776
1579
|
|
|
777
1580
|
# read or input the api token
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
1581
|
+
try:
|
|
1582
|
+
api_token = prepare_api_token(interaction=True, **kwargs)
|
|
1583
|
+
except Abort:
|
|
1584
|
+
console.print("[yellow]Abort[/yellow]")
|
|
1585
|
+
exit(0)
|
|
1586
|
+
except RecceConfigException:
|
|
1587
|
+
show_invalid_api_token_message()
|
|
1588
|
+
exit(1)
|
|
785
1589
|
|
|
786
|
-
auth_options = {
|
|
1590
|
+
auth_options = {"api_token": api_token}
|
|
787
1591
|
|
|
788
1592
|
# load local state
|
|
789
|
-
state_loader = create_state_loader(
|
|
790
|
-
|
|
1593
|
+
state_loader = create_state_loader(
|
|
1594
|
+
review_mode=True, cloud_mode=False, state_file=state_file, cloud_options=cloud_options
|
|
1595
|
+
)
|
|
791
1596
|
|
|
792
1597
|
if not state_loader.verify():
|
|
793
1598
|
error, hint = state_loader.error_and_hint
|
|
@@ -808,9 +1613,8 @@ def share(state_file, **kwargs):
|
|
|
808
1613
|
|
|
809
1614
|
try:
|
|
810
1615
|
response = state_manager.share_state(state_file_name, state_loader.state)
|
|
811
|
-
if response.get(
|
|
812
|
-
console.print("[[red]Error[/red]] Failed to share the state.\n"
|
|
813
|
-
f"Reason: {response.get('message')}")
|
|
1616
|
+
if response.get("status") == "error":
|
|
1617
|
+
console.print("[[red]Error[/red]] Failed to share the state.\n" f"Reason: {response.get('message')}")
|
|
814
1618
|
else:
|
|
815
1619
|
console.print(f"Shared Link: {response.get('share_url')}")
|
|
816
1620
|
except RecceCloudException as e:
|
|
@@ -819,40 +1623,212 @@ def share(state_file, **kwargs):
|
|
|
819
1623
|
exit(1)
|
|
820
1624
|
|
|
821
1625
|
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
1626
|
+
snapshot_id_option = click.option(
|
|
1627
|
+
"--snapshot-id",
|
|
1628
|
+
help="The snapshot ID to upload artifacts to cloud.",
|
|
1629
|
+
type=click.STRING,
|
|
1630
|
+
envvar=["RECCE_SNAPSHOT_ID", "RECCE_SESSION_ID"],
|
|
1631
|
+
required=True,
|
|
1632
|
+
)
|
|
1633
|
+
|
|
1634
|
+
session_id_option = click.option(
|
|
1635
|
+
"--session-id",
|
|
1636
|
+
help="The session ID to upload artifacts to cloud.",
|
|
1637
|
+
type=click.STRING,
|
|
1638
|
+
envvar=["RECCE_SESSION_ID", "RECCE_SNAPSHOT_ID"],
|
|
1639
|
+
required=True,
|
|
1640
|
+
)
|
|
1641
|
+
|
|
1642
|
+
target_path_option = click.option(
|
|
1643
|
+
"--target-path",
|
|
1644
|
+
help="dbt artifacts directory for your artifacts.",
|
|
1645
|
+
type=click.STRING,
|
|
1646
|
+
default="target",
|
|
1647
|
+
show_default=True,
|
|
1648
|
+
)
|
|
1649
|
+
|
|
1650
|
+
|
|
1651
|
+
@cli.command(cls=TrackCommand, hidden=True)
|
|
1652
|
+
@add_options([session_id_option, target_path_option])
|
|
1653
|
+
@add_options(recce_cloud_auth_options)
|
|
1654
|
+
@add_options(recce_options)
|
|
1655
|
+
def upload_session(**kwargs):
|
|
1656
|
+
"""
|
|
1657
|
+
Upload target/manifest.json and target/catalog.json to the specific session ID
|
|
1658
|
+
|
|
1659
|
+
Upload the dbt artifacts (manifest.json, catalog.json) to Recce Cloud for the given session ID.
|
|
1660
|
+
This allows you to associate artifacts with a specific session for later use.
|
|
1661
|
+
|
|
1662
|
+
Examples:\n
|
|
1663
|
+
|
|
1664
|
+
\b
|
|
1665
|
+
# Upload artifacts to a session ID
|
|
1666
|
+
recce upload-session --session-id <session-id>
|
|
1667
|
+
|
|
1668
|
+
\b
|
|
1669
|
+
# Upload artifacts from custom target path to a session ID
|
|
1670
|
+
recce upload-session --session-id <session-id> --target-path my-target
|
|
1671
|
+
"""
|
|
829
1672
|
from rich.console import Console
|
|
830
1673
|
|
|
831
1674
|
console = Console()
|
|
832
1675
|
handle_debug_flag(**kwargs)
|
|
833
|
-
is_review = True
|
|
834
|
-
is_cloud = False
|
|
835
|
-
cloud_options = None
|
|
836
|
-
flag = {
|
|
837
|
-
'read_only': True,
|
|
838
|
-
}
|
|
839
|
-
state_loader = create_state_loader(is_review, is_cloud, state_file, cloud_options)
|
|
840
1676
|
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
1677
|
+
# Initialize Recce Config
|
|
1678
|
+
RecceConfig(config_file=kwargs.get("config"))
|
|
1679
|
+
|
|
1680
|
+
try:
|
|
1681
|
+
api_token = prepare_api_token(**kwargs)
|
|
1682
|
+
except RecceConfigException:
|
|
1683
|
+
show_invalid_api_token_message()
|
|
845
1684
|
exit(1)
|
|
846
1685
|
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
1686
|
+
session_id = kwargs.get("session_id")
|
|
1687
|
+
target_path = kwargs.get("target_path")
|
|
1688
|
+
|
|
1689
|
+
try:
|
|
1690
|
+
rc = upload_artifacts_to_session(
|
|
1691
|
+
target_path, session_id=session_id, token=api_token, debug=kwargs.get("debug", False)
|
|
1692
|
+
)
|
|
1693
|
+
console.rule("Uploaded Successfully")
|
|
1694
|
+
console.print(
|
|
1695
|
+
f'Uploaded dbt artifacts to Recce Cloud for session ID "{session_id}" from "{os.path.abspath(target_path)}"'
|
|
1696
|
+
)
|
|
1697
|
+
except Exception as e:
|
|
1698
|
+
console.rule("Failed to Upload Session", style="red")
|
|
1699
|
+
console.print(f"[[red]Error[/red]] Failed to upload the dbt artifacts to the session {session_id}.")
|
|
1700
|
+
console.print(f"Reason: {e}")
|
|
1701
|
+
rc = 1
|
|
1702
|
+
return rc
|
|
1703
|
+
|
|
1704
|
+
|
|
1705
|
+
# Backward compatibility for `recce snapshot` command
|
|
1706
|
+
@cli.command(
|
|
1707
|
+
cls=TrackCommand,
|
|
1708
|
+
hidden=True,
|
|
1709
|
+
deprecated=True,
|
|
1710
|
+
help="Upload target/manifest.json and target/catalog.json to the specific snapshot ID",
|
|
1711
|
+
)
|
|
1712
|
+
@add_options([snapshot_id_option, target_path_option])
|
|
1713
|
+
@add_options(recce_cloud_auth_options)
|
|
1714
|
+
@add_options(recce_options)
|
|
1715
|
+
def snapshot(**kwargs):
|
|
1716
|
+
kwargs["session_id"] = kwargs.get("snapshot_id")
|
|
1717
|
+
return upload_session(**kwargs)
|
|
1718
|
+
|
|
1719
|
+
|
|
1720
|
+
@cli.command(hidden=True, cls=TrackCommand)
|
|
1721
|
+
@click.argument("state_file", required=True)
|
|
1722
|
+
@click.option("--host", default="localhost", show_default=True, help="The host to bind to.")
|
|
1723
|
+
@click.option("--port", default=8000, show_default=True, help="The port to bind to.", type=int)
|
|
1724
|
+
@click.option("--lifetime", default=0, show_default=True, help="The lifetime of the server in seconds.", type=int)
|
|
1725
|
+
@click.option("--share-url", help="The share URL triggers this instance.", type=click.STRING, envvar="RECCE_SHARE_URL")
|
|
1726
|
+
@click.pass_context
|
|
1727
|
+
def read_only(ctx, state_file=None, **kwargs):
|
|
1728
|
+
# Invoke `recce server --mode read-only <state_file> ...
|
|
1729
|
+
kwargs["mode"] = RecceServerMode.read_only
|
|
1730
|
+
ctx.invoke(server, state_file=state_file, **kwargs)
|
|
1731
|
+
|
|
1732
|
+
|
|
1733
|
+
@cli.command(cls=TrackCommand)
|
|
1734
|
+
@click.option("--sse", is_flag=True, default=False, help="Start in HTTP/SSE mode instead of stdio mode")
|
|
1735
|
+
@click.option("--host", default="localhost", help="Host to bind to in SSE mode (default: localhost)")
|
|
1736
|
+
@click.option("--port", default=8000, type=int, help="Port to bind to in SSE mode (default: 8000)")
|
|
1737
|
+
@add_options(dbt_related_options)
|
|
1738
|
+
@add_options(sqlmesh_related_options)
|
|
1739
|
+
@add_options(recce_options)
|
|
1740
|
+
@add_options(recce_dbt_artifact_dir_options)
|
|
1741
|
+
@add_options(recce_cloud_options)
|
|
1742
|
+
@add_options(recce_cloud_auth_options)
|
|
1743
|
+
@add_options(recce_hidden_options)
|
|
1744
|
+
def mcp_server(sse, host, port, **kwargs):
|
|
1745
|
+
"""
|
|
1746
|
+
[Experiment] Start the Recce MCP (Model Context Protocol) server
|
|
1747
|
+
|
|
1748
|
+
The MCP server provides an interface for AI assistants and tools to interact
|
|
1749
|
+
with Recce's data validation capabilities. By default, it uses stdio for
|
|
1750
|
+
communication. Use --sse to enable HTTP/Server-Sent Events mode instead.
|
|
1751
|
+
|
|
1752
|
+
Available tools:
|
|
1753
|
+
- get_lineage_diff: Get lineage differences between environments
|
|
1754
|
+
- row_count_diff: Compare row counts between environments
|
|
1755
|
+
- query: Execute SQL queries with dbt templating
|
|
1756
|
+
- query_diff: Compare query results between environments
|
|
1757
|
+
- profile_diff: Generate statistical profiles and compare
|
|
1758
|
+
|
|
1759
|
+
Examples:\n
|
|
1760
|
+
|
|
1761
|
+
\b
|
|
1762
|
+
# Start the MCP server in stdio mode (default)
|
|
1763
|
+
recce mcp-server
|
|
1764
|
+
|
|
1765
|
+
\b
|
|
1766
|
+
# Start in HTTP/SSE mode on default port 8000
|
|
1767
|
+
recce mcp-server --sse
|
|
1768
|
+
|
|
1769
|
+
\b
|
|
1770
|
+
# Start in HTTP/SSE mode with custom host and port
|
|
1771
|
+
recce mcp-server --sse --host 0.0.0.0 --port 9000
|
|
1772
|
+
|
|
1773
|
+
\b
|
|
1774
|
+
# Start with custom dbt configuration
|
|
1775
|
+
recce mcp-server --target prod --project-dir ./my_project
|
|
1776
|
+
|
|
1777
|
+
SSE Connection URL (when using --sse): http://<host>:<port>/sse
|
|
1778
|
+
"""
|
|
1779
|
+
from rich.console import Console
|
|
1780
|
+
|
|
1781
|
+
console = Console()
|
|
1782
|
+
try:
|
|
1783
|
+
# Import here to avoid import errors if mcp is not installed
|
|
1784
|
+
from recce.mcp_server import run_mcp_server
|
|
1785
|
+
except ImportError as e:
|
|
1786
|
+
console.print(f"[[red]Error[/red]] Failed to import MCP server: {e}")
|
|
1787
|
+
console.print(r"Please install the MCP package: pip install 'recce\[mcp]'")
|
|
850
1788
|
exit(1)
|
|
851
1789
|
|
|
852
|
-
|
|
853
|
-
|
|
1790
|
+
# Initialize Recce Config
|
|
1791
|
+
RecceConfig(config_file=kwargs.get("config"))
|
|
1792
|
+
|
|
1793
|
+
handle_debug_flag(**kwargs)
|
|
1794
|
+
patch_derived_args(kwargs)
|
|
854
1795
|
|
|
855
|
-
|
|
1796
|
+
# Prepare API token
|
|
1797
|
+
try:
|
|
1798
|
+
api_token = prepare_api_token(**kwargs)
|
|
1799
|
+
kwargs["api_token"] = api_token
|
|
1800
|
+
except RecceConfigException:
|
|
1801
|
+
show_invalid_api_token_message()
|
|
1802
|
+
exit(1)
|
|
1803
|
+
|
|
1804
|
+
# Create state loader using shared function (if cloud mode is enabled)
|
|
1805
|
+
is_cloud = kwargs.get("cloud", False)
|
|
1806
|
+
if is_cloud:
|
|
1807
|
+
state_loader = create_state_loader_by_args(None, **kwargs)
|
|
1808
|
+
kwargs["state_loader"] = state_loader
|
|
1809
|
+
|
|
1810
|
+
try:
|
|
1811
|
+
if sse:
|
|
1812
|
+
console.print(f"Starting Recce MCP Server in HTTP/SSE mode on {host}:{port}...")
|
|
1813
|
+
console.print(f"SSE endpoint: http://{host}:{port}/sse")
|
|
1814
|
+
console.print("Available tools: get_lineage_diff, row_count_diff, query, query_diff, profile_diff")
|
|
1815
|
+
else:
|
|
1816
|
+
console.print("Starting Recce MCP Server in stdio mode...")
|
|
1817
|
+
console.print("Available tools: get_lineage_diff, row_count_diff, query, query_diff, profile_diff")
|
|
1818
|
+
|
|
1819
|
+
# Run the server (stdio or SSE based on --sse flag)
|
|
1820
|
+
asyncio.run(run_mcp_server(sse=sse, host=host, port=port, **kwargs))
|
|
1821
|
+
except (asyncio.CancelledError, KeyboardInterrupt):
|
|
1822
|
+
# Graceful shutdown (e.g., Ctrl+C)
|
|
1823
|
+
console.print("[yellow]MCP Server interrupted[/yellow]")
|
|
1824
|
+
exit(0)
|
|
1825
|
+
except Exception as e:
|
|
1826
|
+
console.print(f"[[red]Error[/red]] Failed to start MCP server: {e}")
|
|
1827
|
+
if kwargs.get("debug"):
|
|
1828
|
+
import traceback
|
|
1829
|
+
|
|
1830
|
+
traceback.print_exc()
|
|
1831
|
+
exit(1)
|
|
856
1832
|
|
|
857
1833
|
|
|
858
1834
|
if __name__ == "__main__":
|