recce-nightly 0.62.0.20250417__py3-none-any.whl → 1.30.0.20251221__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of recce-nightly might be problematic. Click here for more details.
- recce/VERSION +1 -1
- recce/__init__.py +27 -22
- recce/adapter/base.py +11 -14
- recce/adapter/dbt_adapter/__init__.py +845 -461
- recce/adapter/dbt_adapter/dbt_version.py +3 -0
- recce/adapter/sqlmesh_adapter.py +24 -35
- recce/apis/check_api.py +59 -42
- recce/apis/check_events_api.py +353 -0
- recce/apis/check_func.py +41 -35
- recce/apis/run_api.py +25 -19
- recce/apis/run_func.py +64 -25
- recce/artifact.py +119 -51
- recce/cli.py +1301 -324
- recce/config.py +43 -34
- recce/connect_to_cloud.py +138 -0
- recce/core.py +55 -47
- recce/data/404/index.html +2 -0
- recce/data/404.html +2 -1
- recce/data/__next.@lineage.!KHNsb3Qp.__PAGE__.txt +7 -0
- recce/data/__next.@lineage.!KHNsb3Qp.txt +4 -0
- recce/data/__next.__PAGE__.txt +6 -0
- recce/data/__next._full.txt +32 -0
- recce/data/__next._head.txt +8 -0
- recce/data/__next._index.txt +14 -0
- recce/data/__next._tree.txt +8 -0
- recce/data/_next/static/chunks/025a7e3e3f9f40ae.js +1 -0
- recce/data/_next/static/chunks/0ce56d67ef5779ca.js +4 -0
- recce/data/_next/static/chunks/1a6a78780155dac7.js +48 -0
- recce/data/_next/static/chunks/1de8485918b9182a.css +2 -0
- recce/data/_next/static/chunks/1e4b1b50d1e34993.js +1 -0
- recce/data/_next/static/chunks/206d5d181e4c738e.js +1 -0
- recce/data/_next/static/chunks/2c357efc34c5b859.js +25 -0
- recce/data/_next/static/chunks/2e9d95d2d48c479c.js +1 -0
- recce/data/_next/static/chunks/2f016dc4a3edad2e.js +2 -0
- recce/data/_next/static/chunks/313251962d698f7c.js +1 -0
- recce/data/_next/static/chunks/3a9f021f38eb5574.css +1 -0
- recce/data/_next/static/chunks/40079da8d2b8f651.js +1 -0
- recce/data/_next/static/chunks/4599182bffb64661.js +38 -0
- recce/data/_next/static/chunks/4e62f6e184173580.js +1 -0
- recce/data/_next/static/chunks/5c4dfb0d09eaa401.js +1 -0
- recce/data/_next/static/chunks/69e4f06ccfdfc3ac.js +1 -0
- recce/data/_next/static/chunks/6b206cb4707d6bee.js +1 -0
- recce/data/_next/static/chunks/6d8557f062aa4386.css +1 -0
- recce/data/_next/static/chunks/7fbe3650bd83b6b5.js +1 -0
- recce/data/_next/static/chunks/83fa823a825674f6.js +1 -0
- recce/data/_next/static/chunks/848a6c9b5f55f7ed.js +1 -0
- recce/data/_next/static/chunks/859462b0858aef88.css +2 -0
- recce/data/_next/static/chunks/923964f18c87d0f1.css +1 -0
- recce/data/_next/static/chunks/939390f911895d7c.js +48 -0
- recce/data/_next/static/chunks/99a9817237a07f43.js +1 -0
- recce/data/_next/static/chunks/9fed8b4b2b924054.js +5 -0
- recce/data/_next/static/chunks/b6949f6c5892110c.js +1 -0
- recce/data/_next/static/chunks/b851a1d3f8149828.js +1 -0
- recce/data/_next/static/chunks/c734f9ad957de0b4.js +1 -0
- recce/data/_next/static/chunks/cdde321b0ec75717.js +2 -0
- recce/data/_next/static/chunks/d0f91117d77ff844.css +1 -0
- recce/data/_next/static/chunks/d6c8667911c2500f.js +1 -0
- recce/data/_next/static/chunks/da8dab68c02752cf.js +74 -0
- recce/data/_next/static/chunks/dc074049c9d12d97.js +109 -0
- recce/data/_next/static/chunks/ee7f1a8227342421.js +1 -0
- recce/data/_next/static/chunks/fa2f4e56c2fccc73.js +1 -0
- recce/data/_next/static/chunks/turbopack-1fad664f62979b93.js +3 -0
- recce/data/_next/static/media/favicon.a8d38d84.ico +0 -0
- recce/data/_next/static/media/montserrat-cyrillic-800-normal.d80d830d.woff2 +0 -0
- recce/data/_next/static/media/montserrat-cyrillic-800-normal.f9d58125.woff +0 -0
- recce/data/_next/static/media/montserrat-cyrillic-ext-800-normal.076c2a93.woff2 +0 -0
- recce/data/_next/static/media/montserrat-cyrillic-ext-800-normal.a4fa76b5.woff +0 -0
- recce/data/_next/static/media/montserrat-latin-800-normal.cde454cc.woff2 +0 -0
- recce/data/_next/static/media/montserrat-latin-800-normal.d5761935.woff +0 -0
- recce/data/_next/static/media/montserrat-latin-ext-800-normal.40ec0659.woff2 +0 -0
- recce/data/_next/static/media/montserrat-latin-ext-800-normal.b671449b.woff +0 -0
- recce/data/_next/static/media/montserrat-vietnamese-800-normal.9f7b8541.woff +0 -0
- recce/data/_next/static/media/montserrat-vietnamese-800-normal.f9eb854e.woff2 +0 -0
- recce/data/_next/static/nX-Uz0AH6Tc6hIQUFGqaB/_buildManifest.js +11 -0
- recce/data/_next/static/nX-Uz0AH6Tc6hIQUFGqaB/_clientMiddlewareManifest.json +1 -0
- recce/data/_not-found/__next._full.txt +24 -0
- recce/data/_not-found/__next._head.txt +8 -0
- recce/data/_not-found/__next._index.txt +13 -0
- recce/data/_not-found/__next._not-found.__PAGE__.txt +5 -0
- recce/data/_not-found/__next._not-found.txt +4 -0
- recce/data/_not-found/__next._tree.txt +6 -0
- recce/data/_not-found/index.html +2 -0
- recce/data/_not-found/index.txt +24 -0
- recce/data/auth_callback.html +68 -0
- recce/data/checks/__next.@lineage.__DEFAULT__.txt +7 -0
- recce/data/checks/__next._full.txt +39 -0
- recce/data/checks/__next._head.txt +8 -0
- recce/data/checks/__next._index.txt +14 -0
- recce/data/checks/__next._tree.txt +8 -0
- recce/data/checks/__next.checks.__PAGE__.txt +10 -0
- recce/data/checks/__next.checks.txt +4 -0
- recce/data/checks/index.html +2 -0
- recce/data/checks/index.txt +39 -0
- recce/data/imgs/reload-image.svg +4 -0
- recce/data/index.html +2 -27
- recce/data/index.txt +32 -7
- recce/data/lineage/__next.@lineage.__DEFAULT__.txt +7 -0
- recce/data/lineage/__next._full.txt +39 -0
- recce/data/lineage/__next._head.txt +8 -0
- recce/data/lineage/__next._index.txt +14 -0
- recce/data/lineage/__next._tree.txt +8 -0
- recce/data/lineage/__next.lineage.__PAGE__.txt +10 -0
- recce/data/lineage/__next.lineage.txt +4 -0
- recce/data/lineage/index.html +2 -0
- recce/data/lineage/index.txt +39 -0
- recce/data/query/__next.@lineage.__DEFAULT__.txt +7 -0
- recce/data/query/__next._full.txt +37 -0
- recce/data/query/__next._head.txt +8 -0
- recce/data/query/__next._index.txt +14 -0
- recce/data/query/__next._tree.txt +8 -0
- recce/data/query/__next.query.__PAGE__.txt +9 -0
- recce/data/query/__next.query.txt +4 -0
- recce/data/query/index.html +2 -0
- recce/data/query/index.txt +37 -0
- recce/diff.py +6 -12
- recce/event/CONFIG.bak +1 -0
- recce/event/__init__.py +86 -74
- recce/event/collector.py +33 -22
- recce/event/track.py +49 -27
- recce/exceptions.py +1 -1
- recce/git.py +7 -7
- recce/github.py +57 -53
- recce/mcp_server.py +725 -0
- recce/models/__init__.py +4 -1
- recce/models/check.py +438 -21
- recce/models/run.py +1 -0
- recce/models/types.py +134 -28
- recce/pull_request.py +27 -25
- recce/run.py +179 -122
- recce/server.py +394 -104
- recce/state/__init__.py +31 -0
- recce/state/cloud.py +644 -0
- recce/state/const.py +26 -0
- recce/state/local.py +56 -0
- recce/state/state.py +119 -0
- recce/state/state_loader.py +174 -0
- recce/summary.py +196 -149
- recce/tasks/__init__.py +19 -3
- recce/tasks/core.py +11 -13
- recce/tasks/dataframe.py +82 -18
- recce/tasks/histogram.py +69 -34
- recce/tasks/lineage.py +2 -2
- recce/tasks/profile.py +152 -86
- recce/tasks/query.py +180 -89
- recce/tasks/rowcount.py +37 -31
- recce/tasks/schema.py +18 -15
- recce/tasks/top_k.py +35 -35
- recce/tasks/utils.py +147 -0
- recce/tasks/valuediff.py +247 -155
- recce/util/__init__.py +3 -0
- recce/util/api_token.py +80 -0
- recce/util/breaking.py +105 -100
- recce/util/cll.py +274 -219
- recce/util/cloud/__init__.py +15 -0
- recce/util/cloud/base.py +115 -0
- recce/util/cloud/check_events.py +190 -0
- recce/util/cloud/checks.py +242 -0
- recce/util/io.py +22 -17
- recce/util/lineage.py +65 -16
- recce/util/logger.py +1 -1
- recce/util/onboarding_state.py +45 -0
- recce/util/perf_tracking.py +85 -0
- recce/util/recce_cloud.py +347 -72
- recce/util/singleton.py +4 -4
- recce/util/startup_perf.py +121 -0
- recce/yaml/__init__.py +7 -10
- recce_nightly-1.30.0.20251221.dist-info/METADATA +195 -0
- recce_nightly-1.30.0.20251221.dist-info/RECORD +183 -0
- {recce_nightly-0.62.0.20250417.dist-info → recce_nightly-1.30.0.20251221.dist-info}/WHEEL +1 -2
- recce/data/_next/static/chunks/1f229bf6-d9fe92e56db8d93b.js +0 -1
- recce/data/_next/static/chunks/29e3cc0d-8c150e37dff9631b.js +0 -1
- recce/data/_next/static/chunks/36e1c10d-bb0210cbd6573a8d.js +0 -1
- recce/data/_next/static/chunks/3998a672-eaad84bdd88cc73e.js +0 -1
- recce/data/_next/static/chunks/450c323b-1bb5db526e54435a.js +0 -1
- recce/data/_next/static/chunks/47d8844f-79a1b53c66a7d7ec.js +0 -1
- recce/data/_next/static/chunks/500-e51c92a025a51234.js +0 -65
- recce/data/_next/static/chunks/6dc81886-c94b9b91bc2c3caf.js +0 -1
- recce/data/_next/static/chunks/700-3b65fc3666820d00.js +0 -2
- recce/data/_next/static/chunks/7a8a3e83-d7fa409d97b38b2b.js +0 -1
- recce/data/_next/static/chunks/7f27ae6c-413f6b869a04183a.js +0 -1
- recce/data/_next/static/chunks/9746af58-d74bef4d03eea6ab.js +0 -1
- recce/data/_next/static/chunks/a30376cd-7d806e1602f2dc3a.js +0 -1
- recce/data/_next/static/chunks/app/_not-found/page-8a886fa0855c3105.js +0 -1
- recce/data/_next/static/chunks/app/layout-9102e22cb73f74d6.js +0 -1
- recce/data/_next/static/chunks/app/page-9adc25782272ed2e.js +0 -1
- recce/data/_next/static/chunks/b63b1b3f-7395c74e11a14e95.js +0 -1
- recce/data/_next/static/chunks/c132bf7d-8102037f9ccf372a.js +0 -1
- recce/data/_next/static/chunks/c1ceaa8b-a1e442154d23515e.js +0 -1
- recce/data/_next/static/chunks/cd9f8d63-cf0d5a7b0f7a92e8.js +0 -54
- recce/data/_next/static/chunks/ce84277d-f42c2c58049cea2d.js +0 -1
- recce/data/_next/static/chunks/e24bf851-0f8cbc99656833e7.js +0 -1
- recce/data/_next/static/chunks/fee69bc6-f17d36c080742e74.js +0 -1
- recce/data/_next/static/chunks/framework-ded83d71b51ce901.js +0 -1
- recce/data/_next/static/chunks/main-a0859f1f36d0aa6c.js +0 -1
- recce/data/_next/static/chunks/main-app-0225a2255968e566.js +0 -1
- recce/data/_next/static/chunks/pages/_app-d5672bf3d8b6371b.js +0 -1
- recce/data/_next/static/chunks/pages/_error-ed75be3f25588548.js +0 -1
- recce/data/_next/static/chunks/webpack-567d72f0bc0820d5.js +0 -1
- recce/data/_next/static/css/c9ecb46a4b21c126.css +0 -14
- recce/data/_next/static/media/montserrat-cyrillic-800-normal.22628180.woff2 +0 -0
- recce/data/_next/static/media/montserrat-cyrillic-800-normal.31d693bb.woff +0 -0
- recce/data/_next/static/media/montserrat-cyrillic-ext-800-normal.7e2c1e62.woff +0 -0
- recce/data/_next/static/media/montserrat-cyrillic-ext-800-normal.94a63aea.woff2 +0 -0
- recce/data/_next/static/media/montserrat-latin-800-normal.6f8fa298.woff2 +0 -0
- recce/data/_next/static/media/montserrat-latin-800-normal.97e20d5e.woff +0 -0
- recce/data/_next/static/media/montserrat-latin-ext-800-normal.013b84f9.woff2 +0 -0
- recce/data/_next/static/media/montserrat-latin-ext-800-normal.aff52ab0.woff +0 -0
- recce/data/_next/static/media/montserrat-vietnamese-800-normal.5f21869b.woff +0 -0
- recce/data/_next/static/media/montserrat-vietnamese-800-normal.c0035377.woff2 +0 -0
- recce/data/_next/static/qiyFlux77VkhxiceAJe_F/_buildManifest.js +0 -1
- recce/state.py +0 -753
- recce_nightly-0.62.0.20250417.dist-info/METADATA +0 -311
- recce_nightly-0.62.0.20250417.dist-info/RECORD +0 -139
- recce_nightly-0.62.0.20250417.dist-info/top_level.txt +0 -2
- tests/__init__.py +0 -0
- tests/adapter/__init__.py +0 -0
- tests/adapter/dbt_adapter/__init__.py +0 -0
- tests/adapter/dbt_adapter/conftest.py +0 -13
- tests/adapter/dbt_adapter/dbt_test_helper.py +0 -283
- tests/adapter/dbt_adapter/test_dbt_adapter.py +0 -40
- tests/adapter/dbt_adapter/test_dbt_cll.py +0 -102
- tests/adapter/dbt_adapter/test_selector.py +0 -177
- tests/tasks/__init__.py +0 -0
- tests/tasks/conftest.py +0 -4
- tests/tasks/test_histogram.py +0 -137
- tests/tasks/test_lineage.py +0 -42
- tests/tasks/test_preset_checks.py +0 -50
- tests/tasks/test_profile.py +0 -73
- tests/tasks/test_query.py +0 -151
- tests/tasks/test_row_count.py +0 -116
- tests/tasks/test_schema.py +0 -99
- tests/tasks/test_top_k.py +0 -73
- tests/tasks/test_valuediff.py +0 -74
- tests/test_cli.py +0 -122
- tests/test_config.py +0 -45
- tests/test_core.py +0 -27
- tests/test_dbt.py +0 -36
- tests/test_pull_request.py +0 -130
- tests/test_server.py +0 -98
- tests/test_state.py +0 -123
- tests/test_summary.py +0 -57
- /recce/data/_next/static/chunks/{polyfills-42372ed130431b0a.js → a6dad97d9634a72d.js} +0 -0
- /recce/data/_next/static/{qiyFlux77VkhxiceAJe_F → nX-Uz0AH6Tc6hIQUFGqaB}/_ssgManifest.js +0 -0
- {recce_nightly-0.62.0.20250417.dist-info → recce_nightly-1.30.0.20251221.dist-info}/entry_points.txt +0 -0
- {recce_nightly-0.62.0.20250417.dist-info → recce_nightly-1.30.0.20251221.dist-info}/licenses/LICENSE +0 -0
recce/cli.py
CHANGED
|
@@ -5,17 +5,40 @@ from typing import List
|
|
|
5
5
|
|
|
6
6
|
import click
|
|
7
7
|
import uvicorn
|
|
8
|
+
from click import Abort
|
|
8
9
|
|
|
9
10
|
from recce import event
|
|
10
|
-
from recce.artifact import
|
|
11
|
-
|
|
12
|
-
|
|
11
|
+
from recce.artifact import (
|
|
12
|
+
delete_dbt_artifacts,
|
|
13
|
+
download_dbt_artifacts,
|
|
14
|
+
upload_artifacts_to_session,
|
|
15
|
+
upload_dbt_artifacts,
|
|
16
|
+
)
|
|
17
|
+
from recce.config import RECCE_CONFIG_FILE, RECCE_ERROR_LOG_FILE, RecceConfig
|
|
18
|
+
from recce.connect_to_cloud import (
|
|
19
|
+
generate_key_pair,
|
|
20
|
+
prepare_connection_url,
|
|
21
|
+
run_one_time_http_server,
|
|
22
|
+
)
|
|
23
|
+
from recce.exceptions import RecceConfigException
|
|
13
24
|
from recce.git import current_branch, current_default_branch
|
|
14
|
-
from recce.run import
|
|
15
|
-
from recce.
|
|
25
|
+
from recce.run import check_github_ci_env, cli_run
|
|
26
|
+
from recce.server import RecceServerMode
|
|
27
|
+
from recce.state import (
|
|
28
|
+
CloudStateLoader,
|
|
29
|
+
FileStateLoader,
|
|
30
|
+
RecceCloudStateManager,
|
|
31
|
+
RecceShareStateManager,
|
|
32
|
+
)
|
|
16
33
|
from recce.summary import generate_markdown_summary
|
|
34
|
+
from recce.util.api_token import prepare_api_token, show_invalid_api_token_message
|
|
17
35
|
from recce.util.logger import CustomFormatter
|
|
18
|
-
from recce.util.
|
|
36
|
+
from recce.util.onboarding_state import update_onboarding_state
|
|
37
|
+
from recce.util.recce_cloud import (
|
|
38
|
+
RecceCloudException,
|
|
39
|
+
)
|
|
40
|
+
from recce.util.startup_perf import track_timing
|
|
41
|
+
|
|
19
42
|
from .core import RecceContext, set_default_context
|
|
20
43
|
from .event.track import TrackCommand
|
|
21
44
|
|
|
@@ -24,11 +47,17 @@ event.init()
|
|
|
24
47
|
|
|
25
48
|
def create_state_loader(review_mode, cloud_mode, state_file, cloud_options):
|
|
26
49
|
from rich.console import Console
|
|
50
|
+
|
|
27
51
|
console = Console()
|
|
28
52
|
|
|
29
53
|
try:
|
|
30
|
-
|
|
31
|
-
|
|
54
|
+
state_loader = (
|
|
55
|
+
CloudStateLoader(review_mode=review_mode, cloud_options=cloud_options)
|
|
56
|
+
if cloud_mode
|
|
57
|
+
else FileStateLoader(review_mode=review_mode, state_file=state_file)
|
|
58
|
+
)
|
|
59
|
+
state_loader.load()
|
|
60
|
+
return state_loader
|
|
32
61
|
except RecceCloudException as e:
|
|
33
62
|
console.print("[[red]Error[/red]] Failed to load recce state file")
|
|
34
63
|
console.print(f"Reason: {e.reason}")
|
|
@@ -39,13 +68,92 @@ def create_state_loader(review_mode, cloud_mode, state_file, cloud_options):
|
|
|
39
68
|
exit(1)
|
|
40
69
|
|
|
41
70
|
|
|
71
|
+
def patch_derived_args(args):
|
|
72
|
+
"""
|
|
73
|
+
Patch derived args based on other args.
|
|
74
|
+
"""
|
|
75
|
+
if args.get("session_id") or args.get("share_url"):
|
|
76
|
+
args["cloud"] = True
|
|
77
|
+
args["review"] = True
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@track_timing("state_loader_init")
|
|
81
|
+
def create_state_loader_by_args(state_file=None, **kwargs):
|
|
82
|
+
"""
|
|
83
|
+
Create a state loader based on CLI arguments.
|
|
84
|
+
|
|
85
|
+
This function handles the cloud options logic that is shared between
|
|
86
|
+
server and mcp-server commands.
|
|
87
|
+
|
|
88
|
+
Args:
|
|
89
|
+
state_file: Optional path to state file
|
|
90
|
+
**kwargs: CLI arguments including api_token, cloud, review, session_id, share_url, etc.
|
|
91
|
+
|
|
92
|
+
Returns:
|
|
93
|
+
state_loader: The created state loader instance
|
|
94
|
+
"""
|
|
95
|
+
from rich.console import Console
|
|
96
|
+
|
|
97
|
+
console = Console()
|
|
98
|
+
|
|
99
|
+
api_token = kwargs.get("api_token")
|
|
100
|
+
is_review = kwargs.get("review", False)
|
|
101
|
+
is_cloud = kwargs.get("cloud", False)
|
|
102
|
+
cloud_options = None
|
|
103
|
+
|
|
104
|
+
# Handle share_url and session_id
|
|
105
|
+
share_url = kwargs.get("share_url")
|
|
106
|
+
session_id = kwargs.get("session_id")
|
|
107
|
+
|
|
108
|
+
if share_url:
|
|
109
|
+
share_id = share_url.split("/")[-1]
|
|
110
|
+
if not share_id:
|
|
111
|
+
console.print("[[red]Error[/red]] Invalid share URL format.")
|
|
112
|
+
exit(1)
|
|
113
|
+
|
|
114
|
+
if is_cloud:
|
|
115
|
+
# Cloud mode
|
|
116
|
+
if share_url:
|
|
117
|
+
cloud_options = {
|
|
118
|
+
"host": kwargs.get("state_file_host"),
|
|
119
|
+
"api_token": api_token,
|
|
120
|
+
"share_id": share_id,
|
|
121
|
+
}
|
|
122
|
+
elif session_id:
|
|
123
|
+
cloud_options = {
|
|
124
|
+
"host": kwargs.get("state_file_host"),
|
|
125
|
+
"api_token": api_token,
|
|
126
|
+
"session_id": session_id,
|
|
127
|
+
}
|
|
128
|
+
else:
|
|
129
|
+
cloud_options = {
|
|
130
|
+
"host": kwargs.get("state_file_host"),
|
|
131
|
+
"github_token": kwargs.get("cloud_token"),
|
|
132
|
+
"password": kwargs.get("password"),
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
# Create state loader
|
|
136
|
+
state_loader = create_state_loader(is_review, is_cloud, state_file, cloud_options)
|
|
137
|
+
|
|
138
|
+
return state_loader
|
|
139
|
+
|
|
140
|
+
|
|
42
141
|
def handle_debug_flag(**kwargs):
|
|
43
|
-
if kwargs.get(
|
|
142
|
+
if kwargs.get("debug"):
|
|
44
143
|
import logging
|
|
144
|
+
|
|
45
145
|
ch = logging.StreamHandler()
|
|
46
146
|
ch.setFormatter(CustomFormatter())
|
|
47
147
|
logging.basicConfig(handlers=[ch], level=logging.DEBUG)
|
|
48
148
|
|
|
149
|
+
# Explicitly set uvicorn logger to DEBUG level
|
|
150
|
+
uvicorn_logger = logging.getLogger("uvicorn")
|
|
151
|
+
uvicorn_logger.setLevel(logging.DEBUG)
|
|
152
|
+
|
|
153
|
+
# Set all child loggers to DEBUG as well
|
|
154
|
+
for handler in uvicorn_logger.handlers:
|
|
155
|
+
handler.setLevel(logging.DEBUG)
|
|
156
|
+
|
|
49
157
|
|
|
50
158
|
def add_options(options):
|
|
51
159
|
def _add_options(func):
|
|
@@ -57,43 +165,109 @@ def add_options(options):
|
|
|
57
165
|
|
|
58
166
|
|
|
59
167
|
dbt_related_options = [
|
|
60
|
-
click.option(
|
|
61
|
-
click.option(
|
|
62
|
-
click.option(
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
168
|
+
click.option("--target", "-t", help="Which target to load for the given profile.", type=click.STRING),
|
|
169
|
+
click.option("--profile", help="Which existing profile to load.", type=click.STRING),
|
|
170
|
+
click.option(
|
|
171
|
+
"--project-dir",
|
|
172
|
+
help="Which directory to look in for the dbt_project.yml file.",
|
|
173
|
+
type=click.Path(),
|
|
174
|
+
envvar="DBT_PROJECT_DIR",
|
|
175
|
+
),
|
|
176
|
+
click.option(
|
|
177
|
+
"--profiles-dir",
|
|
178
|
+
help="Which directory to look in for the profiles.yml file.",
|
|
179
|
+
type=click.Path(),
|
|
180
|
+
envvar="DBT_PROFILES_DIR",
|
|
181
|
+
),
|
|
66
182
|
]
|
|
67
183
|
|
|
68
184
|
sqlmesh_related_options = [
|
|
69
|
-
click.option(
|
|
70
|
-
click.option(
|
|
71
|
-
click.option(
|
|
185
|
+
click.option("--sqlmesh", is_flag=True, help="Use SQLMesh ", hidden=True),
|
|
186
|
+
click.option("--sqlmesh-envs", is_flag=False, help="SQLMesh envs to compare. SOURCE:TARGET", hidden=True),
|
|
187
|
+
click.option("--sqlmesh-config", is_flag=False, help="SQLMesh config name to use", hidden=True),
|
|
72
188
|
]
|
|
73
189
|
|
|
74
190
|
recce_options = [
|
|
75
|
-
click.option(
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
191
|
+
click.option(
|
|
192
|
+
"--config",
|
|
193
|
+
help="Path to the recce config file.",
|
|
194
|
+
type=click.Path(),
|
|
195
|
+
default=RECCE_CONFIG_FILE,
|
|
196
|
+
show_default=True,
|
|
197
|
+
),
|
|
198
|
+
click.option(
|
|
199
|
+
"--error-log", help="Path to the error log file.", type=click.Path(), default=RECCE_ERROR_LOG_FILE, hidden=True
|
|
200
|
+
),
|
|
201
|
+
click.option("--debug", is_flag=True, help="Enable debug mode.", hidden=True),
|
|
80
202
|
]
|
|
81
203
|
|
|
82
204
|
recce_cloud_options = [
|
|
83
|
-
click.option(
|
|
84
|
-
click.option(
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
205
|
+
click.option("--cloud", is_flag=True, help="Fetch the state file from cloud."),
|
|
206
|
+
click.option(
|
|
207
|
+
"--cloud-token", help="The GitHub token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN"
|
|
208
|
+
),
|
|
209
|
+
click.option(
|
|
210
|
+
"--state-file-host",
|
|
211
|
+
help="The host to fetch the state file from.",
|
|
212
|
+
type=click.STRING,
|
|
213
|
+
envvar="RECCE_STATE_FILE_HOST",
|
|
214
|
+
default="",
|
|
215
|
+
hidden=True,
|
|
216
|
+
),
|
|
217
|
+
click.option(
|
|
218
|
+
"--password",
|
|
219
|
+
"-p",
|
|
220
|
+
help="The password to encrypt the state file in cloud.",
|
|
221
|
+
type=click.STRING,
|
|
222
|
+
envvar="RECCE_STATE_PASSWORD",
|
|
223
|
+
),
|
|
224
|
+
]
|
|
225
|
+
|
|
226
|
+
recce_cloud_auth_options = [
|
|
227
|
+
click.option(
|
|
228
|
+
"--api-token",
|
|
229
|
+
help="The personal token generated by Recce Cloud.",
|
|
230
|
+
type=click.STRING,
|
|
231
|
+
envvar="RECCE_API_TOKEN",
|
|
232
|
+
)
|
|
90
233
|
]
|
|
91
234
|
|
|
92
235
|
recce_dbt_artifact_dir_options = [
|
|
93
|
-
click.option(
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
236
|
+
click.option(
|
|
237
|
+
"--target-path",
|
|
238
|
+
help="dbt artifacts directory for your development branch.",
|
|
239
|
+
type=click.STRING,
|
|
240
|
+
default="target",
|
|
241
|
+
),
|
|
242
|
+
click.option(
|
|
243
|
+
"--target-base-path",
|
|
244
|
+
help="dbt artifacts directory to be used as the base for the comparison.",
|
|
245
|
+
type=click.STRING,
|
|
246
|
+
default="target-base",
|
|
247
|
+
),
|
|
248
|
+
]
|
|
249
|
+
|
|
250
|
+
recce_hidden_options = [
|
|
251
|
+
click.option(
|
|
252
|
+
"--mode",
|
|
253
|
+
envvar="RECCE_SERVER_MODE",
|
|
254
|
+
type=click.Choice(RecceServerMode.available_members(), case_sensitive=False),
|
|
255
|
+
hidden=True,
|
|
256
|
+
),
|
|
257
|
+
click.option(
|
|
258
|
+
"--share-url",
|
|
259
|
+
help="The share URL triggers this instance.",
|
|
260
|
+
type=click.STRING,
|
|
261
|
+
envvar="RECCE_SHARE_URL",
|
|
262
|
+
hidden=True,
|
|
263
|
+
),
|
|
264
|
+
click.option(
|
|
265
|
+
"--session-id",
|
|
266
|
+
help="The session ID triggers this instance.",
|
|
267
|
+
type=click.STRING,
|
|
268
|
+
envvar=["RECCE_SESSION_ID", "RECCE_SNAPSHOT_ID"], # Backward compatibility with RECCE_SNAPSHOT_ID
|
|
269
|
+
hidden=True,
|
|
270
|
+
),
|
|
97
271
|
]
|
|
98
272
|
|
|
99
273
|
|
|
@@ -105,8 +279,9 @@ def _execute_sql(context, sql_template, base=False):
|
|
|
105
279
|
exit(1)
|
|
106
280
|
|
|
107
281
|
from recce.adapter.dbt_adapter import DbtAdapter
|
|
282
|
+
|
|
108
283
|
dbt_adapter: DbtAdapter = context.adapter
|
|
109
|
-
with dbt_adapter.connection_named(
|
|
284
|
+
with dbt_adapter.connection_named("recce"):
|
|
110
285
|
sql = dbt_adapter.generate_sql(sql_template, base)
|
|
111
286
|
response, result = dbt_adapter.execute(sql, fetch=True, auto_begin=True)
|
|
112
287
|
table = result
|
|
@@ -119,13 +294,15 @@ def _execute_sql(context, sql_template, base=False):
|
|
|
119
294
|
def cli(ctx, **kwargs):
|
|
120
295
|
"""Recce: Data validation toolkit for comprehensive PR review"""
|
|
121
296
|
from rich.console import Console
|
|
297
|
+
|
|
122
298
|
from recce import __is_recce_outdated__, __latest_version__
|
|
299
|
+
|
|
123
300
|
if __is_recce_outdated__ is True:
|
|
124
|
-
error_console = Console(stderr=True, style=
|
|
301
|
+
error_console = Console(stderr=True, style="bold")
|
|
125
302
|
error_console.print(
|
|
126
303
|
f"[[yellow]Update Available[/yellow]] A new version of Recce {__latest_version__} is available.",
|
|
127
304
|
)
|
|
128
|
-
error_console.print("Please update using the command: 'pip install --upgrade recce'.", end=
|
|
305
|
+
error_console.print("Please update using the command: 'pip install --upgrade recce'.", end="\n\n")
|
|
129
306
|
|
|
130
307
|
|
|
131
308
|
@cli.command(cls=TrackCommand)
|
|
@@ -134,12 +311,110 @@ def version():
|
|
|
134
311
|
Show version information
|
|
135
312
|
"""
|
|
136
313
|
from recce import __version__
|
|
314
|
+
|
|
137
315
|
print(__version__)
|
|
138
316
|
|
|
139
317
|
|
|
318
|
+
@cli.command(cls=TrackCommand)
|
|
319
|
+
@add_options(dbt_related_options)
|
|
320
|
+
@add_options(recce_dbt_artifact_dir_options)
|
|
321
|
+
def debug(**kwargs):
|
|
322
|
+
"""
|
|
323
|
+
Diagnose and verify Recce setup for the development and the base environments
|
|
324
|
+
"""
|
|
325
|
+
|
|
326
|
+
from rich.console import Console
|
|
327
|
+
|
|
328
|
+
from recce.adapter.dbt_adapter import DbtAdapter
|
|
329
|
+
from recce.core import load_context
|
|
330
|
+
|
|
331
|
+
console = Console()
|
|
332
|
+
|
|
333
|
+
def check_artifacts(env_name, target_path):
|
|
334
|
+
console.rule(f"{env_name} Environment", style="orange3")
|
|
335
|
+
if not target_path.is_dir():
|
|
336
|
+
console.print(f"[[red]MISS[/red]] Directory not found: {target_path}")
|
|
337
|
+
return [False, False, False]
|
|
338
|
+
|
|
339
|
+
console.print(f"[[green]OK[/green]] Directory exists: {target_path}")
|
|
340
|
+
|
|
341
|
+
manifest_path = target_path / "manifest.json"
|
|
342
|
+
manifest_is_ready = manifest_path.is_file()
|
|
343
|
+
if manifest_is_ready:
|
|
344
|
+
console.print(f"[[green]OK[/green]] Manifest JSON file exists : {manifest_path}")
|
|
345
|
+
else:
|
|
346
|
+
console.print(f"[[red]MISS[/red]] Manifest JSON file not found: {manifest_path}")
|
|
347
|
+
|
|
348
|
+
catalog_path = target_path / "catalog.json"
|
|
349
|
+
catalog_is_ready = catalog_path.is_file()
|
|
350
|
+
if catalog_is_ready:
|
|
351
|
+
console.print(f"[[green]OK[/green]] Catalog JSON file exists: {catalog_path}")
|
|
352
|
+
else:
|
|
353
|
+
console.print(f"[[red]MISS[/red]] Catalog JSON file not found: {catalog_path}")
|
|
354
|
+
|
|
355
|
+
return [True, manifest_is_ready, catalog_is_ready]
|
|
356
|
+
|
|
357
|
+
project_dir_path = Path(kwargs.get("project_dir") or "./")
|
|
358
|
+
target_path = project_dir_path.joinpath(Path(kwargs.get("target_path", "target")))
|
|
359
|
+
target_base_path = project_dir_path.joinpath(Path(kwargs.get("target_base_path", "target-base")))
|
|
360
|
+
|
|
361
|
+
curr_is_ready = check_artifacts("Development", target_path)
|
|
362
|
+
base_is_ready = check_artifacts("Base", target_base_path)
|
|
363
|
+
|
|
364
|
+
console.rule("Warehouse Connection", style="orange3")
|
|
365
|
+
conn_is_ready = True
|
|
366
|
+
try:
|
|
367
|
+
context_kwargs = {**kwargs, "target_base_path": kwargs.get("target_path")}
|
|
368
|
+
ctx = load_context(**context_kwargs)
|
|
369
|
+
dbt_adapter: DbtAdapter = ctx.adapter
|
|
370
|
+
sql = dbt_adapter.generate_sql("select 1", False)
|
|
371
|
+
dbt_adapter.execute(sql, fetch=True, auto_begin=True)
|
|
372
|
+
console.print("[[green]OK[/green]] Connection test")
|
|
373
|
+
except Exception:
|
|
374
|
+
conn_is_ready = False
|
|
375
|
+
console.print("[[red]FAIL[/red]] Connection test")
|
|
376
|
+
|
|
377
|
+
console.rule("Result", style="orange3")
|
|
378
|
+
if all(curr_is_ready) and all(base_is_ready) and conn_is_ready:
|
|
379
|
+
console.print("[[green]OK[/green]] Ready to launch! Type 'recce server'.")
|
|
380
|
+
elif all(curr_is_ready) and conn_is_ready:
|
|
381
|
+
console.print("[[orange3]OK[/orange3]] Ready to launch with [i]limited features[/i]. Type 'recce server'.")
|
|
382
|
+
|
|
383
|
+
if not curr_is_ready[0]:
|
|
384
|
+
console.print(
|
|
385
|
+
"[[orange3]TIP[/orange3]] Run dbt or overwrite the default directory of the development environment with '--target-path'."
|
|
386
|
+
)
|
|
387
|
+
else:
|
|
388
|
+
if not curr_is_ready[1]:
|
|
389
|
+
console.print(
|
|
390
|
+
"[[orange3]TIP[/orange3]] 'dbt run' to generate the manifest JSON file for the development environment."
|
|
391
|
+
)
|
|
392
|
+
if not curr_is_ready[2]:
|
|
393
|
+
console.print(
|
|
394
|
+
"[[orange3]TIP[/orange3]] 'dbt docs generate' to generate the catalog JSON file for the development environment."
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
if not base_is_ready[0]:
|
|
398
|
+
console.print(
|
|
399
|
+
"[[orange3]TIP[/orange3]] Run dbt with '--target-path target-base' or overwrite the default directory of the base environment with '--target-base-path'."
|
|
400
|
+
)
|
|
401
|
+
else:
|
|
402
|
+
if not base_is_ready[1]:
|
|
403
|
+
console.print(
|
|
404
|
+
"[[orange3]TIP[/orange3]] 'dbt run --target-path target-base' to generate the manifest JSON file for the base environment."
|
|
405
|
+
)
|
|
406
|
+
if not base_is_ready[2]:
|
|
407
|
+
console.print(
|
|
408
|
+
"[[orange3]TIP[/orange3]] 'dbt docs generate --target-path target-base' to generate the catalog JSON file for the base environment."
|
|
409
|
+
)
|
|
410
|
+
|
|
411
|
+
if not conn_is_ready:
|
|
412
|
+
console.print("[[orange3]TIP[/orange3]] Run 'dbt debug' to check the connection.")
|
|
413
|
+
|
|
414
|
+
|
|
140
415
|
@cli.command(hidden=True, cls=TrackCommand)
|
|
141
|
-
@click.option(
|
|
142
|
-
@click.option(
|
|
416
|
+
@click.option("--sql", help="Sql template to query", required=True)
|
|
417
|
+
@click.option("--base", is_flag=True, help="Run the query on the base environment")
|
|
143
418
|
@add_options(dbt_related_options)
|
|
144
419
|
def query(sql, base: bool = False, **kwargs):
|
|
145
420
|
"""
|
|
@@ -155,20 +430,25 @@ def query(sql, base: bool = False, **kwargs):
|
|
|
155
430
|
"""
|
|
156
431
|
context = RecceContext.load(**kwargs)
|
|
157
432
|
result = _execute_sql(context, sql, base=base)
|
|
158
|
-
print(result.to_string(na_rep=
|
|
433
|
+
print(result.to_string(na_rep="-", index=False))
|
|
159
434
|
|
|
160
435
|
|
|
161
436
|
def _split_comma_separated(ctx, param, value):
|
|
162
|
-
return value.split(
|
|
437
|
+
return value.split(",") if value else None
|
|
163
438
|
|
|
164
439
|
|
|
165
440
|
@cli.command(hidden=True, cls=TrackCommand)
|
|
166
|
-
@click.option(
|
|
167
|
-
@click.option(
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
441
|
+
@click.option("--sql", help="Sql template to query.", required=True)
|
|
442
|
+
@click.option(
|
|
443
|
+
"--primary-keys",
|
|
444
|
+
type=click.STRING,
|
|
445
|
+
help="Comma-separated list of primary key columns.",
|
|
446
|
+
callback=_split_comma_separated,
|
|
447
|
+
)
|
|
448
|
+
@click.option("--keep-shape", is_flag=True, help="Keep unchanged columns. Otherwise, unchanged columns are hidden.")
|
|
449
|
+
@click.option(
|
|
450
|
+
"--keep-equal", is_flag=True, help='Keep values that are equal. Otherwise, equal values are shown as "-".'
|
|
451
|
+
)
|
|
172
452
|
@add_options(dbt_related_options)
|
|
173
453
|
def diff(sql, primary_keys: List[str] = None, keep_shape: bool = False, keep_equal: bool = False, **kwargs):
|
|
174
454
|
"""
|
|
@@ -189,26 +469,34 @@ def diff(sql, primary_keys: List[str] = None, keep_shape: bool = False, keep_equ
|
|
|
189
469
|
after.set_index(primary_keys, inplace=True)
|
|
190
470
|
|
|
191
471
|
before_aligned, after_aligned = before.align(after)
|
|
192
|
-
diff = before_aligned.compare(
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
print(diff.to_string(na_rep='-') if not diff.empty else 'no changes')
|
|
472
|
+
diff = before_aligned.compare(
|
|
473
|
+
after_aligned, result_names=("base", "current"), keep_equal=keep_equal, keep_shape=keep_shape
|
|
474
|
+
)
|
|
475
|
+
print(diff.to_string(na_rep="-") if not diff.empty else "no changes")
|
|
197
476
|
|
|
198
477
|
|
|
199
478
|
@cli.command(cls=TrackCommand)
|
|
200
|
-
@click.argument(
|
|
201
|
-
@click.option(
|
|
202
|
-
@click.option(
|
|
203
|
-
@click.option(
|
|
204
|
-
@click.option(
|
|
205
|
-
|
|
479
|
+
@click.argument("state_file", required=False)
|
|
480
|
+
@click.option("--host", default="localhost", show_default=True, help="The host to bind to.")
|
|
481
|
+
@click.option("--port", default=8000, show_default=True, help="The port to bind to.", type=int)
|
|
482
|
+
@click.option("--lifetime", default=0, show_default=True, help="The lifetime of the server in seconds.", type=int)
|
|
483
|
+
@click.option(
|
|
484
|
+
"--idle-timeout",
|
|
485
|
+
default=0,
|
|
486
|
+
show_default=True,
|
|
487
|
+
help="The idle timeout in seconds. If 0, idle timeout is disabled. Maximum value is capped by lifetime.",
|
|
488
|
+
type=int,
|
|
489
|
+
)
|
|
490
|
+
@click.option("--review", is_flag=True, help="Open the state file in the review mode.")
|
|
491
|
+
@click.option("--single-env", is_flag=True, help="Launch in single environment mode directly.")
|
|
206
492
|
@add_options(dbt_related_options)
|
|
207
493
|
@add_options(sqlmesh_related_options)
|
|
208
494
|
@add_options(recce_options)
|
|
209
495
|
@add_options(recce_dbt_artifact_dir_options)
|
|
210
496
|
@add_options(recce_cloud_options)
|
|
211
|
-
|
|
497
|
+
@add_options(recce_cloud_auth_options)
|
|
498
|
+
@add_options(recce_hidden_options)
|
|
499
|
+
def server(host, port, lifetime, idle_timeout=0, state_file=None, **kwargs):
|
|
212
500
|
"""
|
|
213
501
|
Launch the recce server
|
|
214
502
|
|
|
@@ -229,59 +517,84 @@ def server(host, port, state_file=None, **kwargs):
|
|
|
229
517
|
recce server --review recce_state.json
|
|
230
518
|
|
|
231
519
|
\b
|
|
232
|
-
# Launch the server
|
|
520
|
+
# Launch the server using the state from the PR of your current branch. (Requires GitHub token)
|
|
521
|
+
export GITHUB_TOKEN=<your-github-token>
|
|
233
522
|
recce server --cloud
|
|
234
523
|
recce server --review --cloud
|
|
235
524
|
|
|
236
525
|
"""
|
|
237
526
|
|
|
238
|
-
from .server import app, AppState
|
|
239
527
|
from rich.console import Console
|
|
528
|
+
from rich.prompt import Confirm
|
|
529
|
+
|
|
530
|
+
from .server import AppState, app
|
|
240
531
|
|
|
241
|
-
RecceConfig(config_file=kwargs.get(
|
|
532
|
+
RecceConfig(config_file=kwargs.get("config"))
|
|
533
|
+
|
|
534
|
+
# Initialize startup performance tracking
|
|
535
|
+
from recce.util.startup_perf import StartupPerfTracker, set_startup_tracker
|
|
536
|
+
|
|
537
|
+
startup_tracker = StartupPerfTracker()
|
|
538
|
+
set_startup_tracker(startup_tracker)
|
|
242
539
|
|
|
243
540
|
handle_debug_flag(**kwargs)
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
541
|
+
patch_derived_args(kwargs)
|
|
542
|
+
|
|
543
|
+
server_mode = kwargs.get("mode") if kwargs.get("mode") else RecceServerMode.server
|
|
544
|
+
is_review = kwargs.get("review", False)
|
|
545
|
+
is_cloud = kwargs.get("cloud", False)
|
|
546
|
+
startup_tracker.set_cloud_mode(is_cloud)
|
|
248
547
|
flag = {
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
548
|
+
"single_env_onboarding": False,
|
|
549
|
+
"show_relaunch_hint": False,
|
|
550
|
+
"preview": False,
|
|
551
|
+
"read_only": False,
|
|
252
552
|
}
|
|
253
|
-
|
|
254
|
-
cloud_options = {
|
|
255
|
-
'host': kwargs.get('state_file_host'),
|
|
256
|
-
'token': kwargs.get('cloud_token'),
|
|
257
|
-
'password': kwargs.get('password'),
|
|
258
|
-
}
|
|
259
|
-
cloud_onboarding_state = get_recce_cloud_onboarding_state(kwargs.get('cloud_token'))
|
|
260
|
-
flag['show_onboarding_guide'] = False if cloud_onboarding_state == 'completed' else True
|
|
261
|
-
|
|
262
|
-
auth_options = {}
|
|
263
|
-
api_token = kwargs.get('api_token') if kwargs.get('api_token') else get_recce_api_token()
|
|
264
|
-
auth_options['api_token'] = api_token
|
|
265
|
-
|
|
266
|
-
# Check Single Environment Onboarding Mode if the review mode is False
|
|
267
|
-
if not os.path.isdir(kwargs.get('target_base_path')) and is_review is False:
|
|
268
|
-
# Mark as single env onboarding mode if user provides the target-path only
|
|
269
|
-
flag['single_env_onboarding'] = True
|
|
270
|
-
flag['show_relaunch_hint'] = True
|
|
271
|
-
target_path = kwargs.get('target_path')
|
|
272
|
-
target_base_path = kwargs.get('target_base_path')
|
|
273
|
-
# Use the target path as the base path
|
|
274
|
-
kwargs['target_base_path'] = target_path
|
|
553
|
+
console = Console()
|
|
275
554
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
555
|
+
# Prepare API token
|
|
556
|
+
try:
|
|
557
|
+
api_token = prepare_api_token(**kwargs)
|
|
558
|
+
kwargs["api_token"] = api_token
|
|
559
|
+
except RecceConfigException:
|
|
560
|
+
show_invalid_api_token_message()
|
|
561
|
+
exit(1)
|
|
562
|
+
auth_options = {
|
|
563
|
+
"api_token": api_token,
|
|
564
|
+
}
|
|
283
565
|
|
|
284
|
-
|
|
566
|
+
# Check Single Environment Onboarding Mode if not in cloud mode and not in review mode
|
|
567
|
+
if not is_cloud and not is_review:
|
|
568
|
+
project_dir_path = Path(kwargs.get("project_dir") or "./")
|
|
569
|
+
target_base_path = project_dir_path.joinpath(Path(kwargs.get("target_base_path", "target-base")))
|
|
570
|
+
if not target_base_path.is_dir():
|
|
571
|
+
# Mark as single env onboarding mode if user provides the target-path only
|
|
572
|
+
flag["single_env_onboarding"] = True
|
|
573
|
+
flag["show_relaunch_hint"] = True
|
|
574
|
+
# Use the target path as the base path
|
|
575
|
+
kwargs["target_base_path"] = kwargs.get("target_path")
|
|
576
|
+
|
|
577
|
+
# Server mode:
|
|
578
|
+
#
|
|
579
|
+
# It's used to determine the features disabled in the Web UI. Only used in the cloud-managed recce instances.
|
|
580
|
+
#
|
|
581
|
+
# Read-Only: No run query, no checklist
|
|
582
|
+
# Preview (Metadata-Only): No run query
|
|
583
|
+
if server_mode == RecceServerMode.preview:
|
|
584
|
+
flag["preview"] = True
|
|
585
|
+
elif server_mode == RecceServerMode.read_only:
|
|
586
|
+
flag["read_only"] = True
|
|
587
|
+
|
|
588
|
+
# Onboarding State logic update here
|
|
589
|
+
update_onboarding_state(api_token, flag.get("single_env_onboarding"))
|
|
590
|
+
|
|
591
|
+
# Create state loader using shared function
|
|
592
|
+
from recce.util.startup_perf import get_startup_tracker
|
|
593
|
+
|
|
594
|
+
state_loader = create_state_loader_by_args(state_file, **kwargs)
|
|
595
|
+
|
|
596
|
+
if (tracker := get_startup_tracker()) and hasattr(state_loader, "catalog"):
|
|
597
|
+
tracker.set_catalog_type(state_loader.catalog)
|
|
285
598
|
|
|
286
599
|
if not state_loader.verify():
|
|
287
600
|
error, hint = state_loader.error_and_hint
|
|
@@ -289,42 +602,111 @@ def server(host, port, state_file=None, **kwargs):
|
|
|
289
602
|
console.print(f"{hint}")
|
|
290
603
|
exit(1)
|
|
291
604
|
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
605
|
+
try:
|
|
606
|
+
result, message = RecceContext.verify_required_artifacts(**kwargs)
|
|
607
|
+
except Exception as e:
|
|
608
|
+
result = False
|
|
609
|
+
error_type = type(e).__name__
|
|
610
|
+
error_message = str(e)
|
|
611
|
+
message = f"{error_type}: {error_message}"
|
|
298
612
|
if not result:
|
|
613
|
+
console.rule("Notice", style="orange3")
|
|
299
614
|
console.print(f"[[red]Error[/red]] {message}")
|
|
300
615
|
exit(1)
|
|
301
616
|
|
|
302
|
-
|
|
617
|
+
if state_loader.review_mode:
|
|
618
|
+
console.rule("Recce Server : Review Mode")
|
|
619
|
+
elif flag.get("single_env_onboarding"):
|
|
620
|
+
# Show warning message
|
|
621
|
+
console.rule("Notice", style="orange3")
|
|
622
|
+
console.print(
|
|
623
|
+
"Recce will launch with limited features (no environment comparison).\n"
|
|
624
|
+
"\n"
|
|
625
|
+
"For full functionality, set up a base environment first.\n"
|
|
626
|
+
"Setup help: 'recce debug' or https://docs.datarecce.io/configure-diff/\n"
|
|
627
|
+
)
|
|
628
|
+
|
|
629
|
+
single_env_flag = kwargs.get("single_env", False)
|
|
630
|
+
if not single_env_flag:
|
|
631
|
+
lanch_in_single_env = Confirm.ask("Continue to launch Recce?")
|
|
632
|
+
if not lanch_in_single_env:
|
|
633
|
+
exit(0)
|
|
634
|
+
|
|
635
|
+
console.rule("Recce Server : Limited Features")
|
|
636
|
+
else:
|
|
637
|
+
console.rule("Recce Server")
|
|
638
|
+
|
|
639
|
+
# Validate idle_timeout: cap at lifetime if it exceeds lifetime
|
|
640
|
+
if idle_timeout > 0:
|
|
641
|
+
# If lifetime is set (> 0) and idle_timeout exceeds it, cap to lifetime
|
|
642
|
+
if lifetime > 0 and idle_timeout > lifetime:
|
|
643
|
+
effective_idle_timeout = lifetime
|
|
644
|
+
console.print(
|
|
645
|
+
f"[[yellow]Warning[/yellow]] idle_timeout ({idle_timeout}s) exceeds lifetime ({lifetime}s). "
|
|
646
|
+
f"Capping idle_timeout to {effective_idle_timeout}s."
|
|
647
|
+
)
|
|
648
|
+
else:
|
|
649
|
+
# Use idle_timeout as-is (either lifetime is 0, or idle_timeout <= lifetime)
|
|
650
|
+
effective_idle_timeout = idle_timeout
|
|
651
|
+
else:
|
|
652
|
+
# idle_timeout is 0 or negative, disable idle timeout
|
|
653
|
+
effective_idle_timeout = 0
|
|
654
|
+
|
|
655
|
+
state = AppState(
|
|
656
|
+
command=server_mode,
|
|
657
|
+
state_loader=state_loader,
|
|
658
|
+
kwargs=kwargs,
|
|
659
|
+
flag=flag,
|
|
660
|
+
auth_options=auth_options,
|
|
661
|
+
lifetime=lifetime,
|
|
662
|
+
idle_timeout=effective_idle_timeout,
|
|
663
|
+
share_url=kwargs.get("share_url"),
|
|
664
|
+
organization_name=os.environ.get("RECCE_SESSION_ORGANIZATION_NAME"),
|
|
665
|
+
web_url=os.environ.get("RECCE_CLOUD_WEB_URL"),
|
|
666
|
+
)
|
|
303
667
|
app.state = state
|
|
304
668
|
|
|
305
|
-
|
|
669
|
+
if server_mode == RecceServerMode.read_only:
|
|
670
|
+
set_default_context(RecceContext.load(**kwargs, state_loader=state_loader))
|
|
671
|
+
|
|
672
|
+
uvicorn.run(app, host=host, port=port, lifespan="on")
|
|
306
673
|
|
|
307
674
|
|
|
308
|
-
DEFAULT_RECCE_STATE_FILE =
|
|
675
|
+
DEFAULT_RECCE_STATE_FILE = "recce_state.json"
|
|
309
676
|
|
|
310
677
|
|
|
311
678
|
@cli.command(cls=TrackCommand)
|
|
312
|
-
@click.option(
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
@click.option(
|
|
322
|
-
|
|
679
|
+
@click.option(
|
|
680
|
+
"-o",
|
|
681
|
+
"--output",
|
|
682
|
+
help="Path of the output state file.",
|
|
683
|
+
type=click.Path(),
|
|
684
|
+
default=DEFAULT_RECCE_STATE_FILE,
|
|
685
|
+
show_default=True,
|
|
686
|
+
)
|
|
687
|
+
@click.option("--state-file", help="Path of the import state file.", type=click.Path())
|
|
688
|
+
@click.option("--summary", help="Path of the summary markdown file.", type=click.Path())
|
|
689
|
+
@click.option("--skip-query", is_flag=True, help="Skip running the queries for the checks.")
|
|
690
|
+
@click.option("--skip-check", is_flag=True, help="Skip running the checks.")
|
|
691
|
+
@click.option(
|
|
692
|
+
"--git-current-branch",
|
|
693
|
+
help="The git branch of the current environment.",
|
|
694
|
+
type=click.STRING,
|
|
695
|
+
envvar="GITHUB_HEAD_REF",
|
|
696
|
+
)
|
|
697
|
+
@click.option(
|
|
698
|
+
"--git-base-branch", help="The git branch of the base environment.", type=click.STRING, envvar="GITHUB_BASE_REF"
|
|
699
|
+
)
|
|
700
|
+
@click.option(
|
|
701
|
+
"--github-pull-request-url", help="The github pull request url to use for the lineage.", type=click.STRING
|
|
702
|
+
)
|
|
323
703
|
@add_options(dbt_related_options)
|
|
324
704
|
@add_options(sqlmesh_related_options)
|
|
325
705
|
@add_options(recce_options)
|
|
326
706
|
@add_options(recce_dbt_artifact_dir_options)
|
|
327
707
|
@add_options(recce_cloud_options)
|
|
708
|
+
@add_options(recce_cloud_auth_options)
|
|
709
|
+
@add_options(recce_hidden_options)
|
|
328
710
|
def run(output, **kwargs):
|
|
329
711
|
"""
|
|
330
712
|
Run recce and output the state file
|
|
@@ -345,25 +727,32 @@ def run(output, **kwargs):
|
|
|
345
727
|
|
|
346
728
|
"""
|
|
347
729
|
from rich.console import Console
|
|
730
|
+
|
|
348
731
|
handle_debug_flag(**kwargs)
|
|
349
732
|
console = Console()
|
|
350
733
|
is_github_action, pr_url = check_github_ci_env(**kwargs)
|
|
351
734
|
if is_github_action is True and pr_url is not None:
|
|
352
|
-
kwargs[
|
|
735
|
+
kwargs["github_pull_request_url"] = pr_url
|
|
353
736
|
|
|
354
737
|
# Initialize Recce Config
|
|
355
|
-
RecceConfig(config_file=kwargs.get(
|
|
738
|
+
RecceConfig(config_file=kwargs.get("config"))
|
|
356
739
|
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
740
|
+
patch_derived_args(kwargs)
|
|
741
|
+
# Remove share_url from kwargs to avoid affecting state loader creation
|
|
742
|
+
kwargs.pop("share_url", None)
|
|
743
|
+
|
|
744
|
+
state_file = kwargs.pop("state_file", None)
|
|
745
|
+
|
|
746
|
+
# Prepare API token
|
|
747
|
+
try:
|
|
748
|
+
api_token = prepare_api_token(**kwargs)
|
|
749
|
+
kwargs["api_token"] = api_token
|
|
750
|
+
except RecceConfigException:
|
|
751
|
+
show_invalid_api_token_message()
|
|
752
|
+
exit(1)
|
|
364
753
|
|
|
365
|
-
|
|
366
|
-
|
|
754
|
+
# Create state loader using shared function
|
|
755
|
+
state_loader = create_state_loader_by_args(state_file, **kwargs)
|
|
367
756
|
|
|
368
757
|
if not state_loader.verify():
|
|
369
758
|
error, hint = state_loader.error_and_hint
|
|
@@ -378,7 +767,7 @@ def run(output, **kwargs):
|
|
|
378
767
|
|
|
379
768
|
# Verify the output state file path
|
|
380
769
|
try:
|
|
381
|
-
if os.path.isdir(output) or output.endswith(
|
|
770
|
+
if os.path.isdir(output) or output.endswith("/"):
|
|
382
771
|
|
|
383
772
|
output_dir = Path(output)
|
|
384
773
|
# Create the directory if not exists
|
|
@@ -386,7 +775,8 @@ def run(output, **kwargs):
|
|
|
386
775
|
output = os.path.join(output, DEFAULT_RECCE_STATE_FILE)
|
|
387
776
|
console.print(
|
|
388
777
|
f"[[yellow]Warning[/yellow]] The path '{output_dir}' is a directory. "
|
|
389
|
-
f"The state file will be saved as '{output}'."
|
|
778
|
+
f"The state file will be saved as '{output}'."
|
|
779
|
+
)
|
|
390
780
|
else:
|
|
391
781
|
# Create the parent directory if not exists
|
|
392
782
|
output_dir = Path(output).parent
|
|
@@ -400,30 +790,43 @@ def run(output, **kwargs):
|
|
|
400
790
|
|
|
401
791
|
|
|
402
792
|
@cli.command(cls=TrackCommand)
|
|
403
|
-
@click.argument(
|
|
404
|
-
@click.option(
|
|
405
|
-
|
|
406
|
-
|
|
793
|
+
@click.argument("state_file", required=False)
|
|
794
|
+
@click.option(
|
|
795
|
+
"--format",
|
|
796
|
+
"-f",
|
|
797
|
+
help="Output format. Currently only markdown is supported.",
|
|
798
|
+
type=click.Choice(["markdown", "mermaid", "check"], case_sensitive=False),
|
|
799
|
+
default="markdown",
|
|
800
|
+
show_default=True,
|
|
801
|
+
hidden=True,
|
|
802
|
+
)
|
|
407
803
|
@add_options(dbt_related_options)
|
|
408
804
|
@add_options(recce_options)
|
|
409
805
|
@add_options(recce_cloud_options)
|
|
410
806
|
def summary(state_file, **kwargs):
|
|
411
807
|
"""
|
|
412
|
-
|
|
808
|
+
Generate a summary of the recce state file
|
|
413
809
|
"""
|
|
414
810
|
from rich.console import Console
|
|
811
|
+
|
|
415
812
|
from .core import load_context
|
|
813
|
+
|
|
416
814
|
handle_debug_flag(**kwargs)
|
|
417
815
|
console = Console()
|
|
418
|
-
cloud_mode = kwargs.get(
|
|
419
|
-
cloud_options =
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
816
|
+
cloud_mode = kwargs.get("cloud", False)
|
|
817
|
+
cloud_options = (
|
|
818
|
+
{
|
|
819
|
+
"host": kwargs.get("state_file_host"),
|
|
820
|
+
"github_token": kwargs.get("cloud_token"),
|
|
821
|
+
"password": kwargs.get("password"),
|
|
822
|
+
}
|
|
823
|
+
if cloud_mode
|
|
824
|
+
else None
|
|
825
|
+
)
|
|
424
826
|
|
|
425
|
-
state_loader = create_state_loader(
|
|
426
|
-
|
|
827
|
+
state_loader = create_state_loader(
|
|
828
|
+
review_mode=True, cloud_mode=cloud_mode, state_file=state_file, cloud_options=cloud_options
|
|
829
|
+
)
|
|
427
830
|
|
|
428
831
|
if not state_loader.verify():
|
|
429
832
|
error, hint = state_loader.error_and_hint
|
|
@@ -438,56 +841,93 @@ def summary(state_file, **kwargs):
|
|
|
438
841
|
console.print(f"{e}")
|
|
439
842
|
exit(1)
|
|
440
843
|
|
|
441
|
-
output = generate_markdown_summary(ctx, summary_format=kwargs.get(
|
|
844
|
+
output = generate_markdown_summary(ctx, summary_format=kwargs.get("format"))
|
|
442
845
|
print(output)
|
|
443
846
|
|
|
444
847
|
|
|
445
|
-
@cli.
|
|
848
|
+
@cli.command(cls=TrackCommand)
|
|
849
|
+
def connect_to_cloud():
|
|
850
|
+
"""
|
|
851
|
+
Connect OSS to Cloud
|
|
852
|
+
"""
|
|
853
|
+
import webbrowser
|
|
854
|
+
|
|
855
|
+
from rich.console import Console
|
|
856
|
+
|
|
857
|
+
console = Console()
|
|
858
|
+
|
|
859
|
+
# Prepare RSA keys for connecting to cloud
|
|
860
|
+
private_key, public_key = generate_key_pair()
|
|
861
|
+
|
|
862
|
+
connect_url, callback_port = prepare_connection_url(public_key)
|
|
863
|
+
console.rule("Connecting to Recce Cloud")
|
|
864
|
+
console.print("Attempting to automatically open the Recce Cloud authorization page in your default browser.")
|
|
865
|
+
console.print("If the browser does not open, please open the following URL:")
|
|
866
|
+
console.print(connect_url)
|
|
867
|
+
webbrowser.open(connect_url)
|
|
868
|
+
|
|
869
|
+
# Launch a callback HTTP server for fetching the api-token
|
|
870
|
+
run_one_time_http_server(private_key, port=callback_port)
|
|
871
|
+
|
|
872
|
+
|
|
873
|
+
@cli.group("cloud", short_help="Manage Recce Cloud state file.")
|
|
446
874
|
def cloud(**kwargs):
|
|
447
875
|
# Manage Recce Cloud.
|
|
448
876
|
pass
|
|
449
877
|
|
|
450
878
|
|
|
451
879
|
@cloud.command(cls=TrackCommand)
|
|
452
|
-
@click.option(
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
880
|
+
@click.option("--cloud-token", help="The GitHub token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN")
|
|
881
|
+
@click.option(
|
|
882
|
+
"--state-file-host",
|
|
883
|
+
help="The host to fetch the state file from.",
|
|
884
|
+
type=click.STRING,
|
|
885
|
+
envvar="RECCE_STATE_FILE_HOST",
|
|
886
|
+
default="",
|
|
887
|
+
hidden=True,
|
|
888
|
+
)
|
|
889
|
+
@click.option(
|
|
890
|
+
"--password",
|
|
891
|
+
"-p",
|
|
892
|
+
help="The password to encrypt the state file in cloud.",
|
|
893
|
+
type=click.STRING,
|
|
894
|
+
envvar="RECCE_STATE_PASSWORD",
|
|
895
|
+
)
|
|
896
|
+
@click.option("--force", "-f", help="Bypasses the confirmation prompt. Purge the state file directly.", is_flag=True)
|
|
459
897
|
@add_options(recce_options)
|
|
460
898
|
def purge(**kwargs):
|
|
461
899
|
"""
|
|
462
|
-
|
|
900
|
+
Purge the state file from cloud
|
|
463
901
|
"""
|
|
464
902
|
from rich.console import Console
|
|
903
|
+
|
|
465
904
|
handle_debug_flag(**kwargs)
|
|
466
905
|
console = Console()
|
|
467
906
|
state_loader = None
|
|
468
907
|
cloud_options = {
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
908
|
+
"host": kwargs.get("state_file_host"),
|
|
909
|
+
"github_token": kwargs.get("cloud_token"),
|
|
910
|
+
"password": kwargs.get("password"),
|
|
472
911
|
}
|
|
473
|
-
force_to_purge = kwargs.get(
|
|
912
|
+
force_to_purge = kwargs.get("force", False)
|
|
474
913
|
|
|
475
914
|
try:
|
|
476
|
-
console.rule(
|
|
477
|
-
state_loader =
|
|
478
|
-
|
|
915
|
+
console.rule("Check Recce State from Cloud")
|
|
916
|
+
state_loader = create_state_loader(
|
|
917
|
+
review_mode=False, cloud_mode=True, state_file=None, cloud_options=cloud_options
|
|
918
|
+
)
|
|
479
919
|
except Exception:
|
|
480
920
|
console.print("[[yellow]Skip[/yellow]] Cannot access existing state file from cloud. Purge it directly.")
|
|
481
921
|
|
|
482
922
|
if state_loader is None:
|
|
483
923
|
try:
|
|
484
|
-
if force_to_purge is True or click.confirm(
|
|
924
|
+
if force_to_purge is True or click.confirm("\nDo you want to purge the state file?"):
|
|
485
925
|
rc, err_msg = RecceCloudStateManager(cloud_options).purge_cloud_state()
|
|
486
926
|
if rc is True:
|
|
487
|
-
console.rule(
|
|
927
|
+
console.rule("Purged Successfully")
|
|
488
928
|
else:
|
|
489
|
-
console.rule(
|
|
490
|
-
console.print(f
|
|
929
|
+
console.rule("Failed to Purge", style="red")
|
|
930
|
+
console.print(f"Reason: {err_msg}")
|
|
491
931
|
|
|
492
932
|
except click.exceptions.Abort:
|
|
493
933
|
pass
|
|
@@ -498,21 +938,21 @@ def purge(**kwargs):
|
|
|
498
938
|
console.print("[[yellow]Skip[/yellow]] No state file found in cloud.")
|
|
499
939
|
return 0
|
|
500
940
|
|
|
501
|
-
pr_info = info.get(
|
|
502
|
-
console.print(
|
|
503
|
-
console.print(
|
|
504
|
-
console.print(f
|
|
505
|
-
console.print(f
|
|
941
|
+
pr_info = info.get("pull_request")
|
|
942
|
+
console.print("[green]State File hosted by[/green]", info.get("source"))
|
|
943
|
+
console.print("[green]GitHub Repository[/green]", info.get("pull_request").repository)
|
|
944
|
+
console.print(f"[green]GitHub Pull Request[/green]\n{pr_info.title} #{pr_info.id}")
|
|
945
|
+
console.print(f"Branch merged into [blue]{pr_info.base_branch}[/blue] from [blue]{pr_info.branch}[/blue]")
|
|
506
946
|
console.print(pr_info.url)
|
|
507
947
|
|
|
508
948
|
try:
|
|
509
|
-
if force_to_purge is True or click.confirm(
|
|
949
|
+
if force_to_purge is True or click.confirm("\nDo you want to purge the state file?"):
|
|
510
950
|
response = state_loader.purge()
|
|
511
951
|
if response is True:
|
|
512
|
-
console.rule(
|
|
952
|
+
console.rule("Purged Successfully")
|
|
513
953
|
else:
|
|
514
|
-
console.rule(
|
|
515
|
-
console.print(f
|
|
954
|
+
console.rule("Failed to Purge", style="red")
|
|
955
|
+
console.print(f"Reason: {state_loader.error_message}")
|
|
516
956
|
except click.exceptions.Abort:
|
|
517
957
|
pass
|
|
518
958
|
|
|
@@ -520,32 +960,43 @@ def purge(**kwargs):
|
|
|
520
960
|
|
|
521
961
|
|
|
522
962
|
@cloud.command(cls=TrackCommand)
|
|
523
|
-
@click.argument(
|
|
524
|
-
@click.option(
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
963
|
+
@click.argument("state_file", type=click.Path(exists=True))
|
|
964
|
+
@click.option("--cloud-token", help="The GitHub token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN")
|
|
965
|
+
@click.option(
|
|
966
|
+
"--state-file-host",
|
|
967
|
+
help="The host to fetch the state file from.",
|
|
968
|
+
type=click.STRING,
|
|
969
|
+
envvar="RECCE_STATE_FILE_HOST",
|
|
970
|
+
default="",
|
|
971
|
+
hidden=True,
|
|
972
|
+
)
|
|
973
|
+
@click.option(
|
|
974
|
+
"--password",
|
|
975
|
+
"-p",
|
|
976
|
+
help="The password to encrypt the state file in cloud.",
|
|
977
|
+
type=click.STRING,
|
|
978
|
+
envvar="RECCE_STATE_PASSWORD",
|
|
979
|
+
)
|
|
530
980
|
@add_options(recce_options)
|
|
531
981
|
def upload(state_file, **kwargs):
|
|
532
982
|
"""
|
|
533
|
-
|
|
983
|
+
Upload the state file to cloud
|
|
534
984
|
"""
|
|
535
985
|
from rich.console import Console
|
|
536
986
|
|
|
537
987
|
handle_debug_flag(**kwargs)
|
|
538
988
|
cloud_options = {
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
989
|
+
"host": kwargs.get("state_file_host"),
|
|
990
|
+
"github_token": kwargs.get("cloud_token"),
|
|
991
|
+
"password": kwargs.get("password"),
|
|
542
992
|
}
|
|
543
993
|
|
|
544
994
|
console = Console()
|
|
545
995
|
|
|
546
996
|
# load local state
|
|
547
|
-
state_loader = create_state_loader(
|
|
548
|
-
|
|
997
|
+
state_loader = create_state_loader(
|
|
998
|
+
review_mode=False, cloud_mode=False, state_file=state_file, cloud_options=cloud_options
|
|
999
|
+
)
|
|
549
1000
|
|
|
550
1001
|
if not state_loader.verify():
|
|
551
1002
|
error, hint = state_loader.error_and_hint
|
|
@@ -563,34 +1014,50 @@ def upload(state_file, **kwargs):
|
|
|
563
1014
|
|
|
564
1015
|
cloud_state_file_exists = state_manager.check_cloud_state_exists()
|
|
565
1016
|
|
|
566
|
-
if cloud_state_file_exists and not click.confirm(
|
|
1017
|
+
if cloud_state_file_exists and not click.confirm("\nDo you want to overwrite the existing state file?"):
|
|
567
1018
|
return 0
|
|
568
1019
|
|
|
569
1020
|
console.print(state_manager.upload_state_to_cloud(state_loader.state))
|
|
570
1021
|
|
|
571
1022
|
|
|
572
1023
|
@cloud.command(cls=TrackCommand)
|
|
573
|
-
@click.option(
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
1024
|
+
@click.option(
|
|
1025
|
+
"-o",
|
|
1026
|
+
"--output",
|
|
1027
|
+
help="Path of the downloaded state file.",
|
|
1028
|
+
type=click.STRING,
|
|
1029
|
+
default=DEFAULT_RECCE_STATE_FILE,
|
|
1030
|
+
show_default=True,
|
|
1031
|
+
)
|
|
1032
|
+
@click.option("--cloud-token", help="The GitHub token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN")
|
|
1033
|
+
@click.option(
|
|
1034
|
+
"--state-file-host",
|
|
1035
|
+
help="The host to fetch the state file from.",
|
|
1036
|
+
type=click.STRING,
|
|
1037
|
+
envvar="RECCE_STATE_FILE_HOST",
|
|
1038
|
+
default="",
|
|
1039
|
+
hidden=True,
|
|
1040
|
+
)
|
|
1041
|
+
@click.option(
|
|
1042
|
+
"--password",
|
|
1043
|
+
"-p",
|
|
1044
|
+
help="The password to encrypt the state file in cloud.",
|
|
1045
|
+
type=click.STRING,
|
|
1046
|
+
envvar="RECCE_STATE_PASSWORD",
|
|
1047
|
+
)
|
|
581
1048
|
@add_options(recce_options)
|
|
582
1049
|
def download(**kwargs):
|
|
583
1050
|
"""
|
|
584
|
-
|
|
1051
|
+
Download the state file to cloud
|
|
585
1052
|
"""
|
|
586
1053
|
from rich.console import Console
|
|
587
1054
|
|
|
588
1055
|
handle_debug_flag(**kwargs)
|
|
589
|
-
filepath = kwargs.get(
|
|
1056
|
+
filepath = kwargs.get("output")
|
|
590
1057
|
cloud_options = {
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
1058
|
+
"host": kwargs.get("state_file_host"),
|
|
1059
|
+
"github_token": kwargs.get("cloud_token"),
|
|
1060
|
+
"password": kwargs.get("password"),
|
|
594
1061
|
}
|
|
595
1062
|
|
|
596
1063
|
console = Console()
|
|
@@ -606,7 +1073,7 @@ def download(**kwargs):
|
|
|
606
1073
|
cloud_state_file_exists = state_manager.check_cloud_state_exists()
|
|
607
1074
|
|
|
608
1075
|
if not cloud_state_file_exists:
|
|
609
|
-
console.print(
|
|
1076
|
+
console.print("[yellow]Skip[/yellow] No state file found in cloud.")
|
|
610
1077
|
return 0
|
|
611
1078
|
|
|
612
1079
|
state_manager.download_state_from_cloud(filepath)
|
|
@@ -614,41 +1081,60 @@ def download(**kwargs):
|
|
|
614
1081
|
|
|
615
1082
|
|
|
616
1083
|
@cloud.command(cls=TrackCommand)
|
|
617
|
-
@click.option(
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
1084
|
+
@click.option("--cloud-token", help="The GitHub token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN")
|
|
1085
|
+
@click.option(
|
|
1086
|
+
"--branch",
|
|
1087
|
+
"-b",
|
|
1088
|
+
help="The branch of the provided artifacts.",
|
|
1089
|
+
type=click.STRING,
|
|
1090
|
+
envvar="GITHUB_HEAD_REF",
|
|
1091
|
+
default=current_branch(),
|
|
1092
|
+
show_default=True,
|
|
1093
|
+
)
|
|
1094
|
+
@click.option(
|
|
1095
|
+
"--target-path",
|
|
1096
|
+
help="dbt artifacts directory for your artifacts.",
|
|
1097
|
+
type=click.STRING,
|
|
1098
|
+
default="target",
|
|
1099
|
+
show_default=True,
|
|
1100
|
+
)
|
|
1101
|
+
@click.option(
|
|
1102
|
+
"--password",
|
|
1103
|
+
"-p",
|
|
1104
|
+
help="The password to encrypt the dbt artifacts in cloud.",
|
|
1105
|
+
type=click.STRING,
|
|
1106
|
+
envvar="RECCE_STATE_PASSWORD",
|
|
1107
|
+
required=True,
|
|
1108
|
+
)
|
|
625
1109
|
@add_options(recce_options)
|
|
626
1110
|
def upload_artifacts(**kwargs):
|
|
627
1111
|
"""
|
|
628
|
-
|
|
1112
|
+
Upload the dbt artifacts to cloud
|
|
629
1113
|
|
|
630
|
-
|
|
631
|
-
|
|
1114
|
+
Upload the dbt artifacts (metadata.json, catalog.json) to Recce Cloud for the given branch.
|
|
1115
|
+
The password is used to encrypt the dbt artifacts in the cloud. You will need the password to download the dbt artifacts.
|
|
632
1116
|
|
|
633
|
-
|
|
634
|
-
|
|
1117
|
+
By default, the artifacts are uploaded to the current branch. You can specify the branch using the --branch option.
|
|
1118
|
+
The target path is set to 'target' by default. You can specify the target path using the --target-path option.
|
|
635
1119
|
"""
|
|
636
1120
|
from rich.console import Console
|
|
1121
|
+
|
|
637
1122
|
console = Console()
|
|
638
|
-
cloud_token = kwargs.get(
|
|
639
|
-
password = kwargs.get(
|
|
640
|
-
target_path = kwargs.get(
|
|
641
|
-
branch = kwargs.get(
|
|
1123
|
+
cloud_token = kwargs.get("cloud_token")
|
|
1124
|
+
password = kwargs.get("password")
|
|
1125
|
+
target_path = kwargs.get("target_path")
|
|
1126
|
+
branch = kwargs.get("branch")
|
|
642
1127
|
|
|
643
1128
|
try:
|
|
644
|
-
rc = upload_dbt_artifacts(
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
console.rule(
|
|
1129
|
+
rc = upload_dbt_artifacts(
|
|
1130
|
+
target_path, branch=branch, token=cloud_token, password=password, debug=kwargs.get("debug", False)
|
|
1131
|
+
)
|
|
1132
|
+
console.rule("Uploaded Successfully")
|
|
648
1133
|
console.print(
|
|
649
|
-
f'Uploaded dbt artifacts to Recce Cloud for branch "{branch}" from "{os.path.abspath(target_path)}"'
|
|
1134
|
+
f'Uploaded dbt artifacts to Recce Cloud for branch "{branch}" from "{os.path.abspath(target_path)}"'
|
|
1135
|
+
)
|
|
650
1136
|
except Exception as e:
|
|
651
|
-
console.rule(
|
|
1137
|
+
console.rule("Failed to Upload", style="red")
|
|
652
1138
|
console.print("[[red]Error[/red]] Failed to upload the dbt artifacts to cloud.")
|
|
653
1139
|
console.print(f"Reason: {e}")
|
|
654
1140
|
rc = 1
|
|
@@ -657,22 +1143,32 @@ def upload_artifacts(**kwargs):
|
|
|
657
1143
|
|
|
658
1144
|
def _download_artifacts(branch, cloud_token, console, kwargs, password, target_path):
|
|
659
1145
|
try:
|
|
660
|
-
rc = download_dbt_artifacts(
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
1146
|
+
rc = download_dbt_artifacts(
|
|
1147
|
+
target_path,
|
|
1148
|
+
branch=branch,
|
|
1149
|
+
token=cloud_token,
|
|
1150
|
+
password=password,
|
|
1151
|
+
force=kwargs.get("force", False),
|
|
1152
|
+
debug=kwargs.get("debug", False),
|
|
1153
|
+
)
|
|
1154
|
+
console.rule("Downloaded Successfully")
|
|
664
1155
|
console.print(
|
|
665
|
-
f'Downloaded dbt artifacts from Recce Cloud for branch "{branch}" to "{os.path.abspath(target_path)}"'
|
|
1156
|
+
f'Downloaded dbt artifacts from Recce Cloud for branch "{branch}" to "{os.path.abspath(target_path)}"'
|
|
1157
|
+
)
|
|
666
1158
|
except Exception as e:
|
|
667
|
-
console.rule(
|
|
1159
|
+
console.rule("Failed to Download", style="red")
|
|
668
1160
|
console.print("[[red]Error[/red]] Failed to download the dbt artifacts from cloud.")
|
|
669
1161
|
reason = str(e)
|
|
670
1162
|
|
|
671
|
-
if
|
|
1163
|
+
if (
|
|
1164
|
+
"Requests specifying Server Side Encryption with Customer provided keys must provide the correct secret key"
|
|
1165
|
+
in reason
|
|
1166
|
+
):
|
|
672
1167
|
console.print("Reason: Decryption failed due to incorrect password.")
|
|
673
1168
|
console.print(
|
|
674
|
-
"Please provide the correct password to decrypt the dbt artifacts. Or re-upload the dbt artifacts with a new password."
|
|
675
|
-
|
|
1169
|
+
"Please provide the correct password to decrypt the dbt artifacts. Or re-upload the dbt artifacts with a new password."
|
|
1170
|
+
)
|
|
1171
|
+
elif "The specified key does not exist" in reason:
|
|
676
1172
|
console.print("Reason: The dbt artifacts is not found in the cloud.")
|
|
677
1173
|
console.print("Please upload the dbt artifacts to the cloud before downloading it.")
|
|
678
1174
|
else:
|
|
@@ -682,90 +1178,415 @@ def _download_artifacts(branch, cloud_token, console, kwargs, password, target_p
|
|
|
682
1178
|
|
|
683
1179
|
|
|
684
1180
|
@cloud.command(cls=TrackCommand)
|
|
685
|
-
@click.option(
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
1181
|
+
@click.option("--cloud-token", help="The GitHub token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN")
|
|
1182
|
+
@click.option(
|
|
1183
|
+
"--branch",
|
|
1184
|
+
"-b",
|
|
1185
|
+
help="The branch of the selected artifacts.",
|
|
1186
|
+
type=click.STRING,
|
|
1187
|
+
envvar="GITHUB_BASE_REF",
|
|
1188
|
+
default=current_branch(),
|
|
1189
|
+
show_default=True,
|
|
1190
|
+
)
|
|
1191
|
+
@click.option(
|
|
1192
|
+
"--target-path",
|
|
1193
|
+
help="The dbt artifacts directory for your artifacts.",
|
|
1194
|
+
type=click.STRING,
|
|
1195
|
+
default="target",
|
|
1196
|
+
show_default=True,
|
|
1197
|
+
)
|
|
1198
|
+
@click.option(
|
|
1199
|
+
"--password",
|
|
1200
|
+
"-p",
|
|
1201
|
+
help="The password to decrypt the dbt artifacts in cloud.",
|
|
1202
|
+
type=click.STRING,
|
|
1203
|
+
envvar="RECCE_STATE_PASSWORD",
|
|
1204
|
+
required=True,
|
|
1205
|
+
)
|
|
1206
|
+
@click.option("--force", "-f", help="Bypasses the confirmation prompt. Download the artifacts directly.", is_flag=True)
|
|
695
1207
|
@add_options(recce_options)
|
|
696
1208
|
def download_artifacts(**kwargs):
|
|
697
1209
|
"""
|
|
698
|
-
|
|
1210
|
+
Download the dbt artifacts from cloud
|
|
699
1211
|
|
|
700
|
-
|
|
701
|
-
|
|
1212
|
+
Download the dbt artifacts (metadata.json, catalog.json) from Recce Cloud for the given branch.
|
|
1213
|
+
The password is used to decrypt the dbt artifacts in the cloud.
|
|
702
1214
|
|
|
703
|
-
|
|
704
|
-
|
|
1215
|
+
By default, the artifacts are downloaded from the current branch. You can specify the branch using the --branch option.
|
|
1216
|
+
The target path is set to 'target' by default. You can specify the target path using the --target-path option.
|
|
705
1217
|
"""
|
|
706
1218
|
from rich.console import Console
|
|
1219
|
+
|
|
707
1220
|
console = Console()
|
|
708
|
-
cloud_token = kwargs.get(
|
|
709
|
-
password = kwargs.get(
|
|
710
|
-
target_path = kwargs.get(
|
|
711
|
-
branch = kwargs.get(
|
|
1221
|
+
cloud_token = kwargs.get("cloud_token")
|
|
1222
|
+
password = kwargs.get("password")
|
|
1223
|
+
target_path = kwargs.get("target_path")
|
|
1224
|
+
branch = kwargs.get("branch")
|
|
712
1225
|
return _download_artifacts(branch, cloud_token, console, kwargs, password, target_path)
|
|
713
1226
|
|
|
714
1227
|
|
|
715
1228
|
@cloud.command(cls=TrackCommand)
|
|
716
|
-
@click.option(
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
1229
|
+
@click.option("--cloud-token", help="The GitHub token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN")
|
|
1230
|
+
@click.option(
|
|
1231
|
+
"--branch",
|
|
1232
|
+
"-b",
|
|
1233
|
+
help="The branch of the selected artifacts.",
|
|
1234
|
+
type=click.STRING,
|
|
1235
|
+
envvar="GITHUB_BASE_REF",
|
|
1236
|
+
default=current_default_branch(),
|
|
1237
|
+
show_default=True,
|
|
1238
|
+
)
|
|
1239
|
+
@click.option(
|
|
1240
|
+
"--target-path",
|
|
1241
|
+
help="The dbt artifacts directory for your artifacts.",
|
|
1242
|
+
type=click.STRING,
|
|
1243
|
+
default="target-base",
|
|
1244
|
+
show_default=True,
|
|
1245
|
+
)
|
|
1246
|
+
@click.option(
|
|
1247
|
+
"--password",
|
|
1248
|
+
"-p",
|
|
1249
|
+
help="The password to decrypt the dbt artifacts in cloud.",
|
|
1250
|
+
type=click.STRING,
|
|
1251
|
+
envvar="RECCE_STATE_PASSWORD",
|
|
1252
|
+
required=True,
|
|
1253
|
+
)
|
|
1254
|
+
@click.option("--force", "-f", help="Bypasses the confirmation prompt. Download the artifacts directly.", is_flag=True)
|
|
726
1255
|
@add_options(recce_options)
|
|
727
1256
|
def download_base_artifacts(**kwargs):
|
|
728
1257
|
"""
|
|
729
|
-
|
|
1258
|
+
Download the base dbt artifacts from cloud
|
|
730
1259
|
|
|
731
|
-
|
|
732
|
-
|
|
1260
|
+
Download the base dbt artifacts (metadata.json, catalog.json) from Recce Cloud.
|
|
1261
|
+
This is useful when you start to set up the base dbt artifacts for the first time.
|
|
733
1262
|
|
|
734
|
-
|
|
1263
|
+
Please make sure you have uploaded the dbt artifacts before downloading them.
|
|
735
1264
|
"""
|
|
736
1265
|
from rich.console import Console
|
|
1266
|
+
|
|
737
1267
|
console = Console()
|
|
738
|
-
cloud_token = kwargs.get(
|
|
739
|
-
password = kwargs.get(
|
|
740
|
-
target_path = kwargs.get(
|
|
741
|
-
branch = kwargs.get(
|
|
1268
|
+
cloud_token = kwargs.get("cloud_token")
|
|
1269
|
+
password = kwargs.get("password")
|
|
1270
|
+
target_path = kwargs.get("target_path")
|
|
1271
|
+
branch = kwargs.get("branch")
|
|
1272
|
+
# If recce can't infer default branch from "GITHUB_BASE_REF" and current_default_branch()
|
|
1273
|
+
if branch is None:
|
|
1274
|
+
console.print(
|
|
1275
|
+
"[[red]Error[/red]] Please provide your base branch name with '--branch' to download the base " "artifacts."
|
|
1276
|
+
)
|
|
1277
|
+
exit(1)
|
|
1278
|
+
|
|
742
1279
|
return _download_artifacts(branch, cloud_token, console, kwargs, password, target_path)
|
|
743
1280
|
|
|
744
1281
|
|
|
745
|
-
@
|
|
1282
|
+
@cloud.command(cls=TrackCommand)
|
|
1283
|
+
@click.option("--cloud-token", help="The GitHub token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN")
|
|
1284
|
+
@click.option(
|
|
1285
|
+
"--branch",
|
|
1286
|
+
"-b",
|
|
1287
|
+
help="The branch to delete artifacts from.",
|
|
1288
|
+
type=click.STRING,
|
|
1289
|
+
envvar="GITHUB_HEAD_REF",
|
|
1290
|
+
default=current_branch(),
|
|
1291
|
+
show_default=True,
|
|
1292
|
+
)
|
|
1293
|
+
@click.option("--force", "-f", help="Bypasses the confirmation prompt. Delete the artifacts directly.", is_flag=True)
|
|
1294
|
+
@add_options(recce_options)
|
|
1295
|
+
def delete_artifacts(**kwargs):
|
|
1296
|
+
"""
|
|
1297
|
+
Delete the dbt artifacts from cloud
|
|
1298
|
+
|
|
1299
|
+
Delete the dbt artifacts (metadata.json, catalog.json) from Recce Cloud for the given branch.
|
|
1300
|
+
This will permanently remove the artifacts from the cloud storage.
|
|
1301
|
+
|
|
1302
|
+
By default, the artifacts are deleted from the current branch. You can specify the branch using the --branch option.
|
|
1303
|
+
"""
|
|
1304
|
+
from rich.console import Console
|
|
1305
|
+
|
|
1306
|
+
console = Console()
|
|
1307
|
+
cloud_token = kwargs.get("cloud_token")
|
|
1308
|
+
branch = kwargs.get("branch")
|
|
1309
|
+
force = kwargs.get("force", False)
|
|
1310
|
+
|
|
1311
|
+
if not force:
|
|
1312
|
+
if not click.confirm(f'Do you want to delete artifacts from branch "{branch}"?'):
|
|
1313
|
+
console.print("Deletion cancelled.")
|
|
1314
|
+
return 0
|
|
1315
|
+
|
|
1316
|
+
try:
|
|
1317
|
+
delete_dbt_artifacts(branch=branch, token=cloud_token, debug=kwargs.get("debug", False))
|
|
1318
|
+
console.print(f"[[green]Success[/green]] Artifacts deleted from branch: {branch}")
|
|
1319
|
+
return 0
|
|
1320
|
+
except click.exceptions.Abort:
|
|
1321
|
+
pass
|
|
1322
|
+
except RecceCloudException as e:
|
|
1323
|
+
console.print("[[red]Error[/red]] Failed to delete the dbt artifacts from cloud.")
|
|
1324
|
+
console.print(f"Reason: {e.reason}")
|
|
1325
|
+
exit(1)
|
|
1326
|
+
except Exception as e:
|
|
1327
|
+
console.print("[[red]Error[/red]] Failed to delete the dbt artifacts from cloud.")
|
|
1328
|
+
console.print(f"Reason: {e}")
|
|
1329
|
+
exit(1)
|
|
1330
|
+
|
|
1331
|
+
|
|
1332
|
+
@cloud.command(cls=TrackCommand, name="list-organizations")
|
|
1333
|
+
@click.option("--api-token", help="The Recce Cloud API token.", type=click.STRING, envvar="RECCE_API_TOKEN")
|
|
1334
|
+
@add_options(recce_options)
|
|
1335
|
+
def list_organizations(**kwargs):
|
|
1336
|
+
"""
|
|
1337
|
+
List organizations from Recce Cloud
|
|
1338
|
+
|
|
1339
|
+
Lists all organizations that the authenticated user has access to.
|
|
1340
|
+
"""
|
|
1341
|
+
from rich.console import Console
|
|
1342
|
+
from rich.table import Table
|
|
1343
|
+
|
|
1344
|
+
console = Console()
|
|
1345
|
+
handle_debug_flag(**kwargs)
|
|
1346
|
+
|
|
1347
|
+
try:
|
|
1348
|
+
api_token = prepare_api_token(**kwargs)
|
|
1349
|
+
except RecceConfigException:
|
|
1350
|
+
show_invalid_api_token_message()
|
|
1351
|
+
exit(1)
|
|
1352
|
+
|
|
1353
|
+
try:
|
|
1354
|
+
from recce.util.recce_cloud import RecceCloud
|
|
1355
|
+
|
|
1356
|
+
cloud = RecceCloud(api_token)
|
|
1357
|
+
organizations = cloud.list_organizations()
|
|
1358
|
+
|
|
1359
|
+
if not organizations:
|
|
1360
|
+
console.print("No organizations found.")
|
|
1361
|
+
return
|
|
1362
|
+
|
|
1363
|
+
table = Table(title="Organizations")
|
|
1364
|
+
table.add_column("ID", style="cyan")
|
|
1365
|
+
table.add_column("Name", style="green")
|
|
1366
|
+
table.add_column("Display Name", style="yellow")
|
|
1367
|
+
|
|
1368
|
+
for org in organizations:
|
|
1369
|
+
table.add_row(str(org.get("id", "")), org.get("name", ""), org.get("display_name", ""))
|
|
1370
|
+
|
|
1371
|
+
console.print(table)
|
|
1372
|
+
|
|
1373
|
+
except RecceCloudException as e:
|
|
1374
|
+
console.print(f"[[red]Error[/red]] {e}")
|
|
1375
|
+
exit(1)
|
|
1376
|
+
except Exception as e:
|
|
1377
|
+
console.print(f"[[red]Error[/red]] {e}")
|
|
1378
|
+
exit(1)
|
|
1379
|
+
|
|
1380
|
+
|
|
1381
|
+
@cloud.command(cls=TrackCommand, name="list-projects")
|
|
1382
|
+
@click.option(
|
|
1383
|
+
"--organization",
|
|
1384
|
+
"-o",
|
|
1385
|
+
help="Organization ID (can also be set via RECCE_ORGANIZATION_ID environment variable)",
|
|
1386
|
+
type=click.STRING,
|
|
1387
|
+
envvar="RECCE_ORGANIZATION_ID",
|
|
1388
|
+
)
|
|
1389
|
+
@click.option("--api-token", help="The Recce Cloud API token.", type=click.STRING, envvar="RECCE_API_TOKEN")
|
|
1390
|
+
@add_options(recce_options)
|
|
1391
|
+
def list_projects(**kwargs):
|
|
1392
|
+
"""
|
|
1393
|
+
List projects from Recce Cloud
|
|
1394
|
+
|
|
1395
|
+
Lists all projects in the specified organization that the authenticated user has access to.
|
|
1396
|
+
|
|
1397
|
+
Examples:
|
|
1398
|
+
|
|
1399
|
+
# Using environment variable
|
|
1400
|
+
export RECCE_ORGANIZATION_ID=8
|
|
1401
|
+
recce cloud list-projects
|
|
1402
|
+
|
|
1403
|
+
# Using command line argument
|
|
1404
|
+
recce cloud list-projects --organization 8
|
|
1405
|
+
|
|
1406
|
+
# Override environment variable
|
|
1407
|
+
export RECCE_ORGANIZATION_ID=8
|
|
1408
|
+
recce cloud list-projects --organization 10
|
|
1409
|
+
"""
|
|
1410
|
+
from rich.console import Console
|
|
1411
|
+
from rich.table import Table
|
|
1412
|
+
|
|
1413
|
+
console = Console()
|
|
1414
|
+
handle_debug_flag(**kwargs)
|
|
1415
|
+
|
|
1416
|
+
try:
|
|
1417
|
+
api_token = prepare_api_token(**kwargs)
|
|
1418
|
+
except RecceConfigException:
|
|
1419
|
+
show_invalid_api_token_message()
|
|
1420
|
+
exit(1)
|
|
1421
|
+
|
|
1422
|
+
organization = kwargs.get("organization")
|
|
1423
|
+
if not organization:
|
|
1424
|
+
console.print("[[red]Error[/red]] Organization ID is required. Please provide it via:")
|
|
1425
|
+
console.print(" --organization <id> or set RECCE_ORGANIZATION_ID environment variable")
|
|
1426
|
+
exit(1)
|
|
1427
|
+
|
|
1428
|
+
try:
|
|
1429
|
+
from recce.util.recce_cloud import RecceCloud
|
|
1430
|
+
|
|
1431
|
+
cloud = RecceCloud(api_token)
|
|
1432
|
+
projects = cloud.list_projects(organization)
|
|
1433
|
+
|
|
1434
|
+
if not projects:
|
|
1435
|
+
console.print(f"No projects found in organization {organization}.")
|
|
1436
|
+
return
|
|
1437
|
+
|
|
1438
|
+
table = Table(title=f"Projects in Organization {organization}")
|
|
1439
|
+
table.add_column("ID", style="cyan")
|
|
1440
|
+
table.add_column("Name", style="green")
|
|
1441
|
+
table.add_column("Display Name", style="yellow")
|
|
1442
|
+
|
|
1443
|
+
for project in projects:
|
|
1444
|
+
table.add_row(str(project.get("id", "")), project.get("name", ""), project.get("display_name", ""))
|
|
1445
|
+
|
|
1446
|
+
console.print(table)
|
|
1447
|
+
|
|
1448
|
+
except RecceCloudException as e:
|
|
1449
|
+
console.print(f"[[red]Error[/red]] {e}")
|
|
1450
|
+
exit(1)
|
|
1451
|
+
except Exception as e:
|
|
1452
|
+
console.print(f"[[red]Error[/red]] {e}")
|
|
1453
|
+
exit(1)
|
|
1454
|
+
|
|
1455
|
+
|
|
1456
|
+
@cloud.command(cls=TrackCommand, name="list-sessions")
|
|
1457
|
+
@click.option(
|
|
1458
|
+
"--organization",
|
|
1459
|
+
"-o",
|
|
1460
|
+
help="Organization ID (can also be set via RECCE_ORGANIZATION_ID environment variable)",
|
|
1461
|
+
type=click.STRING,
|
|
1462
|
+
envvar="RECCE_ORGANIZATION_ID",
|
|
1463
|
+
)
|
|
1464
|
+
@click.option(
|
|
1465
|
+
"--project",
|
|
1466
|
+
"-p",
|
|
1467
|
+
help="Project ID (can also be set via RECCE_PROJECT_ID environment variable)",
|
|
1468
|
+
type=click.STRING,
|
|
1469
|
+
envvar="RECCE_PROJECT_ID",
|
|
1470
|
+
)
|
|
1471
|
+
@click.option("--api-token", help="The Recce Cloud API token.", type=click.STRING, envvar="RECCE_API_TOKEN")
|
|
1472
|
+
@add_options(recce_options)
|
|
1473
|
+
def list_sessions(**kwargs):
|
|
1474
|
+
"""
|
|
1475
|
+
List sessions from Recce Cloud
|
|
1476
|
+
|
|
1477
|
+
Lists all sessions in the specified project that the authenticated user has access to.
|
|
1478
|
+
|
|
1479
|
+
Examples:
|
|
1480
|
+
|
|
1481
|
+
# Using environment variables
|
|
1482
|
+
export RECCE_ORGANIZATION_ID=8
|
|
1483
|
+
export RECCE_PROJECT_ID=7
|
|
1484
|
+
recce cloud list-sessions
|
|
1485
|
+
|
|
1486
|
+
# Using command line arguments
|
|
1487
|
+
recce cloud list-sessions --organization 8 --project 7
|
|
1488
|
+
|
|
1489
|
+
# Mixed usage (env + CLI override)
|
|
1490
|
+
export RECCE_ORGANIZATION_ID=8
|
|
1491
|
+
recce cloud list-sessions --project 7
|
|
1492
|
+
|
|
1493
|
+
# Override environment variables
|
|
1494
|
+
export RECCE_ORGANIZATION_ID=8
|
|
1495
|
+
export RECCE_PROJECT_ID=7
|
|
1496
|
+
recce cloud list-sessions --organization 10 --project 9
|
|
1497
|
+
"""
|
|
1498
|
+
from rich.console import Console
|
|
1499
|
+
from rich.table import Table
|
|
1500
|
+
|
|
1501
|
+
console = Console()
|
|
1502
|
+
handle_debug_flag(**kwargs)
|
|
1503
|
+
|
|
1504
|
+
try:
|
|
1505
|
+
api_token = prepare_api_token(**kwargs)
|
|
1506
|
+
except RecceConfigException:
|
|
1507
|
+
show_invalid_api_token_message()
|
|
1508
|
+
exit(1)
|
|
1509
|
+
|
|
1510
|
+
organization = kwargs.get("organization")
|
|
1511
|
+
project = kwargs.get("project")
|
|
1512
|
+
|
|
1513
|
+
# Validate required parameters
|
|
1514
|
+
if not organization:
|
|
1515
|
+
console.print("[[red]Error[/red]] Organization ID is required. Please provide it via:")
|
|
1516
|
+
console.print(" --organization <id> or set RECCE_ORGANIZATION_ID environment variable")
|
|
1517
|
+
exit(1)
|
|
1518
|
+
|
|
1519
|
+
if not project:
|
|
1520
|
+
console.print("[[red]Error[/red]] Project ID is required. Please provide it via:")
|
|
1521
|
+
console.print(" --project <id> or set RECCE_PROJECT_ID environment variable")
|
|
1522
|
+
exit(1)
|
|
1523
|
+
|
|
1524
|
+
try:
|
|
1525
|
+
from recce.util.recce_cloud import RecceCloud
|
|
1526
|
+
|
|
1527
|
+
cloud = RecceCloud(api_token)
|
|
1528
|
+
sessions = cloud.list_sessions(organization, project)
|
|
1529
|
+
|
|
1530
|
+
if not sessions:
|
|
1531
|
+
console.print(f"No sessions found in project {project}.")
|
|
1532
|
+
return
|
|
1533
|
+
|
|
1534
|
+
table = Table(title=f"Sessions in Project {project}")
|
|
1535
|
+
table.add_column("ID", style="cyan")
|
|
1536
|
+
table.add_column("Name", style="green")
|
|
1537
|
+
table.add_column("Is Base", style="yellow")
|
|
1538
|
+
|
|
1539
|
+
for session in sessions:
|
|
1540
|
+
is_base = "✓" if session.get("is_base", False) else ""
|
|
1541
|
+
table.add_row(session.get("id", ""), session.get("name", ""), is_base)
|
|
1542
|
+
|
|
1543
|
+
console.print(table)
|
|
1544
|
+
|
|
1545
|
+
except RecceCloudException as e:
|
|
1546
|
+
console.print(f"[[red]Error[/red]] {e}")
|
|
1547
|
+
exit(1)
|
|
1548
|
+
except Exception as e:
|
|
1549
|
+
console.print(f"[[red]Error[/red]] {e}")
|
|
1550
|
+
exit(1)
|
|
1551
|
+
|
|
1552
|
+
|
|
1553
|
+
@cli.group("github", short_help="GitHub related commands", hidden=True)
|
|
746
1554
|
def github(**kwargs):
|
|
747
1555
|
pass
|
|
748
1556
|
|
|
749
1557
|
|
|
750
|
-
@github.command(
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
1558
|
+
@github.command(
|
|
1559
|
+
cls=TrackCommand, short_help="Download the artifacts from the GitHub repository based on the current Pull Request."
|
|
1560
|
+
)
|
|
1561
|
+
@click.option(
|
|
1562
|
+
"--github-token",
|
|
1563
|
+
help="The github token to use for accessing GitHub repo.",
|
|
1564
|
+
type=click.STRING,
|
|
1565
|
+
envvar="GITHUB_TOKEN",
|
|
1566
|
+
)
|
|
1567
|
+
@click.option(
|
|
1568
|
+
"--github-repo",
|
|
1569
|
+
help="The github repo to use for accessing GitHub repo.",
|
|
1570
|
+
type=click.STRING,
|
|
1571
|
+
envvar="GITHUB_REPOSITORY",
|
|
1572
|
+
)
|
|
756
1573
|
def artifact(**kwargs):
|
|
757
1574
|
from recce.github import recce_ci_artifact
|
|
1575
|
+
|
|
758
1576
|
return recce_ci_artifact(**kwargs)
|
|
759
1577
|
|
|
760
1578
|
|
|
761
|
-
@cli.command(cls=TrackCommand
|
|
762
|
-
@click.argument(
|
|
763
|
-
@click.option(
|
|
764
|
-
|
|
765
|
-
|
|
1579
|
+
@cli.command(cls=TrackCommand)
|
|
1580
|
+
@click.argument("state_file", type=click.Path(exists=True))
|
|
1581
|
+
@click.option(
|
|
1582
|
+
"--api-token",
|
|
1583
|
+
help="The personal token generated by Recce Cloud.",
|
|
1584
|
+
type=click.STRING,
|
|
1585
|
+
envvar="RECCE_API_TOKEN",
|
|
1586
|
+
)
|
|
766
1587
|
def share(state_file, **kwargs):
|
|
767
1588
|
"""
|
|
768
|
-
|
|
1589
|
+
Share the state file
|
|
769
1590
|
"""
|
|
770
1591
|
from rich.console import Console
|
|
771
1592
|
|
|
@@ -774,23 +1595,21 @@ def share(state_file, **kwargs):
|
|
|
774
1595
|
cloud_options = None
|
|
775
1596
|
|
|
776
1597
|
# read or input the api token
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
1598
|
+
try:
|
|
1599
|
+
api_token = prepare_api_token(interaction=True, **kwargs)
|
|
1600
|
+
except Abort:
|
|
1601
|
+
console.print("[yellow]Abort[/yellow]")
|
|
1602
|
+
exit(0)
|
|
1603
|
+
except RecceConfigException:
|
|
1604
|
+
show_invalid_api_token_message()
|
|
780
1605
|
exit(1)
|
|
781
1606
|
|
|
782
|
-
|
|
783
|
-
console.print("Please login Recce Cloud and copy the API token from the setting page.\n"
|
|
784
|
-
f"{RECCE_CLOUD_API_HOST}/settings#tokens\n"
|
|
785
|
-
"You can also edit it in the recce profiles yaml file later.")
|
|
786
|
-
api_token = click.prompt('Your Recce API token', type=str, hide_input=True, show_default=False)
|
|
787
|
-
update_user_profile({'api_token': api_token})
|
|
788
|
-
|
|
789
|
-
auth_options = {'api_token': api_token}
|
|
1607
|
+
auth_options = {"api_token": api_token}
|
|
790
1608
|
|
|
791
1609
|
# load local state
|
|
792
|
-
state_loader = create_state_loader(
|
|
793
|
-
|
|
1610
|
+
state_loader = create_state_loader(
|
|
1611
|
+
review_mode=True, cloud_mode=False, state_file=state_file, cloud_options=cloud_options
|
|
1612
|
+
)
|
|
794
1613
|
|
|
795
1614
|
if not state_loader.verify():
|
|
796
1615
|
error, hint = state_loader.error_and_hint
|
|
@@ -811,9 +1630,8 @@ def share(state_file, **kwargs):
|
|
|
811
1630
|
|
|
812
1631
|
try:
|
|
813
1632
|
response = state_manager.share_state(state_file_name, state_loader.state)
|
|
814
|
-
if response.get(
|
|
815
|
-
console.print("[[red]Error[/red]] Failed to share the state.\n"
|
|
816
|
-
f"Reason: {response.get('message')}")
|
|
1633
|
+
if response.get("status") == "error":
|
|
1634
|
+
console.print("[[red]Error[/red]] Failed to share the state.\n" f"Reason: {response.get('message')}")
|
|
817
1635
|
else:
|
|
818
1636
|
console.print(f"Shared Link: {response.get('share_url')}")
|
|
819
1637
|
except RecceCloudException as e:
|
|
@@ -822,40 +1640,199 @@ def share(state_file, **kwargs):
|
|
|
822
1640
|
exit(1)
|
|
823
1641
|
|
|
824
1642
|
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
1643
|
+
snapshot_id_option = click.option(
|
|
1644
|
+
"--snapshot-id",
|
|
1645
|
+
help="The snapshot ID to upload artifacts to cloud.",
|
|
1646
|
+
type=click.STRING,
|
|
1647
|
+
envvar=["RECCE_SNAPSHOT_ID", "RECCE_SESSION_ID"],
|
|
1648
|
+
required=True,
|
|
1649
|
+
)
|
|
1650
|
+
|
|
1651
|
+
session_id_option = click.option(
|
|
1652
|
+
"--session-id",
|
|
1653
|
+
help="The session ID to upload artifacts to cloud.",
|
|
1654
|
+
type=click.STRING,
|
|
1655
|
+
envvar=["RECCE_SESSION_ID", "RECCE_SNAPSHOT_ID"],
|
|
1656
|
+
required=True,
|
|
1657
|
+
)
|
|
1658
|
+
|
|
1659
|
+
target_path_option = click.option(
|
|
1660
|
+
"--target-path",
|
|
1661
|
+
help="dbt artifacts directory for your artifacts.",
|
|
1662
|
+
type=click.STRING,
|
|
1663
|
+
default="target",
|
|
1664
|
+
show_default=True,
|
|
1665
|
+
)
|
|
1666
|
+
|
|
830
1667
|
|
|
831
|
-
|
|
1668
|
+
@cli.command(cls=TrackCommand, hidden=True)
|
|
1669
|
+
@add_options([session_id_option, target_path_option])
|
|
1670
|
+
@add_options(recce_cloud_auth_options)
|
|
1671
|
+
@add_options(recce_options)
|
|
1672
|
+
def upload_session(**kwargs):
|
|
1673
|
+
"""
|
|
1674
|
+
Upload target/manifest.json and target/catalog.json to the specific session ID
|
|
1675
|
+
|
|
1676
|
+
Upload the dbt artifacts (manifest.json, catalog.json) to Recce Cloud for the given session ID.
|
|
1677
|
+
This allows you to associate artifacts with a specific session for later use.
|
|
1678
|
+
|
|
1679
|
+
Examples:\n
|
|
1680
|
+
|
|
1681
|
+
\b
|
|
1682
|
+
# Upload artifacts to a session ID
|
|
1683
|
+
recce upload-session --session-id <session-id>
|
|
1684
|
+
|
|
1685
|
+
\b
|
|
1686
|
+
# Upload artifacts from custom target path to a session ID
|
|
1687
|
+
recce upload-session --session-id <session-id> --target-path my-target
|
|
1688
|
+
"""
|
|
832
1689
|
from rich.console import Console
|
|
833
1690
|
|
|
834
1691
|
console = Console()
|
|
835
1692
|
handle_debug_flag(**kwargs)
|
|
836
|
-
is_review = True
|
|
837
|
-
is_cloud = False
|
|
838
|
-
cloud_options = None
|
|
839
|
-
flag = {
|
|
840
|
-
'read_only': True,
|
|
841
|
-
}
|
|
842
|
-
state_loader = create_state_loader(is_review, is_cloud, state_file, cloud_options)
|
|
843
1693
|
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
1694
|
+
# Initialize Recce Config
|
|
1695
|
+
RecceConfig(config_file=kwargs.get("config"))
|
|
1696
|
+
|
|
1697
|
+
try:
|
|
1698
|
+
api_token = prepare_api_token(**kwargs)
|
|
1699
|
+
except RecceConfigException:
|
|
1700
|
+
show_invalid_api_token_message()
|
|
848
1701
|
exit(1)
|
|
849
1702
|
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
1703
|
+
session_id = kwargs.get("session_id")
|
|
1704
|
+
target_path = kwargs.get("target_path")
|
|
1705
|
+
|
|
1706
|
+
try:
|
|
1707
|
+
rc = upload_artifacts_to_session(
|
|
1708
|
+
target_path, session_id=session_id, token=api_token, debug=kwargs.get("debug", False)
|
|
1709
|
+
)
|
|
1710
|
+
console.rule("Uploaded Successfully")
|
|
1711
|
+
console.print(
|
|
1712
|
+
f'Uploaded dbt artifacts to Recce Cloud for session ID "{session_id}" from "{os.path.abspath(target_path)}"'
|
|
1713
|
+
)
|
|
1714
|
+
except Exception as e:
|
|
1715
|
+
console.rule("Failed to Upload Session", style="red")
|
|
1716
|
+
console.print(f"[[red]Error[/red]] Failed to upload the dbt artifacts to the session {session_id}.")
|
|
1717
|
+
console.print(f"Reason: {e}")
|
|
1718
|
+
rc = 1
|
|
1719
|
+
return rc
|
|
1720
|
+
|
|
1721
|
+
|
|
1722
|
+
# Backward compatibility for `recce snapshot` command
|
|
1723
|
+
@cli.command(
|
|
1724
|
+
cls=TrackCommand,
|
|
1725
|
+
hidden=True,
|
|
1726
|
+
deprecated=True,
|
|
1727
|
+
help="Upload target/manifest.json and target/catalog.json to the specific snapshot ID",
|
|
1728
|
+
)
|
|
1729
|
+
@add_options([snapshot_id_option, target_path_option])
|
|
1730
|
+
@add_options(recce_cloud_auth_options)
|
|
1731
|
+
@add_options(recce_options)
|
|
1732
|
+
def snapshot(**kwargs):
|
|
1733
|
+
kwargs["session_id"] = kwargs.get("snapshot_id")
|
|
1734
|
+
return upload_session(**kwargs)
|
|
1735
|
+
|
|
1736
|
+
|
|
1737
|
+
@cli.command(hidden=True, cls=TrackCommand)
|
|
1738
|
+
@click.argument("state_file", required=True)
|
|
1739
|
+
@click.option("--host", default="localhost", show_default=True, help="The host to bind to.")
|
|
1740
|
+
@click.option("--port", default=8000, show_default=True, help="The port to bind to.", type=int)
|
|
1741
|
+
@click.option("--lifetime", default=0, show_default=True, help="The lifetime of the server in seconds.", type=int)
|
|
1742
|
+
@click.option("--share-url", help="The share URL triggers this instance.", type=click.STRING, envvar="RECCE_SHARE_URL")
|
|
1743
|
+
@click.pass_context
|
|
1744
|
+
def read_only(ctx, state_file=None, **kwargs):
|
|
1745
|
+
# Invoke `recce server --mode read-only <state_file> ...
|
|
1746
|
+
kwargs["mode"] = RecceServerMode.read_only
|
|
1747
|
+
ctx.invoke(server, state_file=state_file, **kwargs)
|
|
1748
|
+
|
|
1749
|
+
|
|
1750
|
+
@cli.command(cls=TrackCommand)
|
|
1751
|
+
@click.option("--sse", is_flag=True, default=False, help="Start in HTTP/SSE mode instead of stdio mode")
|
|
1752
|
+
@click.option("--host", default="localhost", help="Host to bind to in SSE mode (default: localhost)")
|
|
1753
|
+
@click.option("--port", default=8000, type=int, help="Port to bind to in SSE mode (default: 8000)")
|
|
1754
|
+
@add_options(dbt_related_options)
|
|
1755
|
+
@add_options(sqlmesh_related_options)
|
|
1756
|
+
@add_options(recce_options)
|
|
1757
|
+
@add_options(recce_dbt_artifact_dir_options)
|
|
1758
|
+
@add_options(recce_cloud_options)
|
|
1759
|
+
@add_options(recce_cloud_auth_options)
|
|
1760
|
+
@add_options(recce_hidden_options)
|
|
1761
|
+
def mcp_server(sse, host, port, **kwargs):
|
|
1762
|
+
"""
|
|
1763
|
+
Start the Recce MCP (Model Context Protocol) server
|
|
1764
|
+
|
|
1765
|
+
The MCP server provides an interface for AI assistants and tools to interact
|
|
1766
|
+
with Recce's data validation capabilities. By default, it uses stdio for
|
|
1767
|
+
communication. Use --sse to enable HTTP/Server-Sent Events mode instead.
|
|
1768
|
+
|
|
1769
|
+
Examples:\n
|
|
1770
|
+
|
|
1771
|
+
\b
|
|
1772
|
+
# Start the MCP server in stdio mode (default)
|
|
1773
|
+
recce mcp-server
|
|
1774
|
+
|
|
1775
|
+
\b
|
|
1776
|
+
# Start in HTTP/SSE mode on default port 8000
|
|
1777
|
+
recce mcp-server --sse
|
|
1778
|
+
|
|
1779
|
+
\b
|
|
1780
|
+
# Start in HTTP/SSE mode with custom host and port
|
|
1781
|
+
recce mcp-server --sse --host 0.0.0.0 --port 9000
|
|
1782
|
+
|
|
1783
|
+
SSE Connection URL (when using --sse): http://<host>:<port>/sse
|
|
1784
|
+
"""
|
|
1785
|
+
from rich.console import Console
|
|
1786
|
+
|
|
1787
|
+
console = Console()
|
|
1788
|
+
try:
|
|
1789
|
+
# Import here to avoid import errors if mcp is not installed
|
|
1790
|
+
from recce.mcp_server import run_mcp_server
|
|
1791
|
+
except ImportError as e:
|
|
1792
|
+
console.print(f"[[red]Error[/red]] Failed to import MCP server: {e}")
|
|
1793
|
+
console.print(r"Please install the MCP package: pip install 'recce\[mcp]'")
|
|
1794
|
+
exit(1)
|
|
1795
|
+
|
|
1796
|
+
# Initialize Recce Config
|
|
1797
|
+
RecceConfig(config_file=kwargs.get("config"))
|
|
1798
|
+
|
|
1799
|
+
handle_debug_flag(**kwargs)
|
|
1800
|
+
patch_derived_args(kwargs)
|
|
1801
|
+
|
|
1802
|
+
# Prepare API token
|
|
1803
|
+
try:
|
|
1804
|
+
api_token = prepare_api_token(**kwargs)
|
|
1805
|
+
kwargs["api_token"] = api_token
|
|
1806
|
+
except RecceConfigException:
|
|
1807
|
+
show_invalid_api_token_message()
|
|
853
1808
|
exit(1)
|
|
854
1809
|
|
|
855
|
-
|
|
856
|
-
|
|
1810
|
+
# Create state loader using shared function (if cloud mode is enabled)
|
|
1811
|
+
is_cloud = kwargs.get("cloud", False)
|
|
1812
|
+
if is_cloud:
|
|
1813
|
+
state_loader = create_state_loader_by_args(None, **kwargs)
|
|
1814
|
+
kwargs["state_loader"] = state_loader
|
|
857
1815
|
|
|
858
|
-
|
|
1816
|
+
try:
|
|
1817
|
+
if sse:
|
|
1818
|
+
console.print(f"Starting Recce MCP Server in HTTP/SSE mode on {host}:{port}...")
|
|
1819
|
+
console.print(f"SSE endpoint: http://{host}:{port}/sse")
|
|
1820
|
+
else:
|
|
1821
|
+
console.print("Starting Recce MCP Server in stdio mode...")
|
|
1822
|
+
|
|
1823
|
+
# Run the server (stdio or SSE based on --sse flag)
|
|
1824
|
+
asyncio.run(run_mcp_server(sse=sse, host=host, port=port, **kwargs))
|
|
1825
|
+
except (asyncio.CancelledError, KeyboardInterrupt):
|
|
1826
|
+
# Graceful shutdown (e.g., Ctrl+C)
|
|
1827
|
+
console.print("[yellow]MCP Server interrupted[/yellow]")
|
|
1828
|
+
exit(0)
|
|
1829
|
+
except Exception as e:
|
|
1830
|
+
console.print(f"[[red]Error[/red]] Failed to start MCP server: {e}")
|
|
1831
|
+
if kwargs.get("debug"):
|
|
1832
|
+
import traceback
|
|
1833
|
+
|
|
1834
|
+
traceback.print_exc()
|
|
1835
|
+
exit(1)
|
|
859
1836
|
|
|
860
1837
|
|
|
861
1838
|
if __name__ == "__main__":
|