recce-nightly 1.2.0.20250506__py3-none-any.whl → 1.4.0.20250514__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of recce-nightly might be problematic. Click here for more details.
- recce/VERSION +1 -1
- recce/__init__.py +22 -22
- recce/adapter/base.py +11 -14
- recce/adapter/dbt_adapter/__init__.py +355 -316
- recce/adapter/dbt_adapter/dbt_version.py +3 -0
- recce/adapter/sqlmesh_adapter.py +24 -35
- recce/apis/check_api.py +39 -28
- recce/apis/check_func.py +33 -27
- recce/apis/run_api.py +25 -19
- recce/apis/run_func.py +29 -23
- recce/artifact.py +44 -49
- recce/cli.py +484 -285
- recce/config.py +42 -33
- recce/core.py +52 -44
- recce/data/404.html +1 -1
- recce/data/_next/static/chunks/{368-7587b306577df275.js → 778-aef312bffb4c0312.js} +15 -15
- recce/data/_next/static/chunks/8d700b6a.ed11a130057c7a47.js +1 -0
- recce/data/_next/static/chunks/app/layout-c713a2829d3279e4.js +1 -0
- recce/data/_next/static/chunks/app/page-7086764277331fcb.js +1 -0
- recce/data/_next/static/chunks/{cd9f8d63-cf0d5a7b0f7a92e8.js → cd9f8d63-e020f408095ed77c.js} +3 -3
- recce/data/_next/static/chunks/webpack-b787cb1a4f2293de.js +1 -0
- recce/data/_next/static/css/88b8abc134cfd59a.css +3 -0
- recce/data/index.html +2 -2
- recce/data/index.txt +2 -2
- recce/diff.py +6 -12
- recce/event/__init__.py +74 -72
- recce/event/collector.py +27 -20
- recce/event/track.py +39 -27
- recce/exceptions.py +1 -1
- recce/git.py +7 -7
- recce/github.py +57 -53
- recce/models/__init__.py +1 -1
- recce/models/check.py +6 -7
- recce/models/run.py +1 -0
- recce/models/types.py +27 -27
- recce/pull_request.py +26 -24
- recce/run.py +148 -111
- recce/server.py +105 -88
- recce/state.py +209 -177
- recce/summary.py +168 -143
- recce/tasks/__init__.py +3 -3
- recce/tasks/core.py +11 -13
- recce/tasks/dataframe.py +19 -17
- recce/tasks/histogram.py +69 -34
- recce/tasks/lineage.py +2 -2
- recce/tasks/profile.py +152 -86
- recce/tasks/query.py +139 -87
- recce/tasks/rowcount.py +33 -30
- recce/tasks/schema.py +14 -14
- recce/tasks/top_k.py +35 -35
- recce/tasks/valuediff.py +216 -152
- recce/util/breaking.py +77 -84
- recce/util/cll.py +55 -51
- recce/util/io.py +19 -17
- recce/util/logger.py +1 -1
- recce/util/recce_cloud.py +70 -72
- recce/util/singleton.py +4 -4
- recce/yaml/__init__.py +7 -10
- {recce_nightly-1.2.0.20250506.dist-info → recce_nightly-1.4.0.20250514.dist-info}/METADATA +5 -2
- recce_nightly-1.4.0.20250514.dist-info/RECORD +143 -0
- {recce_nightly-1.2.0.20250506.dist-info → recce_nightly-1.4.0.20250514.dist-info}/WHEEL +1 -1
- tests/adapter/dbt_adapter/conftest.py +1 -0
- tests/adapter/dbt_adapter/dbt_test_helper.py +28 -18
- tests/adapter/dbt_adapter/test_dbt_adapter.py +0 -15
- tests/adapter/dbt_adapter/test_dbt_cll.py +39 -32
- tests/adapter/dbt_adapter/test_selector.py +22 -21
- tests/tasks/test_histogram.py +58 -66
- tests/tasks/test_lineage.py +36 -23
- tests/tasks/test_preset_checks.py +45 -31
- tests/tasks/test_profile.py +340 -15
- tests/tasks/test_query.py +40 -40
- tests/tasks/test_row_count.py +65 -46
- tests/tasks/test_schema.py +65 -42
- tests/tasks/test_top_k.py +22 -18
- tests/tasks/test_valuediff.py +43 -32
- tests/test_cli.py +71 -58
- tests/test_config.py +7 -9
- tests/test_core.py +5 -3
- tests/test_dbt.py +7 -7
- tests/test_pull_request.py +1 -1
- tests/test_server.py +19 -13
- tests/test_state.py +40 -27
- tests/test_summary.py +18 -14
- recce/data/_next/static/chunks/8d700b6a-f0b1f6b9e0d97ce2.js +0 -1
- recce/data/_next/static/chunks/app/layout-9102e22cb73f74d6.js +0 -1
- recce/data/_next/static/chunks/app/page-cee661090afbd6aa.js +0 -1
- recce/data/_next/static/chunks/webpack-567d72f0bc0820d5.js +0 -1
- recce_nightly-1.2.0.20250506.dist-info/RECORD +0 -142
- /recce/data/_next/static/{Kcbs3GEIyH2LxgLYat0es → E_HPXsXdrqHg2YEHmU3mK}/_buildManifest.js +0 -0
- /recce/data/_next/static/{Kcbs3GEIyH2LxgLYat0es → E_HPXsXdrqHg2YEHmU3mK}/_ssgManifest.js +0 -0
- {recce_nightly-1.2.0.20250506.dist-info → recce_nightly-1.4.0.20250514.dist-info}/entry_points.txt +0 -0
- {recce_nightly-1.2.0.20250506.dist-info → recce_nightly-1.4.0.20250514.dist-info}/licenses/LICENSE +0 -0
- {recce_nightly-1.2.0.20250506.dist-info → recce_nightly-1.4.0.20250514.dist-info}/top_level.txt +0 -0
recce/cli.py
CHANGED
|
@@ -7,15 +7,19 @@ import click
|
|
|
7
7
|
import uvicorn
|
|
8
8
|
|
|
9
9
|
from recce import event
|
|
10
|
-
from recce.artifact import
|
|
11
|
-
from recce.config import
|
|
10
|
+
from recce.artifact import download_dbt_artifacts, upload_dbt_artifacts
|
|
11
|
+
from recce.config import RECCE_CONFIG_FILE, RECCE_ERROR_LOG_FILE, RecceConfig
|
|
12
12
|
from recce.event import get_recce_api_token, update_user_profile
|
|
13
13
|
from recce.git import current_branch, current_default_branch
|
|
14
|
-
from recce.run import
|
|
15
|
-
from recce.state import
|
|
14
|
+
from recce.run import check_github_ci_env, cli_run
|
|
15
|
+
from recce.state import RecceCloudStateManager, RecceShareStateManager, RecceStateLoader
|
|
16
16
|
from recce.summary import generate_markdown_summary
|
|
17
17
|
from recce.util.logger import CustomFormatter
|
|
18
|
-
from recce.util.recce_cloud import
|
|
18
|
+
from recce.util.recce_cloud import (
|
|
19
|
+
RECCE_CLOUD_API_HOST,
|
|
20
|
+
RecceCloudException,
|
|
21
|
+
get_recce_cloud_onboarding_state,
|
|
22
|
+
)
|
|
19
23
|
from .core import RecceContext, set_default_context
|
|
20
24
|
from .event.track import TrackCommand
|
|
21
25
|
|
|
@@ -24,11 +28,13 @@ event.init()
|
|
|
24
28
|
|
|
25
29
|
def create_state_loader(review_mode, cloud_mode, state_file, cloud_options):
|
|
26
30
|
from rich.console import Console
|
|
31
|
+
|
|
27
32
|
console = Console()
|
|
28
33
|
|
|
29
34
|
try:
|
|
30
|
-
return RecceStateLoader(
|
|
31
|
-
|
|
35
|
+
return RecceStateLoader(
|
|
36
|
+
review_mode=review_mode, cloud_mode=cloud_mode, state_file=state_file, cloud_options=cloud_options
|
|
37
|
+
)
|
|
32
38
|
except RecceCloudException as e:
|
|
33
39
|
console.print("[[red]Error[/red]] Failed to load recce state file")
|
|
34
40
|
console.print(f"Reason: {e.reason}")
|
|
@@ -40,8 +46,9 @@ def create_state_loader(review_mode, cloud_mode, state_file, cloud_options):
|
|
|
40
46
|
|
|
41
47
|
|
|
42
48
|
def handle_debug_flag(**kwargs):
|
|
43
|
-
if kwargs.get(
|
|
49
|
+
if kwargs.get("debug"):
|
|
44
50
|
import logging
|
|
51
|
+
|
|
45
52
|
ch = logging.StreamHandler()
|
|
46
53
|
ch.setFormatter(CustomFormatter())
|
|
47
54
|
logging.basicConfig(handlers=[ch], level=logging.DEBUG)
|
|
@@ -57,43 +64,75 @@ def add_options(options):
|
|
|
57
64
|
|
|
58
65
|
|
|
59
66
|
dbt_related_options = [
|
|
60
|
-
click.option(
|
|
61
|
-
click.option(
|
|
62
|
-
click.option(
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
67
|
+
click.option("--target", "-t", help="Which target to load for the given profile.", type=click.STRING),
|
|
68
|
+
click.option("--profile", help="Which existing profile to load.", type=click.STRING),
|
|
69
|
+
click.option(
|
|
70
|
+
"--project-dir",
|
|
71
|
+
help="Which directory to look in for the dbt_project.yml file.",
|
|
72
|
+
type=click.Path(),
|
|
73
|
+
envvar="DBT_PROJECT_DIR",
|
|
74
|
+
),
|
|
75
|
+
click.option(
|
|
76
|
+
"--profiles-dir",
|
|
77
|
+
help="Which directory to look in for the profiles.yml file.",
|
|
78
|
+
type=click.Path(),
|
|
79
|
+
envvar="DBT_PROFILES_DIR",
|
|
80
|
+
),
|
|
66
81
|
]
|
|
67
82
|
|
|
68
83
|
sqlmesh_related_options = [
|
|
69
|
-
click.option(
|
|
70
|
-
click.option(
|
|
71
|
-
click.option(
|
|
84
|
+
click.option("--sqlmesh", is_flag=True, help="Use SQLMesh ", hidden=True),
|
|
85
|
+
click.option("--sqlmesh-envs", is_flag=False, help="SQLMesh envs to compare. SOURCE:TARGET", hidden=True),
|
|
86
|
+
click.option("--sqlmesh-config", is_flag=False, help="SQLMesh config name to use", hidden=True),
|
|
72
87
|
]
|
|
73
88
|
|
|
74
89
|
recce_options = [
|
|
75
|
-
click.option(
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
90
|
+
click.option(
|
|
91
|
+
"--config",
|
|
92
|
+
help="Path to the recce config file.",
|
|
93
|
+
type=click.Path(),
|
|
94
|
+
default=RECCE_CONFIG_FILE,
|
|
95
|
+
show_default=True,
|
|
96
|
+
),
|
|
97
|
+
click.option(
|
|
98
|
+
"--error-log", help="Path to the error log file.", type=click.Path(), default=RECCE_ERROR_LOG_FILE, hidden=True
|
|
99
|
+
),
|
|
100
|
+
click.option("--debug", is_flag=True, help="Enable debug mode.", hidden=True),
|
|
80
101
|
]
|
|
81
102
|
|
|
82
103
|
recce_cloud_options = [
|
|
83
|
-
click.option(
|
|
84
|
-
click.option(
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
104
|
+
click.option("--cloud", is_flag=True, help="Fetch the state file from cloud."),
|
|
105
|
+
click.option("--cloud-token", help="The token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN"),
|
|
106
|
+
click.option(
|
|
107
|
+
"--state-file-host",
|
|
108
|
+
help="The host to fetch the state file from.",
|
|
109
|
+
type=click.STRING,
|
|
110
|
+
envvar="RECCE_STATE_FILE_HOST",
|
|
111
|
+
default="",
|
|
112
|
+
hidden=True,
|
|
113
|
+
),
|
|
114
|
+
click.option(
|
|
115
|
+
"--password",
|
|
116
|
+
"-p",
|
|
117
|
+
help="The password to encrypt the state file in cloud.",
|
|
118
|
+
type=click.STRING,
|
|
119
|
+
envvar="RECCE_STATE_PASSWORD",
|
|
120
|
+
),
|
|
90
121
|
]
|
|
91
122
|
|
|
92
123
|
recce_dbt_artifact_dir_options = [
|
|
93
|
-
click.option(
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
124
|
+
click.option(
|
|
125
|
+
"--target-path",
|
|
126
|
+
help="dbt artifacts directory for your development branch.",
|
|
127
|
+
type=click.STRING,
|
|
128
|
+
default="target",
|
|
129
|
+
),
|
|
130
|
+
click.option(
|
|
131
|
+
"--target-base-path",
|
|
132
|
+
help="dbt artifacts directory to be used as the base for the comparison.",
|
|
133
|
+
type=click.STRING,
|
|
134
|
+
default="target-base",
|
|
135
|
+
),
|
|
97
136
|
]
|
|
98
137
|
|
|
99
138
|
|
|
@@ -105,8 +144,9 @@ def _execute_sql(context, sql_template, base=False):
|
|
|
105
144
|
exit(1)
|
|
106
145
|
|
|
107
146
|
from recce.adapter.dbt_adapter import DbtAdapter
|
|
147
|
+
|
|
108
148
|
dbt_adapter: DbtAdapter = context.adapter
|
|
109
|
-
with dbt_adapter.connection_named(
|
|
149
|
+
with dbt_adapter.connection_named("recce"):
|
|
110
150
|
sql = dbt_adapter.generate_sql(sql_template, base)
|
|
111
151
|
response, result = dbt_adapter.execute(sql, fetch=True, auto_begin=True)
|
|
112
152
|
table = result
|
|
@@ -119,13 +159,15 @@ def _execute_sql(context, sql_template, base=False):
|
|
|
119
159
|
def cli(ctx, **kwargs):
|
|
120
160
|
"""Recce: Data validation toolkit for comprehensive PR review"""
|
|
121
161
|
from rich.console import Console
|
|
162
|
+
|
|
122
163
|
from recce import __is_recce_outdated__, __latest_version__
|
|
164
|
+
|
|
123
165
|
if __is_recce_outdated__ is True:
|
|
124
|
-
error_console = Console(stderr=True, style=
|
|
166
|
+
error_console = Console(stderr=True, style="bold")
|
|
125
167
|
error_console.print(
|
|
126
168
|
f"[[yellow]Update Available[/yellow]] A new version of Recce {__latest_version__} is available.",
|
|
127
169
|
)
|
|
128
|
-
error_console.print("Please update using the command: 'pip install --upgrade recce'.", end=
|
|
170
|
+
error_console.print("Please update using the command: 'pip install --upgrade recce'.", end="\n\n")
|
|
129
171
|
|
|
130
172
|
|
|
131
173
|
@cli.command(cls=TrackCommand)
|
|
@@ -134,12 +176,13 @@ def version():
|
|
|
134
176
|
Show version information
|
|
135
177
|
"""
|
|
136
178
|
from recce import __version__
|
|
179
|
+
|
|
137
180
|
print(__version__)
|
|
138
181
|
|
|
139
182
|
|
|
140
183
|
@cli.command(hidden=True, cls=TrackCommand)
|
|
141
|
-
@click.option(
|
|
142
|
-
@click.option(
|
|
184
|
+
@click.option("--sql", help="Sql template to query", required=True)
|
|
185
|
+
@click.option("--base", is_flag=True, help="Run the query on the base environment")
|
|
143
186
|
@add_options(dbt_related_options)
|
|
144
187
|
def query(sql, base: bool = False, **kwargs):
|
|
145
188
|
"""
|
|
@@ -155,20 +198,25 @@ def query(sql, base: bool = False, **kwargs):
|
|
|
155
198
|
"""
|
|
156
199
|
context = RecceContext.load(**kwargs)
|
|
157
200
|
result = _execute_sql(context, sql, base=base)
|
|
158
|
-
print(result.to_string(na_rep=
|
|
201
|
+
print(result.to_string(na_rep="-", index=False))
|
|
159
202
|
|
|
160
203
|
|
|
161
204
|
def _split_comma_separated(ctx, param, value):
|
|
162
|
-
return value.split(
|
|
205
|
+
return value.split(",") if value else None
|
|
163
206
|
|
|
164
207
|
|
|
165
208
|
@cli.command(hidden=True, cls=TrackCommand)
|
|
166
|
-
@click.option(
|
|
167
|
-
@click.option(
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
209
|
+
@click.option("--sql", help="Sql template to query.", required=True)
|
|
210
|
+
@click.option(
|
|
211
|
+
"--primary-keys",
|
|
212
|
+
type=click.STRING,
|
|
213
|
+
help="Comma-separated list of primary key columns.",
|
|
214
|
+
callback=_split_comma_separated,
|
|
215
|
+
)
|
|
216
|
+
@click.option("--keep-shape", is_flag=True, help="Keep unchanged columns. Otherwise, unchanged columns are hidden.")
|
|
217
|
+
@click.option(
|
|
218
|
+
"--keep-equal", is_flag=True, help='Keep values that are equal. Otherwise, equal values are shown as "-".'
|
|
219
|
+
)
|
|
172
220
|
@add_options(dbt_related_options)
|
|
173
221
|
def diff(sql, primary_keys: List[str] = None, keep_shape: bool = False, keep_equal: bool = False, **kwargs):
|
|
174
222
|
"""
|
|
@@ -189,21 +237,19 @@ def diff(sql, primary_keys: List[str] = None, keep_shape: bool = False, keep_equ
|
|
|
189
237
|
after.set_index(primary_keys, inplace=True)
|
|
190
238
|
|
|
191
239
|
before_aligned, after_aligned = before.align(after)
|
|
192
|
-
diff = before_aligned.compare(
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
print(diff.to_string(na_rep='-') if not diff.empty else 'no changes')
|
|
240
|
+
diff = before_aligned.compare(
|
|
241
|
+
after_aligned, result_names=("base", "current"), keep_equal=keep_equal, keep_shape=keep_shape
|
|
242
|
+
)
|
|
243
|
+
print(diff.to_string(na_rep="-") if not diff.empty else "no changes")
|
|
197
244
|
|
|
198
245
|
|
|
199
246
|
@cli.command(cls=TrackCommand)
|
|
200
|
-
@click.argument(
|
|
201
|
-
@click.option(
|
|
202
|
-
@click.option(
|
|
203
|
-
@click.option(
|
|
204
|
-
@click.option(
|
|
205
|
-
@click.option(
|
|
206
|
-
envvar='RECCE_API_TOKEN')
|
|
247
|
+
@click.argument("state_file", required=False)
|
|
248
|
+
@click.option("--host", default="localhost", show_default=True, help="The host to bind to.")
|
|
249
|
+
@click.option("--port", default=8000, show_default=True, help="The port to bind to.", type=int)
|
|
250
|
+
@click.option("--lifetime", default=0, show_default=True, help="The lifetime of the server in seconds.", type=int)
|
|
251
|
+
@click.option("--review", is_flag=True, help="Open the state file in the review mode.")
|
|
252
|
+
@click.option("--api-token", help="The token used by Recce Cloud API.", type=click.STRING, envvar="RECCE_API_TOKEN")
|
|
207
253
|
@add_options(dbt_related_options)
|
|
208
254
|
@add_options(sqlmesh_related_options)
|
|
209
255
|
@add_options(recce_options)
|
|
@@ -236,50 +282,49 @@ def server(host, port, lifetime, state_file=None, **kwargs):
|
|
|
236
282
|
|
|
237
283
|
"""
|
|
238
284
|
|
|
239
|
-
from .server import app, AppState
|
|
240
285
|
from rich.console import Console
|
|
241
286
|
|
|
242
|
-
|
|
287
|
+
from .server import AppState, app
|
|
288
|
+
|
|
289
|
+
RecceConfig(config_file=kwargs.get("config"))
|
|
243
290
|
|
|
244
291
|
handle_debug_flag(**kwargs)
|
|
245
|
-
is_review = kwargs.get(
|
|
246
|
-
is_cloud = kwargs.get(
|
|
292
|
+
is_review = kwargs.get("review", False)
|
|
293
|
+
is_cloud = kwargs.get("cloud", False)
|
|
247
294
|
console = Console()
|
|
248
295
|
cloud_options = None
|
|
249
|
-
flag = {
|
|
250
|
-
'show_onboarding_guide': True,
|
|
251
|
-
'single_env_onboarding': False,
|
|
252
|
-
'show_relaunch_hint': False
|
|
253
|
-
}
|
|
296
|
+
flag = {"show_onboarding_guide": True, "single_env_onboarding": False, "show_relaunch_hint": False}
|
|
254
297
|
if is_cloud:
|
|
255
298
|
cloud_options = {
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
299
|
+
"host": kwargs.get("state_file_host"),
|
|
300
|
+
"token": kwargs.get("cloud_token"),
|
|
301
|
+
"password": kwargs.get("password"),
|
|
259
302
|
}
|
|
260
|
-
cloud_onboarding_state = get_recce_cloud_onboarding_state(kwargs.get(
|
|
261
|
-
flag[
|
|
303
|
+
cloud_onboarding_state = get_recce_cloud_onboarding_state(kwargs.get("cloud_token"))
|
|
304
|
+
flag["show_onboarding_guide"] = False if cloud_onboarding_state == "completed" else True
|
|
262
305
|
|
|
263
306
|
auth_options = {}
|
|
264
|
-
api_token = kwargs.get(
|
|
265
|
-
auth_options[
|
|
307
|
+
api_token = kwargs.get("api_token") if kwargs.get("api_token") else get_recce_api_token()
|
|
308
|
+
auth_options["api_token"] = api_token
|
|
266
309
|
|
|
267
310
|
# Check Single Environment Onboarding Mode if the review mode is False
|
|
268
|
-
if not os.path.isdir(kwargs.get(
|
|
311
|
+
if not os.path.isdir(kwargs.get("target_base_path")) and is_review is False:
|
|
269
312
|
# Mark as single env onboarding mode if user provides the target-path only
|
|
270
|
-
flag[
|
|
271
|
-
flag[
|
|
272
|
-
target_path = kwargs.get(
|
|
273
|
-
target_base_path = kwargs.get(
|
|
313
|
+
flag["single_env_onboarding"] = True
|
|
314
|
+
flag["show_relaunch_hint"] = True
|
|
315
|
+
target_path = kwargs.get("target_path")
|
|
316
|
+
target_base_path = kwargs.get("target_base_path")
|
|
274
317
|
# Use the target path as the base path
|
|
275
|
-
kwargs[
|
|
318
|
+
kwargs["target_base_path"] = target_path
|
|
276
319
|
|
|
277
320
|
# Show warning message
|
|
278
|
-
console.rule(
|
|
279
|
-
console.print(
|
|
280
|
-
console.print(
|
|
281
|
-
|
|
282
|
-
|
|
321
|
+
console.rule("Notice", style="orange3")
|
|
322
|
+
console.print("Recce is launching in single environment mode with limited functionality.")
|
|
323
|
+
console.print(
|
|
324
|
+
"For full functionality, prepare a base set of dbt artifacts to compare against in "
|
|
325
|
+
f"'{target_base_path}'."
|
|
326
|
+
)
|
|
327
|
+
console.print("https://docs.datarecce.io/get-started/#prepare-dbt-artifacts")
|
|
283
328
|
console.print()
|
|
284
329
|
|
|
285
330
|
state_loader = create_state_loader(is_review, is_cloud, state_file, cloud_options)
|
|
@@ -300,28 +345,46 @@ def server(host, port, lifetime, state_file=None, **kwargs):
|
|
|
300
345
|
console.print(f"[[red]Error[/red]] {message}")
|
|
301
346
|
exit(1)
|
|
302
347
|
|
|
303
|
-
state = AppState(
|
|
304
|
-
|
|
348
|
+
state = AppState(
|
|
349
|
+
command="server",
|
|
350
|
+
state_loader=state_loader,
|
|
351
|
+
kwargs=kwargs,
|
|
352
|
+
flag=flag,
|
|
353
|
+
auth_options=auth_options,
|
|
354
|
+
lifetime=lifetime,
|
|
355
|
+
)
|
|
305
356
|
app.state = state
|
|
306
357
|
|
|
307
|
-
uvicorn.run(app, host=host, port=port, lifespan=
|
|
358
|
+
uvicorn.run(app, host=host, port=port, lifespan="on")
|
|
308
359
|
|
|
309
360
|
|
|
310
|
-
DEFAULT_RECCE_STATE_FILE =
|
|
361
|
+
DEFAULT_RECCE_STATE_FILE = "recce_state.json"
|
|
311
362
|
|
|
312
363
|
|
|
313
364
|
@cli.command(cls=TrackCommand)
|
|
314
|
-
@click.option(
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
@click.option(
|
|
324
|
-
|
|
365
|
+
@click.option(
|
|
366
|
+
"-o",
|
|
367
|
+
"--output",
|
|
368
|
+
help="Path of the output state file.",
|
|
369
|
+
type=click.Path(),
|
|
370
|
+
default=DEFAULT_RECCE_STATE_FILE,
|
|
371
|
+
show_default=True,
|
|
372
|
+
)
|
|
373
|
+
@click.option("--state-file", help="Path of the import state file.", type=click.Path())
|
|
374
|
+
@click.option("--summary", help="Path of the summary markdown file.", type=click.Path())
|
|
375
|
+
@click.option("--skip-query", is_flag=True, help="Skip running the queries for the checks.")
|
|
376
|
+
@click.option(
|
|
377
|
+
"--git-current-branch",
|
|
378
|
+
help="The git branch of the current environment.",
|
|
379
|
+
type=click.STRING,
|
|
380
|
+
envvar="GITHUB_HEAD_REF",
|
|
381
|
+
)
|
|
382
|
+
@click.option(
|
|
383
|
+
"--git-base-branch", help="The git branch of the base environment.", type=click.STRING, envvar="GITHUB_BASE_REF"
|
|
384
|
+
)
|
|
385
|
+
@click.option(
|
|
386
|
+
"--github-pull-request-url", help="The github pull request url to use for the lineage.", type=click.STRING
|
|
387
|
+
)
|
|
325
388
|
@add_options(dbt_related_options)
|
|
326
389
|
@add_options(sqlmesh_related_options)
|
|
327
390
|
@add_options(recce_options)
|
|
@@ -347,25 +410,31 @@ def run(output, **kwargs):
|
|
|
347
410
|
|
|
348
411
|
"""
|
|
349
412
|
from rich.console import Console
|
|
413
|
+
|
|
350
414
|
handle_debug_flag(**kwargs)
|
|
351
415
|
console = Console()
|
|
352
416
|
is_github_action, pr_url = check_github_ci_env(**kwargs)
|
|
353
417
|
if is_github_action is True and pr_url is not None:
|
|
354
|
-
kwargs[
|
|
418
|
+
kwargs["github_pull_request_url"] = pr_url
|
|
355
419
|
|
|
356
420
|
# Initialize Recce Config
|
|
357
|
-
RecceConfig(config_file=kwargs.get(
|
|
358
|
-
|
|
359
|
-
cloud_mode = kwargs.get(
|
|
360
|
-
state_file = kwargs.get(
|
|
361
|
-
cloud_options =
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
421
|
+
RecceConfig(config_file=kwargs.get("config"))
|
|
422
|
+
|
|
423
|
+
cloud_mode = kwargs.get("cloud", False)
|
|
424
|
+
state_file = kwargs.get("state_file")
|
|
425
|
+
cloud_options = (
|
|
426
|
+
{
|
|
427
|
+
"host": kwargs.get("state_file_host"),
|
|
428
|
+
"token": kwargs.get("cloud_token"),
|
|
429
|
+
"password": kwargs.get("password"),
|
|
430
|
+
}
|
|
431
|
+
if cloud_mode
|
|
432
|
+
else None
|
|
433
|
+
)
|
|
366
434
|
|
|
367
|
-
state_loader = create_state_loader(
|
|
368
|
-
|
|
435
|
+
state_loader = create_state_loader(
|
|
436
|
+
review_mode=False, cloud_mode=cloud_mode, state_file=state_file, cloud_options=cloud_options
|
|
437
|
+
)
|
|
369
438
|
|
|
370
439
|
if not state_loader.verify():
|
|
371
440
|
error, hint = state_loader.error_and_hint
|
|
@@ -380,7 +449,7 @@ def run(output, **kwargs):
|
|
|
380
449
|
|
|
381
450
|
# Verify the output state file path
|
|
382
451
|
try:
|
|
383
|
-
if os.path.isdir(output) or output.endswith(
|
|
452
|
+
if os.path.isdir(output) or output.endswith("/"):
|
|
384
453
|
|
|
385
454
|
output_dir = Path(output)
|
|
386
455
|
# Create the directory if not exists
|
|
@@ -388,7 +457,8 @@ def run(output, **kwargs):
|
|
|
388
457
|
output = os.path.join(output, DEFAULT_RECCE_STATE_FILE)
|
|
389
458
|
console.print(
|
|
390
459
|
f"[[yellow]Warning[/yellow]] The path '{output_dir}' is a directory. "
|
|
391
|
-
f"The state file will be saved as '{output}'."
|
|
460
|
+
f"The state file will be saved as '{output}'."
|
|
461
|
+
)
|
|
392
462
|
else:
|
|
393
463
|
# Create the parent directory if not exists
|
|
394
464
|
output_dir = Path(output).parent
|
|
@@ -402,30 +472,43 @@ def run(output, **kwargs):
|
|
|
402
472
|
|
|
403
473
|
|
|
404
474
|
@cli.command(cls=TrackCommand)
|
|
405
|
-
@click.argument(
|
|
406
|
-
@click.option(
|
|
407
|
-
|
|
408
|
-
|
|
475
|
+
@click.argument("state_file", required=False)
|
|
476
|
+
@click.option(
|
|
477
|
+
"--format",
|
|
478
|
+
"-f",
|
|
479
|
+
help="Output format. Currently only markdown is supported.",
|
|
480
|
+
type=click.Choice(["markdown", "mermaid", "check"], case_sensitive=False),
|
|
481
|
+
default="markdown",
|
|
482
|
+
show_default=True,
|
|
483
|
+
hidden=True,
|
|
484
|
+
)
|
|
409
485
|
@add_options(dbt_related_options)
|
|
410
486
|
@add_options(recce_options)
|
|
411
487
|
@add_options(recce_cloud_options)
|
|
412
488
|
def summary(state_file, **kwargs):
|
|
413
489
|
"""
|
|
414
|
-
|
|
490
|
+
Generate a summary of the recce state file
|
|
415
491
|
"""
|
|
416
492
|
from rich.console import Console
|
|
493
|
+
|
|
417
494
|
from .core import load_context
|
|
495
|
+
|
|
418
496
|
handle_debug_flag(**kwargs)
|
|
419
497
|
console = Console()
|
|
420
|
-
cloud_mode = kwargs.get(
|
|
421
|
-
cloud_options =
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
498
|
+
cloud_mode = kwargs.get("cloud", False)
|
|
499
|
+
cloud_options = (
|
|
500
|
+
{
|
|
501
|
+
"host": kwargs.get("state_file_host"),
|
|
502
|
+
"token": kwargs.get("cloud_token"),
|
|
503
|
+
"password": kwargs.get("password"),
|
|
504
|
+
}
|
|
505
|
+
if cloud_mode
|
|
506
|
+
else None
|
|
507
|
+
)
|
|
426
508
|
|
|
427
|
-
state_loader = create_state_loader(
|
|
428
|
-
|
|
509
|
+
state_loader = create_state_loader(
|
|
510
|
+
review_mode=True, cloud_mode=cloud_mode, state_file=state_file, cloud_options=cloud_options
|
|
511
|
+
)
|
|
429
512
|
|
|
430
513
|
if not state_loader.verify():
|
|
431
514
|
error, hint = state_loader.error_and_hint
|
|
@@ -440,56 +523,68 @@ def summary(state_file, **kwargs):
|
|
|
440
523
|
console.print(f"{e}")
|
|
441
524
|
exit(1)
|
|
442
525
|
|
|
443
|
-
output = generate_markdown_summary(ctx, summary_format=kwargs.get(
|
|
526
|
+
output = generate_markdown_summary(ctx, summary_format=kwargs.get("format"))
|
|
444
527
|
print(output)
|
|
445
528
|
|
|
446
529
|
|
|
447
|
-
@cli.group(
|
|
530
|
+
@cli.group("cloud", short_help="Manage Recce Cloud state file.")
|
|
448
531
|
def cloud(**kwargs):
|
|
449
532
|
# Manage Recce Cloud.
|
|
450
533
|
pass
|
|
451
534
|
|
|
452
535
|
|
|
453
536
|
@cloud.command(cls=TrackCommand)
|
|
454
|
-
@click.option(
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
537
|
+
@click.option("--cloud-token", help="The token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN")
|
|
538
|
+
@click.option(
|
|
539
|
+
"--state-file-host",
|
|
540
|
+
help="The host to fetch the state file from.",
|
|
541
|
+
type=click.STRING,
|
|
542
|
+
envvar="RECCE_STATE_FILE_HOST",
|
|
543
|
+
default="",
|
|
544
|
+
hidden=True,
|
|
545
|
+
)
|
|
546
|
+
@click.option(
|
|
547
|
+
"--password",
|
|
548
|
+
"-p",
|
|
549
|
+
help="The password to encrypt the state file in cloud.",
|
|
550
|
+
type=click.STRING,
|
|
551
|
+
envvar="RECCE_STATE_PASSWORD",
|
|
552
|
+
)
|
|
553
|
+
@click.option("--force", "-f", help="Bypasses the confirmation prompt. Purge the state file directly.", is_flag=True)
|
|
461
554
|
@add_options(recce_options)
|
|
462
555
|
def purge(**kwargs):
|
|
463
556
|
"""
|
|
464
|
-
|
|
557
|
+
Purge the state file from cloud
|
|
465
558
|
"""
|
|
466
559
|
from rich.console import Console
|
|
560
|
+
|
|
467
561
|
handle_debug_flag(**kwargs)
|
|
468
562
|
console = Console()
|
|
469
563
|
state_loader = None
|
|
470
564
|
cloud_options = {
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
565
|
+
"host": kwargs.get("state_file_host"),
|
|
566
|
+
"token": kwargs.get("cloud_token"),
|
|
567
|
+
"password": kwargs.get("password"),
|
|
474
568
|
}
|
|
475
|
-
force_to_purge = kwargs.get(
|
|
569
|
+
force_to_purge = kwargs.get("force", False)
|
|
476
570
|
|
|
477
571
|
try:
|
|
478
|
-
console.rule(
|
|
479
|
-
state_loader = RecceStateLoader(
|
|
480
|
-
|
|
572
|
+
console.rule("Check Recce State from Cloud")
|
|
573
|
+
state_loader = RecceStateLoader(
|
|
574
|
+
review_mode=False, cloud_mode=True, state_file=None, cloud_options=cloud_options
|
|
575
|
+
)
|
|
481
576
|
except Exception:
|
|
482
577
|
console.print("[[yellow]Skip[/yellow]] Cannot access existing state file from cloud. Purge it directly.")
|
|
483
578
|
|
|
484
579
|
if state_loader is None:
|
|
485
580
|
try:
|
|
486
|
-
if force_to_purge is True or click.confirm(
|
|
581
|
+
if force_to_purge is True or click.confirm("\nDo you want to purge the state file?"):
|
|
487
582
|
rc, err_msg = RecceCloudStateManager(cloud_options).purge_cloud_state()
|
|
488
583
|
if rc is True:
|
|
489
|
-
console.rule(
|
|
584
|
+
console.rule("Purged Successfully")
|
|
490
585
|
else:
|
|
491
|
-
console.rule(
|
|
492
|
-
console.print(f
|
|
586
|
+
console.rule("Failed to Purge", style="red")
|
|
587
|
+
console.print(f"Reason: {err_msg}")
|
|
493
588
|
|
|
494
589
|
except click.exceptions.Abort:
|
|
495
590
|
pass
|
|
@@ -500,21 +595,21 @@ def purge(**kwargs):
|
|
|
500
595
|
console.print("[[yellow]Skip[/yellow]] No state file found in cloud.")
|
|
501
596
|
return 0
|
|
502
597
|
|
|
503
|
-
pr_info = info.get(
|
|
504
|
-
console.print(
|
|
505
|
-
console.print(
|
|
506
|
-
console.print(f
|
|
507
|
-
console.print(f
|
|
598
|
+
pr_info = info.get("pull_request")
|
|
599
|
+
console.print("[green]State File hosted by[/green]", info.get("source"))
|
|
600
|
+
console.print("[green]GitHub Repository[/green]", info.get("pull_request").repository)
|
|
601
|
+
console.print(f"[green]GitHub Pull Request[/green]\n{pr_info.title} #{pr_info.id}")
|
|
602
|
+
console.print(f"Branch merged into [blue]{pr_info.base_branch}[/blue] from [blue]{pr_info.branch}[/blue]")
|
|
508
603
|
console.print(pr_info.url)
|
|
509
604
|
|
|
510
605
|
try:
|
|
511
|
-
if force_to_purge is True or click.confirm(
|
|
606
|
+
if force_to_purge is True or click.confirm("\nDo you want to purge the state file?"):
|
|
512
607
|
response = state_loader.purge()
|
|
513
608
|
if response is True:
|
|
514
|
-
console.rule(
|
|
609
|
+
console.rule("Purged Successfully")
|
|
515
610
|
else:
|
|
516
|
-
console.rule(
|
|
517
|
-
console.print(f
|
|
611
|
+
console.rule("Failed to Purge", style="red")
|
|
612
|
+
console.print(f"Reason: {state_loader.error_message}")
|
|
518
613
|
except click.exceptions.Abort:
|
|
519
614
|
pass
|
|
520
615
|
|
|
@@ -522,32 +617,43 @@ def purge(**kwargs):
|
|
|
522
617
|
|
|
523
618
|
|
|
524
619
|
@cloud.command(cls=TrackCommand)
|
|
525
|
-
@click.argument(
|
|
526
|
-
@click.option(
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
620
|
+
@click.argument("state_file", type=click.Path(exists=True))
|
|
621
|
+
@click.option("--cloud-token", help="The token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN")
|
|
622
|
+
@click.option(
|
|
623
|
+
"--state-file-host",
|
|
624
|
+
help="The host to fetch the state file from.",
|
|
625
|
+
type=click.STRING,
|
|
626
|
+
envvar="RECCE_STATE_FILE_HOST",
|
|
627
|
+
default="",
|
|
628
|
+
hidden=True,
|
|
629
|
+
)
|
|
630
|
+
@click.option(
|
|
631
|
+
"--password",
|
|
632
|
+
"-p",
|
|
633
|
+
help="The password to encrypt the state file in cloud.",
|
|
634
|
+
type=click.STRING,
|
|
635
|
+
envvar="RECCE_STATE_PASSWORD",
|
|
636
|
+
)
|
|
532
637
|
@add_options(recce_options)
|
|
533
638
|
def upload(state_file, **kwargs):
|
|
534
639
|
"""
|
|
535
|
-
|
|
640
|
+
Upload the state file to cloud
|
|
536
641
|
"""
|
|
537
642
|
from rich.console import Console
|
|
538
643
|
|
|
539
644
|
handle_debug_flag(**kwargs)
|
|
540
645
|
cloud_options = {
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
646
|
+
"host": kwargs.get("state_file_host"),
|
|
647
|
+
"token": kwargs.get("cloud_token"),
|
|
648
|
+
"password": kwargs.get("password"),
|
|
544
649
|
}
|
|
545
650
|
|
|
546
651
|
console = Console()
|
|
547
652
|
|
|
548
653
|
# load local state
|
|
549
|
-
state_loader = create_state_loader(
|
|
550
|
-
|
|
654
|
+
state_loader = create_state_loader(
|
|
655
|
+
review_mode=False, cloud_mode=False, state_file=state_file, cloud_options=cloud_options
|
|
656
|
+
)
|
|
551
657
|
|
|
552
658
|
if not state_loader.verify():
|
|
553
659
|
error, hint = state_loader.error_and_hint
|
|
@@ -565,34 +671,50 @@ def upload(state_file, **kwargs):
|
|
|
565
671
|
|
|
566
672
|
cloud_state_file_exists = state_manager.check_cloud_state_exists()
|
|
567
673
|
|
|
568
|
-
if cloud_state_file_exists and not click.confirm(
|
|
674
|
+
if cloud_state_file_exists and not click.confirm("\nDo you want to overwrite the existing state file?"):
|
|
569
675
|
return 0
|
|
570
676
|
|
|
571
677
|
console.print(state_manager.upload_state_to_cloud(state_loader.state))
|
|
572
678
|
|
|
573
679
|
|
|
574
680
|
@cloud.command(cls=TrackCommand)
|
|
575
|
-
@click.option(
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
681
|
+
@click.option(
|
|
682
|
+
"-o",
|
|
683
|
+
"--output",
|
|
684
|
+
help="Path of the downloaded state file.",
|
|
685
|
+
type=click.STRING,
|
|
686
|
+
default=DEFAULT_RECCE_STATE_FILE,
|
|
687
|
+
show_default=True,
|
|
688
|
+
)
|
|
689
|
+
@click.option("--cloud-token", help="The token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN")
|
|
690
|
+
@click.option(
|
|
691
|
+
"--state-file-host",
|
|
692
|
+
help="The host to fetch the state file from.",
|
|
693
|
+
type=click.STRING,
|
|
694
|
+
envvar="RECCE_STATE_FILE_HOST",
|
|
695
|
+
default="",
|
|
696
|
+
hidden=True,
|
|
697
|
+
)
|
|
698
|
+
@click.option(
|
|
699
|
+
"--password",
|
|
700
|
+
"-p",
|
|
701
|
+
help="The password to encrypt the state file in cloud.",
|
|
702
|
+
type=click.STRING,
|
|
703
|
+
envvar="RECCE_STATE_PASSWORD",
|
|
704
|
+
)
|
|
583
705
|
@add_options(recce_options)
|
|
584
706
|
def download(**kwargs):
|
|
585
707
|
"""
|
|
586
|
-
|
|
708
|
+
Download the state file to cloud
|
|
587
709
|
"""
|
|
588
710
|
from rich.console import Console
|
|
589
711
|
|
|
590
712
|
handle_debug_flag(**kwargs)
|
|
591
|
-
filepath = kwargs.get(
|
|
713
|
+
filepath = kwargs.get("output")
|
|
592
714
|
cloud_options = {
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
715
|
+
"host": kwargs.get("state_file_host"),
|
|
716
|
+
"token": kwargs.get("cloud_token"),
|
|
717
|
+
"password": kwargs.get("password"),
|
|
596
718
|
}
|
|
597
719
|
|
|
598
720
|
console = Console()
|
|
@@ -608,7 +730,7 @@ def download(**kwargs):
|
|
|
608
730
|
cloud_state_file_exists = state_manager.check_cloud_state_exists()
|
|
609
731
|
|
|
610
732
|
if not cloud_state_file_exists:
|
|
611
|
-
console.print(
|
|
733
|
+
console.print("[yellow]Skip[/yellow] No state file found in cloud.")
|
|
612
734
|
return 0
|
|
613
735
|
|
|
614
736
|
state_manager.download_state_from_cloud(filepath)
|
|
@@ -616,41 +738,60 @@ def download(**kwargs):
|
|
|
616
738
|
|
|
617
739
|
|
|
618
740
|
@cloud.command(cls=TrackCommand)
|
|
619
|
-
@click.option(
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
741
|
+
@click.option("--cloud-token", help="The token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN")
|
|
742
|
+
@click.option(
|
|
743
|
+
"--branch",
|
|
744
|
+
"-b",
|
|
745
|
+
help="The branch of the provided artifacts.",
|
|
746
|
+
type=click.STRING,
|
|
747
|
+
envvar="GITHUB_HEAD_REF",
|
|
748
|
+
default=current_branch(),
|
|
749
|
+
show_default=True,
|
|
750
|
+
)
|
|
751
|
+
@click.option(
|
|
752
|
+
"--target-path",
|
|
753
|
+
help="dbt artifacts directory for your artifacts.",
|
|
754
|
+
type=click.STRING,
|
|
755
|
+
default="target",
|
|
756
|
+
show_default=True,
|
|
757
|
+
)
|
|
758
|
+
@click.option(
|
|
759
|
+
"--password",
|
|
760
|
+
"-p",
|
|
761
|
+
help="The password to encrypt the dbt artifacts in cloud.",
|
|
762
|
+
type=click.STRING,
|
|
763
|
+
envvar="RECCE_STATE_PASSWORD",
|
|
764
|
+
required=True,
|
|
765
|
+
)
|
|
627
766
|
@add_options(recce_options)
|
|
628
767
|
def upload_artifacts(**kwargs):
|
|
629
768
|
"""
|
|
630
|
-
|
|
769
|
+
Upload the dbt artifacts to cloud
|
|
631
770
|
|
|
632
|
-
|
|
633
|
-
|
|
771
|
+
Upload the dbt artifacts (metadata.json, catalog.json) to Recce Cloud for the given branch.
|
|
772
|
+
The password is used to encrypt the dbt artifacts in the cloud. You will need the password to download the dbt artifacts.
|
|
634
773
|
|
|
635
|
-
|
|
636
|
-
|
|
774
|
+
By default, the artifacts are uploaded to the current branch. You can specify the branch using the --branch option.
|
|
775
|
+
The target path is set to 'target' by default. You can specify the target path using the --target-path option.
|
|
637
776
|
"""
|
|
638
777
|
from rich.console import Console
|
|
778
|
+
|
|
639
779
|
console = Console()
|
|
640
|
-
cloud_token = kwargs.get(
|
|
641
|
-
password = kwargs.get(
|
|
642
|
-
target_path = kwargs.get(
|
|
643
|
-
branch = kwargs.get(
|
|
780
|
+
cloud_token = kwargs.get("cloud_token")
|
|
781
|
+
password = kwargs.get("password")
|
|
782
|
+
target_path = kwargs.get("target_path")
|
|
783
|
+
branch = kwargs.get("branch")
|
|
644
784
|
|
|
645
785
|
try:
|
|
646
|
-
rc = upload_dbt_artifacts(
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
console.rule(
|
|
786
|
+
rc = upload_dbt_artifacts(
|
|
787
|
+
target_path, branch=branch, token=cloud_token, password=password, debug=kwargs.get("debug", False)
|
|
788
|
+
)
|
|
789
|
+
console.rule("Uploaded Successfully")
|
|
650
790
|
console.print(
|
|
651
|
-
f'Uploaded dbt artifacts to Recce Cloud for branch "{branch}" from "{os.path.abspath(target_path)}"'
|
|
791
|
+
f'Uploaded dbt artifacts to Recce Cloud for branch "{branch}" from "{os.path.abspath(target_path)}"'
|
|
792
|
+
)
|
|
652
793
|
except Exception as e:
|
|
653
|
-
console.rule(
|
|
794
|
+
console.rule("Failed to Upload", style="red")
|
|
654
795
|
console.print("[[red]Error[/red]] Failed to upload the dbt artifacts to cloud.")
|
|
655
796
|
console.print(f"Reason: {e}")
|
|
656
797
|
rc = 1
|
|
@@ -659,22 +800,32 @@ def upload_artifacts(**kwargs):
|
|
|
659
800
|
|
|
660
801
|
def _download_artifacts(branch, cloud_token, console, kwargs, password, target_path):
|
|
661
802
|
try:
|
|
662
|
-
rc = download_dbt_artifacts(
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
803
|
+
rc = download_dbt_artifacts(
|
|
804
|
+
target_path,
|
|
805
|
+
branch=branch,
|
|
806
|
+
token=cloud_token,
|
|
807
|
+
password=password,
|
|
808
|
+
force=kwargs.get("force", False),
|
|
809
|
+
debug=kwargs.get("debug", False),
|
|
810
|
+
)
|
|
811
|
+
console.rule("Downloaded Successfully")
|
|
666
812
|
console.print(
|
|
667
|
-
f'Downloaded dbt artifacts from Recce Cloud for branch "{branch}" to "{os.path.abspath(target_path)}"'
|
|
813
|
+
f'Downloaded dbt artifacts from Recce Cloud for branch "{branch}" to "{os.path.abspath(target_path)}"'
|
|
814
|
+
)
|
|
668
815
|
except Exception as e:
|
|
669
|
-
console.rule(
|
|
816
|
+
console.rule("Failed to Download", style="red")
|
|
670
817
|
console.print("[[red]Error[/red]] Failed to download the dbt artifacts from cloud.")
|
|
671
818
|
reason = str(e)
|
|
672
819
|
|
|
673
|
-
if
|
|
820
|
+
if (
|
|
821
|
+
"Requests specifying Server Side Encryption with Customer provided keys must provide the correct secret key"
|
|
822
|
+
in reason
|
|
823
|
+
):
|
|
674
824
|
console.print("Reason: Decryption failed due to incorrect password.")
|
|
675
825
|
console.print(
|
|
676
|
-
"Please provide the correct password to decrypt the dbt artifacts. Or re-upload the dbt artifacts with a new password."
|
|
677
|
-
|
|
826
|
+
"Please provide the correct password to decrypt the dbt artifacts. Or re-upload the dbt artifacts with a new password."
|
|
827
|
+
)
|
|
828
|
+
elif "The specified key does not exist" in reason:
|
|
678
829
|
console.print("Reason: The dbt artifacts is not found in the cloud.")
|
|
679
830
|
console.print("Please upload the dbt artifacts to the cloud before downloading it.")
|
|
680
831
|
else:
|
|
@@ -684,89 +835,132 @@ def _download_artifacts(branch, cloud_token, console, kwargs, password, target_p
|
|
|
684
835
|
|
|
685
836
|
|
|
686
837
|
@cloud.command(cls=TrackCommand)
|
|
687
|
-
@click.option(
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
838
|
+
@click.option("--cloud-token", help="The token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN")
|
|
839
|
+
@click.option(
|
|
840
|
+
"--branch",
|
|
841
|
+
"-b",
|
|
842
|
+
help="The branch of the selected artifacts.",
|
|
843
|
+
type=click.STRING,
|
|
844
|
+
envvar="GITHUB_BASE_REF",
|
|
845
|
+
default=current_branch(),
|
|
846
|
+
show_default=True,
|
|
847
|
+
)
|
|
848
|
+
@click.option(
|
|
849
|
+
"--target-path",
|
|
850
|
+
help="The dbt artifacts directory for your artifacts.",
|
|
851
|
+
type=click.STRING,
|
|
852
|
+
default="target",
|
|
853
|
+
show_default=True,
|
|
854
|
+
)
|
|
855
|
+
@click.option(
|
|
856
|
+
"--password",
|
|
857
|
+
"-p",
|
|
858
|
+
help="The password to decrypt the dbt artifacts in cloud.",
|
|
859
|
+
type=click.STRING,
|
|
860
|
+
envvar="RECCE_STATE_PASSWORD",
|
|
861
|
+
required=True,
|
|
862
|
+
)
|
|
863
|
+
@click.option("--force", "-f", help="Bypasses the confirmation prompt. Download the artifacts directly.", is_flag=True)
|
|
697
864
|
@add_options(recce_options)
|
|
698
865
|
def download_artifacts(**kwargs):
|
|
699
866
|
"""
|
|
700
|
-
|
|
867
|
+
Download the dbt artifacts from cloud
|
|
701
868
|
|
|
702
|
-
|
|
703
|
-
|
|
869
|
+
Download the dbt artifacts (metadata.json, catalog.json) from Recce Cloud for the given branch.
|
|
870
|
+
The password is used to decrypt the dbt artifacts in the cloud.
|
|
704
871
|
|
|
705
|
-
|
|
706
|
-
|
|
872
|
+
By default, the artifacts are downloaded from the current branch. You can specify the branch using the --branch option.
|
|
873
|
+
The target path is set to 'target' by default. You can specify the target path using the --target-path option.
|
|
707
874
|
"""
|
|
708
875
|
from rich.console import Console
|
|
876
|
+
|
|
709
877
|
console = Console()
|
|
710
|
-
cloud_token = kwargs.get(
|
|
711
|
-
password = kwargs.get(
|
|
712
|
-
target_path = kwargs.get(
|
|
713
|
-
branch = kwargs.get(
|
|
878
|
+
cloud_token = kwargs.get("cloud_token")
|
|
879
|
+
password = kwargs.get("password")
|
|
880
|
+
target_path = kwargs.get("target_path")
|
|
881
|
+
branch = kwargs.get("branch")
|
|
714
882
|
return _download_artifacts(branch, cloud_token, console, kwargs, password, target_path)
|
|
715
883
|
|
|
716
884
|
|
|
717
885
|
@cloud.command(cls=TrackCommand)
|
|
718
|
-
@click.option(
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
886
|
+
@click.option("--cloud-token", help="The token used by Recce Cloud.", type=click.STRING, envvar="GITHUB_TOKEN")
|
|
887
|
+
@click.option(
|
|
888
|
+
"--branch",
|
|
889
|
+
"-b",
|
|
890
|
+
help="The branch of the selected artifacts.",
|
|
891
|
+
type=click.STRING,
|
|
892
|
+
envvar="GITHUB_BASE_REF",
|
|
893
|
+
default=current_default_branch(),
|
|
894
|
+
show_default=True,
|
|
895
|
+
)
|
|
896
|
+
@click.option(
|
|
897
|
+
"--target-path",
|
|
898
|
+
help="The dbt artifacts directory for your artifacts.",
|
|
899
|
+
type=click.STRING,
|
|
900
|
+
default="target-base",
|
|
901
|
+
show_default=True,
|
|
902
|
+
)
|
|
903
|
+
@click.option(
|
|
904
|
+
"--password",
|
|
905
|
+
"-p",
|
|
906
|
+
help="The password to decrypt the dbt artifacts in cloud.",
|
|
907
|
+
type=click.STRING,
|
|
908
|
+
envvar="RECCE_STATE_PASSWORD",
|
|
909
|
+
required=True,
|
|
910
|
+
)
|
|
911
|
+
@click.option("--force", "-f", help="Bypasses the confirmation prompt. Download the artifacts directly.", is_flag=True)
|
|
728
912
|
@add_options(recce_options)
|
|
729
913
|
def download_base_artifacts(**kwargs):
|
|
730
914
|
"""
|
|
731
|
-
|
|
915
|
+
Download the base dbt artifacts from cloud
|
|
732
916
|
|
|
733
|
-
|
|
734
|
-
|
|
917
|
+
Download the base dbt artifacts (metadata.json, catalog.json) from Recce Cloud.
|
|
918
|
+
This is useful when you start to set up the base dbt artifacts for the first time.
|
|
735
919
|
|
|
736
|
-
|
|
920
|
+
Please make sure you have uploaded the dbt artifacts before downloading them.
|
|
737
921
|
"""
|
|
738
922
|
from rich.console import Console
|
|
923
|
+
|
|
739
924
|
console = Console()
|
|
740
|
-
cloud_token = kwargs.get(
|
|
741
|
-
password = kwargs.get(
|
|
742
|
-
target_path = kwargs.get(
|
|
743
|
-
branch = kwargs.get(
|
|
925
|
+
cloud_token = kwargs.get("cloud_token")
|
|
926
|
+
password = kwargs.get("password")
|
|
927
|
+
target_path = kwargs.get("target_path")
|
|
928
|
+
branch = kwargs.get("branch")
|
|
744
929
|
return _download_artifacts(branch, cloud_token, console, kwargs, password, target_path)
|
|
745
930
|
|
|
746
931
|
|
|
747
|
-
@cli.group(
|
|
932
|
+
@cli.group("github", short_help="GitHub related commands", hidden=True)
|
|
748
933
|
def github(**kwargs):
|
|
749
934
|
pass
|
|
750
935
|
|
|
751
936
|
|
|
752
|
-
@github.command(
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
937
|
+
@github.command(
|
|
938
|
+
cls=TrackCommand, short_help="Download the artifacts from the GitHub repository based on the current Pull Request."
|
|
939
|
+
)
|
|
940
|
+
@click.option(
|
|
941
|
+
"--github-token",
|
|
942
|
+
help="The github token to use for accessing GitHub repo.",
|
|
943
|
+
type=click.STRING,
|
|
944
|
+
envvar="GITHUB_TOKEN",
|
|
945
|
+
)
|
|
946
|
+
@click.option(
|
|
947
|
+
"--github-repo",
|
|
948
|
+
help="The github repo to use for accessing GitHub repo.",
|
|
949
|
+
type=click.STRING,
|
|
950
|
+
envvar="GITHUB_REPOSITORY",
|
|
951
|
+
)
|
|
758
952
|
def artifact(**kwargs):
|
|
759
953
|
from recce.github import recce_ci_artifact
|
|
954
|
+
|
|
760
955
|
return recce_ci_artifact(**kwargs)
|
|
761
956
|
|
|
762
957
|
|
|
763
958
|
@cli.command(cls=TrackCommand)
|
|
764
|
-
@click.argument(
|
|
765
|
-
@click.option(
|
|
766
|
-
envvar='RECCE_API_TOKEN')
|
|
959
|
+
@click.argument("state_file", type=click.Path(exists=True))
|
|
960
|
+
@click.option("--api-token", help="The token used by Recce Cloud API.", type=click.STRING, envvar="RECCE_API_TOKEN")
|
|
767
961
|
def share(state_file, **kwargs):
|
|
768
962
|
"""
|
|
769
|
-
|
|
963
|
+
Share the state file
|
|
770
964
|
"""
|
|
771
965
|
from rich.console import Console
|
|
772
966
|
|
|
@@ -775,19 +969,22 @@ def share(state_file, **kwargs):
|
|
|
775
969
|
cloud_options = None
|
|
776
970
|
|
|
777
971
|
# read or input the api token
|
|
778
|
-
api_token = kwargs.get(
|
|
972
|
+
api_token = kwargs.get("api_token") if kwargs.get("api_token") else get_recce_api_token()
|
|
779
973
|
if api_token is None:
|
|
780
|
-
console.print(
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
974
|
+
console.print(
|
|
975
|
+
"An API token is required to this. This can be obtained in your user account settings.\n"
|
|
976
|
+
f"{RECCE_CLOUD_API_HOST}/settings#tokens\n"
|
|
977
|
+
"Your API token will be added to '~/.recce/profile.yml' for more convenient sharing."
|
|
978
|
+
)
|
|
979
|
+
api_token = click.prompt("Your Recce API token", type=str, hide_input=True, show_default=False)
|
|
980
|
+
update_user_profile({"api_token": api_token})
|
|
785
981
|
|
|
786
|
-
auth_options = {
|
|
982
|
+
auth_options = {"api_token": api_token}
|
|
787
983
|
|
|
788
984
|
# load local state
|
|
789
|
-
state_loader = create_state_loader(
|
|
790
|
-
|
|
985
|
+
state_loader = create_state_loader(
|
|
986
|
+
review_mode=True, cloud_mode=False, state_file=state_file, cloud_options=cloud_options
|
|
987
|
+
)
|
|
791
988
|
|
|
792
989
|
if not state_loader.verify():
|
|
793
990
|
error, hint = state_loader.error_and_hint
|
|
@@ -808,9 +1005,8 @@ def share(state_file, **kwargs):
|
|
|
808
1005
|
|
|
809
1006
|
try:
|
|
810
1007
|
response = state_manager.share_state(state_file_name, state_loader.state)
|
|
811
|
-
if response.get(
|
|
812
|
-
console.print("[[red]Error[/red]] Failed to share the state.\n"
|
|
813
|
-
f"Reason: {response.get('message')}")
|
|
1008
|
+
if response.get("status") == "error":
|
|
1009
|
+
console.print("[[red]Error[/red]] Failed to share the state.\n" f"Reason: {response.get('message')}")
|
|
814
1010
|
else:
|
|
815
1011
|
console.print(f"Shared Link: {response.get('share_url')}")
|
|
816
1012
|
except RecceCloudException as e:
|
|
@@ -820,21 +1016,23 @@ def share(state_file, **kwargs):
|
|
|
820
1016
|
|
|
821
1017
|
|
|
822
1018
|
@cli.command(hidden=True, cls=TrackCommand)
|
|
823
|
-
@click.argument(
|
|
824
|
-
@click.option(
|
|
825
|
-
@click.option(
|
|
826
|
-
@click.option(
|
|
1019
|
+
@click.argument("state_file", required=True)
|
|
1020
|
+
@click.option("--host", default="localhost", show_default=True, help="The host to bind to.")
|
|
1021
|
+
@click.option("--port", default=8000, show_default=True, help="The port to bind to.", type=int)
|
|
1022
|
+
@click.option("--lifetime", default=0, show_default=True, help="The lifetime of the server in seconds.", type=int)
|
|
1023
|
+
@click.option("--share-url", help="The share URL triggers this instance.", type=click.STRING, envvar="RECCE_SHARE_URL")
|
|
827
1024
|
def read_only(host, port, lifetime, state_file=None, **kwargs):
|
|
828
|
-
from .server import app, AppState
|
|
829
1025
|
from rich.console import Console
|
|
830
1026
|
|
|
1027
|
+
from .server import AppState, app
|
|
1028
|
+
|
|
831
1029
|
console = Console()
|
|
832
1030
|
handle_debug_flag(**kwargs)
|
|
833
1031
|
is_review = True
|
|
834
1032
|
is_cloud = False
|
|
835
1033
|
cloud_options = None
|
|
836
1034
|
flag = {
|
|
837
|
-
|
|
1035
|
+
"read_only": True,
|
|
838
1036
|
}
|
|
839
1037
|
state_loader = create_state_loader(is_review, is_cloud, state_file, cloud_options)
|
|
840
1038
|
|
|
@@ -849,10 +1047,11 @@ def read_only(host, port, lifetime, state_file=None, **kwargs):
|
|
|
849
1047
|
console.print(f"[[red]Error[/red]] {message}")
|
|
850
1048
|
exit(1)
|
|
851
1049
|
|
|
852
|
-
app.state = AppState(command=
|
|
1050
|
+
app.state = AppState(command="read_only", state_loader=state_loader, kwargs=kwargs, flag=flag, lifetime=lifetime,
|
|
1051
|
+
share_url=kwargs.get("share_url"))
|
|
853
1052
|
set_default_context(RecceContext.load(**kwargs, review=is_review, state_loader=state_loader))
|
|
854
1053
|
|
|
855
|
-
uvicorn.run(app, host=host, port=port, lifespan=
|
|
1054
|
+
uvicorn.run(app, host=host, port=port, lifespan="on")
|
|
856
1055
|
|
|
857
1056
|
|
|
858
1057
|
if __name__ == "__main__":
|