tinybird 0.0.1.dev19__py3-none-any.whl → 0.0.1.dev20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/client.py +2 -2
- tinybird/config.py +1 -1
- tinybird/feedback_manager.py +8 -30
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/modules/auth.py +1 -1
- tinybird/tb/modules/build.py +10 -78
- tinybird/tb/modules/cicd.py +1 -1
- tinybird/tb/modules/cli.py +9 -837
- tinybird/tb/modules/common.py +1 -55
- tinybird/tb/modules/connection.py +1 -1
- tinybird/tb/modules/create.py +18 -7
- tinybird/tb/modules/datafile/build.py +142 -971
- tinybird/tb/modules/datafile/build_common.py +1 -1
- tinybird/tb/modules/datafile/build_datasource.py +1 -1
- tinybird/tb/modules/datafile/build_pipe.py +1 -1
- tinybird/tb/modules/datafile/common.py +11 -11
- tinybird/tb/modules/datafile/diff.py +1 -1
- tinybird/tb/modules/datafile/fixture.py +1 -1
- tinybird/tb/modules/datafile/format_common.py +0 -7
- tinybird/tb/modules/datafile/format_datasource.py +0 -2
- tinybird/tb/modules/datafile/format_pipe.py +0 -2
- tinybird/tb/modules/datafile/parse_datasource.py +1 -1
- tinybird/tb/modules/datafile/parse_pipe.py +1 -1
- tinybird/tb/modules/datafile/pull.py +1 -1
- tinybird/tb/modules/datasource.py +4 -75
- tinybird/tb/modules/feedback_manager.py +1048 -0
- tinybird/tb/modules/fmt.py +1 -1
- tinybird/tb/modules/job.py +1 -1
- tinybird/tb/modules/llm.py +6 -4
- tinybird/tb/modules/local.py +26 -21
- tinybird/tb/modules/local_common.py +2 -1
- tinybird/tb/modules/login.py +8 -8
- tinybird/tb/modules/mock.py +18 -7
- tinybird/tb/modules/pipe.py +4 -126
- tinybird/tb/modules/{build_shell.py → shell.py} +9 -21
- tinybird/tb/modules/table.py +88 -5
- tinybird/tb/modules/tag.py +2 -2
- tinybird/tb/modules/test.py +8 -11
- tinybird/tb/modules/tinyunit/tinyunit.py +1 -1
- tinybird/tb/modules/token.py +2 -2
- tinybird/tb/modules/watch.py +72 -0
- tinybird/tb/modules/workspace.py +1 -1
- tinybird/tb/modules/workspace_members.py +1 -1
- {tinybird-0.0.1.dev19.dist-info → tinybird-0.0.1.dev20.dist-info}/METADATA +1 -1
- tinybird-0.0.1.dev20.dist-info/RECORD +76 -0
- tinybird-0.0.1.dev19.dist-info/RECORD +0 -74
- {tinybird-0.0.1.dev19.dist-info → tinybird-0.0.1.dev20.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev19.dist-info → tinybird-0.0.1.dev20.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev19.dist-info → tinybird-0.0.1.dev20.dist-info}/top_level.txt +0 -0
tinybird/tb/modules/cli.py
CHANGED
|
@@ -7,64 +7,48 @@ import json
|
|
|
7
7
|
import logging
|
|
8
8
|
import os
|
|
9
9
|
import pprint
|
|
10
|
-
import re
|
|
11
|
-
import shutil
|
|
12
|
-
import sys
|
|
13
10
|
from os import getcwd
|
|
14
11
|
from pathlib import Path
|
|
15
|
-
from typing import Any, Callable,
|
|
12
|
+
from typing import Any, Callable, Iterable, List, Optional, Tuple, Union
|
|
16
13
|
|
|
17
14
|
import click
|
|
18
15
|
import humanfriendly
|
|
19
16
|
from click import Context
|
|
20
|
-
from packaging import version
|
|
21
17
|
|
|
22
|
-
import tinybird.context as context
|
|
23
18
|
from tinybird.client import (
|
|
24
19
|
AuthException,
|
|
25
20
|
AuthNoTokenException,
|
|
26
|
-
DoesNotExistException,
|
|
27
21
|
TinyB,
|
|
28
22
|
)
|
|
29
|
-
from tinybird.config import SUPPORTED_CONNECTORS,
|
|
30
|
-
from tinybird.feedback_manager import FeedbackManager
|
|
23
|
+
from tinybird.config import SUPPORTED_CONNECTORS, get_config
|
|
31
24
|
from tinybird.tb import __cli__
|
|
32
25
|
from tinybird.tb.modules.common import (
|
|
33
|
-
OLDEST_ROLLBACK,
|
|
34
26
|
CatchAuthExceptions,
|
|
35
27
|
CLIException,
|
|
36
28
|
_get_tb_client,
|
|
37
29
|
coro,
|
|
38
|
-
create_tb_client,
|
|
39
30
|
echo_safe_format_table,
|
|
40
31
|
get_current_main_workspace,
|
|
41
32
|
getenv_bool,
|
|
42
|
-
is_major_semver,
|
|
43
|
-
is_post_semver,
|
|
44
33
|
load_connector_config,
|
|
45
|
-
remove_release,
|
|
46
34
|
try_update_config_with_remote,
|
|
47
|
-
wait_job,
|
|
48
35
|
)
|
|
49
36
|
from tinybird.tb.modules.config import CLIConfig
|
|
50
|
-
from tinybird.tb.modules.datafile.build import build_graph
|
|
37
|
+
from tinybird.tb.modules.datafile.build import build_graph
|
|
51
38
|
from tinybird.tb.modules.datafile.common import (
|
|
52
39
|
Datafile,
|
|
53
40
|
DatafileSyntaxError,
|
|
54
41
|
get_project_filenames,
|
|
55
|
-
get_resource_versions,
|
|
56
|
-
has_internal_datafiles,
|
|
57
42
|
)
|
|
58
43
|
from tinybird.tb.modules.datafile.diff import diff_command
|
|
59
44
|
from tinybird.tb.modules.datafile.exceptions import (
|
|
60
|
-
AlreadyExistsException,
|
|
61
45
|
ParseException,
|
|
62
46
|
)
|
|
63
47
|
from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
|
|
64
48
|
from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
|
|
65
49
|
from tinybird.tb.modules.datafile.pull import folder_pull
|
|
50
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
66
51
|
from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
67
|
-
from tinybird.tb.modules.telemetry import add_telemetry_event
|
|
68
52
|
|
|
69
53
|
__old_click_echo = click.echo
|
|
70
54
|
__old_click_secho = click.secho
|
|
@@ -234,7 +218,7 @@ async def cli(
|
|
|
234
218
|
load_connector_config(ctx, connector, debug, check_uninstalled=True)
|
|
235
219
|
|
|
236
220
|
|
|
237
|
-
@cli.command()
|
|
221
|
+
@cli.command(hidden=True)
|
|
238
222
|
@click.argument("filenames", type=click.Path(exists=True), nargs=-1, default=None)
|
|
239
223
|
@click.option("--debug", is_flag=True, default=False, help="Print internal representation")
|
|
240
224
|
def check(filenames: List[str], debug: bool) -> None:
|
|
@@ -273,208 +257,14 @@ def check(filenames: List[str], debug: bool) -> None:
|
|
|
273
257
|
|
|
274
258
|
except DatafileSyntaxError as e:
|
|
275
259
|
# TODO(eclbg): add the filename to the error message
|
|
276
|
-
raise CLIException(e)
|
|
260
|
+
raise CLIException(str(e))
|
|
277
261
|
except ParseException as e:
|
|
278
262
|
raise CLIException(FeedbackManager.error_exception(error=e))
|
|
279
263
|
|
|
280
264
|
process(filenames=filenames)
|
|
281
265
|
|
|
282
266
|
|
|
283
|
-
@cli.command()
|
|
284
|
-
@click.option(
|
|
285
|
-
"--dry-run",
|
|
286
|
-
is_flag=True,
|
|
287
|
-
default=False,
|
|
288
|
-
help="Run the command without creating resources on the Tinybird account or any side effect",
|
|
289
|
-
)
|
|
290
|
-
@click.option(
|
|
291
|
-
"--check/--no-check", is_flag=True, default=True, help="Enable/Disable output checking, enabled by default"
|
|
292
|
-
)
|
|
293
|
-
@click.option("--push-deps", is_flag=True, default=False, help="Push dependencies, disabled by default")
|
|
294
|
-
@click.option(
|
|
295
|
-
"--only-changes",
|
|
296
|
-
is_flag=True,
|
|
297
|
-
default=False,
|
|
298
|
-
help="Push only the resources that have changed compared to the destination workspace",
|
|
299
|
-
)
|
|
300
|
-
@click.option(
|
|
301
|
-
"--debug",
|
|
302
|
-
is_flag=True,
|
|
303
|
-
default=False,
|
|
304
|
-
help="Prints internal representation, can be combined with any command to get more information.",
|
|
305
|
-
)
|
|
306
|
-
@click.option("-f", "--force", is_flag=True, default=False, help="Override pipes when they already exist")
|
|
307
|
-
@click.option(
|
|
308
|
-
"--override-datasource",
|
|
309
|
-
is_flag=True,
|
|
310
|
-
default=False,
|
|
311
|
-
help="When pushing a pipe with a Materialized node if the target Data Source exists it will try to override it.",
|
|
312
|
-
)
|
|
313
|
-
@click.option("--populate", is_flag=True, default=False, help="Populate materialized nodes when pushing them")
|
|
314
|
-
@click.option(
|
|
315
|
-
"--subset",
|
|
316
|
-
type=float,
|
|
317
|
-
default=None,
|
|
318
|
-
help="Populate with a subset percent of the data (limited to a maximum of 2M rows), this is useful to quickly test a materialized node with some data. The subset must be greater than 0 and lower than 0.1. A subset of 0.1 means a 10 percent of the data in the source Data Source will be used to populate the materialized view. Use it together with --populate, it has precedence over --sql-condition",
|
|
319
|
-
)
|
|
320
|
-
@click.option(
|
|
321
|
-
"--sql-condition",
|
|
322
|
-
type=str,
|
|
323
|
-
default=None,
|
|
324
|
-
help="Populate with a SQL condition to be applied to the trigger Data Source of the Materialized View. For instance, `--sql-condition='date == toYYYYMM(now())'` it'll populate taking all the rows from the trigger Data Source which `date` is the current month. Use it together with --populate. --sql-condition is not taken into account if the --subset param is present. Including in the ``sql_condition`` any column present in the Data Source ``engine_sorting_key`` will make the populate job process less data.",
|
|
325
|
-
)
|
|
326
|
-
@click.option(
|
|
327
|
-
"--unlink-on-populate-error",
|
|
328
|
-
is_flag=True,
|
|
329
|
-
default=False,
|
|
330
|
-
help="If the populate job fails the Materialized View is unlinked and new data won't be ingested in the Materialized View. First time a populate job fails, the Materialized View is always unlinked.",
|
|
331
|
-
)
|
|
332
|
-
@click.option("--fixtures", is_flag=True, default=False, help="Append fixtures to data sources")
|
|
333
|
-
@click.option(
|
|
334
|
-
"--wait",
|
|
335
|
-
is_flag=True,
|
|
336
|
-
default=False,
|
|
337
|
-
help="To be used along with --populate command. Waits for populate jobs to finish, showing a progress bar. Disabled by default.",
|
|
338
|
-
)
|
|
339
|
-
@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
|
|
340
|
-
@click.option(
|
|
341
|
-
"--only-response-times", is_flag=True, default=False, help="Checks only response times, when --force push a pipe"
|
|
342
|
-
)
|
|
343
|
-
@click.argument("filenames", type=click.Path(exists=True), nargs=-1, default=None)
|
|
344
|
-
@click.option("--workspace_map", nargs=2, type=str, multiple=True)
|
|
345
|
-
@click.option(
|
|
346
|
-
"--workspace",
|
|
347
|
-
nargs=2,
|
|
348
|
-
type=str,
|
|
349
|
-
multiple=True,
|
|
350
|
-
help="add a workspace path to the list of external workspaces, usage: --workspace name path/to/folder",
|
|
351
|
-
)
|
|
352
|
-
@click.option(
|
|
353
|
-
"--no-versions",
|
|
354
|
-
is_flag=True,
|
|
355
|
-
default=False,
|
|
356
|
-
help="when set, resource dependency versions are not used, it pushes the dependencies as-is",
|
|
357
|
-
)
|
|
358
|
-
@click.option(
|
|
359
|
-
"-l", "--limit", type=click.IntRange(0, 100), default=0, required=False, help="Number of requests to validate"
|
|
360
|
-
)
|
|
361
|
-
@click.option(
|
|
362
|
-
"--sample-by-params",
|
|
363
|
-
type=click.IntRange(1, 100),
|
|
364
|
-
default=1,
|
|
365
|
-
required=False,
|
|
366
|
-
help="When set, we will aggregate the pipe_stats_rt requests by extractURLParameterNames(assumeNotNull(url)) and for each combination we will take a sample of N requests",
|
|
367
|
-
)
|
|
368
|
-
@click.option(
|
|
369
|
-
"-ff", "--failfast", is_flag=True, default=False, help="When set, the checker will exit as soon one test fails"
|
|
370
|
-
)
|
|
371
|
-
@click.option(
|
|
372
|
-
"--ignore-order", is_flag=True, default=False, help="When set, the checker will ignore the order of list properties"
|
|
373
|
-
)
|
|
374
|
-
@click.option(
|
|
375
|
-
"--validate-processed-bytes",
|
|
376
|
-
is_flag=True,
|
|
377
|
-
default=False,
|
|
378
|
-
help="When set, the checker will validate that the new version doesn't process more than 25% than the current version",
|
|
379
|
-
)
|
|
380
|
-
@click.option(
|
|
381
|
-
"--check-requests-from-main",
|
|
382
|
-
is_flag=True,
|
|
383
|
-
default=False,
|
|
384
|
-
help="When set, the checker will get Main Workspace requests",
|
|
385
|
-
hidden=True,
|
|
386
|
-
)
|
|
387
|
-
@click.option(
|
|
388
|
-
"--folder",
|
|
389
|
-
default=".",
|
|
390
|
-
help="Folder from where to execute the command. By default the current folder",
|
|
391
|
-
hidden=True,
|
|
392
|
-
type=click.types.STRING,
|
|
393
|
-
)
|
|
394
|
-
@click.option(
|
|
395
|
-
"--user_token",
|
|
396
|
-
is_flag=False,
|
|
397
|
-
default=None,
|
|
398
|
-
help="The user token is required for sharing a datasource that contains the SHARED_WITH entry.",
|
|
399
|
-
type=click.types.STRING,
|
|
400
|
-
)
|
|
401
|
-
@click.pass_context
|
|
402
|
-
@coro
|
|
403
|
-
async def push(
|
|
404
|
-
ctx: Context,
|
|
405
|
-
filenames: Optional[List[str]],
|
|
406
|
-
dry_run: bool,
|
|
407
|
-
check: bool,
|
|
408
|
-
push_deps: bool,
|
|
409
|
-
only_changes: bool,
|
|
410
|
-
debug: bool,
|
|
411
|
-
force: bool,
|
|
412
|
-
override_datasource: bool,
|
|
413
|
-
populate: bool,
|
|
414
|
-
subset: Optional[float],
|
|
415
|
-
sql_condition: Optional[str],
|
|
416
|
-
unlink_on_populate_error: bool,
|
|
417
|
-
fixtures: bool,
|
|
418
|
-
wait: bool,
|
|
419
|
-
yes: bool,
|
|
420
|
-
only_response_times: bool,
|
|
421
|
-
workspace_map,
|
|
422
|
-
workspace,
|
|
423
|
-
no_versions: bool,
|
|
424
|
-
limit: int,
|
|
425
|
-
sample_by_params: int,
|
|
426
|
-
failfast: bool,
|
|
427
|
-
ignore_order: bool,
|
|
428
|
-
validate_processed_bytes: bool,
|
|
429
|
-
check_requests_from_main: bool,
|
|
430
|
-
folder: str,
|
|
431
|
-
user_token: Optional[str],
|
|
432
|
-
) -> None:
|
|
433
|
-
"""Push files to Tinybird."""
|
|
434
|
-
|
|
435
|
-
ignore_sql_errors = FeatureFlags.ignore_sql_errors()
|
|
436
|
-
context.disable_template_security_validation.set(True)
|
|
437
|
-
|
|
438
|
-
is_internal = has_internal_datafiles(folder)
|
|
439
|
-
|
|
440
|
-
await folder_push(
|
|
441
|
-
create_tb_client(ctx),
|
|
442
|
-
filenames,
|
|
443
|
-
dry_run,
|
|
444
|
-
check,
|
|
445
|
-
push_deps,
|
|
446
|
-
only_changes,
|
|
447
|
-
debug=debug,
|
|
448
|
-
force=force,
|
|
449
|
-
override_datasource=override_datasource,
|
|
450
|
-
populate=populate,
|
|
451
|
-
populate_subset=subset,
|
|
452
|
-
populate_condition=sql_condition,
|
|
453
|
-
unlink_on_populate_error=unlink_on_populate_error,
|
|
454
|
-
upload_fixtures=fixtures,
|
|
455
|
-
wait=wait,
|
|
456
|
-
ignore_sql_errors=ignore_sql_errors,
|
|
457
|
-
skip_confirmation=yes,
|
|
458
|
-
only_response_times=only_response_times,
|
|
459
|
-
workspace_map=dict(workspace_map),
|
|
460
|
-
workspace_lib_paths=workspace,
|
|
461
|
-
no_versions=no_versions,
|
|
462
|
-
run_tests=False,
|
|
463
|
-
tests_to_run=limit,
|
|
464
|
-
tests_sample_by_params=sample_by_params,
|
|
465
|
-
tests_failfast=failfast,
|
|
466
|
-
tests_ignore_order=ignore_order,
|
|
467
|
-
tests_validate_processed_bytes=validate_processed_bytes,
|
|
468
|
-
tests_check_requests_from_branch=check_requests_from_main,
|
|
469
|
-
folder=folder,
|
|
470
|
-
config=CLIConfig.get_project_config(),
|
|
471
|
-
user_token=user_token,
|
|
472
|
-
is_internal=is_internal,
|
|
473
|
-
)
|
|
474
|
-
return
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
@cli.command()
|
|
267
|
+
@cli.command(hidden=True)
|
|
478
268
|
@click.option(
|
|
479
269
|
"--folder", default=None, type=click.Path(exists=True, file_okay=False), help="Folder where files will be placed"
|
|
480
270
|
)
|
|
@@ -491,7 +281,7 @@ async def pull(ctx: Context, folder: str, force: bool, fmt: bool) -> None:
|
|
|
491
281
|
return await folder_pull(client, folder, force, fmt=fmt)
|
|
492
282
|
|
|
493
283
|
|
|
494
|
-
@cli.command()
|
|
284
|
+
@cli.command(hidden=True)
|
|
495
285
|
@click.option("--no-deps", is_flag=True, default=False, help="Print only data sources with no pipes using them")
|
|
496
286
|
@click.option("--match", default=None, help="Retrieve any resource matching the pattern")
|
|
497
287
|
@click.option("--pipe", default=None, help="Retrieve any resource used by pipe")
|
|
@@ -643,11 +433,6 @@ async def sql(
|
|
|
643
433
|
f"SELECT * FROM ({query}) LIMIT {rows_limit} FORMAT {req_format}", pipeline=pipeline
|
|
644
434
|
)
|
|
645
435
|
elif pipe and node:
|
|
646
|
-
datasources: List[Dict[str, Any]] = await client.datasources()
|
|
647
|
-
pipes: List[Dict[str, Any]] = await client.pipes()
|
|
648
|
-
|
|
649
|
-
existing_resources: List[str] = [x["name"] for x in datasources] + [x["name"] for x in pipes]
|
|
650
|
-
resource_versions = get_resource_versions(existing_resources)
|
|
651
436
|
filenames = [pipe]
|
|
652
437
|
|
|
653
438
|
# build graph to get new versions for all the files involved in the query
|
|
@@ -660,22 +445,6 @@ async def sql(
|
|
|
660
445
|
skip_connectors=True,
|
|
661
446
|
)
|
|
662
447
|
|
|
663
|
-
# update existing versions
|
|
664
|
-
latest_datasource_versions = resource_versions.copy()
|
|
665
|
-
|
|
666
|
-
for dep in dependencies_graph.to_run.values():
|
|
667
|
-
ds = dep["resource_name"]
|
|
668
|
-
if dep["version"] is not None:
|
|
669
|
-
latest_datasource_versions[ds] = dep["version"]
|
|
670
|
-
|
|
671
|
-
# build the graph again with the rigth version
|
|
672
|
-
dependencies_graph = await build_graph(
|
|
673
|
-
filenames,
|
|
674
|
-
client,
|
|
675
|
-
dir_path=".",
|
|
676
|
-
resource_versions=latest_datasource_versions,
|
|
677
|
-
verbose=False,
|
|
678
|
-
)
|
|
679
448
|
query = ""
|
|
680
449
|
for _, elem in dependencies_graph.to_run.items():
|
|
681
450
|
for _node in elem["nodes"]:
|
|
@@ -717,291 +486,6 @@ async def sql(
|
|
|
717
486
|
click.echo(FeedbackManager.info_no_rows())
|
|
718
487
|
|
|
719
488
|
|
|
720
|
-
@cli.command(
|
|
721
|
-
name="materialize",
|
|
722
|
-
short_help="Given a local Pipe datafile (.pipe) and a node name it generates the target Data Source and materialized Pipe ready to be pushed and guides you through the process to create the materialized view",
|
|
723
|
-
)
|
|
724
|
-
@click.argument("filename", type=click.Path(exists=True))
|
|
725
|
-
@click.argument("target_datasource", default=None, required=False)
|
|
726
|
-
@click.option("--push-deps", is_flag=True, default=False, help="Push dependencies, disabled by default")
|
|
727
|
-
@click.option("--workspace_map", nargs=2, type=str, multiple=True, hidden=True)
|
|
728
|
-
@click.option(
|
|
729
|
-
"--workspace",
|
|
730
|
-
nargs=2,
|
|
731
|
-
type=str,
|
|
732
|
-
multiple=True,
|
|
733
|
-
help="add a workspace path to the list of external workspaces, usage: --workspace name path/to/folder",
|
|
734
|
-
)
|
|
735
|
-
@click.option(
|
|
736
|
-
"--no-versions",
|
|
737
|
-
is_flag=True,
|
|
738
|
-
default=False,
|
|
739
|
-
help="when set, resource dependency versions are not used, it pushes the dependencies as-is",
|
|
740
|
-
)
|
|
741
|
-
@click.option("--verbose", is_flag=True, default=False, help="Prints more log")
|
|
742
|
-
@click.option("--force-populate", default=None, required=False, help="subset or full", hidden=True)
|
|
743
|
-
@click.option(
|
|
744
|
-
"--unlink-on-populate-error",
|
|
745
|
-
is_flag=True,
|
|
746
|
-
default=False,
|
|
747
|
-
help="If the populate job fails the Materialized View is unlinked and new data won't be ingested in the Materialized View. First time a populate job fails, the Materialized View is always unlinked.",
|
|
748
|
-
)
|
|
749
|
-
@click.option("--override-pipe", is_flag=True, default=False, help="Override pipe if exists or prompt", hidden=True)
|
|
750
|
-
@click.option(
|
|
751
|
-
"--override-datasource", is_flag=True, default=False, help="Override data source if exists or prompt", hidden=True
|
|
752
|
-
)
|
|
753
|
-
@click.pass_context
|
|
754
|
-
@coro
|
|
755
|
-
async def materialize(
|
|
756
|
-
ctx: Context,
|
|
757
|
-
filename: str,
|
|
758
|
-
push_deps: bool,
|
|
759
|
-
workspace_map: List[str],
|
|
760
|
-
workspace: List[str],
|
|
761
|
-
no_versions: bool,
|
|
762
|
-
verbose: bool,
|
|
763
|
-
force_populate: Optional[str],
|
|
764
|
-
unlink_on_populate_error: bool,
|
|
765
|
-
override_pipe: bool,
|
|
766
|
-
override_datasource: bool,
|
|
767
|
-
target_datasource: Optional[str] = None,
|
|
768
|
-
) -> None:
|
|
769
|
-
deprecation_notice = FeedbackManager.warning_deprecated(
|
|
770
|
-
warning="'tb materialize' is deprecated. To create a Materialized View in a guided way you can use the UI: `Create Pipe` and use the `Create Materialized View` wizard. Finally download the resulting `.pipe` and `.datasource` files."
|
|
771
|
-
)
|
|
772
|
-
click.echo(deprecation_notice)
|
|
773
|
-
cl = create_tb_client(ctx)
|
|
774
|
-
|
|
775
|
-
async def _try_push_pipe_to_analyze(pipe_name):
|
|
776
|
-
try:
|
|
777
|
-
to_run = await folder_push(
|
|
778
|
-
cl,
|
|
779
|
-
filenames=[filename],
|
|
780
|
-
dry_run=False,
|
|
781
|
-
check=False,
|
|
782
|
-
push_deps=push_deps,
|
|
783
|
-
debug=False,
|
|
784
|
-
force=False,
|
|
785
|
-
workspace_map=dict(workspace_map),
|
|
786
|
-
workspace_lib_paths=workspace,
|
|
787
|
-
no_versions=no_versions,
|
|
788
|
-
run_tests=False,
|
|
789
|
-
as_standard=True,
|
|
790
|
-
raise_on_exists=True,
|
|
791
|
-
verbose=verbose,
|
|
792
|
-
)
|
|
793
|
-
except AlreadyExistsException as e:
|
|
794
|
-
if "Datasource" in str(e):
|
|
795
|
-
click.echo(str(e))
|
|
796
|
-
return
|
|
797
|
-
if override_pipe or click.confirm(FeedbackManager.info_pipe_exists(name=pipe_name)):
|
|
798
|
-
to_run = await folder_push(
|
|
799
|
-
cl,
|
|
800
|
-
filenames=[filename],
|
|
801
|
-
dry_run=False,
|
|
802
|
-
check=False,
|
|
803
|
-
push_deps=push_deps,
|
|
804
|
-
debug=False,
|
|
805
|
-
force=True,
|
|
806
|
-
workspace_map=dict(workspace_map),
|
|
807
|
-
workspace_lib_paths=workspace,
|
|
808
|
-
no_versions=no_versions,
|
|
809
|
-
run_tests=False,
|
|
810
|
-
as_standard=True,
|
|
811
|
-
verbose=verbose,
|
|
812
|
-
)
|
|
813
|
-
else:
|
|
814
|
-
return
|
|
815
|
-
except click.ClickException as ex:
|
|
816
|
-
# HACK: By now, datafile raises click.ClickException instead of
|
|
817
|
-
# CLIException to avoid circular imports. Thats we need to trace
|
|
818
|
-
# the error here.
|
|
819
|
-
#
|
|
820
|
-
# Once we do a big refactor in datafile, we can get rid of this
|
|
821
|
-
# snippet.
|
|
822
|
-
msg: str = str(ex)
|
|
823
|
-
add_telemetry_event("datafile_error", error=msg)
|
|
824
|
-
click.echo(msg)
|
|
825
|
-
|
|
826
|
-
return to_run
|
|
827
|
-
|
|
828
|
-
def _choose_node_name(pipe):
|
|
829
|
-
node = pipe["nodes"][0]
|
|
830
|
-
materialized_nodes = [node for node in pipe["nodes"] if node["type"].lower() == "materialized"]
|
|
831
|
-
|
|
832
|
-
if len(materialized_nodes) == 1:
|
|
833
|
-
node = materialized_nodes[0]
|
|
834
|
-
|
|
835
|
-
if len(pipe["nodes"]) > 1 and len(materialized_nodes) != 1:
|
|
836
|
-
for index, node in enumerate(pipe["nodes"], start=1):
|
|
837
|
-
click.echo(f" [{index}] Materialize node with name => {node['name']}")
|
|
838
|
-
option = click.prompt(FeedbackManager.prompt_choose_node(), default=len(pipe["nodes"]))
|
|
839
|
-
node = pipe["nodes"][option - 1]
|
|
840
|
-
node_name = node["name"]
|
|
841
|
-
return node, node_name
|
|
842
|
-
|
|
843
|
-
def _choose_target_datasource_name(pipe, node, node_name):
|
|
844
|
-
return target_datasource or node.get("datasource", None) or f'mv_{pipe["resource_name"]}_{node_name}'
|
|
845
|
-
|
|
846
|
-
def _save_local_backup_pipe(pipe):
|
|
847
|
-
pipe_bak = f"{filename}_bak"
|
|
848
|
-
shutil.copyfile(filename, pipe_bak)
|
|
849
|
-
pipe_file_name = f"{pipe['resource_name']}.pipe"
|
|
850
|
-
click.echo(FeedbackManager.info_pipe_backup_created(name=pipe_bak))
|
|
851
|
-
return pipe_file_name
|
|
852
|
-
|
|
853
|
-
def _save_local_datasource(datasource_name, ds_datafile):
|
|
854
|
-
base = Path("datasources")
|
|
855
|
-
if not base.exists():
|
|
856
|
-
base = Path()
|
|
857
|
-
file_name = f"{datasource_name}.datasource"
|
|
858
|
-
f = base / file_name
|
|
859
|
-
with open(f"{f}", "w") as file:
|
|
860
|
-
file.write(ds_datafile)
|
|
861
|
-
|
|
862
|
-
click.echo(FeedbackManager.success_generated_local_file(file=f))
|
|
863
|
-
return f
|
|
864
|
-
|
|
865
|
-
async def _try_push_datasource(datasource_name, f):
|
|
866
|
-
exists = False
|
|
867
|
-
try:
|
|
868
|
-
exists = await cl.get_datasource(datasource_name)
|
|
869
|
-
except Exception:
|
|
870
|
-
pass
|
|
871
|
-
|
|
872
|
-
if exists:
|
|
873
|
-
click.echo(FeedbackManager.info_materialize_push_datasource_exists(name=f.name))
|
|
874
|
-
if override_datasource or click.confirm(FeedbackManager.info_materialize_push_datasource_override(name=f)):
|
|
875
|
-
try:
|
|
876
|
-
await cl.datasource_delete(datasource_name, force=True)
|
|
877
|
-
except DoesNotExistException:
|
|
878
|
-
pass
|
|
879
|
-
|
|
880
|
-
filename = str(f.absolute())
|
|
881
|
-
to_run = await folder_push(
|
|
882
|
-
cl,
|
|
883
|
-
filenames=[filename],
|
|
884
|
-
push_deps=push_deps,
|
|
885
|
-
workspace_map=dict(workspace_map),
|
|
886
|
-
workspace_lib_paths=workspace,
|
|
887
|
-
no_versions=no_versions,
|
|
888
|
-
verbose=verbose,
|
|
889
|
-
)
|
|
890
|
-
return to_run
|
|
891
|
-
|
|
892
|
-
def _save_local_pipe(pipe_file_name, pipe_datafile, pipe):
|
|
893
|
-
base = Path("pipes")
|
|
894
|
-
if not base.exists():
|
|
895
|
-
base = Path()
|
|
896
|
-
f_pipe = base / pipe_file_name
|
|
897
|
-
|
|
898
|
-
with open(f"{f_pipe}", "w") as file:
|
|
899
|
-
if pipe["version"] is not None and pipe["version"] >= 0:
|
|
900
|
-
pipe_datafile = f"VERSION {pipe['version']} \n {pipe_datafile}"
|
|
901
|
-
matches = re.findall(r"([^\s\.]*__v\d+)", pipe_datafile)
|
|
902
|
-
for match in set(matches):
|
|
903
|
-
m = match.split("__v")[0]
|
|
904
|
-
if m in pipe_datafile:
|
|
905
|
-
pipe_datafile = pipe_datafile.replace(match, m)
|
|
906
|
-
file.write(pipe_datafile)
|
|
907
|
-
|
|
908
|
-
click.echo(FeedbackManager.success_generated_local_file(file=f_pipe))
|
|
909
|
-
return f_pipe
|
|
910
|
-
|
|
911
|
-
async def _try_push_pipe(f_pipe):
|
|
912
|
-
if override_pipe:
|
|
913
|
-
option = 2
|
|
914
|
-
else:
|
|
915
|
-
click.echo(FeedbackManager.info_materialize_push_pipe_skip(name=f_pipe.name))
|
|
916
|
-
click.echo(FeedbackManager.info_materialize_push_pipe_override(name=f_pipe.name))
|
|
917
|
-
option = click.prompt(FeedbackManager.prompt_choose(), default=1)
|
|
918
|
-
force = True
|
|
919
|
-
check = option == 1
|
|
920
|
-
|
|
921
|
-
filename = str(f_pipe.absolute())
|
|
922
|
-
to_run = await folder_push(
|
|
923
|
-
cl,
|
|
924
|
-
filenames=[filename],
|
|
925
|
-
dry_run=False,
|
|
926
|
-
check=check,
|
|
927
|
-
push_deps=push_deps,
|
|
928
|
-
debug=False,
|
|
929
|
-
force=force,
|
|
930
|
-
unlink_on_populate_error=unlink_on_populate_error,
|
|
931
|
-
workspace_map=dict(workspace_map),
|
|
932
|
-
workspace_lib_paths=workspace,
|
|
933
|
-
no_versions=no_versions,
|
|
934
|
-
run_tests=False,
|
|
935
|
-
verbose=verbose,
|
|
936
|
-
)
|
|
937
|
-
return to_run
|
|
938
|
-
|
|
939
|
-
async def _populate(pipe, node_name, f_pipe):
|
|
940
|
-
if force_populate or click.confirm(FeedbackManager.prompt_populate(file=f_pipe)):
|
|
941
|
-
if not force_populate:
|
|
942
|
-
click.echo(FeedbackManager.info_materialize_populate_partial())
|
|
943
|
-
click.echo(FeedbackManager.info_materialize_populate_full())
|
|
944
|
-
option = click.prompt(FeedbackManager.prompt_choose(), default=1)
|
|
945
|
-
else:
|
|
946
|
-
option = 1 if force_populate == "subset" else 2
|
|
947
|
-
populate = False
|
|
948
|
-
populate_subset = False
|
|
949
|
-
if option == 1:
|
|
950
|
-
populate_subset = 0.1
|
|
951
|
-
populate = True
|
|
952
|
-
elif option == 2:
|
|
953
|
-
populate = True
|
|
954
|
-
|
|
955
|
-
if populate:
|
|
956
|
-
response = await cl.populate_node(
|
|
957
|
-
pipe["name"],
|
|
958
|
-
node_name,
|
|
959
|
-
populate_subset=populate_subset,
|
|
960
|
-
unlink_on_populate_error=unlink_on_populate_error,
|
|
961
|
-
)
|
|
962
|
-
if "job" not in response:
|
|
963
|
-
raise CLIException(response)
|
|
964
|
-
|
|
965
|
-
job_url = response.get("job", {}).get("job_url", None)
|
|
966
|
-
job_id = response.get("job", {}).get("job_id", None)
|
|
967
|
-
click.echo(FeedbackManager.info_populate_job_url(url=job_url))
|
|
968
|
-
wait_populate = True
|
|
969
|
-
if wait_populate:
|
|
970
|
-
await wait_job(cl, job_id, job_url, "Populating")
|
|
971
|
-
|
|
972
|
-
click.echo(FeedbackManager.warning_beta_tester())
|
|
973
|
-
pipe_name = os.path.basename(filename).rsplit(".", 1)[0]
|
|
974
|
-
click.echo(FeedbackManager.info_before_push_materialize(name=filename))
|
|
975
|
-
try:
|
|
976
|
-
# extracted the materialize logic to local functions so the workflow is more readable
|
|
977
|
-
to_run = await _try_push_pipe_to_analyze(pipe_name)
|
|
978
|
-
|
|
979
|
-
if to_run is None:
|
|
980
|
-
return
|
|
981
|
-
|
|
982
|
-
pipe = to_run[pipe_name.split("/")[-1]]
|
|
983
|
-
node, node_name = _choose_node_name(pipe)
|
|
984
|
-
datasource_name = _choose_target_datasource_name(pipe, node, node_name)
|
|
985
|
-
|
|
986
|
-
click.echo(FeedbackManager.info_before_materialize(name=pipe["name"]))
|
|
987
|
-
analysis = await cl.analyze_pipe_node(pipe["name"], node, datasource_name=datasource_name)
|
|
988
|
-
ds_datafile = analysis["analysis"]["datasource"]["datafile"]
|
|
989
|
-
pipe_datafile = analysis["analysis"]["pipe"]["datafile"]
|
|
990
|
-
|
|
991
|
-
pipe_file_name = _save_local_backup_pipe(pipe)
|
|
992
|
-
f = _save_local_datasource(datasource_name, ds_datafile)
|
|
993
|
-
await _try_push_datasource(datasource_name, f)
|
|
994
|
-
|
|
995
|
-
f_pipe = _save_local_pipe(pipe_file_name, pipe_datafile, pipe)
|
|
996
|
-
await _try_push_pipe(f_pipe)
|
|
997
|
-
await _populate(pipe, node_name, f_pipe)
|
|
998
|
-
click.echo(FeedbackManager.success_created_matview(name=datasource_name))
|
|
999
|
-
except AuthNoTokenException:
|
|
1000
|
-
raise
|
|
1001
|
-
except Exception as e:
|
|
1002
|
-
raise CLIException(FeedbackManager.error_exception(error=str(e)))
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
489
|
def __patch_click_output():
|
|
1006
490
|
import re
|
|
1007
491
|
|
|
@@ -1036,316 +520,4 @@ def __patch_click_output():
|
|
|
1036
520
|
|
|
1037
521
|
def __unpatch_click_output():
|
|
1038
522
|
click.echo = __old_click_echo
|
|
1039
|
-
click.secho =
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
@cli.command(short_help="Learn how to include info about the CLI in your shell PROMPT")
|
|
1043
|
-
@click.pass_context
|
|
1044
|
-
@coro
|
|
1045
|
-
async def prompt(_ctx: Context) -> None:
|
|
1046
|
-
click.secho("Follow these instructions => https://www.tinybird.co/docs/cli.html#configure-the-shell-prompt")
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
@cli.command()
|
|
1050
|
-
@click.option(
|
|
1051
|
-
"--dry-run",
|
|
1052
|
-
is_flag=True,
|
|
1053
|
-
default=False,
|
|
1054
|
-
help="Run the command without creating resources on the Tinybird account or any side effect",
|
|
1055
|
-
)
|
|
1056
|
-
@click.option(
|
|
1057
|
-
"--debug",
|
|
1058
|
-
is_flag=True,
|
|
1059
|
-
default=False,
|
|
1060
|
-
help="Prints internal representation, can be combined with any command to get more information.",
|
|
1061
|
-
)
|
|
1062
|
-
@click.option(
|
|
1063
|
-
"--override-datasource",
|
|
1064
|
-
is_flag=True,
|
|
1065
|
-
default=False,
|
|
1066
|
-
help="When pushing a pipe with a Materialized node if the target Data Source exists it will try to override it.",
|
|
1067
|
-
)
|
|
1068
|
-
@click.option("--populate", is_flag=True, default=False, help="Populate materialized nodes when pushing them")
|
|
1069
|
-
@click.option(
|
|
1070
|
-
"--subset",
|
|
1071
|
-
type=float,
|
|
1072
|
-
default=None,
|
|
1073
|
-
help="Populate with a subset percent of the data (limited to a maximum of 2M rows), this is useful to quickly test a materialized node with some data. The subset must be greater than 0 and lower than 0.1. A subset of 0.1 means a 10 percent of the data in the source Data Source will be used to populate the materialized view. Use it together with --populate, it has precedence over --sql-condition",
|
|
1074
|
-
)
|
|
1075
|
-
@click.option(
|
|
1076
|
-
"--sql-condition",
|
|
1077
|
-
type=str,
|
|
1078
|
-
default=None,
|
|
1079
|
-
help="Populate with a SQL condition to be applied to the trigger Data Source of the Materialized View. For instance, `--sql-condition='date == toYYYYMM(now())'` it'll populate taking all the rows from the trigger Data Source which `date` is the current month. Use it together with --populate. --sql-condition is not taken into account if the --subset param is present. Including in the ``sql_condition`` any column present in the Data Source ``engine_sorting_key`` will make the populate job process less data.",
|
|
1080
|
-
)
|
|
1081
|
-
@click.option(
|
|
1082
|
-
"--unlink-on-populate-error",
|
|
1083
|
-
is_flag=True,
|
|
1084
|
-
default=False,
|
|
1085
|
-
help="If the populate job fails the Materialized View is unlinked and new data won't be ingested in the Materialized View. First time a populate job fails, the Materialized View is always unlinked.",
|
|
1086
|
-
)
|
|
1087
|
-
@click.option(
|
|
1088
|
-
"--wait",
|
|
1089
|
-
is_flag=True,
|
|
1090
|
-
default=False,
|
|
1091
|
-
help="To be used along with --populate command. Waits for populate jobs to finish, showing a progress bar. Disabled by default.",
|
|
1092
|
-
)
|
|
1093
|
-
@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
|
|
1094
|
-
@click.option("--workspace_map", nargs=2, type=str, multiple=True)
|
|
1095
|
-
@click.option(
|
|
1096
|
-
"--workspace",
|
|
1097
|
-
nargs=2,
|
|
1098
|
-
type=str,
|
|
1099
|
-
multiple=True,
|
|
1100
|
-
help="add a workspace path to the list of external workspaces, usage: --workspace name path/to/folder",
|
|
1101
|
-
)
|
|
1102
|
-
@click.option(
|
|
1103
|
-
"--folder",
|
|
1104
|
-
default=".",
|
|
1105
|
-
help="Folder from where to execute the command. By default the current folder",
|
|
1106
|
-
hidden=True,
|
|
1107
|
-
type=click.types.STRING,
|
|
1108
|
-
)
|
|
1109
|
-
@click.option(
|
|
1110
|
-
"--user_token",
|
|
1111
|
-
is_flag=False,
|
|
1112
|
-
default=None,
|
|
1113
|
-
help="The user token is required for sharing a datasource that contains the SHARED_WITH entry.",
|
|
1114
|
-
type=click.types.STRING,
|
|
1115
|
-
)
|
|
1116
|
-
@click.option(
|
|
1117
|
-
"--fork-downstream",
|
|
1118
|
-
is_flag=True,
|
|
1119
|
-
default=False,
|
|
1120
|
-
help="Creates new versions of the dependent downstream resources to the changed resources.",
|
|
1121
|
-
hidden=True,
|
|
1122
|
-
)
|
|
1123
|
-
@click.option(
|
|
1124
|
-
"--fork",
|
|
1125
|
-
is_flag=True,
|
|
1126
|
-
default=False,
|
|
1127
|
-
help="Creates new versions of the changed resources. Use --fork-downstream to fork also the downstream dependencies of the changed resources.",
|
|
1128
|
-
hidden=True,
|
|
1129
|
-
)
|
|
1130
|
-
# this is added to use tb deploy in dry run mode. It's temprary => https://gitlab.com/tinybird/analytics/-/issues/12551
|
|
1131
|
-
@click.option(
|
|
1132
|
-
"--use-main",
|
|
1133
|
-
is_flag=True,
|
|
1134
|
-
default=False,
|
|
1135
|
-
help="Use main commit instead of release commit",
|
|
1136
|
-
hidden=True,
|
|
1137
|
-
)
|
|
1138
|
-
@click.option(
|
|
1139
|
-
"--skip-head-outdated",
|
|
1140
|
-
is_flag=True,
|
|
1141
|
-
default=False,
|
|
1142
|
-
help="Allows to deploy any commit without checking if the branch is rebased to the workspace commit",
|
|
1143
|
-
hidden=True,
|
|
1144
|
-
)
|
|
1145
|
-
@click.pass_context
|
|
1146
|
-
@coro
|
|
1147
|
-
async def deploy(
|
|
1148
|
-
ctx: Context,
|
|
1149
|
-
dry_run: bool,
|
|
1150
|
-
debug: bool,
|
|
1151
|
-
override_datasource: bool,
|
|
1152
|
-
populate: bool,
|
|
1153
|
-
subset: Optional[float],
|
|
1154
|
-
sql_condition: Optional[str],
|
|
1155
|
-
unlink_on_populate_error: bool,
|
|
1156
|
-
wait: bool,
|
|
1157
|
-
yes: bool,
|
|
1158
|
-
workspace_map,
|
|
1159
|
-
workspace,
|
|
1160
|
-
folder: str,
|
|
1161
|
-
user_token: Optional[str],
|
|
1162
|
-
fork_downstream: bool,
|
|
1163
|
-
fork: bool,
|
|
1164
|
-
use_main: bool,
|
|
1165
|
-
skip_head_outdated: bool,
|
|
1166
|
-
) -> None:
|
|
1167
|
-
"""Deploy in Tinybird pushing resources changed from last git commit deployed.
|
|
1168
|
-
|
|
1169
|
-
Usage: tb deploy
|
|
1170
|
-
"""
|
|
1171
|
-
try:
|
|
1172
|
-
ignore_sql_errors = FeatureFlags.ignore_sql_errors()
|
|
1173
|
-
context.disable_template_security_validation.set(True)
|
|
1174
|
-
|
|
1175
|
-
is_internal = has_internal_datafiles(folder)
|
|
1176
|
-
|
|
1177
|
-
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
1178
|
-
config = CLIConfig.get_project_config()
|
|
1179
|
-
workspaces: List[Dict[str, Any]] = (await client.user_workspaces_and_branches()).get("workspaces", [])
|
|
1180
|
-
current_ws: Dict[str, Any] = next(
|
|
1181
|
-
(workspace for workspace in workspaces if config and workspace.get("id", ".") == config.get("id", "..")), {}
|
|
1182
|
-
)
|
|
1183
|
-
|
|
1184
|
-
semver = config.get("semver")
|
|
1185
|
-
auto_promote = getenv_bool("TB_AUTO_PROMOTE", True)
|
|
1186
|
-
release = current_ws.get("release", {})
|
|
1187
|
-
current_semver: Optional[str] = None
|
|
1188
|
-
if release and isinstance(release, dict):
|
|
1189
|
-
current_semver = release.get("semver")
|
|
1190
|
-
|
|
1191
|
-
if not current_semver:
|
|
1192
|
-
click.echo(FeedbackManager.error_init_release(workspace=current_ws.get("name")))
|
|
1193
|
-
sys.exit(1)
|
|
1194
|
-
|
|
1195
|
-
release_created = False
|
|
1196
|
-
new_release = False
|
|
1197
|
-
check_backfill_required = False
|
|
1198
|
-
|
|
1199
|
-
# semver is now optional, if not sent force the bump
|
|
1200
|
-
if not semver:
|
|
1201
|
-
semver = current_semver
|
|
1202
|
-
else:
|
|
1203
|
-
click.echo(FeedbackManager.warning_deprecated_releases())
|
|
1204
|
-
|
|
1205
|
-
if semver and current_semver:
|
|
1206
|
-
new_version = version.parse(semver.split("-snapshot")[0])
|
|
1207
|
-
current_version = version.parse(current_semver.split("-snapshot")[0])
|
|
1208
|
-
show_feedback = new_version != current_version
|
|
1209
|
-
if show_feedback:
|
|
1210
|
-
if dry_run:
|
|
1211
|
-
click.echo(
|
|
1212
|
-
FeedbackManager.info_dry_releases_detected(current_semver=current_version, semver=new_version)
|
|
1213
|
-
)
|
|
1214
|
-
else:
|
|
1215
|
-
click.echo(
|
|
1216
|
-
FeedbackManager.info_releases_detected(current_semver=current_version, semver=new_version)
|
|
1217
|
-
)
|
|
1218
|
-
|
|
1219
|
-
if new_version == current_version:
|
|
1220
|
-
if current_version.post is None:
|
|
1221
|
-
semver = str(current_version) + "-1"
|
|
1222
|
-
new_version = version.Version(semver)
|
|
1223
|
-
else:
|
|
1224
|
-
semver = f"{current_version.major}.{current_version.minor}.{current_version.micro}-{current_version.post + 1}"
|
|
1225
|
-
new_version = version.Version(semver)
|
|
1226
|
-
|
|
1227
|
-
if is_post_semver(new_version, current_version):
|
|
1228
|
-
if show_feedback:
|
|
1229
|
-
if dry_run:
|
|
1230
|
-
click.echo(FeedbackManager.info_dry_local_release(version=new_version))
|
|
1231
|
-
else:
|
|
1232
|
-
click.echo(FeedbackManager.info_local_release(version=new_version))
|
|
1233
|
-
yes = True
|
|
1234
|
-
auto_promote = False
|
|
1235
|
-
new_release = False
|
|
1236
|
-
elif is_major_semver(new_version, current_version):
|
|
1237
|
-
if show_feedback:
|
|
1238
|
-
if dry_run:
|
|
1239
|
-
click.echo(FeedbackManager.info_dry_major_release(version=new_version))
|
|
1240
|
-
else:
|
|
1241
|
-
click.echo(FeedbackManager.info_major_release(version=new_version))
|
|
1242
|
-
auto_promote = False
|
|
1243
|
-
new_release = True
|
|
1244
|
-
else:
|
|
1245
|
-
# allows TB_AUTO_PROMOTE=0 so release is left in preview
|
|
1246
|
-
auto_promote = getenv_bool("TB_AUTO_PROMOTE", True)
|
|
1247
|
-
if auto_promote:
|
|
1248
|
-
if show_feedback:
|
|
1249
|
-
if dry_run:
|
|
1250
|
-
click.echo(
|
|
1251
|
-
FeedbackManager.info_dry_minor_patch_release_with_autopromote(version=new_version)
|
|
1252
|
-
)
|
|
1253
|
-
else:
|
|
1254
|
-
click.echo(FeedbackManager.info_minor_patch_release_with_autopromote(version=new_version))
|
|
1255
|
-
else:
|
|
1256
|
-
if show_feedback:
|
|
1257
|
-
if dry_run:
|
|
1258
|
-
click.echo(FeedbackManager.info_dry_minor_patch_release_no_autopromote(version=new_version))
|
|
1259
|
-
else:
|
|
1260
|
-
click.echo(FeedbackManager.info_minor_patch_release_no_autopromote(version=new_version))
|
|
1261
|
-
new_release = True
|
|
1262
|
-
|
|
1263
|
-
if new_release:
|
|
1264
|
-
release_created = True
|
|
1265
|
-
fork_downstream = True
|
|
1266
|
-
# allows TB_CHECK_BACKFILL_REQUIRED=0 so it is not checked
|
|
1267
|
-
check_backfill_required = getenv_bool("TB_CHECK_BACKFILL_REQUIRED", True)
|
|
1268
|
-
try:
|
|
1269
|
-
tb_client = create_tb_client(ctx)
|
|
1270
|
-
await folder_push(
|
|
1271
|
-
tb_client=tb_client,
|
|
1272
|
-
dry_run=dry_run,
|
|
1273
|
-
check=False,
|
|
1274
|
-
push_deps=True,
|
|
1275
|
-
debug=debug,
|
|
1276
|
-
force=True,
|
|
1277
|
-
git_release=True,
|
|
1278
|
-
override_datasource=override_datasource,
|
|
1279
|
-
populate=populate,
|
|
1280
|
-
populate_subset=subset,
|
|
1281
|
-
populate_condition=sql_condition,
|
|
1282
|
-
unlink_on_populate_error=unlink_on_populate_error,
|
|
1283
|
-
upload_fixtures=False,
|
|
1284
|
-
wait=wait,
|
|
1285
|
-
ignore_sql_errors=ignore_sql_errors,
|
|
1286
|
-
skip_confirmation=yes,
|
|
1287
|
-
workspace_map=dict(workspace_map),
|
|
1288
|
-
workspace_lib_paths=workspace,
|
|
1289
|
-
run_tests=False,
|
|
1290
|
-
folder=folder,
|
|
1291
|
-
config=config,
|
|
1292
|
-
user_token=user_token,
|
|
1293
|
-
fork_downstream=fork_downstream,
|
|
1294
|
-
fork=fork,
|
|
1295
|
-
is_internal=is_internal,
|
|
1296
|
-
release_created=release_created,
|
|
1297
|
-
auto_promote=auto_promote,
|
|
1298
|
-
check_backfill_required=check_backfill_required,
|
|
1299
|
-
use_main=use_main,
|
|
1300
|
-
check_outdated=not skip_head_outdated,
|
|
1301
|
-
)
|
|
1302
|
-
except Exception as e:
|
|
1303
|
-
if release_created and not dry_run:
|
|
1304
|
-
await client.release_failed(config["id"], semver)
|
|
1305
|
-
raise e
|
|
1306
|
-
|
|
1307
|
-
if release_created:
|
|
1308
|
-
try:
|
|
1309
|
-
if not dry_run:
|
|
1310
|
-
await client.release_preview(config["id"], semver)
|
|
1311
|
-
click.echo(FeedbackManager.success_release_preview(semver=semver))
|
|
1312
|
-
else:
|
|
1313
|
-
click.echo(FeedbackManager.success_dry_release_preview(semver=semver))
|
|
1314
|
-
if auto_promote:
|
|
1315
|
-
try:
|
|
1316
|
-
force_remove = getenv_bool("TB_FORCE_REMOVE_OLDEST_ROLLBACK", False)
|
|
1317
|
-
if dry_run:
|
|
1318
|
-
click.echo(FeedbackManager.warning_dry_remove_oldest_rollback(semver=semver))
|
|
1319
|
-
else:
|
|
1320
|
-
click.echo(FeedbackManager.warning_remove_oldest_rollback(semver=semver))
|
|
1321
|
-
try:
|
|
1322
|
-
await remove_release(dry_run, config, OLDEST_ROLLBACK, client, force=force_remove)
|
|
1323
|
-
except Exception as e:
|
|
1324
|
-
click.echo(FeedbackManager.error_remove_oldest_rollback(error=str(e), semver=semver))
|
|
1325
|
-
sys.exit(1)
|
|
1326
|
-
|
|
1327
|
-
if not dry_run:
|
|
1328
|
-
release = await client.release_promote(config["id"], semver)
|
|
1329
|
-
click.echo(FeedbackManager.success_release_promote(semver=semver))
|
|
1330
|
-
click.echo(FeedbackManager.success_git_release(release_commit=release["commit"]))
|
|
1331
|
-
else:
|
|
1332
|
-
click.echo(FeedbackManager.success_dry_release_promote(semver=semver))
|
|
1333
|
-
click.echo(FeedbackManager.success_dry_git_release(release_commit=release["commit"]))
|
|
1334
|
-
except Exception as e:
|
|
1335
|
-
raise CLIException(FeedbackManager.error_exception(error=str(e)))
|
|
1336
|
-
except Exception as e:
|
|
1337
|
-
raise CLIException(FeedbackManager.error_exception(error=str(e)))
|
|
1338
|
-
|
|
1339
|
-
if not new_release:
|
|
1340
|
-
if dry_run:
|
|
1341
|
-
if show_feedback:
|
|
1342
|
-
click.echo(FeedbackManager.success_dry_release_update(semver=current_semver, new_semver=semver))
|
|
1343
|
-
else:
|
|
1344
|
-
client.semver = None
|
|
1345
|
-
await client.update_release_semver(config["id"], current_semver, semver)
|
|
1346
|
-
if show_feedback:
|
|
1347
|
-
click.echo(FeedbackManager.success_release_update(semver=current_semver, new_semver=semver))
|
|
1348
|
-
except AuthNoTokenException:
|
|
1349
|
-
raise
|
|
1350
|
-
except Exception as e:
|
|
1351
|
-
raise CLIException(str(e))
|
|
523
|
+
click.secho = __old_click_secho
|