tinybird-cli 6.5.7.dev0__py3-none-any.whl → 6.5.8.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tinybird/__cli__.py +2 -2
- tinybird/ch_utils/constants.py +1 -0
- tinybird/ch_utils/engine.py +90 -54
- tinybird/client.py +58 -46
- tinybird/config.py +0 -4
- tinybird/datafile_common.py +67 -44
- tinybird/datatypes.py +34 -12
- tinybird/feedback_manager.py +5 -0
- tinybird/git_settings.py +1 -3
- tinybird/sql.py +3 -10
- tinybird/sql_template.py +121 -98
- tinybird/sql_template_fmt.py +10 -15
- tinybird/sql_toolset.py +63 -11
- tinybird/syncasync.py +9 -54
- tinybird/tb_cli_modules/branch.py +4 -25
- tinybird/tb_cli_modules/cicd.py +2 -0
- tinybird/tb_cli_modules/cli.py +5 -9
- tinybird/tb_cli_modules/common.py +30 -97
- tinybird/tb_cli_modules/config.py +1 -2
- tinybird/tb_cli_modules/datasource.py +2 -2
- tinybird/tb_cli_modules/pipe.py +49 -34
- tinybird/tb_cli_modules/tinyunit/tinyunit.py +4 -4
- tinybird/tb_cli_modules/token.py +1 -2
- tinybird/tb_cli_modules/workspace.py +11 -13
- tinybird/tornado_template.py +227 -83
- tinybird/utils/__init__.py +0 -0
- tinybird/utils/bools.py +19 -0
- {tinybird_cli-6.5.7.dev0.dist-info → tinybird_cli-6.5.8.dev0.dist-info}/METADATA +9 -1
- tinybird_cli-6.5.8.dev0.dist-info/RECORD +45 -0
- tinybird_cli-6.5.7.dev0.dist-info/RECORD +0 -43
- {tinybird_cli-6.5.7.dev0.dist-info → tinybird_cli-6.5.8.dev0.dist-info}/WHEEL +0 -0
- {tinybird_cli-6.5.7.dev0.dist-info → tinybird_cli-6.5.8.dev0.dist-info}/entry_points.txt +0 -0
- {tinybird_cli-6.5.7.dev0.dist-info → tinybird_cli-6.5.8.dev0.dist-info}/top_level.txt +0 -0
tinybird/syncasync.py
CHANGED
|
@@ -142,8 +142,7 @@ class Local:
|
|
|
142
142
|
storage = self._get_storage()
|
|
143
143
|
if key in storage:
|
|
144
144
|
return storage[key]
|
|
145
|
-
|
|
146
|
-
raise AttributeError(f"{self!r} object has no attribute {key!r}")
|
|
145
|
+
raise AttributeError(f"{self!r} object has no attribute {key!r}")
|
|
147
146
|
|
|
148
147
|
def __setattr__(self, key, value):
|
|
149
148
|
if key in ("_context_refs", "_thread_critical", "_thread_lock", "_attr_name"):
|
|
@@ -151,6 +150,7 @@ class Local:
|
|
|
151
150
|
with self._thread_lock:
|
|
152
151
|
storage = self._get_storage()
|
|
153
152
|
storage[key] = value
|
|
153
|
+
return None
|
|
154
154
|
|
|
155
155
|
def __delattr__(self, key):
|
|
156
156
|
with self._thread_lock:
|
|
@@ -252,56 +252,12 @@ def _iscoroutinefunction_or_partial(func: Any) -> bool:
|
|
|
252
252
|
# this to exist. Code taken from CPython.
|
|
253
253
|
if sys.version_info >= (3, 8):
|
|
254
254
|
return asyncio.iscoroutinefunction(func)
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
func = func.func
|
|
260
|
-
|
|
261
|
-
return asyncio.iscoroutinefunction(func)
|
|
255
|
+
while inspect.ismethod(func):
|
|
256
|
+
func = func.__func__
|
|
257
|
+
while isinstance(func, functools.partial):
|
|
258
|
+
func = func.func
|
|
262
259
|
|
|
263
|
-
|
|
264
|
-
class ThreadSensitiveContext:
|
|
265
|
-
"""Async context manager to manage context for thread sensitive mode
|
|
266
|
-
This context manager controls which thread pool executor is used when in
|
|
267
|
-
thread sensitive mode. By default, a single thread pool executor is shared
|
|
268
|
-
within a process.
|
|
269
|
-
In Python 3.7+, the ThreadSensitiveContext() context manager may be used to
|
|
270
|
-
specify a thread pool per context.
|
|
271
|
-
In Python 3.6, usage of this context manager has no effect.
|
|
272
|
-
This context manager is re-entrant, so only the outer-most call to
|
|
273
|
-
ThreadSensitiveContext will set the context.
|
|
274
|
-
"""
|
|
275
|
-
|
|
276
|
-
def __init__(self):
|
|
277
|
-
self.token = None
|
|
278
|
-
|
|
279
|
-
if contextvars:
|
|
280
|
-
|
|
281
|
-
async def __aenter__(self):
|
|
282
|
-
try:
|
|
283
|
-
SyncToAsync.thread_sensitive_context.get()
|
|
284
|
-
except LookupError:
|
|
285
|
-
self.token = SyncToAsync.thread_sensitive_context.set(self)
|
|
286
|
-
|
|
287
|
-
return self
|
|
288
|
-
|
|
289
|
-
async def __aexit__(self, exc, value, tb):
|
|
290
|
-
if not self.token:
|
|
291
|
-
return
|
|
292
|
-
|
|
293
|
-
executor = SyncToAsync.context_to_thread_executor.pop(self, None)
|
|
294
|
-
if executor:
|
|
295
|
-
executor.shutdown()
|
|
296
|
-
SyncToAsync.thread_sensitive_context.reset(self.token)
|
|
297
|
-
|
|
298
|
-
else:
|
|
299
|
-
|
|
300
|
-
async def __aenter__(self):
|
|
301
|
-
return self
|
|
302
|
-
|
|
303
|
-
async def __aexit__(self, exc, value, tb):
|
|
304
|
-
pass
|
|
260
|
+
return asyncio.iscoroutinefunction(func)
|
|
305
261
|
|
|
306
262
|
|
|
307
263
|
class AsyncToSync:
|
|
@@ -660,9 +616,8 @@ class SyncToAsync:
|
|
|
660
616
|
if hasattr(asyncio, "current_task"):
|
|
661
617
|
# Python 3.7 and up
|
|
662
618
|
return asyncio.current_task()
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
return asyncio.Task.current_task()
|
|
619
|
+
# Python 3.6
|
|
620
|
+
return asyncio.Task.current_task()
|
|
666
621
|
except RuntimeError:
|
|
667
622
|
return None
|
|
668
623
|
|
|
@@ -264,8 +264,7 @@ async def release_rm(semver: str, oldest_rollback: bool, force: bool, yes: bool,
|
|
|
264
264
|
if not oldest_rollback_semver:
|
|
265
265
|
click.echo(FeedbackManager.info_release_no_rollback())
|
|
266
266
|
return
|
|
267
|
-
|
|
268
|
-
semver = oldest_rollback_semver
|
|
267
|
+
semver = oldest_rollback_semver
|
|
269
268
|
await print_release_summary(config, semver, info=True, dry_run=dry_run)
|
|
270
269
|
if dry_run or yes or click.confirm(FeedbackManager.warning_confirm_delete_release(semver=semver)):
|
|
271
270
|
try:
|
|
@@ -555,17 +554,9 @@ async def data_branch(last_partition: bool, all: bool, wait: bool) -> None:
|
|
|
555
554
|
default=False,
|
|
556
555
|
help="Flag to skip execution of regression tests. This is handy for CI branches where regression might be flaky",
|
|
557
556
|
)
|
|
558
|
-
@click.option(
|
|
559
|
-
"--main",
|
|
560
|
-
is_flag=True,
|
|
561
|
-
default=False,
|
|
562
|
-
help="Run regression tests in the main Branch. For this flag to work all the resources in the Branch pipe endpoints need to exist in the main Branch.",
|
|
563
|
-
)
|
|
564
557
|
@click.pass_context
|
|
565
558
|
@coro
|
|
566
|
-
async def regression_tests(
|
|
567
|
-
ctx, filename: str, wait: bool, skip_regression_tests: Optional[bool] = False, main: Optional[bool] = False
|
|
568
|
-
):
|
|
559
|
+
async def regression_tests(ctx, filename: str, wait: bool, skip_regression_tests: Optional[bool] = False):
|
|
569
560
|
"""Regression test commands for Branches"""
|
|
570
561
|
if skip_regression_tests:
|
|
571
562
|
click.echo(FeedbackManager.warning_regression_skipped())
|
|
@@ -591,9 +582,7 @@ async def regression_tests(
|
|
|
591
582
|
if current_main_workspace["id"] == config["id"]:
|
|
592
583
|
raise CLIException(FeedbackManager.error_not_allowed_in_main_branch())
|
|
593
584
|
try:
|
|
594
|
-
response = await client.branch_regression_tests_file(
|
|
595
|
-
config["id"], regression_tests_commands, run_in_main=main
|
|
596
|
-
)
|
|
585
|
+
response = await client.branch_regression_tests_file(config["id"], regression_tests_commands)
|
|
597
586
|
if "job" not in response:
|
|
598
587
|
raise CLIBranchException(str(response))
|
|
599
588
|
job_id = response["job"]["job_id"]
|
|
@@ -605,7 +594,7 @@ async def regression_tests(
|
|
|
605
594
|
except Exception as e:
|
|
606
595
|
raise CLIBranchException(FeedbackManager.error_exception(error=str(e)))
|
|
607
596
|
elif not ctx.invoked_subcommand:
|
|
608
|
-
await _run_regression(type="coverage", wait=wait
|
|
597
|
+
await _run_regression(type="coverage", wait=wait)
|
|
609
598
|
|
|
610
599
|
|
|
611
600
|
async def _run_regression(
|
|
@@ -621,7 +610,6 @@ async def _run_regression(
|
|
|
621
610
|
failfast: Optional[bool] = False,
|
|
622
611
|
wait: Optional[bool] = False,
|
|
623
612
|
skip: Optional[bool] = False,
|
|
624
|
-
run_in_main: Optional[bool] = False,
|
|
625
613
|
**kwargs,
|
|
626
614
|
):
|
|
627
615
|
if skip:
|
|
@@ -649,7 +637,6 @@ async def _run_regression(
|
|
|
649
637
|
assert_time_increase_percentage=assert_time_increase_percentage,
|
|
650
638
|
assert_bytes_read_increase_percentage=assert_bytes_read_increase_percentage,
|
|
651
639
|
assert_max_time=assert_max_time,
|
|
652
|
-
run_in_main=run_in_main,
|
|
653
640
|
**kwargs,
|
|
654
641
|
)
|
|
655
642
|
if "job" not in response:
|
|
@@ -729,12 +716,6 @@ async def _run_regression(
|
|
|
729
716
|
default=False,
|
|
730
717
|
help="Flag to skip execution of regression tests. This is handy for CI branches where regression might be flaky",
|
|
731
718
|
)
|
|
732
|
-
@click.option(
|
|
733
|
-
"--main",
|
|
734
|
-
is_flag=True,
|
|
735
|
-
default=False,
|
|
736
|
-
help="Run regression tests in the main Branch. For this flag to work all the resources in the Branch pipe endpoints need to exist in the main Branch.",
|
|
737
|
-
)
|
|
738
719
|
@coro
|
|
739
720
|
async def coverage(
|
|
740
721
|
pipe_name: str,
|
|
@@ -748,7 +729,6 @@ async def coverage(
|
|
|
748
729
|
failfast: bool,
|
|
749
730
|
wait: bool,
|
|
750
731
|
skip_regression_tests: Optional[bool] = False,
|
|
751
|
-
main: Optional[bool] = False,
|
|
752
732
|
):
|
|
753
733
|
await _run_regression(
|
|
754
734
|
"coverage",
|
|
@@ -763,7 +743,6 @@ async def coverage(
|
|
|
763
743
|
failfast,
|
|
764
744
|
wait,
|
|
765
745
|
skip_regression_tests,
|
|
766
|
-
run_in_main=main,
|
|
767
746
|
)
|
|
768
747
|
|
|
769
748
|
|
tinybird/tb_cli_modules/cicd.py
CHANGED
|
@@ -327,6 +327,7 @@ class CICDGeneratorBase:
|
|
|
327
327
|
return FeedbackManager.warning_for_cicd_file(
|
|
328
328
|
file_name=cicd_file.file_name, warning_message=cicd_file.warning_message.format(**params)
|
|
329
329
|
)
|
|
330
|
+
return None
|
|
330
331
|
|
|
331
332
|
def is_already_generated(self, path: str) -> bool:
|
|
332
333
|
for cicd_file in self.cicd_files:
|
|
@@ -396,6 +397,7 @@ def ask_provider_interactively():
|
|
|
396
397
|
FeedbackManager.error_git_provider_index(host_index=provider_index, available_options=available_options)
|
|
397
398
|
)
|
|
398
399
|
provider_index = -1
|
|
400
|
+
return None
|
|
399
401
|
|
|
400
402
|
|
|
401
403
|
async def init_cicd(
|
tinybird/tb_cli_modules/cli.py
CHANGED
|
@@ -261,8 +261,7 @@ async def init(
|
|
|
261
261
|
raise CLIGitReleaseException(
|
|
262
262
|
FeedbackManager.error_dottinyb_not_ignored(git_working_dir=f"{cli_git_release.working_dir()}/")
|
|
263
263
|
)
|
|
264
|
-
|
|
265
|
-
click.echo(FeedbackManager.info_dottinyb_already_ignored())
|
|
264
|
+
click.echo(FeedbackManager.info_dottinyb_already_ignored())
|
|
266
265
|
|
|
267
266
|
if (
|
|
268
267
|
os.path.exists(f"{cli_git_release.working_dir()}/.diff_tmp")
|
|
@@ -271,14 +270,13 @@ async def init(
|
|
|
271
270
|
raise CLIGitReleaseException(
|
|
272
271
|
FeedbackManager.error_dotdiff_not_ignored(git_working_dir=f"{cli_git_release.working_dir()}/")
|
|
273
272
|
)
|
|
274
|
-
|
|
275
|
-
click.echo(FeedbackManager.info_dotdifftemp_already_ignored())
|
|
273
|
+
click.echo(FeedbackManager.info_dotdifftemp_already_ignored())
|
|
276
274
|
|
|
277
275
|
if "release" not in current_ws:
|
|
278
276
|
raise CLIGitReleaseException(FeedbackManager.error_no_correct_token_for_init())
|
|
279
277
|
|
|
280
278
|
# If we have a release and we are not overriding the commit, we check if we have a release already
|
|
281
|
-
|
|
279
|
+
if current_ws.get("release") and not override_commit:
|
|
282
280
|
final_response = FeedbackManager.error_release_already_set(
|
|
283
281
|
workspace=current_ws["name"], commit=current_ws["release"]["commit"]
|
|
284
282
|
)
|
|
@@ -1034,7 +1032,7 @@ async def materialize(
|
|
|
1034
1032
|
pass
|
|
1035
1033
|
|
|
1036
1034
|
filename = str(f.absolute())
|
|
1037
|
-
|
|
1035
|
+
return await folder_push(
|
|
1038
1036
|
cl,
|
|
1039
1037
|
filenames=[filename],
|
|
1040
1038
|
push_deps=push_deps,
|
|
@@ -1043,7 +1041,6 @@ async def materialize(
|
|
|
1043
1041
|
no_versions=no_versions,
|
|
1044
1042
|
verbose=verbose,
|
|
1045
1043
|
)
|
|
1046
|
-
return to_run
|
|
1047
1044
|
|
|
1048
1045
|
def _save_local_pipe(pipe_file_name, pipe_datafile, pipe):
|
|
1049
1046
|
base = Path("pipes")
|
|
@@ -1075,7 +1072,7 @@ async def materialize(
|
|
|
1075
1072
|
check = option == 1
|
|
1076
1073
|
|
|
1077
1074
|
filename = str(f_pipe.absolute())
|
|
1078
|
-
|
|
1075
|
+
return await folder_push(
|
|
1079
1076
|
cl,
|
|
1080
1077
|
filenames=[filename],
|
|
1081
1078
|
dry_run=False,
|
|
@@ -1090,7 +1087,6 @@ async def materialize(
|
|
|
1090
1087
|
run_tests=False,
|
|
1091
1088
|
verbose=verbose,
|
|
1092
1089
|
)
|
|
1093
|
-
return to_run
|
|
1094
1090
|
|
|
1095
1091
|
async def _populate(pipe, node_name, f_pipe):
|
|
1096
1092
|
if force_populate or click.confirm(FeedbackManager.prompt_populate(file=f_pipe)):
|
|
@@ -35,7 +35,6 @@ from packaging.version import Version
|
|
|
35
35
|
from tinybird.client import (
|
|
36
36
|
AuthException,
|
|
37
37
|
AuthNoTokenException,
|
|
38
|
-
DoesNotExistException,
|
|
39
38
|
JobException,
|
|
40
39
|
OperationCanNotBePerformed,
|
|
41
40
|
TinyB,
|
|
@@ -194,6 +193,7 @@ class AliasedGroup(click.Group):
|
|
|
194
193
|
cm = click.Group.get_command(self, ctx, cmd_name)
|
|
195
194
|
if cm is not None:
|
|
196
195
|
return cm
|
|
196
|
+
return None
|
|
197
197
|
|
|
198
198
|
def resolve_command(self, ctx, args):
|
|
199
199
|
# always return the command's name, not the alias
|
|
@@ -454,6 +454,7 @@ def ask_for_region_interactively(regions):
|
|
|
454
454
|
available_options = ", ".join(map(str, range(1, len(regions) + 1)))
|
|
455
455
|
click.echo(FeedbackManager.error_region_index(host_index=region_index, available_options=available_options))
|
|
456
456
|
region_index = -1
|
|
457
|
+
return None
|
|
457
458
|
|
|
458
459
|
|
|
459
460
|
def get_region_info(ctx, region=None):
|
|
@@ -728,9 +729,8 @@ async def create_workspace_branch(
|
|
|
728
729
|
job_response = await wait_job(config.get_client(), job_id, job_url, "Branch creation")
|
|
729
730
|
if job_response is None:
|
|
730
731
|
raise CLIException(f"Empty job API response (job_id: {job_id}, job_url: {job_url})")
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
is_summary = "partitions" in response
|
|
732
|
+
response = job_response.get("result", {})
|
|
733
|
+
is_summary = "partitions" in response
|
|
734
734
|
|
|
735
735
|
await switch_workspace(config, branch_name, only_environments=True)
|
|
736
736
|
if is_summary and (bool(last_partition) or bool(all)):
|
|
@@ -757,10 +757,8 @@ async def print_branch_regression_tests_summary(client, job_id, host, response=N
|
|
|
757
757
|
if isinstance(metric, float):
|
|
758
758
|
if is_percentage:
|
|
759
759
|
return f"{round(metric, 3):+} %"
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
else:
|
|
763
|
-
return metric
|
|
760
|
+
return f"{round(metric, 3)} seconds"
|
|
761
|
+
return metric
|
|
764
762
|
|
|
765
763
|
failed = False
|
|
766
764
|
response = await client.job(job_id) if job_id else response or {"progress": []}
|
|
@@ -979,24 +977,6 @@ async def push_data(
|
|
|
979
977
|
if not url:
|
|
980
978
|
raise CLIException(FeedbackManager.error_missing_url(datasource=datasource_name))
|
|
981
979
|
|
|
982
|
-
def cb(res):
|
|
983
|
-
if cb.First: # type: ignore[attr-defined]
|
|
984
|
-
blocks_to_process = len([x for x in res["block_log"] if x["status"] == "idle"])
|
|
985
|
-
if blocks_to_process:
|
|
986
|
-
cb.bar = click.progressbar(label=FeedbackManager.info_progress_blocks(), length=blocks_to_process) # type: ignore[attr-defined]
|
|
987
|
-
cb.bar.update(0) # type: ignore[attr-defined]
|
|
988
|
-
cb.First = False # type: ignore[attr-defined]
|
|
989
|
-
cb.blocks_to_process = blocks_to_process # type: ignore[attr-defined]
|
|
990
|
-
else:
|
|
991
|
-
done = len([x for x in res["block_log"] if x["status"] == "done"])
|
|
992
|
-
if done * 2 > cb.blocks_to_process: # type: ignore[attr-defined]
|
|
993
|
-
cb.bar.label = FeedbackManager.info_progress_current_blocks() # type: ignore[attr-defined]
|
|
994
|
-
cb.bar.update(done - cb.prev_done) # type: ignore[attr-defined]
|
|
995
|
-
cb.prev_done = done # type: ignore[attr-defined]
|
|
996
|
-
|
|
997
|
-
cb.First = True # type: ignore[attr-defined]
|
|
998
|
-
cb.prev_done = 0 # type: ignore[attr-defined]
|
|
999
|
-
|
|
1000
980
|
click.echo(FeedbackManager.info_starting_import_process())
|
|
1001
981
|
|
|
1002
982
|
if isinstance(url, list):
|
|
@@ -1006,18 +986,16 @@ async def push_data(
|
|
|
1006
986
|
|
|
1007
987
|
async def process_url(
|
|
1008
988
|
datasource_name: str, url: str, mode: str, sql_condition: Optional[str], replace_options: Optional[Set[str]]
|
|
1009
|
-
):
|
|
989
|
+
) -> str:
|
|
1010
990
|
parsed = urlparse(url)
|
|
1011
|
-
# Remote URL imports retain the v0 API; local files use v1 body imports.
|
|
1012
991
|
is_remote_url = parsed.scheme in ("http", "https")
|
|
1013
992
|
# poor man's format detection
|
|
1014
993
|
_format = get_format_from_filename_or_url(url)
|
|
1015
994
|
if is_remote_url:
|
|
1016
|
-
res = await client.
|
|
995
|
+
res = await client.datasource_append_url(
|
|
1017
996
|
datasource_name,
|
|
1018
997
|
url,
|
|
1019
998
|
mode=mode,
|
|
1020
|
-
status_callback=cb,
|
|
1021
999
|
sql_condition=sql_condition,
|
|
1022
1000
|
format=_format,
|
|
1023
1001
|
replace_options=replace_options,
|
|
@@ -1032,77 +1010,33 @@ async def push_data(
|
|
|
1032
1010
|
replace_options=replace_options,
|
|
1033
1011
|
)
|
|
1034
1012
|
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
return job_id
|
|
1040
|
-
|
|
1041
|
-
datasource_name = res["datasource"]["name"]
|
|
1042
|
-
try:
|
|
1043
|
-
datasource = await client.get_datasource(datasource_name)
|
|
1044
|
-
except DoesNotExistException:
|
|
1045
|
-
raise CLIException(FeedbackManager.error_datasource_does_not_exist(datasource=datasource_name))
|
|
1046
|
-
except Exception as e:
|
|
1047
|
-
raise CLIException(FeedbackManager.error_exception(error=str(e)))
|
|
1048
|
-
|
|
1049
|
-
total_rows = (datasource.get("statistics", {}) or {}).get("row_count", 0)
|
|
1050
|
-
appended_rows = 0
|
|
1051
|
-
parser = None
|
|
1052
|
-
|
|
1053
|
-
if res.get("error"):
|
|
1054
|
-
raise CLIException(FeedbackManager.error_exception(error=res["error"]))
|
|
1055
|
-
if res.get("errors"):
|
|
1056
|
-
raise CLIException(FeedbackManager.error_exception(error=res["errors"]))
|
|
1057
|
-
if res.get("blocks"):
|
|
1058
|
-
for block in res["blocks"]:
|
|
1059
|
-
if "process_return" in block and block["process_return"] is not None:
|
|
1060
|
-
process_return = block["process_return"][0]
|
|
1061
|
-
parser = process_return["parser"] if process_return.get("parser") else parser
|
|
1062
|
-
if parser and parser != "clickhouse":
|
|
1063
|
-
parser = process_return["parser"]
|
|
1064
|
-
appended_rows += process_return["lines"]
|
|
1065
|
-
|
|
1066
|
-
return parser, total_rows, appended_rows
|
|
1013
|
+
job_id = res.get("id") or res.get("import_id")
|
|
1014
|
+
if not isinstance(job_id, str):
|
|
1015
|
+
raise CLIException("We couldn't confirm that your import started. Please try again.")
|
|
1016
|
+
return job_id
|
|
1067
1017
|
|
|
1068
1018
|
try:
|
|
1069
1019
|
tasks = [process_url(datasource_name, url, mode, sql_condition, replace_options) for url in urls]
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
datasource = await client.get_datasource(datasource_name)
|
|
1078
|
-
total_rows = (datasource.get("statistics", {}) or {}).get("row_count", 0)
|
|
1079
|
-
click.echo(FeedbackManager.success_total_rows(datasource=datasource_name, total_rows=total_rows))
|
|
1080
|
-
if mode == "replace":
|
|
1081
|
-
click.echo(FeedbackManager.success_replaced_datasource(datasource=datasource_name))
|
|
1082
|
-
else:
|
|
1083
|
-
click.echo(FeedbackManager.success_appended_datasource(datasource=datasource_name))
|
|
1084
|
-
click.echo(FeedbackManager.success_progress_blocks())
|
|
1085
|
-
click.echo(FeedbackManager.info_data_pushed(datasource=datasource_name))
|
|
1086
|
-
return None
|
|
1087
|
-
parser, total_rows, appended_rows = list(output)[-1]
|
|
1088
|
-
except AuthNoTokenException:
|
|
1089
|
-
raise
|
|
1090
|
-
except OperationCanNotBePerformed as e:
|
|
1091
|
-
raise CLIException(FeedbackManager.error_operation_can_not_be_performed(error=e))
|
|
1092
|
-
except Exception as e:
|
|
1093
|
-
raise CLIException(FeedbackManager.error_exception(error=e))
|
|
1094
|
-
else:
|
|
1095
|
-
click.echo(FeedbackManager.success_progress_blocks())
|
|
1096
|
-
if mode == "append" and parser and parser != "clickhouse":
|
|
1097
|
-
click.echo(FeedbackManager.success_appended_rows(appended_rows=appended_rows))
|
|
1098
|
-
|
|
1020
|
+
job_ids = list(await gather_with_concurrency(concurrency, *tasks))
|
|
1021
|
+
if not wait:
|
|
1022
|
+
return job_ids
|
|
1023
|
+
for job_id in job_ids:
|
|
1024
|
+
await wait_job_no_ui(client, job_id)
|
|
1025
|
+
datasource = await client.get_datasource(datasource_name)
|
|
1026
|
+
total_rows = (datasource.get("statistics", {}) or {}).get("row_count", 0)
|
|
1099
1027
|
click.echo(FeedbackManager.success_total_rows(datasource=datasource_name, total_rows=total_rows))
|
|
1100
|
-
|
|
1101
1028
|
if mode == "replace":
|
|
1102
1029
|
click.echo(FeedbackManager.success_replaced_datasource(datasource=datasource_name))
|
|
1103
1030
|
else:
|
|
1104
1031
|
click.echo(FeedbackManager.success_appended_datasource(datasource=datasource_name))
|
|
1032
|
+
click.echo(FeedbackManager.success_progress_blocks())
|
|
1105
1033
|
click.echo(FeedbackManager.info_data_pushed(datasource=datasource_name))
|
|
1034
|
+
except AuthNoTokenException:
|
|
1035
|
+
raise
|
|
1036
|
+
except OperationCanNotBePerformed as e:
|
|
1037
|
+
raise CLIException(FeedbackManager.error_operation_can_not_be_performed(error=e))
|
|
1038
|
+
except Exception as e:
|
|
1039
|
+
raise CLIException(FeedbackManager.error_exception(error=e))
|
|
1106
1040
|
|
|
1107
1041
|
return None
|
|
1108
1042
|
|
|
@@ -1243,8 +1177,7 @@ async def switch_workspace(config: CLIConfig, workspace_name_or_id: str, only_en
|
|
|
1243
1177
|
if not workspace:
|
|
1244
1178
|
if only_environments:
|
|
1245
1179
|
raise CLIException(FeedbackManager.error_branch(branch=workspace_name_or_id, cli="tb"))
|
|
1246
|
-
|
|
1247
|
-
raise CLIException(FeedbackManager.error_workspace(workspace=workspace_name_or_id, cli="tb"))
|
|
1180
|
+
raise CLIException(FeedbackManager.error_workspace(workspace=workspace_name_or_id, cli="tb"))
|
|
1248
1181
|
|
|
1249
1182
|
config.set_token(workspace["token"])
|
|
1250
1183
|
config.set_token_for_host(workspace["token"], config.get_host())
|
|
@@ -1371,11 +1304,11 @@ class ConnectionReplacements:
|
|
|
1371
1304
|
"""Maps prompt parameters to API parameters."""
|
|
1372
1305
|
|
|
1373
1306
|
api_params = {}
|
|
1374
|
-
for key in params.
|
|
1307
|
+
for key, value in params.items():
|
|
1375
1308
|
try:
|
|
1376
|
-
api_params[ConnectionReplacements._PARAMS_REPLACEMENTS[service][key]] =
|
|
1309
|
+
api_params[ConnectionReplacements._PARAMS_REPLACEMENTS[service][key]] = value
|
|
1377
1310
|
except KeyError:
|
|
1378
|
-
api_params[key] =
|
|
1311
|
+
api_params[key] = value
|
|
1379
1312
|
|
|
1380
1313
|
api_params["service"] = service
|
|
1381
1314
|
return api_params
|
|
@@ -121,8 +121,7 @@ class CLIConfig:
|
|
|
121
121
|
return self._values[key].origin
|
|
122
122
|
if self._parent:
|
|
123
123
|
return self._parent.get_value_origin(key)
|
|
124
|
-
|
|
125
|
-
return ConfigValueOrigin.NONE
|
|
124
|
+
return ConfigValueOrigin.NONE
|
|
126
125
|
|
|
127
126
|
def persist_to_file(self, override_with_values: Optional["CLIConfig"] = None) -> None:
|
|
128
127
|
if not self._path:
|
|
@@ -385,7 +385,7 @@ async def datasource_truncate(ctx, datasource_name, yes, cascade):
|
|
|
385
385
|
|
|
386
386
|
@datasource.command(name="delete")
|
|
387
387
|
@click.argument("datasource_name")
|
|
388
|
-
@click.option("--sql-condition",
|
|
388
|
+
@click.option("--sql-condition", help="SQL WHERE condition to remove rows", hidden=True, required=True)
|
|
389
389
|
@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
|
|
390
390
|
@click.option(
|
|
391
391
|
"--wait/--no-wait",
|
|
@@ -517,7 +517,7 @@ async def datasource_delete_rows(
|
|
|
517
517
|
)
|
|
518
518
|
)
|
|
519
519
|
break
|
|
520
|
-
|
|
520
|
+
if res["status"] == "error":
|
|
521
521
|
print("\n") # noqa: T201
|
|
522
522
|
raise CLIDatasourceException(FeedbackManager.error_exception(error=res["error"]))
|
|
523
523
|
await asyncio.sleep(1)
|
tinybird/tb_cli_modules/pipe.py
CHANGED
|
@@ -16,7 +16,15 @@ from click import Context
|
|
|
16
16
|
from tinybird import context
|
|
17
17
|
from tinybird.client import AuthNoTokenException, DoesNotExistException, TinyB
|
|
18
18
|
from tinybird.config import DEFAULT_API_HOST, FeatureFlags
|
|
19
|
-
from tinybird.datafile_common import
|
|
19
|
+
from tinybird.datafile_common import (
|
|
20
|
+
PipeNodeTypes,
|
|
21
|
+
PipeTypes,
|
|
22
|
+
folder_push,
|
|
23
|
+
get_name_version,
|
|
24
|
+
local_copy_node_uses_on_demand_compute,
|
|
25
|
+
process_file,
|
|
26
|
+
wait_job,
|
|
27
|
+
)
|
|
20
28
|
from tinybird.feedback_manager import FeedbackManager
|
|
21
29
|
from tinybird.tb_cli_modules.branch import warn_if_in_live
|
|
22
30
|
from tinybird.tb_cli_modules.cli import cli
|
|
@@ -272,7 +280,7 @@ async def pipe_populate(
|
|
|
272
280
|
if not materialized_ids:
|
|
273
281
|
raise CLIPipeException(FeedbackManager.error_populate_no_materialized_in_pipe(pipe=pipe_name))
|
|
274
282
|
|
|
275
|
-
|
|
283
|
+
if len(materialized_ids) > 1:
|
|
276
284
|
raise CLIPipeException(FeedbackManager.error_populate_several_materialized_in_pipe(pipe=pipe_name))
|
|
277
285
|
|
|
278
286
|
node = materialized_ids[0]
|
|
@@ -328,9 +336,8 @@ async def pipe_unlink_output_node(
|
|
|
328
336
|
|
|
329
337
|
if not node_uid:
|
|
330
338
|
raise CLIPipeException(FeedbackManager.error_unlinking_pipe_not_linked(pipe=pipe_name_or_id))
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
click.echo(FeedbackManager.success_pipe_unlinked(pipe=pipe["name"]))
|
|
339
|
+
await client.pipe_unlink_materialized(pipe["name"], node_uid)
|
|
340
|
+
click.echo(FeedbackManager.success_pipe_unlinked(pipe=pipe["name"]))
|
|
334
341
|
|
|
335
342
|
if pipe["type"] == PipeTypes.COPY:
|
|
336
343
|
click.echo(FeedbackManager.info_unlinking_copy_pipe(pipe=pipe["name"]))
|
|
@@ -343,9 +350,8 @@ async def pipe_unlink_output_node(
|
|
|
343
350
|
|
|
344
351
|
if not node_uid:
|
|
345
352
|
raise CLIPipeException(FeedbackManager.error_unlinking_pipe_not_linked(pipe=pipe_name_or_id))
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
click.echo(FeedbackManager.success_pipe_unlinked(pipe=pipe["name"]))
|
|
353
|
+
await client.pipe_remove_copy(pipe["name"], node_uid)
|
|
354
|
+
click.echo(FeedbackManager.success_pipe_unlinked(pipe=pipe["name"]))
|
|
349
355
|
|
|
350
356
|
if pipe["type"] == PipeTypes.DATA_SINK:
|
|
351
357
|
click.echo(FeedbackManager.info_unlinking_sink_pipe(pipe=pipe["name"]))
|
|
@@ -358,9 +364,8 @@ async def pipe_unlink_output_node(
|
|
|
358
364
|
|
|
359
365
|
if not node_uid:
|
|
360
366
|
raise CLIPipeException(FeedbackManager.error_unlinking_pipe_not_linked(pipe=pipe_name_or_id))
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
click.echo(FeedbackManager.success_pipe_unlinked(pipe=pipe["name"]))
|
|
367
|
+
await client.pipe_remove_sink(pipe["name"], node_uid)
|
|
368
|
+
click.echo(FeedbackManager.success_pipe_unlinked(pipe=pipe["name"]))
|
|
364
369
|
|
|
365
370
|
if pipe["type"] == PipeTypes.STREAM:
|
|
366
371
|
click.echo(FeedbackManager.info_unlinking_stream_pipe(pipe=pipe["name"]))
|
|
@@ -368,9 +373,8 @@ async def pipe_unlink_output_node(
|
|
|
368
373
|
|
|
369
374
|
if not node_uid:
|
|
370
375
|
raise CLIPipeException(FeedbackManager.error_unlinking_pipe_not_linked(pipe=pipe_name_or_id))
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
click.echo(FeedbackManager.success_pipe_unlinked(pipe=pipe["name"]))
|
|
376
|
+
await client.pipe_remove_stream(pipe["name"], node_uid)
|
|
377
|
+
click.echo(FeedbackManager.success_pipe_unlinked(pipe=pipe["name"]))
|
|
374
378
|
|
|
375
379
|
except AuthNoTokenException:
|
|
376
380
|
raise
|
|
@@ -753,30 +757,41 @@ async def pipe_copy_run(
|
|
|
753
757
|
|
|
754
758
|
params = dict(key_value.split("=") for key_value in param) if param else {}
|
|
755
759
|
|
|
756
|
-
if yes
|
|
757
|
-
|
|
758
|
-
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
760
|
+
if not yes and not click.confirm(FeedbackManager.warning_confirm_copy_pipe(pipe=pipe_name_or_id)):
|
|
761
|
+
return
|
|
759
762
|
|
|
760
|
-
|
|
761
|
-
|
|
763
|
+
if (
|
|
764
|
+
on_demand_compute
|
|
765
|
+
and not yes
|
|
766
|
+
and local_copy_node_uses_on_demand_compute(".", pipe_name_or_id) is False
|
|
767
|
+
and not click.confirm(
|
|
768
|
+
FeedbackManager.warning_confirm_on_demand_compute_differs_from_datafile(pipe=pipe_name_or_id),
|
|
769
|
+
default=False,
|
|
770
|
+
)
|
|
771
|
+
):
|
|
772
|
+
return
|
|
762
773
|
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
target_datasource_id = response["tags"]["copy_target_datasource"]
|
|
766
|
-
target_datasource = await client.get_datasource(target_datasource_id)
|
|
767
|
-
target_datasource_name = target_datasource["name"]
|
|
768
|
-
click.echo(
|
|
769
|
-
FeedbackManager.success_copy_job_created(target_datasource=target_datasource_name, job_url=job_url)
|
|
770
|
-
)
|
|
774
|
+
click.echo(FeedbackManager.info_copy_job_running(pipe=pipe_name_or_id))
|
|
775
|
+
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
771
776
|
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
click.echo(FeedbackManager.success_data_copied_to_ds(target_datasource=target_datasource_name))
|
|
777
|
+
try:
|
|
778
|
+
response = await client.pipe_run_copy(pipe_name_or_id, params, mode, on_demand_compute=on_demand_compute)
|
|
775
779
|
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
+
job_id = response["job"]["job_id"]
|
|
781
|
+
job_url = response["job"]["job_url"]
|
|
782
|
+
target_datasource_id = response["tags"]["copy_target_datasource"]
|
|
783
|
+
target_datasource = await client.get_datasource(target_datasource_id)
|
|
784
|
+
target_datasource_name = target_datasource["name"]
|
|
785
|
+
click.echo(FeedbackManager.success_copy_job_created(target_datasource=target_datasource_name, job_url=job_url))
|
|
786
|
+
|
|
787
|
+
if wait:
|
|
788
|
+
await wait_job(client, job_id, job_url, "** Copying data")
|
|
789
|
+
click.echo(FeedbackManager.success_data_copied_to_ds(target_datasource=target_datasource_name))
|
|
790
|
+
|
|
791
|
+
except AuthNoTokenException:
|
|
792
|
+
raise
|
|
793
|
+
except Exception as e:
|
|
794
|
+
raise CLIPipeException(FeedbackManager.error_creating_copy_job(error=e))
|
|
780
795
|
|
|
781
796
|
|
|
782
797
|
@pipe_copy.command(name="resume", short_help="Resume a paused copy pipe")
|
|
@@ -102,18 +102,18 @@ class TestResult:
|
|
|
102
102
|
def status(self) -> Status:
|
|
103
103
|
if len(self.data) > 0:
|
|
104
104
|
return FAILED
|
|
105
|
-
|
|
105
|
+
if self.error:
|
|
106
106
|
return Status.build_error_with_msg(self.error)
|
|
107
|
-
|
|
107
|
+
if (
|
|
108
108
|
self.max_bytes_read is not None
|
|
109
109
|
and self.max_elapsed_time is not None
|
|
110
110
|
and self.read_bytes > self.max_bytes_read
|
|
111
111
|
and self.elapsed_time > self.max_elapsed_time
|
|
112
112
|
):
|
|
113
113
|
return PASS_OVER_TIME_AND_READ_BYTES
|
|
114
|
-
|
|
114
|
+
if self.max_bytes_read is not None and self.read_bytes > self.max_bytes_read:
|
|
115
115
|
return PASS_OVER_READ_BYTES
|
|
116
|
-
|
|
116
|
+
if self.max_elapsed_time is not None and self.elapsed_time > self.max_elapsed_time:
|
|
117
117
|
return PASS_OVER_TIME
|
|
118
118
|
return PASS
|
|
119
119
|
|