tinybird-cli 6.5.4.dev0__tar.gz → 6.5.5.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/PKG-INFO +10 -1
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/__cli__.py +2 -2
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/ch_utils/engine.py +6 -25
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/client.py +34 -10
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/config.py +0 -6
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/datafile_common.py +50 -59
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/datatypes.py +0 -4
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/feedback_manager.py +3 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/sql.py +0 -24
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/sql_template.py +92 -45
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/sql_toolset.py +26 -7
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/branch.py +0 -1
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/common.py +15 -17
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/config.py +1 -15
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/datasource.py +68 -4
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -9
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/workspace.py +0 -1
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tornado_template.py +5 -5
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird_cli.egg-info/PKG-INFO +10 -1
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/setup.cfg +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/ch_utils/constants.py +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/check_pypi.py +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/context.py +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/git_settings.py +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/sql_template_fmt.py +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/syncasync.py +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli.py +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/auth.py +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/cicd.py +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/cli.py +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/connection.py +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/exceptions.py +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/fmt.py +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/job.py +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/pipe.py +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/regions.py +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/tag.py +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/telemetry.py +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/test.py +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/token.py +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/workspace_members.py +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird_cli.egg-info/SOURCES.txt +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird_cli.egg-info/dependency_links.txt +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird_cli.egg-info/entry_points.txt +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird_cli.egg-info/requires.txt +0 -0
- {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird_cli.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: tinybird_cli
|
|
3
|
-
Version: 6.5.
|
|
3
|
+
Version: 6.5.5.dev0
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/cli
|
|
6
6
|
Author: Tinybird
|
|
@@ -43,10 +43,19 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
|
|
|
43
43
|
Changelog
|
|
44
44
|
----------
|
|
45
45
|
|
|
46
|
+
6.5.4
|
|
47
|
+
***********
|
|
48
|
+
|
|
49
|
+
- `Added` ``tb datasource append`` and ``tb datasource replace`` support the experimental ``--experimental=use_v1`` option for local CSV, NDJSON, and Parquet files. It submits the import through the v1 endpoint and prints the queued job ID; pass ``--wait`` to wait for completion.
|
|
50
|
+
- `Fixed` Internal improvements to secret template rendering used during datafile parsing.
|
|
51
|
+
- `Fixed` `tb push` now rejects pipes that declare a parameter without a valid name (e.g. a numeric-literal cast like ``{{Float32(1682892000.0)}}``), instead of creating an unusable parameter.
|
|
52
|
+
|
|
46
53
|
6.5.3
|
|
47
54
|
***********
|
|
48
55
|
|
|
56
|
+
- `Removed` `bi_stats_rt` and `bi_stats` from service datasource catalogs and dependency ignore lists.
|
|
49
57
|
- `Added` ``tb datasource delete`` now accepts ``--lightweight-delete``, ``--partition`` and ``--projection-mode``, backed by the ``POST /v1/datasources/{name}/delete`` endpoint that uses ClickHouse's lightweight ``DELETE FROM`` instead of the heavy ``ALTER TABLE ... DELETE`` mutation. Lightweight delete is synchronous by default and returns the affected row count plus a partition-level mutation block; pass ``--no-wait`` to enqueue a job and receive the v0-compatible envelope.
|
|
58
|
+
- `Changed` Upgraded Tornado to 6.5.5.
|
|
50
59
|
|
|
51
60
|
6.5.1
|
|
52
61
|
***********
|
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/cli'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '6.5.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '6.5.5.dev0'
|
|
8
|
+
__revision__ = '5d2bbc5'
|
|
@@ -121,6 +121,12 @@ class TableDetails:
|
|
|
121
121
|
def is_mergetree_family(self) -> bool:
|
|
122
122
|
return self.engine is not None and "mergetree" in self.engine.lower()
|
|
123
123
|
|
|
124
|
+
def is_view(self) -> bool:
|
|
125
|
+
return self.original_engine == "View"
|
|
126
|
+
|
|
127
|
+
def is_merge_final_engine(self) -> bool:
|
|
128
|
+
return self.original_engine == "MergeFinal"
|
|
129
|
+
|
|
124
130
|
def supports_alter_add_column(self) -> bool:
|
|
125
131
|
return self.is_mergetree_family() or (self.engine is not None and self.engine.lower() == "null")
|
|
126
132
|
|
|
@@ -354,31 +360,6 @@ def sorting_key_is_valid(columns: List[Dict[str, Any]], value: Optional[str]) ->
|
|
|
354
360
|
return value
|
|
355
361
|
|
|
356
362
|
|
|
357
|
-
def case_insensitive_check(valid_values: List[str]) -> Callable[[List[Dict[str, Any]], str], Optional[str]]:
|
|
358
|
-
"""
|
|
359
|
-
>>> valid_values = ['ANY', 'ALL']
|
|
360
|
-
>>> checker = case_insensitive_check(valid_values)
|
|
361
|
-
>>> checker([],'ALL')
|
|
362
|
-
|
|
363
|
-
>>> valid_values = ['ANY', 'ALL']
|
|
364
|
-
>>> checker = case_insensitive_check(valid_values)
|
|
365
|
-
>>> checker([],'any')
|
|
366
|
-
|
|
367
|
-
>>> valid_values = ['ANY', 'ALL']
|
|
368
|
-
>>> checker = case_insensitive_check(valid_values)
|
|
369
|
-
>>> checker([],'foo')
|
|
370
|
-
Traceback (most recent call last):
|
|
371
|
-
...
|
|
372
|
-
ValueError: valid values are ANY, ALL
|
|
373
|
-
"""
|
|
374
|
-
|
|
375
|
-
def checker(columns: List[Dict[str, Any]], value: str):
|
|
376
|
-
if value.upper() not in valid_values:
|
|
377
|
-
raise ValueError(f"valid values are {', '.join(valid_values)}")
|
|
378
|
-
|
|
379
|
-
return checker
|
|
380
|
-
|
|
381
|
-
|
|
382
363
|
# [PARTITION BY expr]
|
|
383
364
|
# [ORDER BY expr]
|
|
384
365
|
# [PRIMARY KEY expr]
|
|
@@ -441,6 +441,7 @@ class TinyB:
|
|
|
441
441
|
sql_condition: Optional[str] = None,
|
|
442
442
|
format: str = "csv",
|
|
443
443
|
replace_options: Optional[Set[str]] = None,
|
|
444
|
+
use_v1: bool = False,
|
|
444
445
|
):
|
|
445
446
|
params = {"name": datasource_name, "mode": mode, "format": format, "debug": "blocks_block_log"}
|
|
446
447
|
|
|
@@ -452,20 +453,43 @@ class TinyB:
|
|
|
452
453
|
|
|
453
454
|
async with aiofiles.open(file, "rb") as content:
|
|
454
455
|
file_content = await content.read()
|
|
455
|
-
if format == "csv":
|
|
456
|
-
files = {"csv": ("csv", file_content)}
|
|
457
|
-
else:
|
|
458
|
-
files = {"ndjson": ("ndjson", file_content)}
|
|
459
456
|
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
457
|
+
if use_v1:
|
|
458
|
+
v1_params = {"format": format}
|
|
459
|
+
if sql_condition:
|
|
460
|
+
v1_params["replace_condition"] = sql_condition
|
|
461
|
+
if replace_options:
|
|
462
|
+
for option in list(replace_options):
|
|
463
|
+
v1_params[option] = "true"
|
|
464
|
+
content_types = {
|
|
465
|
+
"csv": "text/csv",
|
|
466
|
+
"ndjson": "application/x-ndjson",
|
|
467
|
+
"parquet": "application/vnd.apache.parquet",
|
|
468
|
+
}
|
|
469
|
+
headers = {"Content-Type": content_types[format]}
|
|
470
|
+
if str(file).endswith(".gz"):
|
|
471
|
+
headers["Content-Encoding"] = "gzip"
|
|
472
|
+
return await self._req(
|
|
473
|
+
f"/v1/datasources/{quote(datasource_name, safe='')}/{mode}?{urlencode(v1_params, safe='')}",
|
|
474
|
+
data=file_content,
|
|
475
|
+
headers=headers,
|
|
463
476
|
method="POST",
|
|
464
477
|
)
|
|
465
|
-
if status_callback:
|
|
466
|
-
status_callback(res)
|
|
467
478
|
|
|
468
|
-
|
|
479
|
+
if format == "csv":
|
|
480
|
+
files = {"csv": ("csv", file_content)}
|
|
481
|
+
else:
|
|
482
|
+
files = {"ndjson": ("ndjson", file_content)}
|
|
483
|
+
|
|
484
|
+
res = await self._req(
|
|
485
|
+
f"v0/datasources?{urlencode(params, safe='')}",
|
|
486
|
+
files=files,
|
|
487
|
+
method="POST",
|
|
488
|
+
)
|
|
489
|
+
if status_callback:
|
|
490
|
+
status_callback(res)
|
|
491
|
+
|
|
492
|
+
return res
|
|
469
493
|
|
|
470
494
|
async def datasource_truncate(self, datasource_name: str):
|
|
471
495
|
return await self._req(f"/v0/datasources/{datasource_name}/truncate", method="POST", data="")
|
|
@@ -112,12 +112,6 @@ async def get_config(
|
|
|
112
112
|
return config
|
|
113
113
|
|
|
114
114
|
|
|
115
|
-
async def write_config(config: Dict[str, Any], dest_file: str = ".tinyb"):
|
|
116
|
-
config_file = Path(getcwd()) / dest_file
|
|
117
|
-
async with aiofiles.open(config_file, "w") as file:
|
|
118
|
-
await file.write(json.dumps(config, indent=4, sort_keys=True))
|
|
119
|
-
|
|
120
|
-
|
|
121
115
|
def get_display_host(ui_host: str):
|
|
122
116
|
return LEGACY_HOSTS.get(ui_host, ui_host)
|
|
123
117
|
|
|
@@ -9,7 +9,7 @@ import aiofiles
|
|
|
9
9
|
from requests import Response
|
|
10
10
|
|
|
11
11
|
try:
|
|
12
|
-
from colorama import
|
|
12
|
+
from colorama import Fore, init
|
|
13
13
|
|
|
14
14
|
init()
|
|
15
15
|
except ImportError: # fallback so that the imported classes always exist
|
|
@@ -18,7 +18,7 @@ except ImportError: # fallback so that the imported classes always exist
|
|
|
18
18
|
def __getattr__(self, name):
|
|
19
19
|
return ""
|
|
20
20
|
|
|
21
|
-
Fore =
|
|
21
|
+
Fore = ColorFallback()
|
|
22
22
|
|
|
23
23
|
import difflib
|
|
24
24
|
import glob
|
|
@@ -94,8 +94,6 @@ INTERNAL_TABLES: Tuple[str, ...] = (
|
|
|
94
94
|
"kafka_ops_log",
|
|
95
95
|
"datasources_storage",
|
|
96
96
|
"endpoint_errors",
|
|
97
|
-
"bi_stats_rt",
|
|
98
|
-
"bi_stats",
|
|
99
97
|
)
|
|
100
98
|
|
|
101
99
|
|
|
@@ -306,10 +304,10 @@ class ValidationException(Exception):
|
|
|
306
304
|
super().__init__(err)
|
|
307
305
|
|
|
308
306
|
|
|
309
|
-
def sizeof_fmt(num:
|
|
307
|
+
def sizeof_fmt(num: float, suffix: str = "b") -> str:
|
|
310
308
|
"""Readable file size
|
|
311
309
|
:param num: Bytes value
|
|
312
|
-
:type num:
|
|
310
|
+
:type num: float
|
|
313
311
|
:param suffix: Unit suffix (optionnal) default = o
|
|
314
312
|
:type suffix: str
|
|
315
313
|
:rtype: str
|
|
@@ -854,9 +852,24 @@ def parse_pipe(
|
|
|
854
852
|
doc = parse(s, basepath=basepath, replace_includes=replace_includes, skip_eval=skip_eval)
|
|
855
853
|
for node in doc.nodes:
|
|
856
854
|
sql = node.get("sql", "")
|
|
857
|
-
|
|
858
|
-
|
|
855
|
+
source_sql = sql.strip()
|
|
856
|
+
if not source_sql:
|
|
857
|
+
raise click.ClickException(
|
|
858
|
+
FeedbackManager.error_parsing_node(node=node["name"], pipe=filename, error="Empty query")
|
|
859
|
+
)
|
|
860
|
+
if source_sql.startswith("%"):
|
|
861
|
+
sql, _, variable_warnings = render_sql_template(source_sql[1:], test_mode=True, name=node["name"])
|
|
859
862
|
doc.warnings = variable_warnings
|
|
863
|
+
if not sql.strip():
|
|
864
|
+
raise click.ClickException(
|
|
865
|
+
FeedbackManager.error_parsing_node(
|
|
866
|
+
node=node["name"],
|
|
867
|
+
pipe=filename,
|
|
868
|
+
error="Template renders to an empty query without parameters",
|
|
869
|
+
)
|
|
870
|
+
)
|
|
871
|
+
else:
|
|
872
|
+
sql = source_sql
|
|
860
873
|
# it'll fail with a ModuleNotFoundError when the toolset is not available but it returns the parsed doc
|
|
861
874
|
from tinybird.sql_toolset import format_sql as toolset_format_sql
|
|
862
875
|
|
|
@@ -868,13 +881,15 @@ def parse_pipe(
|
|
|
868
881
|
)
|
|
869
882
|
)
|
|
870
883
|
except ValueError as e:
|
|
871
|
-
|
|
884
|
+
source_sql = node.get("sql", "")
|
|
885
|
+
source_sql_stripped = source_sql.strip()
|
|
886
|
+
t, template_variables, _ = get_template_and_variables(source_sql, name=node["name"])
|
|
872
887
|
|
|
873
|
-
if
|
|
888
|
+
if not source_sql_stripped.startswith("%") and len(template_variables) > 0:
|
|
874
889
|
raise click.ClickException(FeedbackManager.error_template_start(filename=filename))
|
|
875
890
|
raise click.ClickException(
|
|
876
891
|
FeedbackManager.error_parsing_file(
|
|
877
|
-
filename=filename, lineno="", error=f"{str(e)} + SQL(value error): {
|
|
892
|
+
filename=filename, lineno="", error=f"{str(e)} + SQL(value error): {source_sql}"
|
|
878
893
|
)
|
|
879
894
|
)
|
|
880
895
|
except UnClosedIfError as e:
|
|
@@ -1028,10 +1043,13 @@ def parse(
|
|
|
1028
1043
|
|
|
1029
1044
|
parser_state.current_node["indexes"] = indexes
|
|
1030
1045
|
|
|
1031
|
-
def assign_var(v: str) -> Callable[[VarArg(str), KwArg(Any)], None]:
|
|
1046
|
+
def assign_var(v: str, lowercase: bool = False) -> Callable[[VarArg(str), KwArg(Any)], None]:
|
|
1032
1047
|
def _f(*args: str, **kwargs: Any):
|
|
1033
1048
|
s = _unquote((" ".join(args)).strip())
|
|
1034
|
-
|
|
1049
|
+
val = eval_var(s, skip=skip_eval)
|
|
1050
|
+
if lowercase:
|
|
1051
|
+
val = val.lower()
|
|
1052
|
+
parser_state.current_node[v.lower()] = val
|
|
1035
1053
|
|
|
1036
1054
|
return _f
|
|
1037
1055
|
|
|
@@ -1246,7 +1264,9 @@ def parse(
|
|
|
1246
1264
|
"import_external_datasource": assign_var("import_external_datasource"),
|
|
1247
1265
|
"import_bucket_uri": assign_var("import_bucket_uri"),
|
|
1248
1266
|
"import_from_timestamp": assign_var("import_from_timestamp"),
|
|
1249
|
-
|
|
1267
|
+
# Lowercase: format checks and executor routing downstream expect
|
|
1268
|
+
# lowercase values (ANALYTICS-A8M)
|
|
1269
|
+
"import_format": assign_var("import_format", lowercase=True),
|
|
1250
1270
|
"import_query": assign_var("import_query"),
|
|
1251
1271
|
"import_table_arn": assign_var("import_table_arn"),
|
|
1252
1272
|
"import_export_bucket": assign_var("import_export_bucket"),
|
|
@@ -1750,7 +1770,7 @@ async def process_file(
|
|
|
1750
1770
|
"filename": filename,
|
|
1751
1771
|
"name": name + version,
|
|
1752
1772
|
"nodes": nodes,
|
|
1753
|
-
"deps":
|
|
1773
|
+
"deps": list(set(deps)),
|
|
1754
1774
|
"tokens": doc.tokens,
|
|
1755
1775
|
"description": description,
|
|
1756
1776
|
"warnings": doc.warnings,
|
|
@@ -2082,12 +2102,12 @@ class PipeChecker(unittest.TestCase):
|
|
|
2082
2102
|
|
|
2083
2103
|
if self.ignore_order:
|
|
2084
2104
|
current_data = (
|
|
2085
|
-
sorted(normalize_array(current_data), key=itemgetter(*
|
|
2105
|
+
sorted(normalize_array(current_data), key=itemgetter(*list(current_data[0].keys())))
|
|
2086
2106
|
if len(current_data) > 0
|
|
2087
2107
|
else current_data
|
|
2088
2108
|
)
|
|
2089
2109
|
checker_data = (
|
|
2090
|
-
sorted(normalize_array(checker_data), key=itemgetter(*
|
|
2110
|
+
sorted(normalize_array(checker_data), key=itemgetter(*list(checker_data[0].keys())))
|
|
2091
2111
|
if len(checker_data) > 0
|
|
2092
2112
|
else checker_data
|
|
2093
2113
|
)
|
|
@@ -2202,6 +2222,8 @@ class PipeCheckerRunner:
|
|
|
2202
2222
|
FROM {pipe_stats_rt}
|
|
2203
2223
|
WHERE
|
|
2204
2224
|
pipe_name = '{self.pipe_name}'
|
|
2225
|
+
-- pipe_stats_rt retains 3 months; recent requests are enough for checks
|
|
2226
|
+
AND start_datetime >= now() - INTERVAL 7 DAY
|
|
2205
2227
|
AND url IS NOT NULL
|
|
2206
2228
|
AND extractURLParameter(assumeNotNull(url), 'from') <> 'ui'
|
|
2207
2229
|
AND extractURLParameter(assumeNotNull(url), 'pipe_checker') <> 'true'
|
|
@@ -2229,6 +2251,8 @@ class PipeCheckerRunner:
|
|
|
2229
2251
|
FROM {pipe_stats_rt}
|
|
2230
2252
|
WHERE
|
|
2231
2253
|
pipe_name = '{self.pipe_name}'
|
|
2254
|
+
-- pipe_stats_rt retains 3 months; recent requests are enough for checks
|
|
2255
|
+
AND start_datetime >= now() - INTERVAL 7 DAY
|
|
2232
2256
|
AND url IS NOT NULL
|
|
2233
2257
|
AND extractURLParameter(assumeNotNull(url), 'from') <> 'ui'
|
|
2234
2258
|
AND extractURLParameter(assumeNotNull(url), 'pipe_checker') <> 'true'
|
|
@@ -4141,7 +4165,7 @@ async def build_graph(
|
|
|
4141
4165
|
elif len(warnings) > 1:
|
|
4142
4166
|
click.echo(
|
|
4143
4167
|
FeedbackManager.warning_pipe_restricted_params(
|
|
4144
|
-
words=", ".join(["'{}'"
|
|
4168
|
+
words=", ".join([f"'{param}'" for param in warnings[:-1]]),
|
|
4145
4169
|
last_word=warnings[-1],
|
|
4146
4170
|
)
|
|
4147
4171
|
)
|
|
@@ -4328,7 +4352,7 @@ async def folder_push(
|
|
|
4328
4352
|
|
|
4329
4353
|
workspace_lib_paths = list(workspace_lib_paths)
|
|
4330
4354
|
# include vendor libs without overriding user ones
|
|
4331
|
-
existing_workspaces =
|
|
4355
|
+
existing_workspaces = {x[1] for x in workspace_lib_paths}
|
|
4332
4356
|
vendor_path = Path("vendor")
|
|
4333
4357
|
if vendor_path.exists():
|
|
4334
4358
|
for x in vendor_path.iterdir():
|
|
@@ -4566,7 +4590,7 @@ async def folder_push(
|
|
|
4566
4590
|
|
|
4567
4591
|
if not fork_downstream:
|
|
4568
4592
|
# First, we will deploy the all the resources following the dependency graph except for the endpoints
|
|
4569
|
-
groups =
|
|
4593
|
+
groups = list(toposort(dependencies_graph))
|
|
4570
4594
|
for group in groups:
|
|
4571
4595
|
for name in group:
|
|
4572
4596
|
if name in processed:
|
|
@@ -4592,7 +4616,7 @@ async def folder_push(
|
|
|
4592
4616
|
processed.add(name)
|
|
4593
4617
|
|
|
4594
4618
|
# Then, we will deploy the endpoints that are on the dependency graph
|
|
4595
|
-
groups =
|
|
4619
|
+
groups = list(toposort(endpoints_dep_map))
|
|
4596
4620
|
for group in groups:
|
|
4597
4621
|
for name in group:
|
|
4598
4622
|
if name not in processed:
|
|
@@ -4621,7 +4645,7 @@ async def folder_push(
|
|
|
4621
4645
|
|
|
4622
4646
|
# First, we will deploy the datasources that need to be deployed.
|
|
4623
4647
|
# We need to deploy the datasources from left to right as some datasources might have MV that depend on the column types of previous datasources. Ex: `test_change_column_type_landing_datasource` test
|
|
4624
|
-
groups =
|
|
4648
|
+
groups = list(toposort(dependencies_graph_fork_downstream))
|
|
4625
4649
|
groups.reverse()
|
|
4626
4650
|
for group in groups:
|
|
4627
4651
|
for name in group:
|
|
@@ -4675,7 +4699,7 @@ async def folder_push(
|
|
|
4675
4699
|
# If we have ENDPOINT_A ----> MV_PIPE_B -----> DATASOURCE_B ------> ENDPOINT_C
|
|
4676
4700
|
# Where endpoint A is being used in the MV_PIPE_B, if we only modify the endpoint A
|
|
4677
4701
|
# The dependencies graph will only contain the endpoint A and the MV_PIPE_B, but not the DATASOURCE_B and the ENDPOINT_C
|
|
4678
|
-
groups =
|
|
4702
|
+
groups = list(toposort(dependencies_graph_fork_downstream))
|
|
4679
4703
|
for group in groups:
|
|
4680
4704
|
for name in group:
|
|
4681
4705
|
if name in processed or not is_endpoint(resources_to_run_fork_downstream[name]):
|
|
@@ -4684,7 +4708,7 @@ async def folder_push(
|
|
|
4684
4708
|
endpoints_dep_map[name] = dependencies_graph_fork_downstream[name]
|
|
4685
4709
|
|
|
4686
4710
|
# Now that we have the dependencies of the endpoints, we need to check that the resources has not been deployed yet and only care about the endpoints that depend on endpoints
|
|
4687
|
-
groups =
|
|
4711
|
+
groups = list(toposort(endpoints_dep_map))
|
|
4688
4712
|
|
|
4689
4713
|
# As we have used the forkdownstream graph to get the dependencies of the endpoints, we have all the dependencies of the endpoints
|
|
4690
4714
|
# But we need to deploy the endpoints and the dependencies of the endpoints from left to right
|
|
@@ -4709,7 +4733,7 @@ async def folder_push(
|
|
|
4709
4733
|
# Now we should have the endpoints and datasources deployed, we can deploy the rest of the pipes (copy & sinks)
|
|
4710
4734
|
# We need to rely on the forkdownstream graph as it contains all the modified pipes as well as the dependencies of the pipes
|
|
4711
4735
|
# In this case, we don't need to generate a new graph as we did for the endpoints as the pipes are not going to be used as dependencies and the datasources are already deployed
|
|
4712
|
-
groups =
|
|
4736
|
+
groups = list(toposort(dependencies_graph_fork_downstream))
|
|
4713
4737
|
for group in groups:
|
|
4714
4738
|
for name in group:
|
|
4715
4739
|
if name in processed or is_materialized(resources_to_run_fork_downstream.get(name)):
|
|
@@ -4729,7 +4753,7 @@ async def folder_push(
|
|
|
4729
4753
|
# Finally, we need to deploy the materialized views from right to left.
|
|
4730
4754
|
# We need to rely on the forkdownstream graph as it contains all the modified materialized views as well as the dependencies of the materialized views
|
|
4731
4755
|
# In this case, we don't need to generate a new graph as we did for the endpoints as the pipes are not going to be used as dependencies and the datasources are already deployed
|
|
4732
|
-
groups =
|
|
4756
|
+
groups = list(toposort(dependencies_graph_fork_downstream))
|
|
4733
4757
|
for group in groups:
|
|
4734
4758
|
for name in group:
|
|
4735
4759
|
if name in processed or not is_materialized(resources_to_run_fork_downstream.get(name)):
|
|
@@ -5732,36 +5756,3 @@ def is_file_a_datasource(filename: str) -> bool:
|
|
|
5732
5756
|
return True
|
|
5733
5757
|
|
|
5734
5758
|
return False
|
|
5735
|
-
|
|
5736
|
-
|
|
5737
|
-
def update_connector_params(service: str, ds_params: Dict[str, Any], connector_required_params: List[str]) -> None:
|
|
5738
|
-
"""
|
|
5739
|
-
Update connector parameters for a given service, ensuring required parameters exist.
|
|
5740
|
-
|
|
5741
|
-
:param service: The name of the service (e.g., 's3').
|
|
5742
|
-
:param ds_params: The data source parameters to be checked.
|
|
5743
|
-
:param connector_required_params: The list of required parameters for the connector.
|
|
5744
|
-
:return: None
|
|
5745
|
-
"""
|
|
5746
|
-
|
|
5747
|
-
connector_at_least_one_required_param: List[str] = []
|
|
5748
|
-
|
|
5749
|
-
# Handle the "at least one param" requirement
|
|
5750
|
-
if connector_at_least_one_required_param and not any(
|
|
5751
|
-
key in ds_params for key in connector_at_least_one_required_param
|
|
5752
|
-
):
|
|
5753
|
-
params = [
|
|
5754
|
-
(ImportReplacements.get_datafile_param_for_linker_param(service, param) or param).upper()
|
|
5755
|
-
for param in connector_at_least_one_required_param
|
|
5756
|
-
]
|
|
5757
|
-
click.echo(FeedbackManager.error_updating_connector_missing_at_least_one_param(param=" or ".join(params)))
|
|
5758
|
-
return
|
|
5759
|
-
|
|
5760
|
-
# Handle the mandatory params requirement
|
|
5761
|
-
if not all(key in ds_params for key in connector_required_params):
|
|
5762
|
-
params = [
|
|
5763
|
-
(ImportReplacements.get_datafile_param_for_linker_param(service, param) or param).upper()
|
|
5764
|
-
for param in connector_required_params
|
|
5765
|
-
]
|
|
5766
|
-
click.echo(FeedbackManager.error_updating_connector_missing_params(param=", ".join(params)))
|
|
5767
|
-
return
|
|
@@ -108,10 +108,6 @@ def is_type_date(type_to_check: str) -> bool:
|
|
|
108
108
|
return date_type_pattern.match(type_to_check) is not None
|
|
109
109
|
|
|
110
110
|
|
|
111
|
-
def string_test(x: str) -> bool:
|
|
112
|
-
return True
|
|
113
|
-
|
|
114
|
-
|
|
115
111
|
def date_test(x: str) -> bool:
|
|
116
112
|
return date_pattern.match(x) is not None
|
|
117
113
|
|
|
@@ -864,6 +864,9 @@ class FeedbackManager:
|
|
|
864
864
|
success_total_rows = success_message("** Total rows in {datasource}: {total_rows}")
|
|
865
865
|
success_appended_datasource = success_message("** Data appended to Data Source '{datasource}' successfully!")
|
|
866
866
|
success_replaced_datasource = success_message("** Data replaced in Data Source '{datasource}' successfully!")
|
|
867
|
+
success_import_job_queued = success_message("** {operation} import job queued: {job_id}")
|
|
868
|
+
success_import_job_completed = success_message("** {operation} import completed: {job_id}")
|
|
869
|
+
info_import_job_status = info_message("Check status: tb job details {job_id}")
|
|
867
870
|
success_auth = success_message(
|
|
868
871
|
"** Auth successful! \n** Configuration written to .tinyb file, consider adding it to .gitignore"
|
|
869
872
|
)
|
|
@@ -911,30 +911,6 @@ def engine_replicated_to_local(engine: str) -> str:
|
|
|
911
911
|
return _RE_REPLICATED_MT.sub(_replace, engine.strip())
|
|
912
912
|
|
|
913
913
|
|
|
914
|
-
def engine_patch_replicated_engine(engine: str, engine_full: Optional[str], new_table_name: str) -> Optional[str]:
|
|
915
|
-
"""
|
|
916
|
-
>>> engine_patch_replicated_engine("ReplicatedMergeTree", "ReplicatedMergeTree('/clickhouse/tables/1-1/table_name', 'replica') PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192", 'table_name_staging')
|
|
917
|
-
"ReplicatedMergeTree('/clickhouse/tables/1-1/table_name_staging', 'replica') PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192"
|
|
918
|
-
>>> engine_patch_replicated_engine("ReplicatedMergeTree", "ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/sales_product_rank_rt_replicated_2', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY (purchase_location, sku_rank_lc, date)", 'sales_product_rank_rt_replicated_2_staging')
|
|
919
|
-
"ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/sales_product_rank_rt_replicated_2_staging', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY (purchase_location, sku_rank_lc, date)"
|
|
920
|
-
>>> engine_patch_replicated_engine("ReplicatedMergeTree", None, 't_000') is None
|
|
921
|
-
True
|
|
922
|
-
>>> engine_patch_replicated_engine("Log", "Log()", 't_000')
|
|
923
|
-
'Log()'
|
|
924
|
-
>>> engine_patch_replicated_engine("MergeTree", "MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024", 't_000')
|
|
925
|
-
'MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024'
|
|
926
|
-
"""
|
|
927
|
-
if not engine_full:
|
|
928
|
-
return None
|
|
929
|
-
if engine.lower().startswith("Replicated".lower()):
|
|
930
|
-
parts = _RE_REPLICATED_ENGINE_SPLIT.split(engine_full)
|
|
931
|
-
paths = parts[2].split("/")
|
|
932
|
-
paths[-1] = new_table_name
|
|
933
|
-
zoo_path = "/".join(paths)
|
|
934
|
-
return "".join([*parts[:2], zoo_path, *parts[3:]])
|
|
935
|
-
return engine_full
|
|
936
|
-
|
|
937
|
-
|
|
938
914
|
if __name__ == "__main__":
|
|
939
915
|
print( # noqa: T201
|
|
940
916
|
_parse_table_structure(
|
|
@@ -1396,7 +1396,7 @@ _namespace = {
|
|
|
1396
1396
|
}
|
|
1397
1397
|
|
|
1398
1398
|
|
|
1399
|
-
reserved_vars =
|
|
1399
|
+
reserved_vars = {"_tt_tmp", "_tt_append", "isinstance", "str", "error", "custom_error", *list(vars(builtins))}
|
|
1400
1400
|
for p in DEFAULT_PARAM_NAMES: # we handle these in an specific manner
|
|
1401
1401
|
reserved_vars.discard(p) # `format` is part of builtins
|
|
1402
1402
|
# Allow 'id' to be used as a template parameter - https://gitlab.com/tinybird/analytics/-/issues/19119
|
|
@@ -1542,35 +1542,6 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
|
|
|
1542
1542
|
raise e
|
|
1543
1543
|
|
|
1544
1544
|
|
|
1545
|
-
class CodeWriter:
|
|
1546
|
-
def __init__(self, file, template):
|
|
1547
|
-
self.file = file
|
|
1548
|
-
self.current_template = template
|
|
1549
|
-
self.apply_counter = 0
|
|
1550
|
-
self._indent = 0
|
|
1551
|
-
|
|
1552
|
-
def indent_size(self):
|
|
1553
|
-
return self._indent
|
|
1554
|
-
|
|
1555
|
-
def indent(self):
|
|
1556
|
-
class Indenter:
|
|
1557
|
-
def __enter__(_):
|
|
1558
|
-
self._indent += 1
|
|
1559
|
-
return self
|
|
1560
|
-
|
|
1561
|
-
def __exit__(_, *args):
|
|
1562
|
-
assert self._indent > 0
|
|
1563
|
-
self._indent -= 1
|
|
1564
|
-
|
|
1565
|
-
return Indenter()
|
|
1566
|
-
|
|
1567
|
-
def write_line(self, line, line_number, indent=None):
|
|
1568
|
-
if indent is None:
|
|
1569
|
-
indent = self._indent
|
|
1570
|
-
line_comment = " # %s:%d" % ("<generated>", line_number)
|
|
1571
|
-
print(" " * indent + line + line_comment, file=self.file)
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
1545
|
def get_var_names(t: Template):
|
|
1575
1546
|
"""
|
|
1576
1547
|
Extract variable names from a template.
|
|
@@ -1982,11 +1953,12 @@ def get_var_data(content, node_id=None):
|
|
|
1982
1953
|
"""
|
|
1983
1954
|
|
|
1984
1955
|
def node_to_value(x):
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1956
|
+
# ast.Constant supersedes the deprecated ast.Bytes/ast.Str/ast.Num/
|
|
1957
|
+
# ast.NameConstant node classes. Those were aliases of ast.Constant
|
|
1958
|
+
# since Python 3.8 (emitting DeprecationWarning on access in 3.12) and
|
|
1959
|
+
# were removed in Python 3.14. The parser only ever produces
|
|
1960
|
+
# ast.Constant nodes, so .value holds str/bytes/int/float/bool/None.
|
|
1961
|
+
if type(x) == ast.Constant: # noqa: E721
|
|
1990
1962
|
return x.value
|
|
1991
1963
|
elif type(x) == ast.Name: # noqa: E721
|
|
1992
1964
|
return x.id
|
|
@@ -2000,8 +1972,6 @@ def get_var_data(content, node_id=None):
|
|
|
2000
1972
|
if not r:
|
|
2001
1973
|
r = node_to_value(x.right)
|
|
2002
1974
|
return r
|
|
2003
|
-
elif type(x) == ast.Constant: # noqa: E721
|
|
2004
|
-
return x.value
|
|
2005
1975
|
elif type(x) == ast.UnaryOp and type(x.operand) == ast.Constant: # noqa: E721
|
|
2006
1976
|
if type(x.op) == ast.USub: # noqa: E721
|
|
2007
1977
|
return x.operand.value * -1
|
|
@@ -2020,11 +1990,9 @@ def get_var_data(content, node_id=None):
|
|
|
2020
1990
|
return []
|
|
2021
1991
|
|
|
2022
1992
|
first_elem = x.elts[0]
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
return [elem.n for elem in x.elts]
|
|
2027
|
-
elif type(first_elem) == ast.NameConstant or type(first_elem) == ast.Constant: # noqa: E721
|
|
1993
|
+
# See node_to_value: ast.Constant supersedes the deprecated
|
|
1994
|
+
# ast.Bytes/ast.Str/ast.Num/ast.NameConstant (removed in 3.14).
|
|
1995
|
+
if type(first_elem) == ast.Constant: # noqa: E721
|
|
2028
1996
|
return [elem.value for elem in x.elts]
|
|
2029
1997
|
elif type(first_elem) == ast.Name: # noqa: E721
|
|
2030
1998
|
return [elem.id for elem in x.elts]
|
|
@@ -2295,7 +2263,12 @@ def get_var_names_and_types(t: Template, node_id: Optional[str] = None) -> List[
|
|
|
2295
2263
|
vars_out.extend(
|
|
2296
2264
|
parse_statement_code_if_new_vars(statement_code, skip_names=typed_names | statement_expr_names)
|
|
2297
2265
|
)
|
|
2298
|
-
|
|
2266
|
+
# Inside a {% %} statement a type function with a literal first arg
|
|
2267
|
+
# (e.g. {% set x = Float32(1.5) %}) is a plain value cast: the
|
|
2268
|
+
# assigned variable is the parameter, the literal is not. Drop
|
|
2269
|
+
# these artifacts so they don't surface as bogus params nor get
|
|
2270
|
+
# rejected by validate_template_parameter_names.
|
|
2271
|
+
return [vd for vd in vars_out if isinstance(vd["name"], str) and vd["name"]]
|
|
2299
2272
|
|
|
2300
2273
|
def _n(chunks: list, vars_out: list[dict[str, Any]]) -> None:
|
|
2301
2274
|
for x in chunks:
|
|
@@ -2342,6 +2315,43 @@ def get_var_names_and_types_cached(t: Template):
|
|
|
2342
2315
|
return get_var_names_and_types(t)
|
|
2343
2316
|
|
|
2344
2317
|
|
|
2318
|
+
def validate_template_parameter_names(sql: str, node_id: Optional[str] = None) -> None:
|
|
2319
|
+
"""Reject template parameters that don't have a usable name.
|
|
2320
|
+
|
|
2321
|
+
Type functions such as ``{{Float32(x)}}`` double as casts, so a literal
|
|
2322
|
+
first argument (e.g. ``{{Float32(1682892000.0)}}`` or ``{{String('')}}``)
|
|
2323
|
+
is parsed as a parameter whose name is a number or an empty string instead
|
|
2324
|
+
of an identifier. Such a "parameter" can never be supplied by a caller and
|
|
2325
|
+
breaks parameter metadata consumers, so it must be rejected at write time.
|
|
2326
|
+
|
|
2327
|
+
Raises ``ValueError`` with the same wording used elsewhere for invalid names.
|
|
2328
|
+
|
|
2329
|
+
Templates that fail to parse are left untouched: there are no parameters to
|
|
2330
|
+
validate, and reporting/deferring those errors is the job of the existing
|
|
2331
|
+
render/validation flow (some paths, e.g. ``ignore_sql_errors``, tolerate
|
|
2332
|
+
unparseable SQL on purpose). This keeps the check strictly additive. The
|
|
2333
|
+
only exception is the invalid-name ``ValueError`` that extraction itself
|
|
2334
|
+
raises (e.g. for a list literal used as first argument), which is exactly
|
|
2335
|
+
what this validator exists to report.
|
|
2336
|
+
"""
|
|
2337
|
+
try:
|
|
2338
|
+
params = get_var_names_and_types(Template(sql), node_id=node_id)
|
|
2339
|
+
except SQLTemplateException:
|
|
2340
|
+
# Template-level problem (e.g. a SecurityException converted by
|
|
2341
|
+
# get_var_names_and_types), not an invalid parameter name.
|
|
2342
|
+
return
|
|
2343
|
+
except ValueError:
|
|
2344
|
+
# Extraction itself rejects some invalid names (e.g. a list literal
|
|
2345
|
+
# as first argument) with the same ValueError this function raises.
|
|
2346
|
+
raise
|
|
2347
|
+
except Exception:
|
|
2348
|
+
return
|
|
2349
|
+
for param in params:
|
|
2350
|
+
name = param.get("name")
|
|
2351
|
+
if not isinstance(name, str) or not name:
|
|
2352
|
+
raise ValueError(f'"{name}" can not be used as a variable name')
|
|
2353
|
+
|
|
2354
|
+
|
|
2345
2355
|
def wrap_vars(t, escape_arrays: bool = False):
|
|
2346
2356
|
def _n(chunks, v):
|
|
2347
2357
|
for x in chunks:
|
|
@@ -2943,13 +2953,21 @@ def extract_variables_from_sql(sql: str, params: List[Dict[str, Any]]) -> Dict[s
|
|
|
2943
2953
|
return defaults
|
|
2944
2954
|
|
|
2945
2955
|
|
|
2946
|
-
def render_template_with_secrets(
|
|
2956
|
+
def render_template_with_secrets(
|
|
2957
|
+
name: str,
|
|
2958
|
+
content: str,
|
|
2959
|
+
secrets: Optional[Dict[str, str]] = None,
|
|
2960
|
+
empty_secret_raises: bool = False,
|
|
2961
|
+
) -> str:
|
|
2947
2962
|
"""Renders a template with secrets, allowing for default values.
|
|
2948
2963
|
|
|
2949
2964
|
Args:
|
|
2950
2965
|
name: The name of the template
|
|
2951
2966
|
content: The template content
|
|
2952
2967
|
secrets: A dictionary mapping secret names to their values
|
|
2968
|
+
empty_secret_raises: When True, empty secret values or empty defaults raise
|
|
2969
|
+
instead of returning '""'. Used for sink metadata (Forward deploy) where
|
|
2970
|
+
an empty bucket path must not be baked as s3://"".
|
|
2953
2971
|
|
|
2954
2972
|
Returns:
|
|
2955
2973
|
The rendered template
|
|
@@ -2991,6 +3009,26 @@ def render_template_with_secrets(name: str, content: str, secrets: Optional[Dict
|
|
|
2991
3009
|
Traceback (most recent call last):
|
|
2992
3010
|
...
|
|
2993
3011
|
tinybird.sql_template.SQLTemplateException: Template Syntax Error: Cannot access secret 'MISSING_SECRET'. Check the secret exists in the Workspace and the token has the required scope.
|
|
3012
|
+
|
|
3013
|
+
>>> render_template_with_secrets(
|
|
3014
|
+
... "sink_bucket",
|
|
3015
|
+
... "s3://{{ tb_secret('MISSING_SECRET', '') }}",
|
|
3016
|
+
... secrets = {},
|
|
3017
|
+
... empty_secret_raises=True,
|
|
3018
|
+
... )
|
|
3019
|
+
Traceback (most recent call last):
|
|
3020
|
+
...
|
|
3021
|
+
tinybird.sql_template.SQLTemplateException: Template Syntax Error: Secret 'MISSING_SECRET' resolves to an empty value and cannot be used in sink metadata.
|
|
3022
|
+
|
|
3023
|
+
>>> render_template_with_secrets(
|
|
3024
|
+
... "sink_bucket",
|
|
3025
|
+
... "s3://{{ tb_secret('EMPTY_SECRET', 'fallback') }}",
|
|
3026
|
+
... secrets = {'EMPTY_SECRET': ''},
|
|
3027
|
+
... empty_secret_raises=True,
|
|
3028
|
+
... )
|
|
3029
|
+
Traceback (most recent call last):
|
|
3030
|
+
...
|
|
3031
|
+
tinybird.sql_template.SQLTemplateException: Template Syntax Error: Secret 'EMPTY_SECRET' resolves to an empty value and cannot be used in sink metadata.
|
|
2994
3032
|
"""
|
|
2995
3033
|
if not secrets:
|
|
2996
3034
|
secrets = {}
|
|
@@ -3006,15 +3044,24 @@ def render_template_with_secrets(name: str, content: str, secrets: Optional[Dict
|
|
|
3006
3044
|
The secret value or default
|
|
3007
3045
|
|
|
3008
3046
|
Raises:
|
|
3009
|
-
SQLTemplateException: If the secret is not found and no default is provided
|
|
3047
|
+
SQLTemplateException: If the secret is not found and no default is provided,
|
|
3048
|
+
or if empty_secret_raises is True and the resolved value is empty
|
|
3010
3049
|
"""
|
|
3011
3050
|
if secret_name in secrets:
|
|
3012
3051
|
value = secrets[secret_name]
|
|
3013
3052
|
if isinstance(value, str) and len(value) == 0:
|
|
3053
|
+
if empty_secret_raises:
|
|
3054
|
+
raise SQLTemplateException(
|
|
3055
|
+
f"Secret '{secret_name}' resolves to an empty value and cannot be used in sink metadata."
|
|
3056
|
+
)
|
|
3014
3057
|
return '""'
|
|
3015
3058
|
return value
|
|
3016
3059
|
elif default is not None:
|
|
3017
3060
|
if isinstance(default, str) and len(default) == 0:
|
|
3061
|
+
if empty_secret_raises:
|
|
3062
|
+
raise SQLTemplateException(
|
|
3063
|
+
f"Secret '{secret_name}' resolves to an empty value and cannot be used in sink metadata."
|
|
3064
|
+
)
|
|
3018
3065
|
return '""'
|
|
3019
3066
|
return default
|
|
3020
3067
|
else:
|
|
@@ -158,7 +158,20 @@ def has_unoptimized_join(sql: str, left_table: Optional[Union[Tuple[str, str], T
|
|
|
158
158
|
raise UnoptimizedJoinException(sql)
|
|
159
159
|
|
|
160
160
|
|
|
161
|
-
def
|
|
161
|
+
def _format_fragment_for_mutation_command(fragment: str) -> str:
|
|
162
|
+
"""Normalize a SQL fragment the way CH stores it in the
|
|
163
|
+
`system.mutations.command` column: format the expression with CH's own
|
|
164
|
+
formatter, then escape it as it appears inside the serialized command
|
|
165
|
+
(string literals keep backslash-escaped quotes)."""
|
|
166
|
+
formatted = chquery.format(f"""SELECT {fragment}""").split("SELECT ")[1]
|
|
167
|
+
formatted = formatted.replace("\\", "\\\\").replace("'", "''")
|
|
168
|
+
quoted = chquery.format(f"SELECT '{formatted}'").split("SELECT ")[1]
|
|
169
|
+
return quoted[1:-1]
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def format_where_for_mutation_command(
|
|
173
|
+
where_clause: str, lightweight: bool = False, partition: Optional[str] = None
|
|
174
|
+
) -> str:
|
|
162
175
|
"""
|
|
163
176
|
>>> format_where_for_mutation_command("numnights = 99")
|
|
164
177
|
'DELETE WHERE numnights = 99'
|
|
@@ -172,12 +185,18 @@ def format_where_for_mutation_command(where_clause: str, lightweight: bool = Fal
|
|
|
172
185
|
"DELETE WHERE reservationid = \\\\'\\\\\\\\\\\\'foo\\\\'"
|
|
173
186
|
>>> format_where_for_mutation_command("number < 3", lightweight=True)
|
|
174
187
|
'UPDATE _row_exists = 0 WHERE number < 3'
|
|
188
|
+
>>> format_where_for_mutation_command("number < 3", lightweight=True, partition="201901")
|
|
189
|
+
'UPDATE _row_exists = 0 IN PARTITION 201901 WHERE number < 3'
|
|
190
|
+
>>> format_where_for_mutation_command("number < 3", lightweight=True, partition="'2019-01-01'")
|
|
191
|
+
"UPDATE _row_exists = 0 IN PARTITION \\\\'2019-01-01\\\\' WHERE number < 3"
|
|
175
192
|
"""
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
193
|
+
quoted_condition = _format_fragment_for_mutation_command(where_clause)
|
|
194
|
+
prefix = "UPDATE _row_exists = 0" if lightweight else "DELETE"
|
|
195
|
+
# Partition-scoped mutations store the partition expression in the
|
|
196
|
+
# command (`… IN PARTITION <expr> WHERE <cond>`), normalized by the same
|
|
197
|
+
# CH formatter, so it must be part of the match.
|
|
198
|
+
partition_clause = f" IN PARTITION {_format_fragment_for_mutation_command(partition)}" if partition else ""
|
|
199
|
+
return f"{prefix}{partition_clause} WHERE {quoted_condition}"
|
|
181
200
|
|
|
182
201
|
|
|
183
202
|
# Functions that take table/dictionary names as string literal arguments.
|
|
@@ -624,7 +643,7 @@ def replace_tables(
|
|
|
624
643
|
|
|
625
644
|
if current_replacements:
|
|
626
645
|
# We need to transform the dictionary into something cacheable, so a sorted tuple of tuples it is
|
|
627
|
-
r = tuple(sorted(
|
|
646
|
+
r = tuple(sorted(current_replacements.items()))
|
|
628
647
|
sql = replace_tables_chquery_cached(
|
|
629
648
|
sql,
|
|
630
649
|
r,
|
|
@@ -590,7 +590,6 @@ async def regression_tests(
|
|
|
590
590
|
|
|
591
591
|
if current_main_workspace["id"] == config["id"]:
|
|
592
592
|
raise CLIException(FeedbackManager.error_not_allowed_in_main_branch())
|
|
593
|
-
return
|
|
594
593
|
try:
|
|
595
594
|
response = await client.branch_regression_tests_file(
|
|
596
595
|
config["id"], regression_tests_commands, run_in_main=main
|
|
@@ -175,11 +175,6 @@ def _get_current_workspace_common(
|
|
|
175
175
|
return next((workspace for workspace in workspaces if workspace["id"] == current_workspace_id), None)
|
|
176
176
|
|
|
177
177
|
|
|
178
|
-
async def get_current_environment(client, config):
|
|
179
|
-
workspaces: List[Dict[str, Any]] = (await client.user_workspaces_and_branches()).get("workspaces", [])
|
|
180
|
-
return next((workspace for workspace in workspaces if workspace["id"] == config["id"]), None)
|
|
181
|
-
|
|
182
|
-
|
|
183
178
|
async def get_current_workspace_branches(config: CLIConfig) -> List[Dict[str, Any]]:
|
|
184
179
|
current_main_workspace: Optional[Dict[str, Any]] = await get_current_main_workspace(config)
|
|
185
180
|
if not current_main_workspace:
|
|
@@ -508,12 +503,6 @@ def format_host(host: str, subdomain: Optional[str] = None) -> str:
|
|
|
508
503
|
return host
|
|
509
504
|
|
|
510
505
|
|
|
511
|
-
def region_from_host(region_name_or_host, regions):
|
|
512
|
-
"""Returns the region that matches region_name_or_host"""
|
|
513
|
-
|
|
514
|
-
return next((r for r in regions if _compare_region_host(region_name_or_host, r)), None)
|
|
515
|
-
|
|
516
|
-
|
|
517
506
|
def ask_for_user_token(action: str, ui_host: str) -> str:
|
|
518
507
|
return click.prompt(
|
|
519
508
|
f'\nUse the token called "user token" in order to {action}. Copy it from {ui_host}/tokens and paste it here',
|
|
@@ -974,7 +963,8 @@ async def push_data(
|
|
|
974
963
|
sql_condition: Optional[str] = None,
|
|
975
964
|
replace_options=None,
|
|
976
965
|
concurrency: int = 1,
|
|
977
|
-
|
|
966
|
+
use_v1: bool = False,
|
|
967
|
+
) -> Optional[list[str]]:
|
|
978
968
|
if url and type(url) is tuple:
|
|
979
969
|
url = url[0]
|
|
980
970
|
client: TinyB = ctx.obj["client"]
|
|
@@ -1011,6 +1001,8 @@ async def push_data(
|
|
|
1011
1001
|
datasource_name: str, url: str, mode: str, sql_condition: Optional[str], replace_options: Optional[Set[str]]
|
|
1012
1002
|
):
|
|
1013
1003
|
parsed = urlparse(url)
|
|
1004
|
+
if use_v1 and parsed.scheme in ("http", "https"):
|
|
1005
|
+
raise CLIException("--experimental=use_v1 only supports local files.")
|
|
1014
1006
|
# poor man's format detection
|
|
1015
1007
|
_format = get_format_from_filename_or_url(url)
|
|
1016
1008
|
if parsed.scheme in ("http", "https"):
|
|
@@ -1031,8 +1023,15 @@ async def push_data(
|
|
|
1031
1023
|
sql_condition=sql_condition,
|
|
1032
1024
|
format=_format,
|
|
1033
1025
|
replace_options=replace_options,
|
|
1026
|
+
use_v1=use_v1,
|
|
1034
1027
|
)
|
|
1035
1028
|
|
|
1029
|
+
if use_v1:
|
|
1030
|
+
job_id = res.get("id") or res.get("import_id")
|
|
1031
|
+
if not isinstance(job_id, str):
|
|
1032
|
+
raise CLIException("The v1 import response did not include a job ID.")
|
|
1033
|
+
return job_id
|
|
1034
|
+
|
|
1036
1035
|
datasource_name = res["datasource"]["name"]
|
|
1037
1036
|
try:
|
|
1038
1037
|
datasource = await client.get_datasource(datasource_name)
|
|
@@ -1063,6 +1062,8 @@ async def push_data(
|
|
|
1063
1062
|
try:
|
|
1064
1063
|
tasks = [process_url(datasource_name, url, mode, sql_condition, replace_options) for url in urls]
|
|
1065
1064
|
output = await gather_with_concurrency(concurrency, *tasks)
|
|
1065
|
+
if use_v1:
|
|
1066
|
+
return list(output)
|
|
1066
1067
|
parser, total_rows, appended_rows = list(output)[-1]
|
|
1067
1068
|
except AuthNoTokenException:
|
|
1068
1069
|
raise
|
|
@@ -1083,6 +1084,8 @@ async def push_data(
|
|
|
1083
1084
|
click.echo(FeedbackManager.success_appended_datasource(datasource=datasource_name))
|
|
1084
1085
|
click.echo(FeedbackManager.info_data_pushed(datasource=datasource_name))
|
|
1085
1086
|
|
|
1087
|
+
return None
|
|
1088
|
+
|
|
1086
1089
|
|
|
1087
1090
|
async def sync_data(ctx, datasource_name: str, yes: bool):
|
|
1088
1091
|
client: TinyB = ctx.obj["client"]
|
|
@@ -1121,11 +1124,6 @@ def validate_datasource_name(name):
|
|
|
1121
1124
|
raise CLIException(FeedbackManager.error_datasource_name())
|
|
1122
1125
|
|
|
1123
1126
|
|
|
1124
|
-
def validate_connection_id(connection_id):
|
|
1125
|
-
if not isinstance(connection_id, str) or connection_id == "":
|
|
1126
|
-
raise CLIException(FeedbackManager.error_datasource_connection_id())
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
1127
|
def validate_kafka_topic(topic):
|
|
1130
1128
|
if not isinstance(topic, str):
|
|
1131
1129
|
raise CLIException(FeedbackManager.error_kafka_topic())
|
|
@@ -6,8 +6,6 @@ from enum import Enum
|
|
|
6
6
|
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
7
7
|
from urllib.parse import urlparse
|
|
8
8
|
|
|
9
|
-
from packaging import version
|
|
10
|
-
|
|
11
9
|
import tinybird.client as tbc
|
|
12
10
|
from tinybird.config import CURRENT_VERSION, DEFAULT_API_HOST, DEFAULT_LOCALHOST
|
|
13
11
|
|
|
@@ -34,12 +32,6 @@ class FeatureFlags:
|
|
|
34
32
|
return not ("x.y.z" in CURRENT_VERSION and os.environ.get("TB_CLI_TELEMETRY_SEND_IN_LOCAL", "0") == "0")
|
|
35
33
|
|
|
36
34
|
|
|
37
|
-
def compare_versions(a: str, b: str) -> int:
|
|
38
|
-
va = version.parse(a)
|
|
39
|
-
vb = version.parse(b)
|
|
40
|
-
return -1 if va < vb else (1 if va > vb else 0)
|
|
41
|
-
|
|
42
|
-
|
|
43
35
|
class ConfigValueOrigin(Enum):
|
|
44
36
|
# Sources for config values (environment variables, .tinyb file or default value)
|
|
45
37
|
|
|
@@ -96,7 +88,7 @@ class CLIConfig:
|
|
|
96
88
|
def to_dict(self) -> Dict[str, Any]:
|
|
97
89
|
"""Helper to ease"""
|
|
98
90
|
result: Dict[str, Any] = self._parent.to_dict() if self._parent else {}
|
|
99
|
-
result.update(
|
|
91
|
+
result.update({v.name: deepcopy(v.value) for v in self._values.values()})
|
|
100
92
|
return result
|
|
101
93
|
|
|
102
94
|
def __getitem__(self, key) -> Any:
|
|
@@ -179,12 +171,6 @@ class CLIConfig:
|
|
|
179
171
|
def set_semver(self, semver: Optional[str]) -> None:
|
|
180
172
|
self["semver"] = semver
|
|
181
173
|
|
|
182
|
-
def get_semver(self) -> Optional[str]:
|
|
183
|
-
try:
|
|
184
|
-
return self["semver"]
|
|
185
|
-
except KeyError:
|
|
186
|
-
return None
|
|
187
|
-
|
|
188
174
|
def set_token_for_host(self, token: Optional[str], host: Optional[str]) -> None:
|
|
189
175
|
"""Sets the token for the specified host.
|
|
190
176
|
|
|
@@ -15,7 +15,7 @@ from click import Context
|
|
|
15
15
|
|
|
16
16
|
from tinybird.client import AuthNoTokenException, CanNotBeDeletedException, DoesNotExistException, TinyB
|
|
17
17
|
from tinybird.config import get_display_host
|
|
18
|
-
from tinybird.datafile_common import get_name_version
|
|
18
|
+
from tinybird.datafile_common import get_name_version
|
|
19
19
|
from tinybird.feedback_manager import FeedbackManager
|
|
20
20
|
from tinybird.tb_cli_modules.branch import warn_if_in_live
|
|
21
21
|
from tinybird.tb_cli_modules.cli import cli
|
|
@@ -34,9 +34,12 @@ from tinybird.tb_cli_modules.common import (
|
|
|
34
34
|
validate_kafka_auto_offset_reset,
|
|
35
35
|
validate_kafka_group,
|
|
36
36
|
validate_kafka_topic,
|
|
37
|
+
wait_job,
|
|
37
38
|
)
|
|
38
39
|
from tinybird.tb_cli_modules.config import CLIConfig
|
|
39
|
-
from tinybird.tb_cli_modules.exceptions import CLIDatasourceException
|
|
40
|
+
from tinybird.tb_cli_modules.exceptions import CLIDatasourceException, CLIException
|
|
41
|
+
|
|
42
|
+
EXPERIMENTAL_FEATURE_USE_V1 = "use_v1"
|
|
40
43
|
|
|
41
44
|
|
|
42
45
|
@cli.group()
|
|
@@ -45,6 +48,34 @@ def datasource(ctx):
|
|
|
45
48
|
"""Data Sources commands"""
|
|
46
49
|
|
|
47
50
|
|
|
51
|
+
def _echo_v1_import_jobs_queued(job_ids: list[str], operation: str) -> None:
|
|
52
|
+
for job_id in job_ids:
|
|
53
|
+
click.echo(FeedbackManager.success_import_job_queued(operation=operation, job_id=job_id))
|
|
54
|
+
click.echo(FeedbackManager.info_import_job_status(job_id=job_id))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
async def _wait_for_v1_import_jobs(client: TinyB, job_ids: list[str], operation: str) -> None:
|
|
58
|
+
for job_id in job_ids:
|
|
59
|
+
try:
|
|
60
|
+
await wait_job(client, job_id, f"/v0/jobs/{job_id}", f"{operation} import")
|
|
61
|
+
except CLIException:
|
|
62
|
+
if await _echo_v1_import_job_details(client, job_id):
|
|
63
|
+
raise click.exceptions.Exit(1)
|
|
64
|
+
raise
|
|
65
|
+
click.echo(FeedbackManager.success_import_job_completed(operation=operation, job_id=job_id))
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
async def _echo_v1_import_job_details(client: TinyB, job_id: str) -> bool:
|
|
69
|
+
try:
|
|
70
|
+
job = await client.job(job_id)
|
|
71
|
+
except Exception:
|
|
72
|
+
return False
|
|
73
|
+
click.echo(FeedbackManager.info_job(job=job_id))
|
|
74
|
+
echo_safe_humanfriendly_tables_format_smart_table([job.values()], column_names=job.keys())
|
|
75
|
+
click.echo("\n")
|
|
76
|
+
return True
|
|
77
|
+
|
|
78
|
+
|
|
48
79
|
@datasource.command(name="ls")
|
|
49
80
|
@click.option("--match", default=None, help="Retrieve any resources matching the pattern. eg --match _test")
|
|
50
81
|
@click.option(
|
|
@@ -135,6 +166,13 @@ async def datasource_ls(ctx: Context, match: Optional[str], format_: str):
|
|
|
135
166
|
hidden=True,
|
|
136
167
|
)
|
|
137
168
|
@click.option("--concurrency", help="How many files to submit concurrently", default=1, hidden=True)
|
|
169
|
+
@click.option("--wait", is_flag=True, default=False, help="Wait for a v1 import job to finish.")
|
|
170
|
+
@click.option(
|
|
171
|
+
"--experimental",
|
|
172
|
+
type=click.Choice([EXPERIMENTAL_FEATURE_USE_V1]),
|
|
173
|
+
multiple=True,
|
|
174
|
+
help="Enable an experimental feature. May be specified multiple times.",
|
|
175
|
+
)
|
|
138
176
|
@click.pass_context
|
|
139
177
|
@coro
|
|
140
178
|
async def datasource_append(
|
|
@@ -145,6 +183,8 @@ async def datasource_append(
|
|
|
145
183
|
incremental: Optional[str],
|
|
146
184
|
ignore_empty: bool,
|
|
147
185
|
concurrency: int,
|
|
186
|
+
experimental: tuple[str, ...],
|
|
187
|
+
wait: bool,
|
|
148
188
|
):
|
|
149
189
|
"""
|
|
150
190
|
Appends data to an existing Data Source from URL, local file or a connector
|
|
@@ -157,7 +197,14 @@ async def datasource_append(
|
|
|
157
197
|
|
|
158
198
|
if not url:
|
|
159
199
|
raise CLIDatasourceException(FeedbackManager.error_missing_url(datasource=datasource_name))
|
|
160
|
-
|
|
200
|
+
use_v1 = EXPERIMENTAL_FEATURE_USE_V1 in experimental
|
|
201
|
+
if wait and not use_v1:
|
|
202
|
+
raise CLIDatasourceException("--wait requires --experimental=use_v1.")
|
|
203
|
+
job_ids = await push_data(ctx, datasource_name, url, mode="append", concurrency=concurrency, use_v1=use_v1)
|
|
204
|
+
if use_v1:
|
|
205
|
+
_echo_v1_import_jobs_queued(job_ids or [], "Append")
|
|
206
|
+
if wait:
|
|
207
|
+
await _wait_for_v1_import_jobs(ctx.obj["client"], job_ids or [], "Append")
|
|
161
208
|
|
|
162
209
|
|
|
163
210
|
@datasource.command(name="replace")
|
|
@@ -165,6 +212,13 @@ async def datasource_append(
|
|
|
165
212
|
@click.argument("url", nargs=-1)
|
|
166
213
|
@click.option("--sql-condition", default=None, help="SQL WHERE condition to replace data", hidden=True)
|
|
167
214
|
@click.option("--skip-incompatible-partition-key", is_flag=True, default=False, hidden=True)
|
|
215
|
+
@click.option("--wait", is_flag=True, default=False, help="Wait for a v1 import job to finish.")
|
|
216
|
+
@click.option(
|
|
217
|
+
"--experimental",
|
|
218
|
+
type=click.Choice([EXPERIMENTAL_FEATURE_USE_V1]),
|
|
219
|
+
multiple=True,
|
|
220
|
+
help="Enable an experimental feature. May be specified multiple times.",
|
|
221
|
+
)
|
|
168
222
|
@click.pass_context
|
|
169
223
|
@coro
|
|
170
224
|
async def datasource_replace(
|
|
@@ -173,6 +227,8 @@ async def datasource_replace(
|
|
|
173
227
|
url,
|
|
174
228
|
sql_condition,
|
|
175
229
|
skip_incompatible_partition_key,
|
|
230
|
+
experimental: tuple[str, ...],
|
|
231
|
+
wait: bool,
|
|
176
232
|
):
|
|
177
233
|
"""
|
|
178
234
|
Replaces the data in a data source from a URL, local file or a connector
|
|
@@ -189,14 +245,22 @@ async def datasource_replace(
|
|
|
189
245
|
replace_options = set()
|
|
190
246
|
if skip_incompatible_partition_key:
|
|
191
247
|
replace_options.add("skip_incompatible_partition_key")
|
|
192
|
-
|
|
248
|
+
use_v1 = EXPERIMENTAL_FEATURE_USE_V1 in experimental
|
|
249
|
+
if wait and not use_v1:
|
|
250
|
+
raise CLIDatasourceException("--wait requires --experimental=use_v1.")
|
|
251
|
+
job_ids = await push_data(
|
|
193
252
|
ctx,
|
|
194
253
|
datasource_name,
|
|
195
254
|
url,
|
|
196
255
|
mode="replace",
|
|
197
256
|
sql_condition=sql_condition,
|
|
198
257
|
replace_options=replace_options,
|
|
258
|
+
use_v1=use_v1,
|
|
199
259
|
)
|
|
260
|
+
if use_v1:
|
|
261
|
+
_echo_v1_import_jobs_queued(job_ids or [], "Replace")
|
|
262
|
+
if wait:
|
|
263
|
+
await _wait_for_v1_import_jobs(ctx.obj["client"], job_ids or [], "Replace")
|
|
200
264
|
|
|
201
265
|
|
|
202
266
|
@datasource.command(name="analyze")
|
{tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/tinyunit/tinyunit.py
RENAMED
|
@@ -313,12 +313,3 @@ def test_run_summary(results: List[TestSummaryResults], only_fail: bool = False,
|
|
|
313
313
|
raise CLIException(FeedbackManager.error_some_data_validation_have_failed())
|
|
314
314
|
if total_counts.get(ERROR, 0) > 0:
|
|
315
315
|
raise CLIException(FeedbackManager.error_some_tests_have_errors())
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
def get_bare_url(url: str) -> str:
|
|
319
|
-
if url.startswith("http://"):
|
|
320
|
-
return url[7:]
|
|
321
|
-
elif url.startswith("https://"):
|
|
322
|
-
return url[8:]
|
|
323
|
-
else:
|
|
324
|
-
return url
|
|
@@ -114,7 +114,6 @@ async def clear_workspace(ctx: Context, yes: bool, dry_run: bool) -> None:
|
|
|
114
114
|
if config["id"] == workspace["id"]:
|
|
115
115
|
if workspace.get("is_branch"):
|
|
116
116
|
raise CLIWorkspaceException(FeedbackManager.error_not_allowed_in_branch(cli="tb"))
|
|
117
|
-
return
|
|
118
117
|
else:
|
|
119
118
|
click.echo(FeedbackManager.info_current_workspace())
|
|
120
119
|
table.append(
|
|
@@ -916,10 +916,10 @@ def _parse(reader: _TemplateReader, template, in_block=None, in_loop=None):
|
|
|
916
916
|
|
|
917
917
|
# Intermediate ("else", "elif", etc) blocks
|
|
918
918
|
intermediate_blocks = {
|
|
919
|
-
"else":
|
|
920
|
-
"elif":
|
|
921
|
-
"except":
|
|
922
|
-
"finally":
|
|
919
|
+
"else": {"if", "for", "while", "try"},
|
|
920
|
+
"elif": {"if"},
|
|
921
|
+
"except": {"try"},
|
|
922
|
+
"finally": {"try"},
|
|
923
923
|
}
|
|
924
924
|
allowed_parents = intermediate_blocks.get(operator)
|
|
925
925
|
|
|
@@ -997,7 +997,7 @@ def _parse(reader: _TemplateReader, template, in_block=None, in_loop=None):
|
|
|
997
997
|
|
|
998
998
|
elif operator in ("break", "continue"):
|
|
999
999
|
if not in_loop:
|
|
1000
|
-
reader.raise_parse_error("%s outside %s block" % (operator,
|
|
1000
|
+
reader.raise_parse_error("%s outside %s block" % (operator, {"for", "while"}))
|
|
1001
1001
|
body.chunks.append(_Statement(contents, line))
|
|
1002
1002
|
continue
|
|
1003
1003
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: tinybird_cli
|
|
3
|
-
Version: 6.5.
|
|
3
|
+
Version: 6.5.5.dev0
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/cli
|
|
6
6
|
Author: Tinybird
|
|
@@ -43,10 +43,19 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
|
|
|
43
43
|
Changelog
|
|
44
44
|
----------
|
|
45
45
|
|
|
46
|
+
6.5.4
|
|
47
|
+
***********
|
|
48
|
+
|
|
49
|
+
- `Added` ``tb datasource append`` and ``tb datasource replace`` support the experimental ``--experimental=use_v1`` option for local CSV, NDJSON, and Parquet files. It submits the import through the v1 endpoint and prints the queued job ID; pass ``--wait`` to wait for completion.
|
|
50
|
+
- `Fixed` Internal improvements to secret template rendering used during datafile parsing.
|
|
51
|
+
- `Fixed` `tb push` now rejects pipes that declare a parameter without a valid name (e.g. a numeric-literal cast like ``{{Float32(1682892000.0)}}``), instead of creating an unusable parameter.
|
|
52
|
+
|
|
46
53
|
6.5.3
|
|
47
54
|
***********
|
|
48
55
|
|
|
56
|
+
- `Removed` `bi_stats_rt` and `bi_stats` from service datasource catalogs and dependency ignore lists.
|
|
49
57
|
- `Added` ``tb datasource delete`` now accepts ``--lightweight-delete``, ``--partition`` and ``--projection-mode``, backed by the ``POST /v1/datasources/{name}/delete`` endpoint that uses ClickHouse's lightweight ``DELETE FROM`` instead of the heavy ``ALTER TABLE ... DELETE`` mutation. Lightweight delete is synchronous by default and returns the affected row count plus a partition-level mutation block; pass ``--no-wait`` to enqueue a job and receive the v0-compatible envelope.
|
|
58
|
+
- `Changed` Upgraded Tornado to 6.5.5.
|
|
50
59
|
|
|
51
60
|
6.5.1
|
|
52
61
|
***********
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/workspace_members.py
RENAMED
|
File without changes
|
|
File without changes
|
{tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird_cli.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|