tinybird-cli 6.5.3.dev0__tar.gz → 6.5.5.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/PKG-INFO +15 -1
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/__cli__.py +2 -2
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/ch_utils/engine.py +6 -25
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/client.py +53 -13
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/config.py +0 -6
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/datafile_common.py +50 -59
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/datatypes.py +0 -4
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/feedback_manager.py +8 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/sql.py +0 -24
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/sql_template.py +92 -45
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/sql_toolset.py +28 -6
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/branch.py +0 -1
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/common.py +15 -17
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/config.py +1 -15
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/datasource.py +141 -8
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -9
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/workspace.py +0 -1
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tornado_template.py +5 -5
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird_cli.egg-info/PKG-INFO +15 -1
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/setup.cfg +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/ch_utils/constants.py +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/check_pypi.py +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/context.py +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/git_settings.py +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/sql_template_fmt.py +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/syncasync.py +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli.py +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/auth.py +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/cicd.py +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/cli.py +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/connection.py +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/exceptions.py +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/fmt.py +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/job.py +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/pipe.py +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/regions.py +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/tag.py +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/telemetry.py +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/test.py +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/token.py +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/workspace_members.py +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird_cli.egg-info/SOURCES.txt +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird_cli.egg-info/dependency_links.txt +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird_cli.egg-info/entry_points.txt +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird_cli.egg-info/requires.txt +0 -0
- {tinybird_cli-6.5.3.dev0 → tinybird_cli-6.5.5.dev0}/tinybird_cli.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: tinybird_cli
|
|
3
|
-
Version: 6.5.
|
|
3
|
+
Version: 6.5.5.dev0
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/cli
|
|
6
6
|
Author: Tinybird
|
|
@@ -43,6 +43,20 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
|
|
|
43
43
|
Changelog
|
|
44
44
|
----------
|
|
45
45
|
|
|
46
|
+
6.5.4
|
|
47
|
+
***********
|
|
48
|
+
|
|
49
|
+
- `Added` ``tb datasource append`` and ``tb datasource replace`` support the experimental ``--experimental=use_v1`` option for local CSV, NDJSON, and Parquet files. It submits the import through the v1 endpoint and prints the queued job ID; pass ``--wait`` to wait for completion.
|
|
50
|
+
- `Fixed` Internal improvements to secret template rendering used during datafile parsing.
|
|
51
|
+
- `Fixed` `tb push` now rejects pipes that declare a parameter without a valid name (e.g. a numeric-literal cast like ``{{Float32(1682892000.0)}}``), instead of creating an unusable parameter.
|
|
52
|
+
|
|
53
|
+
6.5.3
|
|
54
|
+
***********
|
|
55
|
+
|
|
56
|
+
- `Removed` `bi_stats_rt` and `bi_stats` from service datasource catalogs and dependency ignore lists.
|
|
57
|
+
- `Added` ``tb datasource delete`` now accepts ``--lightweight-delete``, ``--partition`` and ``--projection-mode``, backed by the ``POST /v1/datasources/{name}/delete`` endpoint that uses ClickHouse's lightweight ``DELETE FROM`` instead of the heavy ``ALTER TABLE ... DELETE`` mutation. Lightweight delete is synchronous by default and returns the affected row count plus a partition-level mutation block; pass ``--no-wait`` to enqueue a job and receive the v0-compatible envelope.
|
|
58
|
+
- `Changed` Upgraded Tornado to 6.5.5.
|
|
59
|
+
|
|
46
60
|
6.5.1
|
|
47
61
|
***********
|
|
48
62
|
|
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/cli'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '6.5.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '6.5.5.dev0'
|
|
8
|
+
__revision__ = '5d2bbc5'
|
|
@@ -121,6 +121,12 @@ class TableDetails:
|
|
|
121
121
|
def is_mergetree_family(self) -> bool:
|
|
122
122
|
return self.engine is not None and "mergetree" in self.engine.lower()
|
|
123
123
|
|
|
124
|
+
def is_view(self) -> bool:
|
|
125
|
+
return self.original_engine == "View"
|
|
126
|
+
|
|
127
|
+
def is_merge_final_engine(self) -> bool:
|
|
128
|
+
return self.original_engine == "MergeFinal"
|
|
129
|
+
|
|
124
130
|
def supports_alter_add_column(self) -> bool:
|
|
125
131
|
return self.is_mergetree_family() or (self.engine is not None and self.engine.lower() == "null")
|
|
126
132
|
|
|
@@ -354,31 +360,6 @@ def sorting_key_is_valid(columns: List[Dict[str, Any]], value: Optional[str]) ->
|
|
|
354
360
|
return value
|
|
355
361
|
|
|
356
362
|
|
|
357
|
-
def case_insensitive_check(valid_values: List[str]) -> Callable[[List[Dict[str, Any]], str], Optional[str]]:
|
|
358
|
-
"""
|
|
359
|
-
>>> valid_values = ['ANY', 'ALL']
|
|
360
|
-
>>> checker = case_insensitive_check(valid_values)
|
|
361
|
-
>>> checker([],'ALL')
|
|
362
|
-
|
|
363
|
-
>>> valid_values = ['ANY', 'ALL']
|
|
364
|
-
>>> checker = case_insensitive_check(valid_values)
|
|
365
|
-
>>> checker([],'any')
|
|
366
|
-
|
|
367
|
-
>>> valid_values = ['ANY', 'ALL']
|
|
368
|
-
>>> checker = case_insensitive_check(valid_values)
|
|
369
|
-
>>> checker([],'foo')
|
|
370
|
-
Traceback (most recent call last):
|
|
371
|
-
...
|
|
372
|
-
ValueError: valid values are ANY, ALL
|
|
373
|
-
"""
|
|
374
|
-
|
|
375
|
-
def checker(columns: List[Dict[str, Any]], value: str):
|
|
376
|
-
if value.upper() not in valid_values:
|
|
377
|
-
raise ValueError(f"valid values are {', '.join(valid_values)}")
|
|
378
|
-
|
|
379
|
-
return checker
|
|
380
|
-
|
|
381
|
-
|
|
382
363
|
# [PARTITION BY expr]
|
|
383
364
|
# [ORDER BY expr]
|
|
384
365
|
# [PRIMARY KEY expr]
|
|
@@ -441,6 +441,7 @@ class TinyB:
|
|
|
441
441
|
sql_condition: Optional[str] = None,
|
|
442
442
|
format: str = "csv",
|
|
443
443
|
replace_options: Optional[Set[str]] = None,
|
|
444
|
+
use_v1: bool = False,
|
|
444
445
|
):
|
|
445
446
|
params = {"name": datasource_name, "mode": mode, "format": format, "debug": "blocks_block_log"}
|
|
446
447
|
|
|
@@ -452,28 +453,67 @@ class TinyB:
|
|
|
452
453
|
|
|
453
454
|
async with aiofiles.open(file, "rb") as content:
|
|
454
455
|
file_content = await content.read()
|
|
455
|
-
if format == "csv":
|
|
456
|
-
files = {"csv": ("csv", file_content)}
|
|
457
|
-
else:
|
|
458
|
-
files = {"ndjson": ("ndjson", file_content)}
|
|
459
456
|
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
457
|
+
if use_v1:
|
|
458
|
+
v1_params = {"format": format}
|
|
459
|
+
if sql_condition:
|
|
460
|
+
v1_params["replace_condition"] = sql_condition
|
|
461
|
+
if replace_options:
|
|
462
|
+
for option in list(replace_options):
|
|
463
|
+
v1_params[option] = "true"
|
|
464
|
+
content_types = {
|
|
465
|
+
"csv": "text/csv",
|
|
466
|
+
"ndjson": "application/x-ndjson",
|
|
467
|
+
"parquet": "application/vnd.apache.parquet",
|
|
468
|
+
}
|
|
469
|
+
headers = {"Content-Type": content_types[format]}
|
|
470
|
+
if str(file).endswith(".gz"):
|
|
471
|
+
headers["Content-Encoding"] = "gzip"
|
|
472
|
+
return await self._req(
|
|
473
|
+
f"/v1/datasources/{quote(datasource_name, safe='')}/{mode}?{urlencode(v1_params, safe='')}",
|
|
474
|
+
data=file_content,
|
|
475
|
+
headers=headers,
|
|
463
476
|
method="POST",
|
|
464
477
|
)
|
|
465
|
-
if status_callback:
|
|
466
|
-
status_callback(res)
|
|
467
478
|
|
|
468
|
-
|
|
479
|
+
if format == "csv":
|
|
480
|
+
files = {"csv": ("csv", file_content)}
|
|
481
|
+
else:
|
|
482
|
+
files = {"ndjson": ("ndjson", file_content)}
|
|
483
|
+
|
|
484
|
+
res = await self._req(
|
|
485
|
+
f"v0/datasources?{urlencode(params, safe='')}",
|
|
486
|
+
files=files,
|
|
487
|
+
method="POST",
|
|
488
|
+
)
|
|
489
|
+
if status_callback:
|
|
490
|
+
status_callback(res)
|
|
491
|
+
|
|
492
|
+
return res
|
|
469
493
|
|
|
470
494
|
async def datasource_truncate(self, datasource_name: str):
|
|
471
495
|
return await self._req(f"/v0/datasources/{datasource_name}/truncate", method="POST", data="")
|
|
472
496
|
|
|
473
|
-
async def datasource_delete_rows(
|
|
474
|
-
|
|
497
|
+
async def datasource_delete_rows(
|
|
498
|
+
self,
|
|
499
|
+
datasource_name: str,
|
|
500
|
+
delete_condition: str,
|
|
501
|
+
dry_run: bool = False,
|
|
502
|
+
lightweight: bool = False,
|
|
503
|
+
wait: bool = True,
|
|
504
|
+
partition: Optional[str] = None,
|
|
505
|
+
projection_mode: Optional[str] = None,
|
|
506
|
+
):
|
|
507
|
+
params: Dict[str, Any] = {"delete_condition": delete_condition}
|
|
508
|
+
if lightweight:
|
|
509
|
+
params["wait"] = "true" if wait else "false"
|
|
510
|
+
if partition:
|
|
511
|
+
params["partition"] = partition
|
|
512
|
+
if projection_mode:
|
|
513
|
+
params["projection_mode"] = projection_mode
|
|
514
|
+
return await self._req(f"/v1/datasources/{datasource_name}/delete", method="POST", data=params)
|
|
475
515
|
if dry_run:
|
|
476
|
-
params
|
|
516
|
+
params["dry_run"] = "true"
|
|
477
517
|
return await self._req(f"/v0/datasources/{datasource_name}/delete", method="POST", data=params)
|
|
478
518
|
|
|
479
519
|
async def datasource_dependencies(
|
|
@@ -112,12 +112,6 @@ async def get_config(
|
|
|
112
112
|
return config
|
|
113
113
|
|
|
114
114
|
|
|
115
|
-
async def write_config(config: Dict[str, Any], dest_file: str = ".tinyb"):
|
|
116
|
-
config_file = Path(getcwd()) / dest_file
|
|
117
|
-
async with aiofiles.open(config_file, "w") as file:
|
|
118
|
-
await file.write(json.dumps(config, indent=4, sort_keys=True))
|
|
119
|
-
|
|
120
|
-
|
|
121
115
|
def get_display_host(ui_host: str):
|
|
122
116
|
return LEGACY_HOSTS.get(ui_host, ui_host)
|
|
123
117
|
|
|
@@ -9,7 +9,7 @@ import aiofiles
|
|
|
9
9
|
from requests import Response
|
|
10
10
|
|
|
11
11
|
try:
|
|
12
|
-
from colorama import
|
|
12
|
+
from colorama import Fore, init
|
|
13
13
|
|
|
14
14
|
init()
|
|
15
15
|
except ImportError: # fallback so that the imported classes always exist
|
|
@@ -18,7 +18,7 @@ except ImportError: # fallback so that the imported classes always exist
|
|
|
18
18
|
def __getattr__(self, name):
|
|
19
19
|
return ""
|
|
20
20
|
|
|
21
|
-
Fore =
|
|
21
|
+
Fore = ColorFallback()
|
|
22
22
|
|
|
23
23
|
import difflib
|
|
24
24
|
import glob
|
|
@@ -94,8 +94,6 @@ INTERNAL_TABLES: Tuple[str, ...] = (
|
|
|
94
94
|
"kafka_ops_log",
|
|
95
95
|
"datasources_storage",
|
|
96
96
|
"endpoint_errors",
|
|
97
|
-
"bi_stats_rt",
|
|
98
|
-
"bi_stats",
|
|
99
97
|
)
|
|
100
98
|
|
|
101
99
|
|
|
@@ -306,10 +304,10 @@ class ValidationException(Exception):
|
|
|
306
304
|
super().__init__(err)
|
|
307
305
|
|
|
308
306
|
|
|
309
|
-
def sizeof_fmt(num:
|
|
307
|
+
def sizeof_fmt(num: float, suffix: str = "b") -> str:
|
|
310
308
|
"""Readable file size
|
|
311
309
|
:param num: Bytes value
|
|
312
|
-
:type num:
|
|
310
|
+
:type num: float
|
|
313
311
|
:param suffix: Unit suffix (optionnal) default = o
|
|
314
312
|
:type suffix: str
|
|
315
313
|
:rtype: str
|
|
@@ -854,9 +852,24 @@ def parse_pipe(
|
|
|
854
852
|
doc = parse(s, basepath=basepath, replace_includes=replace_includes, skip_eval=skip_eval)
|
|
855
853
|
for node in doc.nodes:
|
|
856
854
|
sql = node.get("sql", "")
|
|
857
|
-
|
|
858
|
-
|
|
855
|
+
source_sql = sql.strip()
|
|
856
|
+
if not source_sql:
|
|
857
|
+
raise click.ClickException(
|
|
858
|
+
FeedbackManager.error_parsing_node(node=node["name"], pipe=filename, error="Empty query")
|
|
859
|
+
)
|
|
860
|
+
if source_sql.startswith("%"):
|
|
861
|
+
sql, _, variable_warnings = render_sql_template(source_sql[1:], test_mode=True, name=node["name"])
|
|
859
862
|
doc.warnings = variable_warnings
|
|
863
|
+
if not sql.strip():
|
|
864
|
+
raise click.ClickException(
|
|
865
|
+
FeedbackManager.error_parsing_node(
|
|
866
|
+
node=node["name"],
|
|
867
|
+
pipe=filename,
|
|
868
|
+
error="Template renders to an empty query without parameters",
|
|
869
|
+
)
|
|
870
|
+
)
|
|
871
|
+
else:
|
|
872
|
+
sql = source_sql
|
|
860
873
|
# it'll fail with a ModuleNotFoundError when the toolset is not available but it returns the parsed doc
|
|
861
874
|
from tinybird.sql_toolset import format_sql as toolset_format_sql
|
|
862
875
|
|
|
@@ -868,13 +881,15 @@ def parse_pipe(
|
|
|
868
881
|
)
|
|
869
882
|
)
|
|
870
883
|
except ValueError as e:
|
|
871
|
-
|
|
884
|
+
source_sql = node.get("sql", "")
|
|
885
|
+
source_sql_stripped = source_sql.strip()
|
|
886
|
+
t, template_variables, _ = get_template_and_variables(source_sql, name=node["name"])
|
|
872
887
|
|
|
873
|
-
if
|
|
888
|
+
if not source_sql_stripped.startswith("%") and len(template_variables) > 0:
|
|
874
889
|
raise click.ClickException(FeedbackManager.error_template_start(filename=filename))
|
|
875
890
|
raise click.ClickException(
|
|
876
891
|
FeedbackManager.error_parsing_file(
|
|
877
|
-
filename=filename, lineno="", error=f"{str(e)} + SQL(value error): {
|
|
892
|
+
filename=filename, lineno="", error=f"{str(e)} + SQL(value error): {source_sql}"
|
|
878
893
|
)
|
|
879
894
|
)
|
|
880
895
|
except UnClosedIfError as e:
|
|
@@ -1028,10 +1043,13 @@ def parse(
|
|
|
1028
1043
|
|
|
1029
1044
|
parser_state.current_node["indexes"] = indexes
|
|
1030
1045
|
|
|
1031
|
-
def assign_var(v: str) -> Callable[[VarArg(str), KwArg(Any)], None]:
|
|
1046
|
+
def assign_var(v: str, lowercase: bool = False) -> Callable[[VarArg(str), KwArg(Any)], None]:
|
|
1032
1047
|
def _f(*args: str, **kwargs: Any):
|
|
1033
1048
|
s = _unquote((" ".join(args)).strip())
|
|
1034
|
-
|
|
1049
|
+
val = eval_var(s, skip=skip_eval)
|
|
1050
|
+
if lowercase:
|
|
1051
|
+
val = val.lower()
|
|
1052
|
+
parser_state.current_node[v.lower()] = val
|
|
1035
1053
|
|
|
1036
1054
|
return _f
|
|
1037
1055
|
|
|
@@ -1246,7 +1264,9 @@ def parse(
|
|
|
1246
1264
|
"import_external_datasource": assign_var("import_external_datasource"),
|
|
1247
1265
|
"import_bucket_uri": assign_var("import_bucket_uri"),
|
|
1248
1266
|
"import_from_timestamp": assign_var("import_from_timestamp"),
|
|
1249
|
-
|
|
1267
|
+
# Lowercase: format checks and executor routing downstream expect
|
|
1268
|
+
# lowercase values (ANALYTICS-A8M)
|
|
1269
|
+
"import_format": assign_var("import_format", lowercase=True),
|
|
1250
1270
|
"import_query": assign_var("import_query"),
|
|
1251
1271
|
"import_table_arn": assign_var("import_table_arn"),
|
|
1252
1272
|
"import_export_bucket": assign_var("import_export_bucket"),
|
|
@@ -1750,7 +1770,7 @@ async def process_file(
|
|
|
1750
1770
|
"filename": filename,
|
|
1751
1771
|
"name": name + version,
|
|
1752
1772
|
"nodes": nodes,
|
|
1753
|
-
"deps":
|
|
1773
|
+
"deps": list(set(deps)),
|
|
1754
1774
|
"tokens": doc.tokens,
|
|
1755
1775
|
"description": description,
|
|
1756
1776
|
"warnings": doc.warnings,
|
|
@@ -2082,12 +2102,12 @@ class PipeChecker(unittest.TestCase):
|
|
|
2082
2102
|
|
|
2083
2103
|
if self.ignore_order:
|
|
2084
2104
|
current_data = (
|
|
2085
|
-
sorted(normalize_array(current_data), key=itemgetter(*
|
|
2105
|
+
sorted(normalize_array(current_data), key=itemgetter(*list(current_data[0].keys())))
|
|
2086
2106
|
if len(current_data) > 0
|
|
2087
2107
|
else current_data
|
|
2088
2108
|
)
|
|
2089
2109
|
checker_data = (
|
|
2090
|
-
sorted(normalize_array(checker_data), key=itemgetter(*
|
|
2110
|
+
sorted(normalize_array(checker_data), key=itemgetter(*list(checker_data[0].keys())))
|
|
2091
2111
|
if len(checker_data) > 0
|
|
2092
2112
|
else checker_data
|
|
2093
2113
|
)
|
|
@@ -2202,6 +2222,8 @@ class PipeCheckerRunner:
|
|
|
2202
2222
|
FROM {pipe_stats_rt}
|
|
2203
2223
|
WHERE
|
|
2204
2224
|
pipe_name = '{self.pipe_name}'
|
|
2225
|
+
-- pipe_stats_rt retains 3 months; recent requests are enough for checks
|
|
2226
|
+
AND start_datetime >= now() - INTERVAL 7 DAY
|
|
2205
2227
|
AND url IS NOT NULL
|
|
2206
2228
|
AND extractURLParameter(assumeNotNull(url), 'from') <> 'ui'
|
|
2207
2229
|
AND extractURLParameter(assumeNotNull(url), 'pipe_checker') <> 'true'
|
|
@@ -2229,6 +2251,8 @@ class PipeCheckerRunner:
|
|
|
2229
2251
|
FROM {pipe_stats_rt}
|
|
2230
2252
|
WHERE
|
|
2231
2253
|
pipe_name = '{self.pipe_name}'
|
|
2254
|
+
-- pipe_stats_rt retains 3 months; recent requests are enough for checks
|
|
2255
|
+
AND start_datetime >= now() - INTERVAL 7 DAY
|
|
2232
2256
|
AND url IS NOT NULL
|
|
2233
2257
|
AND extractURLParameter(assumeNotNull(url), 'from') <> 'ui'
|
|
2234
2258
|
AND extractURLParameter(assumeNotNull(url), 'pipe_checker') <> 'true'
|
|
@@ -4141,7 +4165,7 @@ async def build_graph(
|
|
|
4141
4165
|
elif len(warnings) > 1:
|
|
4142
4166
|
click.echo(
|
|
4143
4167
|
FeedbackManager.warning_pipe_restricted_params(
|
|
4144
|
-
words=", ".join(["'{}'"
|
|
4168
|
+
words=", ".join([f"'{param}'" for param in warnings[:-1]]),
|
|
4145
4169
|
last_word=warnings[-1],
|
|
4146
4170
|
)
|
|
4147
4171
|
)
|
|
@@ -4328,7 +4352,7 @@ async def folder_push(
|
|
|
4328
4352
|
|
|
4329
4353
|
workspace_lib_paths = list(workspace_lib_paths)
|
|
4330
4354
|
# include vendor libs without overriding user ones
|
|
4331
|
-
existing_workspaces =
|
|
4355
|
+
existing_workspaces = {x[1] for x in workspace_lib_paths}
|
|
4332
4356
|
vendor_path = Path("vendor")
|
|
4333
4357
|
if vendor_path.exists():
|
|
4334
4358
|
for x in vendor_path.iterdir():
|
|
@@ -4566,7 +4590,7 @@ async def folder_push(
|
|
|
4566
4590
|
|
|
4567
4591
|
if not fork_downstream:
|
|
4568
4592
|
# First, we will deploy the all the resources following the dependency graph except for the endpoints
|
|
4569
|
-
groups =
|
|
4593
|
+
groups = list(toposort(dependencies_graph))
|
|
4570
4594
|
for group in groups:
|
|
4571
4595
|
for name in group:
|
|
4572
4596
|
if name in processed:
|
|
@@ -4592,7 +4616,7 @@ async def folder_push(
|
|
|
4592
4616
|
processed.add(name)
|
|
4593
4617
|
|
|
4594
4618
|
# Then, we will deploy the endpoints that are on the dependency graph
|
|
4595
|
-
groups =
|
|
4619
|
+
groups = list(toposort(endpoints_dep_map))
|
|
4596
4620
|
for group in groups:
|
|
4597
4621
|
for name in group:
|
|
4598
4622
|
if name not in processed:
|
|
@@ -4621,7 +4645,7 @@ async def folder_push(
|
|
|
4621
4645
|
|
|
4622
4646
|
# First, we will deploy the datasources that need to be deployed.
|
|
4623
4647
|
# We need to deploy the datasources from left to right as some datasources might have MV that depend on the column types of previous datasources. Ex: `test_change_column_type_landing_datasource` test
|
|
4624
|
-
groups =
|
|
4648
|
+
groups = list(toposort(dependencies_graph_fork_downstream))
|
|
4625
4649
|
groups.reverse()
|
|
4626
4650
|
for group in groups:
|
|
4627
4651
|
for name in group:
|
|
@@ -4675,7 +4699,7 @@ async def folder_push(
|
|
|
4675
4699
|
# If we have ENDPOINT_A ----> MV_PIPE_B -----> DATASOURCE_B ------> ENDPOINT_C
|
|
4676
4700
|
# Where endpoint A is being used in the MV_PIPE_B, if we only modify the endpoint A
|
|
4677
4701
|
# The dependencies graph will only contain the endpoint A and the MV_PIPE_B, but not the DATASOURCE_B and the ENDPOINT_C
|
|
4678
|
-
groups =
|
|
4702
|
+
groups = list(toposort(dependencies_graph_fork_downstream))
|
|
4679
4703
|
for group in groups:
|
|
4680
4704
|
for name in group:
|
|
4681
4705
|
if name in processed or not is_endpoint(resources_to_run_fork_downstream[name]):
|
|
@@ -4684,7 +4708,7 @@ async def folder_push(
|
|
|
4684
4708
|
endpoints_dep_map[name] = dependencies_graph_fork_downstream[name]
|
|
4685
4709
|
|
|
4686
4710
|
# Now that we have the dependencies of the endpoints, we need to check that the resources has not been deployed yet and only care about the endpoints that depend on endpoints
|
|
4687
|
-
groups =
|
|
4711
|
+
groups = list(toposort(endpoints_dep_map))
|
|
4688
4712
|
|
|
4689
4713
|
# As we have used the forkdownstream graph to get the dependencies of the endpoints, we have all the dependencies of the endpoints
|
|
4690
4714
|
# But we need to deploy the endpoints and the dependencies of the endpoints from left to right
|
|
@@ -4709,7 +4733,7 @@ async def folder_push(
|
|
|
4709
4733
|
# Now we should have the endpoints and datasources deployed, we can deploy the rest of the pipes (copy & sinks)
|
|
4710
4734
|
# We need to rely on the forkdownstream graph as it contains all the modified pipes as well as the dependencies of the pipes
|
|
4711
4735
|
# In this case, we don't need to generate a new graph as we did for the endpoints as the pipes are not going to be used as dependencies and the datasources are already deployed
|
|
4712
|
-
groups =
|
|
4736
|
+
groups = list(toposort(dependencies_graph_fork_downstream))
|
|
4713
4737
|
for group in groups:
|
|
4714
4738
|
for name in group:
|
|
4715
4739
|
if name in processed or is_materialized(resources_to_run_fork_downstream.get(name)):
|
|
@@ -4729,7 +4753,7 @@ async def folder_push(
|
|
|
4729
4753
|
# Finally, we need to deploy the materialized views from right to left.
|
|
4730
4754
|
# We need to rely on the forkdownstream graph as it contains all the modified materialized views as well as the dependencies of the materialized views
|
|
4731
4755
|
# In this case, we don't need to generate a new graph as we did for the endpoints as the pipes are not going to be used as dependencies and the datasources are already deployed
|
|
4732
|
-
groups =
|
|
4756
|
+
groups = list(toposort(dependencies_graph_fork_downstream))
|
|
4733
4757
|
for group in groups:
|
|
4734
4758
|
for name in group:
|
|
4735
4759
|
if name in processed or not is_materialized(resources_to_run_fork_downstream.get(name)):
|
|
@@ -5732,36 +5756,3 @@ def is_file_a_datasource(filename: str) -> bool:
|
|
|
5732
5756
|
return True
|
|
5733
5757
|
|
|
5734
5758
|
return False
|
|
5735
|
-
|
|
5736
|
-
|
|
5737
|
-
def update_connector_params(service: str, ds_params: Dict[str, Any], connector_required_params: List[str]) -> None:
|
|
5738
|
-
"""
|
|
5739
|
-
Update connector parameters for a given service, ensuring required parameters exist.
|
|
5740
|
-
|
|
5741
|
-
:param service: The name of the service (e.g., 's3').
|
|
5742
|
-
:param ds_params: The data source parameters to be checked.
|
|
5743
|
-
:param connector_required_params: The list of required parameters for the connector.
|
|
5744
|
-
:return: None
|
|
5745
|
-
"""
|
|
5746
|
-
|
|
5747
|
-
connector_at_least_one_required_param: List[str] = []
|
|
5748
|
-
|
|
5749
|
-
# Handle the "at least one param" requirement
|
|
5750
|
-
if connector_at_least_one_required_param and not any(
|
|
5751
|
-
key in ds_params for key in connector_at_least_one_required_param
|
|
5752
|
-
):
|
|
5753
|
-
params = [
|
|
5754
|
-
(ImportReplacements.get_datafile_param_for_linker_param(service, param) or param).upper()
|
|
5755
|
-
for param in connector_at_least_one_required_param
|
|
5756
|
-
]
|
|
5757
|
-
click.echo(FeedbackManager.error_updating_connector_missing_at_least_one_param(param=" or ".join(params)))
|
|
5758
|
-
return
|
|
5759
|
-
|
|
5760
|
-
# Handle the mandatory params requirement
|
|
5761
|
-
if not all(key in ds_params for key in connector_required_params):
|
|
5762
|
-
params = [
|
|
5763
|
-
(ImportReplacements.get_datafile_param_for_linker_param(service, param) or param).upper()
|
|
5764
|
-
for param in connector_required_params
|
|
5765
|
-
]
|
|
5766
|
-
click.echo(FeedbackManager.error_updating_connector_missing_params(param=", ".join(params)))
|
|
5767
|
-
return
|
|
@@ -108,10 +108,6 @@ def is_type_date(type_to_check: str) -> bool:
|
|
|
108
108
|
return date_type_pattern.match(type_to_check) is not None
|
|
109
109
|
|
|
110
110
|
|
|
111
|
-
def string_test(x: str) -> bool:
|
|
112
|
-
return True
|
|
113
|
-
|
|
114
|
-
|
|
115
111
|
def date_test(x: str) -> bool:
|
|
116
112
|
return date_pattern.match(x) is not None
|
|
117
113
|
|
|
@@ -864,6 +864,9 @@ class FeedbackManager:
|
|
|
864
864
|
success_total_rows = success_message("** Total rows in {datasource}: {total_rows}")
|
|
865
865
|
success_appended_datasource = success_message("** Data appended to Data Source '{datasource}' successfully!")
|
|
866
866
|
success_replaced_datasource = success_message("** Data replaced in Data Source '{datasource}' successfully!")
|
|
867
|
+
success_import_job_queued = success_message("** {operation} import job queued: {job_id}")
|
|
868
|
+
success_import_job_completed = success_message("** {operation} import completed: {job_id}")
|
|
869
|
+
info_import_job_status = info_message("Check status: tb job details {job_id}")
|
|
867
870
|
success_auth = success_message(
|
|
868
871
|
"** Auth successful! \n** Configuration written to .tinyb file, consider adding it to .gitignore"
|
|
869
872
|
)
|
|
@@ -879,6 +882,11 @@ class FeedbackManager:
|
|
|
879
882
|
success_delete_rows_datasource = success_message(
|
|
880
883
|
"** Data Source '{datasource}' rows deleted matching condition \"{delete_condition}\""
|
|
881
884
|
)
|
|
885
|
+
success_lightweight_delete_rows_datasource = success_message(
|
|
886
|
+
"** Data Source '{datasource}' rows deleted matching condition \"{delete_condition}\""
|
|
887
|
+
"\n Rows affected: {rows_affected}"
|
|
888
|
+
"\n Partitions scanned: {partitions_scanned} (done: {partitions_done}, in progress: {partitions_in_progress})"
|
|
889
|
+
)
|
|
882
890
|
success_dry_run_delete_rows_datasource = success_message(
|
|
883
891
|
"** [DRY RUN] Data Source '{datasource}' rows '{rows}' matching condition \"{delete_condition}\" to be deleted"
|
|
884
892
|
)
|
|
@@ -911,30 +911,6 @@ def engine_replicated_to_local(engine: str) -> str:
|
|
|
911
911
|
return _RE_REPLICATED_MT.sub(_replace, engine.strip())
|
|
912
912
|
|
|
913
913
|
|
|
914
|
-
def engine_patch_replicated_engine(engine: str, engine_full: Optional[str], new_table_name: str) -> Optional[str]:
|
|
915
|
-
"""
|
|
916
|
-
>>> engine_patch_replicated_engine("ReplicatedMergeTree", "ReplicatedMergeTree('/clickhouse/tables/1-1/table_name', 'replica') PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192", 'table_name_staging')
|
|
917
|
-
"ReplicatedMergeTree('/clickhouse/tables/1-1/table_name_staging', 'replica') PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192"
|
|
918
|
-
>>> engine_patch_replicated_engine("ReplicatedMergeTree", "ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/sales_product_rank_rt_replicated_2', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY (purchase_location, sku_rank_lc, date)", 'sales_product_rank_rt_replicated_2_staging')
|
|
919
|
-
"ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/sales_product_rank_rt_replicated_2_staging', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY (purchase_location, sku_rank_lc, date)"
|
|
920
|
-
>>> engine_patch_replicated_engine("ReplicatedMergeTree", None, 't_000') is None
|
|
921
|
-
True
|
|
922
|
-
>>> engine_patch_replicated_engine("Log", "Log()", 't_000')
|
|
923
|
-
'Log()'
|
|
924
|
-
>>> engine_patch_replicated_engine("MergeTree", "MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024", 't_000')
|
|
925
|
-
'MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024'
|
|
926
|
-
"""
|
|
927
|
-
if not engine_full:
|
|
928
|
-
return None
|
|
929
|
-
if engine.lower().startswith("Replicated".lower()):
|
|
930
|
-
parts = _RE_REPLICATED_ENGINE_SPLIT.split(engine_full)
|
|
931
|
-
paths = parts[2].split("/")
|
|
932
|
-
paths[-1] = new_table_name
|
|
933
|
-
zoo_path = "/".join(paths)
|
|
934
|
-
return "".join([*parts[:2], zoo_path, *parts[3:]])
|
|
935
|
-
return engine_full
|
|
936
|
-
|
|
937
|
-
|
|
938
914
|
if __name__ == "__main__":
|
|
939
915
|
print( # noqa: T201
|
|
940
916
|
_parse_table_structure(
|