tinybird-cli 5.20.1.dev0__tar.gz → 5.20.1.dev2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/PKG-INFO +12 -1
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/__cli__.py +2 -2
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/ch_utils/constants.py +14 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/ch_utils/engine.py +5 -2
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/client.py +1 -1
- tinybird_cli-5.20.1.dev0/tinybird/datafile.py → tinybird_cli-5.20.1.dev2/tinybird/datafile_common.py +7 -6
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/feedback_manager.py +1 -1
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/sql.py +11 -1
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/sql_template.py +9 -5
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/sql_toolset.py +118 -5
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/auth.py +2 -3
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/branch.py +3 -4
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/cli.py +9 -11
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/connection.py +1 -1
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/datasource.py +1 -1
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/fmt.py +6 -4
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/pipe.py +1 -1
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +1 -1
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/workspace.py +12 -13
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird_cli.egg-info/PKG-INFO +12 -1
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird_cli.egg-info/SOURCES.txt +1 -1
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/setup.cfg +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/check_pypi.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/config.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/connectors.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/context.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/datatypes.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/git_settings.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/sql_template_fmt.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/syncasync.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/cicd.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/common.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/config.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/exceptions.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/job.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/regions.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/tag.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/telemetry.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/test.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/token.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/workspace_members.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tornado_template.py +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird_cli.egg-info/dependency_links.txt +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird_cli.egg-info/entry_points.txt +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird_cli.egg-info/requires.txt +0 -0
- {tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird_cli.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: tinybird_cli
|
|
3
|
-
Version: 5.20.1.
|
|
3
|
+
Version: 5.20.1.dev2
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/cli
|
|
6
6
|
Author: Tinybird
|
|
@@ -61,6 +61,16 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
|
|
|
61
61
|
Changelog
|
|
62
62
|
----------
|
|
63
63
|
|
|
64
|
+
5.20.1.dev2
|
|
65
|
+
***********
|
|
66
|
+
|
|
67
|
+
- `Fixed` Templating when using `id` parameter and the value was not provided.
|
|
68
|
+
|
|
69
|
+
5.20.1.dev1
|
|
70
|
+
***********
|
|
71
|
+
|
|
72
|
+
- `Added` When trying to push a materialized view with an unoptimized join, the CLI will now warn the user.
|
|
73
|
+
|
|
64
74
|
5.19.0
|
|
65
75
|
***********
|
|
66
76
|
|
|
@@ -198,6 +208,7 @@ Changelog
|
|
|
198
208
|
- `Added` `tb tag` commands to manage tags from the CLI.
|
|
199
209
|
- `Added` support to `TAGS` in `tb fmt`.
|
|
200
210
|
- `Added` support to `TAGS` in `tb pull` and `tb push`. Allows tagging resources for filtering in the UI.
|
|
211
|
+
- `Changed` Improved message when creating a DynamoDB connection
|
|
201
212
|
|
|
202
213
|
5.7.0
|
|
203
214
|
**********
|
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/cli'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '5.20.1.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '5.20.1.dev2'
|
|
8
|
+
__revision__ = 'afe97eb'
|
|
@@ -242,3 +242,17 @@ FORBIDDEN_SQL_KEYWORDS = {
|
|
|
242
242
|
|
|
243
243
|
CH_SETTINGS_JOIN_ALGORITHM_HASH = "hash" # uses 'hash' by default, https://clickhouse.com/docs/en/operations/settings/settings/#settings-join_algorithm
|
|
244
244
|
CH_SETTINGS_JOIN_ALGORITHM_AUTO = "auto,hash"
|
|
245
|
+
|
|
246
|
+
VALID_QUERY_FORMATS = (
|
|
247
|
+
"JSON",
|
|
248
|
+
"CSV",
|
|
249
|
+
"CSVWithNames",
|
|
250
|
+
"TSV",
|
|
251
|
+
"TSVWithNames",
|
|
252
|
+
"PrettyCompact",
|
|
253
|
+
"JSONEachRow",
|
|
254
|
+
"Parquet",
|
|
255
|
+
"JSONStrings",
|
|
256
|
+
"Prometheus",
|
|
257
|
+
"Native",
|
|
258
|
+
)
|
|
@@ -146,10 +146,9 @@ class TableDetails:
|
|
|
146
146
|
def is_replacing_engine(self) -> bool:
|
|
147
147
|
if self.engine:
|
|
148
148
|
engine_lower = self.engine.lower()
|
|
149
|
-
is_aggregating = "aggregatingmergetree" in engine_lower
|
|
150
149
|
is_replacing = "replacingmergetree" in engine_lower
|
|
151
150
|
is_collapsing = "collapsingmergetree" in engine_lower
|
|
152
|
-
return
|
|
151
|
+
return is_replacing or is_collapsing
|
|
153
152
|
return False
|
|
154
153
|
|
|
155
154
|
def diff_ttl(self, new_ttl: str) -> bool:
|
|
@@ -169,6 +168,10 @@ class TableDetails:
|
|
|
169
168
|
@property
|
|
170
169
|
def sorting_key(self) -> Optional[str]:
|
|
171
170
|
_sorting_key = self.details.get("sorting_key", None)
|
|
171
|
+
# TODO: This should use ENABLED_ENGINES to guess if the sorting key is required or not
|
|
172
|
+
# Also checking this and raising an error in a getter is a bit of an anti-pattern,
|
|
173
|
+
# a data source could have a "wrong" sorting key and we won't be able to even show it in the API.
|
|
174
|
+
# All these checks be performed only on creation time.
|
|
172
175
|
if self.is_replacing_engine() and not _sorting_key:
|
|
173
176
|
raise ValueError(f"SORTING_KEY must be defined for the {self.engine} engine")
|
|
174
177
|
if self.is_mergetree_family():
|
|
@@ -552,7 +552,7 @@ class TinyB:
|
|
|
552
552
|
async def analyze_pipe_node(
|
|
553
553
|
self, pipe_name: str, node: Dict[str, Any], dry_run: str = "false", datasource_name: Optional[str] = None
|
|
554
554
|
):
|
|
555
|
-
params = {
|
|
555
|
+
params = {"include_datafile": "true", "dry_run": dry_run, **node.get("params", node)}
|
|
556
556
|
if "mode" in params:
|
|
557
557
|
params.pop("mode")
|
|
558
558
|
node_name = node["params"]["name"] if node.get("params", None) else node["name"]
|
tinybird_cli-5.20.1.dev0/tinybird/datafile.py → tinybird_cli-5.20.1.dev2/tinybird/datafile_common.py
RENAMED
|
@@ -395,7 +395,7 @@ class CLIGitRelease:
|
|
|
395
395
|
|
|
396
396
|
def is_head_outdated(self, current_ws_commit: str) -> bool:
|
|
397
397
|
try:
|
|
398
|
-
return self.repo.is_ancestor(current_ws_commit, self.
|
|
398
|
+
return self.repo.is_ancestor(current_ws_commit, self.head().commit) # type: ignore
|
|
399
399
|
except GitCommandError:
|
|
400
400
|
raise CLIGitReleaseException(
|
|
401
401
|
FeedbackManager.error_in_git_ancestor_commits(
|
|
@@ -405,7 +405,9 @@ class CLIGitRelease:
|
|
|
405
405
|
|
|
406
406
|
def has_untracked_files(self) -> bool:
|
|
407
407
|
current_pwd = os.getcwd()
|
|
408
|
-
working_dir =
|
|
408
|
+
working_dir = (
|
|
409
|
+
f"{self.repo.working_dir}/" if os.getcwd() != self.repo.working_dir else str(self.repo.working_dir)
|
|
410
|
+
)
|
|
409
411
|
project_path = current_pwd[current_pwd.startswith(working_dir) and len(working_dir) :]
|
|
410
412
|
for untracked_file in self.repo.untracked_files:
|
|
411
413
|
for path in PROJECT_PATHS:
|
|
@@ -426,7 +428,7 @@ class CLIGitRelease:
|
|
|
426
428
|
return self.head().commit.hexsha != current_ws_commit
|
|
427
429
|
|
|
428
430
|
def diff(self, current_ws_commit: str) -> List[Diff]:
|
|
429
|
-
return self.repo.commit(current_ws_commit).diff(self.head(), paths=self.paths)
|
|
431
|
+
return self.repo.commit(current_ws_commit).diff(self.head().commit, paths=self.paths)
|
|
430
432
|
|
|
431
433
|
def diff_datafiles(self, current_ws_commit: str) -> List[Diff]:
|
|
432
434
|
# ignore diffs if:
|
|
@@ -1872,8 +1874,7 @@ def drop_token(url: str) -> str:
|
|
|
1872
1874
|
parsed = urlparse(url)
|
|
1873
1875
|
qs = parse_qs(parsed.query)
|
|
1874
1876
|
qs_simplify = {k: v[0] for k, v in qs.items()} # change several arguments to single one
|
|
1875
|
-
|
|
1876
|
-
del qs_simplify["token"]
|
|
1877
|
+
qs_simplify.pop("token", None)
|
|
1877
1878
|
return f"{parsed.scheme}://{parsed.netloc}{parsed.path}?{urlencode(qs_simplify)}"
|
|
1878
1879
|
|
|
1879
1880
|
|
|
@@ -4849,7 +4850,7 @@ def format_include(file_parts: List[str], doc: Datafile, unroll_includes: bool =
|
|
|
4849
4850
|
assert doc.raw is not None
|
|
4850
4851
|
|
|
4851
4852
|
include = [line for line in doc.raw if "INCLUDE" in line and ".incl" in line]
|
|
4852
|
-
if
|
|
4853
|
+
if include:
|
|
4853
4854
|
file_parts.append(include[0])
|
|
4854
4855
|
file_parts.append(DATAFILE_NEW_LINE)
|
|
4855
4856
|
return file_parts
|
|
@@ -491,7 +491,7 @@ Ready? """
|
|
|
491
491
|
"""\n[2] Go to IAM > Policies. Create a new policy with the following permissions. Please, replace {replacements}:\n\n{access_policy}\n\n(The policy has been copied to your clipboard)\n\n"""
|
|
492
492
|
)
|
|
493
493
|
prompt_s3_iamrole_connection_policy_not_copied = prompt_message(
|
|
494
|
-
"""\n[2] Go to IAM > Policies. Create a new policy with the following permissions. Please, copy this policy and replace <bucket> with your bucket name:\n\n{access_policy}\n\n"""
|
|
494
|
+
"""\n[2] Go to IAM > Policies. Create a new policy with the following permissions. Please, copy this policy and replace <bucket> with your bucket name and <table_name> with your table name:\n\n{access_policy}\n\n"""
|
|
495
495
|
)
|
|
496
496
|
prompt_s3_iamrole_connection_role = prompt_message(
|
|
497
497
|
"""\n[3] Go to IAM > Roles. Create a new IAM Role using the following custom trust policy and attach the access policy you just created in the previous step:\n\n{trust_policy}\n\n(The policy has been copied to your clipboard)\n\n"""
|
|
@@ -67,8 +67,18 @@ class TableIndex:
|
|
|
67
67
|
if index_supported_types:
|
|
68
68
|
for supported_type in INDEX_SUPPORTED_TYPES.get(index_supported_types, []):
|
|
69
69
|
# Convert supported type to regex pattern
|
|
70
|
-
# Replace * with \d+ to match any number
|
|
71
70
|
pattern = supported_type.replace("*", r"\d+")
|
|
71
|
+
|
|
72
|
+
# Special handling for complex types that can have parameters
|
|
73
|
+
if supported_type == "Array":
|
|
74
|
+
pattern = r"Array\(.*\)"
|
|
75
|
+
elif supported_type == "Map":
|
|
76
|
+
pattern = r"Map\(.*\)"
|
|
77
|
+
elif supported_type == "LowCardinality":
|
|
78
|
+
pattern = r"LowCardinality\(.*\)"
|
|
79
|
+
elif supported_type == "Nullable":
|
|
80
|
+
pattern = r"Nullable\(.*\)"
|
|
81
|
+
|
|
72
82
|
if re.match(f"^{pattern}$", col_type):
|
|
73
83
|
return
|
|
74
84
|
raise ValueError(
|
|
@@ -435,8 +435,10 @@ def sql_unescape(x, what=""):
|
|
|
435
435
|
"'testing%'"
|
|
436
436
|
>>> sql_unescape('testing%', '$')
|
|
437
437
|
"'testing\\\\%'"
|
|
438
|
+
>>> sql_unescape('testing"')
|
|
439
|
+
'\\'testing"\\''
|
|
438
440
|
"""
|
|
439
|
-
return Expression("'" +
|
|
441
|
+
return Expression("'" + sqlescape_for_string_expression(x).replace(f"\\{what}", what) + "'")
|
|
440
442
|
|
|
441
443
|
|
|
442
444
|
def split_to_array(x, default="", separator: str = ","):
|
|
@@ -1386,6 +1388,7 @@ _namespace = {
|
|
|
1386
1388
|
"close": None,
|
|
1387
1389
|
"print": None,
|
|
1388
1390
|
"input": None,
|
|
1391
|
+
"id": None,
|
|
1389
1392
|
}
|
|
1390
1393
|
|
|
1391
1394
|
|
|
@@ -1657,7 +1660,7 @@ def get_var_data(content, node_id=None):
|
|
|
1657
1660
|
# It will be overriden in later definitions or left as is otherwise.
|
|
1658
1661
|
# args[0] check is used to avoid adding unnamed parameters found in
|
|
1659
1662
|
# templates like: `split_to_array('')`
|
|
1660
|
-
if
|
|
1663
|
+
if args and isinstance(args[0], list):
|
|
1661
1664
|
raise ValueError(f'"{args[0]}" can not be used as a variable name')
|
|
1662
1665
|
if len(args) > 0 and args[0] not in vars and args[0]:
|
|
1663
1666
|
vars[args[0]] = {
|
|
@@ -1669,7 +1672,7 @@ def get_var_data(content, node_id=None):
|
|
|
1669
1672
|
if "default" not in kwargs:
|
|
1670
1673
|
default = kwargs.get("default", args[2] if len(args) > 2 and args[2] else None)
|
|
1671
1674
|
kwargs["default"] = check_default_value(default)
|
|
1672
|
-
if
|
|
1675
|
+
if args:
|
|
1673
1676
|
if isinstance(args[0], list):
|
|
1674
1677
|
raise ValueError(f'"{args[0]}" can not be used as a variable name')
|
|
1675
1678
|
vars[args[0]] = {
|
|
@@ -1678,7 +1681,7 @@ def get_var_data(content, node_id=None):
|
|
|
1678
1681
|
}
|
|
1679
1682
|
elif func in parameter_types:
|
|
1680
1683
|
# avoid variable names to be None
|
|
1681
|
-
if
|
|
1684
|
+
if args and args[0] is not None:
|
|
1682
1685
|
# if this is a cast use the function name to get the type
|
|
1683
1686
|
if "default" not in kwargs:
|
|
1684
1687
|
default = kwargs.get("default", args[1] if len(args) > 1 else None)
|
|
@@ -2018,6 +2021,7 @@ def render_sql_template(
|
|
|
2018
2021
|
test_mode: bool = False,
|
|
2019
2022
|
name: Optional[str] = None,
|
|
2020
2023
|
local_variables: Optional[dict] = None,
|
|
2024
|
+
secrets_in_test_mode: Optional[bool] = True,
|
|
2021
2025
|
) -> Tuple[str, TemplateExecutionResults, list]:
|
|
2022
2026
|
"""
|
|
2023
2027
|
>>> render_sql_template("select * from table where f = {{Float32(foo)}}", { 'foo': -1 })
|
|
@@ -2300,7 +2304,7 @@ def render_sql_template(
|
|
|
2300
2304
|
if secrets:
|
|
2301
2305
|
v.update({"tb_secrets": secrets})
|
|
2302
2306
|
|
|
2303
|
-
if is_tb_secret:
|
|
2307
|
+
if is_tb_secret and secrets_in_test_mode:
|
|
2304
2308
|
v.update({TB_SECRET_IN_TEST_MODE: None})
|
|
2305
2309
|
|
|
2306
2310
|
v.update(type_fns_check)
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import copy
|
|
2
2
|
import logging
|
|
3
|
+
import re
|
|
3
4
|
from collections import defaultdict
|
|
4
5
|
from datetime import datetime
|
|
5
6
|
from functools import lru_cache
|
|
@@ -42,10 +43,88 @@ class InvalidResource(ValueError):
|
|
|
42
43
|
self.table = table
|
|
43
44
|
|
|
44
45
|
|
|
46
|
+
class UnoptimizedJoinException(Exception):
|
|
47
|
+
def __init__(self, sql: str):
|
|
48
|
+
self.sql = sql
|
|
49
|
+
self.msg = f"Materialized node SQL contains a join that is not optimized: {sql}"
|
|
50
|
+
self.documentation = (
|
|
51
|
+
"/docs/work-with-data/optimization/opt201-fix-mistakes#understanding-the-materialized-join-issue"
|
|
52
|
+
)
|
|
53
|
+
super().__init__(self.msg)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
ChQueryTable = Tuple[Optional[str], Optional[str], Optional[str]]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def get_left_table(sql: str, default_database: Optional[str] = None) -> ChQueryTable:
|
|
60
|
+
if default_database is None:
|
|
61
|
+
left_table = chquery.get_left_table(sql)
|
|
62
|
+
else:
|
|
63
|
+
left_table = chquery.get_left_table(sql, default_database=default_database)
|
|
64
|
+
return left_table
|
|
65
|
+
|
|
66
|
+
|
|
45
67
|
def format_sql(sql: str) -> str:
|
|
46
68
|
return chquery.format(sql)
|
|
47
69
|
|
|
48
70
|
|
|
71
|
+
def explain_plan(sql: str) -> str:
|
|
72
|
+
return chquery.explain_ast(sql)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def has_join(sql: str) -> bool:
|
|
76
|
+
return any(line.rstrip().startswith("TableJoin") for line in explain_plan(sql).split())
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def has_unoptimized_join(sql: str, left_table: Optional[Union[Tuple[str, str], Tuple[str, str, str]]] = None) -> None:
|
|
80
|
+
"""
|
|
81
|
+
Check if a SQL query contains an unoptimized join.
|
|
82
|
+
A join is considered optimized if the right table is filtered by the left table's data.
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
sql: The SQL query to check
|
|
86
|
+
left_table: Optional tuple of (database, table) for the left table
|
|
87
|
+
|
|
88
|
+
Raises:
|
|
89
|
+
UnoptimizedJoin: If an unoptimized join is found
|
|
90
|
+
"""
|
|
91
|
+
# TODO: We should check that we are filtering the right table by the left table's data
|
|
92
|
+
# TODO: We should check if using EXPLAIN AST is better than using regex
|
|
93
|
+
|
|
94
|
+
number_of_joins = sum(1 for line in explain_plan(sql).split() if line.rstrip().startswith("TableJoin"))
|
|
95
|
+
if number_of_joins == 0:
|
|
96
|
+
return
|
|
97
|
+
|
|
98
|
+
if not left_table:
|
|
99
|
+
left_table = chquery.get_left_table(sql)
|
|
100
|
+
if not left_table:
|
|
101
|
+
return
|
|
102
|
+
|
|
103
|
+
# Find all JOIN clauses with subqueries
|
|
104
|
+
# This pattern matches anything between JOIN and ON/USING
|
|
105
|
+
join_pattern = r"(?:LEFT\s+|RIGHT\s+|INNER\s+|FULL\s+OUTER\s+)?JOIN\s*\((.*?)\)\s+(?:AS\s+\w+)?\s*(?:ON|USING)"
|
|
106
|
+
|
|
107
|
+
# Find all joins with subqueries
|
|
108
|
+
join_matches = list(re.finditer(join_pattern, sql, re.IGNORECASE | re.DOTALL))
|
|
109
|
+
|
|
110
|
+
if number_of_joins != len(join_matches):
|
|
111
|
+
logging.debug(f"number_of_joins: {number_of_joins}, join_matches: {join_matches}")
|
|
112
|
+
raise UnoptimizedJoinException(sql)
|
|
113
|
+
|
|
114
|
+
# If no joins with subqueries found, probably is an unoptimized join
|
|
115
|
+
if not join_matches:
|
|
116
|
+
raise UnoptimizedJoinException(sql)
|
|
117
|
+
|
|
118
|
+
# Check if the left table is referenced in the subquery
|
|
119
|
+
left_table_ref = f"{left_table[0]}.{left_table[1]}"
|
|
120
|
+
|
|
121
|
+
for match in join_matches:
|
|
122
|
+
subquery = match.group(1) # Get the captured subquery
|
|
123
|
+
logging.debug(f"subquery: {subquery} left_table_ref: {left_table_ref}")
|
|
124
|
+
if left_table_ref not in subquery:
|
|
125
|
+
raise UnoptimizedJoinException(sql)
|
|
126
|
+
|
|
127
|
+
|
|
49
128
|
def format_where_for_mutation_command(where_clause: str) -> str:
|
|
50
129
|
"""
|
|
51
130
|
>>> format_where_for_mutation_command("numnights = 99")
|
|
@@ -148,25 +227,56 @@ def sql_get_used_tables(
|
|
|
148
227
|
|
|
149
228
|
|
|
150
229
|
class ReplacementsDict(dict):
|
|
230
|
+
def __init__(self, *args, enabled_table_functions=None, **kwargs):
|
|
231
|
+
self.enabled_table_functions = enabled_table_functions
|
|
232
|
+
super().__init__(*args, **kwargs)
|
|
233
|
+
|
|
151
234
|
def __getitem__(self, key):
|
|
152
235
|
v = super().__getitem__(key)
|
|
153
236
|
if isinstance(v, tuple):
|
|
154
237
|
k, r = v
|
|
155
238
|
if callable(r):
|
|
156
|
-
r = r()
|
|
239
|
+
r = update_callable_signature(r)(self.enabled_table_functions)
|
|
157
240
|
super().__setitem__(key, (k, r))
|
|
158
241
|
return k, r
|
|
159
242
|
if callable(v):
|
|
160
|
-
v = v()
|
|
243
|
+
v = update_callable_signature(v)(self.enabled_table_functions)
|
|
161
244
|
super().__setitem__(key, v)
|
|
162
245
|
return v
|
|
163
246
|
|
|
164
247
|
|
|
165
|
-
def
|
|
248
|
+
def update_callable_signature(func):
|
|
249
|
+
"""
|
|
250
|
+
Utility function to provide backward compatibility for callable functions
|
|
251
|
+
that don't accept the enabled_table_functions parameter.
|
|
252
|
+
"""
|
|
253
|
+
if callable(func):
|
|
254
|
+
|
|
255
|
+
def wrapper(enabled_table_functions=None):
|
|
256
|
+
# Check if the function accepts the enabled_table_functions parameter
|
|
257
|
+
import inspect
|
|
258
|
+
|
|
259
|
+
sig = inspect.signature(func)
|
|
260
|
+
if len(sig.parameters) == 0:
|
|
261
|
+
# Old-style callable with no parameters
|
|
262
|
+
return func()
|
|
263
|
+
else:
|
|
264
|
+
# New-style callable that accepts enabled_table_functions
|
|
265
|
+
return func(enabled_table_functions)
|
|
266
|
+
|
|
267
|
+
return wrapper
|
|
268
|
+
return func
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def tables_or_sql(replacement: dict, table_functions=False, function_allow_list=None) -> set:
|
|
166
272
|
try:
|
|
167
273
|
return set(
|
|
168
274
|
sql_get_used_tables(
|
|
169
|
-
replacement[1],
|
|
275
|
+
replacement[1],
|
|
276
|
+
default_database=replacement[0],
|
|
277
|
+
raising=True,
|
|
278
|
+
table_functions=table_functions,
|
|
279
|
+
function_allow_list=frozenset(function_allow_list),
|
|
170
280
|
)
|
|
171
281
|
)
|
|
172
282
|
except Exception as e:
|
|
@@ -263,6 +373,7 @@ def replace_tables(
|
|
|
263
373
|
_enabled_table_functions = ENABLED_TABLE_FUNCTIONS
|
|
264
374
|
else:
|
|
265
375
|
_enabled_table_functions = ENABLED_TABLE_FUNCTIONS.union(set(function_allow_list))
|
|
376
|
+
_replacements.enabled_table_functions = frozenset(_enabled_table_functions)
|
|
266
377
|
while _tables:
|
|
267
378
|
table = _tables.pop()
|
|
268
379
|
if len(table) == 3:
|
|
@@ -276,7 +387,9 @@ def replace_tables(
|
|
|
276
387
|
seen_tables.add(table)
|
|
277
388
|
if table in _replacements:
|
|
278
389
|
replacement = _replacements[table]
|
|
279
|
-
dependent_tables = tables_or_sql(
|
|
390
|
+
dependent_tables = tables_or_sql(
|
|
391
|
+
replacement, table_functions=check_functions, function_allow_list=_enabled_table_functions
|
|
392
|
+
)
|
|
280
393
|
deps[table] |= {(d[0], d[1]) for d in dependent_tables}
|
|
281
394
|
for dependent_table in list(dependent_tables):
|
|
282
395
|
if len(dependent_table) == 3:
|
|
@@ -74,9 +74,8 @@ async def auth(ctx: click.Context, token: str, host: str, region: str, connector
|
|
|
74
74
|
# do a clean auth
|
|
75
75
|
if not token and not ctx.parent.params.get("token") and not env_token:
|
|
76
76
|
config.set_token(None)
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
click.echo(FeedbackManager.info_reading_from_env(value="token", envvar="TB_TOKEN"))
|
|
77
|
+
elif env_token and not token:
|
|
78
|
+
click.echo(FeedbackManager.info_reading_from_env(value="token", envvar="TB_TOKEN"))
|
|
80
79
|
|
|
81
80
|
regions: Optional[List[Region]] = None
|
|
82
81
|
try_all_regions = True
|
|
@@ -11,7 +11,7 @@ import aiofiles
|
|
|
11
11
|
import click
|
|
12
12
|
import yaml
|
|
13
13
|
|
|
14
|
-
from tinybird.
|
|
14
|
+
from tinybird.datafile_common import create_release, wait_job
|
|
15
15
|
from tinybird.feedback_manager import FeedbackManager
|
|
16
16
|
from tinybird.tb_cli_modules.cli import cli
|
|
17
17
|
from tinybird.tb_cli_modules.common import (
|
|
@@ -597,9 +597,8 @@ async def regression_tests(
|
|
|
597
597
|
await print_branch_regression_tests_summary(client, job_id, config["host"])
|
|
598
598
|
except Exception as e:
|
|
599
599
|
raise CLIBranchException(FeedbackManager.error_exception(error=str(e)))
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
await _run_regression(type="coverage", wait=wait, run_in_main=main)
|
|
600
|
+
elif not ctx.invoked_subcommand:
|
|
601
|
+
await _run_regression(type="coverage", wait=wait, run_in_main=main)
|
|
603
602
|
|
|
604
603
|
|
|
605
604
|
async def _run_regression(
|
|
@@ -28,7 +28,7 @@ from tinybird.client import (
|
|
|
28
28
|
TinyB,
|
|
29
29
|
)
|
|
30
30
|
from tinybird.config import CURRENT_VERSION, SUPPORTED_CONNECTORS, VERSION, FeatureFlags, get_config
|
|
31
|
-
from tinybird.
|
|
31
|
+
from tinybird.datafile_common import (
|
|
32
32
|
AlreadyExistsException,
|
|
33
33
|
CLIGitRelease,
|
|
34
34
|
CLIGitReleaseException,
|
|
@@ -431,10 +431,9 @@ async def init(
|
|
|
431
431
|
error = True
|
|
432
432
|
else:
|
|
433
433
|
click.echo(FeedbackManager.info_cicd_already_exists(provider=cicd_provider.name))
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
await init_cicd(client, path=cli_git_release.working_dir(), data_project_dir=data_project_dir)
|
|
434
|
+
elif cicd:
|
|
435
|
+
data_project_dir = os.path.relpath(folder, cli_git_release.working_dir())
|
|
436
|
+
await init_cicd(client, path=cli_git_release.working_dir(), data_project_dir=data_project_dir)
|
|
438
437
|
|
|
439
438
|
if final_response:
|
|
440
439
|
if error:
|
|
@@ -1463,12 +1462,11 @@ async def deploy(
|
|
|
1463
1462
|
)
|
|
1464
1463
|
else:
|
|
1465
1464
|
click.echo(FeedbackManager.info_minor_patch_release_with_autopromote(version=new_version))
|
|
1466
|
-
|
|
1467
|
-
if
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
click.echo(FeedbackManager.info_minor_patch_release_no_autopromote(version=new_version))
|
|
1465
|
+
elif show_feedback:
|
|
1466
|
+
if dry_run:
|
|
1467
|
+
click.echo(FeedbackManager.info_dry_minor_patch_release_no_autopromote(version=new_version))
|
|
1468
|
+
else:
|
|
1469
|
+
click.echo(FeedbackManager.info_minor_patch_release_no_autopromote(version=new_version))
|
|
1472
1470
|
new_release = True
|
|
1473
1471
|
|
|
1474
1472
|
if new_release:
|
|
@@ -293,7 +293,7 @@ async def connection_create_snowflake(
|
|
|
293
293
|
if not warehouse:
|
|
294
294
|
warehouses = await client.get_snowflake_warehouses(account, username, password, role) or []
|
|
295
295
|
warehouses_names = [w["name"] for w in warehouses]
|
|
296
|
-
default_warehouse = warehouses_names[0] if
|
|
296
|
+
default_warehouse = warehouses_names[0] if warehouses_names else ""
|
|
297
297
|
warehouse = click.prompt(
|
|
298
298
|
"Warehouse (optional)",
|
|
299
299
|
type=click.types.Choice(warehouses_names, case_sensitive=False),
|
|
@@ -20,7 +20,7 @@ from tinybird.tb_cli_modules.config import CLIConfig
|
|
|
20
20
|
if TYPE_CHECKING:
|
|
21
21
|
from tinybird.connectors import Connector
|
|
22
22
|
|
|
23
|
-
from tinybird.
|
|
23
|
+
from tinybird.datafile_common import get_name_version, wait_job
|
|
24
24
|
from tinybird.feedback_manager import FeedbackManager
|
|
25
25
|
from tinybird.tb_cli_modules.branch import warn_if_in_live
|
|
26
26
|
from tinybird.tb_cli_modules.cli import cli
|
|
@@ -7,7 +7,7 @@ import aiofiles
|
|
|
7
7
|
import click
|
|
8
8
|
from click import Context
|
|
9
9
|
|
|
10
|
-
from tinybird.
|
|
10
|
+
from tinybird.datafile_common import color_diff, format_datasource, format_pipe, is_file_a_datasource, peek
|
|
11
11
|
from tinybird.feedback_manager import FeedbackManager
|
|
12
12
|
from tinybird.tb_cli_modules.cli import cli
|
|
13
13
|
from tinybird.tb_cli_modules.common import coro
|
|
@@ -29,10 +29,11 @@ from tinybird.tb_cli_modules.common import coro
|
|
|
29
29
|
default=False,
|
|
30
30
|
help="Formats local file, prints the diff and exits 1 if different, 0 if equal",
|
|
31
31
|
)
|
|
32
|
+
@click.option("--no-color", is_flag=True, default=False, help="Don't colorize diff")
|
|
32
33
|
@click.pass_context
|
|
33
34
|
@coro
|
|
34
35
|
async def fmt(
|
|
35
|
-
ctx: Context, filenames: List[str], line_length: int, dry_run: bool, yes: bool, diff: bool
|
|
36
|
+
ctx: Context, filenames: List[str], line_length: int, dry_run: bool, yes: bool, diff: bool, no_color: bool
|
|
36
37
|
) -> Optional[str]:
|
|
37
38
|
"""
|
|
38
39
|
Formats a .datasource, .pipe or .incl file
|
|
@@ -66,7 +67,8 @@ async def fmt(
|
|
|
66
67
|
diff_result = difflib.unified_diff(
|
|
67
68
|
lines_file, lines_fmt, fromfile=f"{Path(filename).name} local", tofile="fmt datafile"
|
|
68
69
|
)
|
|
69
|
-
|
|
70
|
+
if not no_color:
|
|
71
|
+
diff_result = color_diff(diff_result)
|
|
70
72
|
not_empty, diff_lines = peek(diff_result)
|
|
71
73
|
if not_empty:
|
|
72
74
|
sys.stdout.writelines(diff_lines)
|
|
@@ -83,7 +85,7 @@ async def fmt(
|
|
|
83
85
|
|
|
84
86
|
click.echo(FeedbackManager.success_generated_local_file(file=filename))
|
|
85
87
|
|
|
86
|
-
if
|
|
88
|
+
if failed:
|
|
87
89
|
click.echo(FeedbackManager.error_failed_to_format_files(number=len(failed)))
|
|
88
90
|
for f in failed:
|
|
89
91
|
click.echo(f"tb fmt {f} --yes")
|
|
@@ -16,7 +16,7 @@ from click import Context
|
|
|
16
16
|
import tinybird.context as context
|
|
17
17
|
from tinybird.client import AuthNoTokenException, DoesNotExistException, TinyB
|
|
18
18
|
from tinybird.config import DEFAULT_API_HOST, FeatureFlags
|
|
19
|
-
from tinybird.
|
|
19
|
+
from tinybird.datafile_common import PipeNodeTypes, PipeTypes, folder_push, get_name_version, process_file, wait_job
|
|
20
20
|
from tinybird.feedback_manager import FeedbackManager
|
|
21
21
|
from tinybird.tb_cli_modules.branch import warn_if_in_live
|
|
22
22
|
from tinybird.tb_cli_modules.cli import cli
|
{tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/tinyunit/tinyunit.py
RENAMED
|
@@ -317,7 +317,7 @@ def test_run_summary(results: List[TestSummaryResults], only_fail: bool = False,
|
|
|
317
317
|
if test.error:
|
|
318
318
|
click.secho(test.error, fg=test.status.color, bold=True, nl=True, err=True)
|
|
319
319
|
|
|
320
|
-
if
|
|
320
|
+
if total_counts:
|
|
321
321
|
click.echo("\nTotals:")
|
|
322
322
|
for key_status, value_total in total_counts.items():
|
|
323
323
|
code_summary = f"Total {key_status.description}: {value_total}"
|
|
@@ -10,7 +10,7 @@ from click import Context
|
|
|
10
10
|
|
|
11
11
|
from tinybird.client import CanNotBeDeletedException, DoesNotExistException, TinyB
|
|
12
12
|
from tinybird.config import get_display_host
|
|
13
|
-
from tinybird.
|
|
13
|
+
from tinybird.datafile_common import PipeTypes
|
|
14
14
|
from tinybird.feedback_manager import FeedbackManager
|
|
15
15
|
from tinybird.tb_cli_modules.cli import cli
|
|
16
16
|
from tinybird.tb_cli_modules.common import (
|
|
@@ -234,20 +234,19 @@ async def create_workspace(
|
|
|
234
234
|
if not organization:
|
|
235
235
|
raise CLIWorkspaceException(FeedbackManager.error_organization_not_found(organization_id=organization_id))
|
|
236
236
|
organization_name = organization.get("name")
|
|
237
|
+
elif len(organizations) == 0:
|
|
238
|
+
click.echo(FeedbackManager.warning_none_organization(ui_host=ui_host))
|
|
239
|
+
elif len(organizations) == 1:
|
|
240
|
+
organization_id = organizations[0].get("id")
|
|
241
|
+
organization_name = organizations[0].get("name")
|
|
237
242
|
else:
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
organization_id =
|
|
242
|
-
organization_name =
|
|
243
|
+
sorted_organizations = sort_organizations_by_user(organizations, user_email=config.get_user_email())
|
|
244
|
+
current_organization = await ask_for_organization_interactively(sorted_organizations)
|
|
245
|
+
if current_organization:
|
|
246
|
+
organization_id = current_organization.get("id")
|
|
247
|
+
organization_name = current_organization.get("name")
|
|
243
248
|
else:
|
|
244
|
-
|
|
245
|
-
current_organization = await ask_for_organization_interactively(sorted_organizations)
|
|
246
|
-
if current_organization:
|
|
247
|
-
organization_id = current_organization.get("id")
|
|
248
|
-
organization_name = current_organization.get("name")
|
|
249
|
-
else:
|
|
250
|
-
return
|
|
249
|
+
return
|
|
251
250
|
|
|
252
251
|
# If we have at least workspace_name, we start the non interactive
|
|
253
252
|
# process, creating an empty workspace
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: tinybird_cli
|
|
3
|
-
Version: 5.20.1.
|
|
3
|
+
Version: 5.20.1.dev2
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/cli
|
|
6
6
|
Author: Tinybird
|
|
@@ -61,6 +61,16 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
|
|
|
61
61
|
Changelog
|
|
62
62
|
----------
|
|
63
63
|
|
|
64
|
+
5.20.1.dev2
|
|
65
|
+
***********
|
|
66
|
+
|
|
67
|
+
- `Fixed` Templating when using `id` parameter and the value was not provided.
|
|
68
|
+
|
|
69
|
+
5.20.1.dev1
|
|
70
|
+
***********
|
|
71
|
+
|
|
72
|
+
- `Added` When trying to push a materialized view with an unoptimized join, the CLI will now warn the user.
|
|
73
|
+
|
|
64
74
|
5.19.0
|
|
65
75
|
***********
|
|
66
76
|
|
|
@@ -198,6 +208,7 @@ Changelog
|
|
|
198
208
|
- `Added` `tb tag` commands to manage tags from the CLI.
|
|
199
209
|
- `Added` support to `TAGS` in `tb fmt`.
|
|
200
210
|
- `Added` support to `TAGS` in `tb pull` and `tb push`. Allows tagging resources for filtering in the UI.
|
|
211
|
+
- `Changed` Improved message when creating a DynamoDB connection
|
|
201
212
|
|
|
202
213
|
5.7.0
|
|
203
214
|
**********
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird/tb_cli_modules/workspace_members.py
RENAMED
|
File without changes
|
|
File without changes
|
{tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird_cli.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
{tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev2}/tinybird_cli.egg-info/entry_points.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|