tinybird 4.6.19.dev0__py3-none-any.whl → 4.6.20.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tinybird/sql.py +3 -4
- tinybird/sql_template.py +52 -71
- tinybird/sql_template_fmt.py +8 -13
- tinybird/sql_toolset.py +3 -5
- tinybird/syncasync.py +9 -11
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/client.py +14 -23
- tinybird/tb/modules/cicd.py +1 -0
- tinybird/tb/modules/cli.py +4 -2
- tinybird/tb/modules/common.py +8 -12
- tinybird/tb/modules/config.py +1 -2
- tinybird/tb/modules/connection_kafka.py +1 -2
- tinybird/tb/modules/connection_s3.py +13 -18
- tinybird/tb/modules/create.py +3 -6
- tinybird/tb/modules/datafile/build.py +2 -3
- tinybird/tb/modules/datafile/build_pipe.py +1 -4
- tinybird/tb/modules/datafile/diff.py +2 -1
- tinybird/tb/modules/datafile/format_common.py +9 -10
- tinybird/tb/modules/datafile/format_connection.py +1 -2
- tinybird/tb/modules/datafile/format_datasource.py +1 -2
- tinybird/tb/modules/datafile/format_pipe.py +1 -2
- tinybird/tb/modules/datafile/pipe_checker.py +6 -7
- tinybird/tb/modules/datafile/playground.py +3 -4
- tinybird/tb/modules/datasource.py +7 -8
- tinybird/tb/modules/deployment_common.py +3 -5
- {tinybird-4.6.19.dev0.dist-info → tinybird-4.6.20.dev0.dist-info}/METADATA +6 -1
- {tinybird-4.6.19.dev0.dist-info → tinybird-4.6.20.dev0.dist-info}/RECORD +30 -30
- {tinybird-4.6.19.dev0.dist-info → tinybird-4.6.20.dev0.dist-info}/WHEEL +0 -0
- {tinybird-4.6.19.dev0.dist-info → tinybird-4.6.20.dev0.dist-info}/entry_points.txt +0 -0
- {tinybird-4.6.19.dev0.dist-info → tinybird-4.6.20.dev0.dist-info}/top_level.txt +0 -0
tinybird/sql.py
CHANGED
|
@@ -675,10 +675,9 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
|
|
|
675
675
|
)
|
|
676
676
|
advance_single_char()
|
|
677
677
|
return schema[begin:i]
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
return get_backticked()
|
|
678
|
+
# backticked name
|
|
679
|
+
advance_single_char()
|
|
680
|
+
return get_backticked()
|
|
682
681
|
|
|
683
682
|
def parse_expr(lookup: Iterable[SyntaxExpr]) -> str:
|
|
684
683
|
nonlocal i, line, pos
|
tinybird/sql_template.py
CHANGED
|
@@ -142,8 +142,7 @@ def transform_type(
|
|
|
142
142
|
if x is None:
|
|
143
143
|
if defined:
|
|
144
144
|
raise SQLTemplateException(REQUIRED_PARAM_NOT_DEFINED, documentation="/cli/advanced-templates.html")
|
|
145
|
-
|
|
146
|
-
return None
|
|
145
|
+
return None
|
|
147
146
|
if tester == "String":
|
|
148
147
|
if x is not None:
|
|
149
148
|
return transform(x)
|
|
@@ -167,12 +166,11 @@ def _and(*args, **kwargs):
|
|
|
167
166
|
tk = k.rsplit("__", 1)
|
|
168
167
|
if len(tk) == 1:
|
|
169
168
|
return "="
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
)
|
|
169
|
+
if tk[1] in operands:
|
|
170
|
+
return operands[tk[1]]
|
|
171
|
+
raise SQLTemplateException(
|
|
172
|
+
f"operand {tk[1]} not supported", documentation="/cli/advanced-templates.html#sql_and"
|
|
173
|
+
)
|
|
176
174
|
|
|
177
175
|
return Expression(
|
|
178
176
|
" and ".join([f"{_name(k)} {_op(k)} {expression_wrapper(v, k)}" for k, v in kwargs.items() if v is not None])
|
|
@@ -277,8 +275,7 @@ def columns(x, default=None, fn=None):
|
|
|
277
275
|
|
|
278
276
|
if fn:
|
|
279
277
|
return Expression(",".join(f"{fn}({str(column(c, c))}) as {c}" for c in _columns))
|
|
280
|
-
|
|
281
|
-
return Expression(",".join(str(column(c, c)) for c in _columns))
|
|
278
|
+
return Expression(",".join(str(column(c, c)) for c in _columns))
|
|
282
279
|
|
|
283
280
|
|
|
284
281
|
def column(x, default=None):
|
|
@@ -368,11 +365,10 @@ def boolean(x, default=None):
|
|
|
368
365
|
if default is None:
|
|
369
366
|
return 0
|
|
370
367
|
return boolean(default)
|
|
371
|
-
|
|
368
|
+
if isinstance(x, Placeholder):
|
|
372
369
|
return boolean(default)
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
return 0
|
|
370
|
+
if isinstance(x, str) and (x == "0" or x.lower() == "false"):
|
|
371
|
+
return 0
|
|
376
372
|
|
|
377
373
|
return int(bool(x))
|
|
378
374
|
|
|
@@ -403,8 +399,7 @@ def array_type(types):
|
|
|
403
399
|
raise SQLTemplateException(
|
|
404
400
|
REQUIRED_PARAM_NOT_DEFINED, documentation="/cli/advanced-templates.html"
|
|
405
401
|
)
|
|
406
|
-
|
|
407
|
-
return None
|
|
402
|
+
return None
|
|
408
403
|
values = []
|
|
409
404
|
list_values = x if type(x) == list else x.split(",") # noqa: E721
|
|
410
405
|
for i, t in enumerate(list_values):
|
|
@@ -744,12 +739,11 @@ def __date_diff(
|
|
|
744
739
|
|
|
745
740
|
if unit == "days":
|
|
746
741
|
return int(diff / 86400)
|
|
747
|
-
|
|
742
|
+
if unit == "hours":
|
|
748
743
|
return int(diff / 3600)
|
|
749
|
-
|
|
744
|
+
if unit == "minutes":
|
|
750
745
|
return int(diff / 60)
|
|
751
|
-
|
|
752
|
-
return int(diff)
|
|
746
|
+
return int(diff)
|
|
753
747
|
except Exception:
|
|
754
748
|
if none_if_error:
|
|
755
749
|
return None
|
|
@@ -1348,9 +1342,9 @@ def escape_single_quote_str(s):
|
|
|
1348
1342
|
def expression_wrapper(x, name, escape_arrays: bool = False):
|
|
1349
1343
|
if type(x) in (unicode_type, bytes, str):
|
|
1350
1344
|
return "'" + sqlescape_for_string_expression(x) + "'"
|
|
1351
|
-
|
|
1345
|
+
if isinstance(x, Placeholder):
|
|
1352
1346
|
return "'__no_value__'"
|
|
1353
|
-
|
|
1347
|
+
if isinstance(x, Comment):
|
|
1354
1348
|
return "-- {x} \n"
|
|
1355
1349
|
if x is None:
|
|
1356
1350
|
truncated_name = name[:20] + "..." if len(name) > 20 else name
|
|
@@ -1361,10 +1355,7 @@ def expression_wrapper(x, name, escape_arrays: bool = False):
|
|
|
1361
1355
|
logging.warning(f"expression_wrapper -> list :{x}:")
|
|
1362
1356
|
|
|
1363
1357
|
try:
|
|
1364
|
-
|
|
1365
|
-
f"[{','.join(escape_single_quote_str(item) if isinstance(item, str) else str(item) for item in x)}]"
|
|
1366
|
-
)
|
|
1367
|
-
return result
|
|
1358
|
+
return f"[{','.join(escape_single_quote_str(item) if isinstance(item, str) else str(item) for item in x)}]"
|
|
1368
1359
|
except Exception as e:
|
|
1369
1360
|
logging.error(f"Error escaping array: {e}")
|
|
1370
1361
|
return x
|
|
@@ -1425,20 +1416,18 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
|
|
|
1425
1416
|
# secret available: Always use workspace secret regardless of test mode
|
|
1426
1417
|
template_execution_results.add_ch_param(x)
|
|
1427
1418
|
return Symbol("{" + sqlescape(x) + ": String}")
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
f"Cannot access secret '{x}'. Check the secret exists in the Workspace and the token has the required scope."
|
|
1441
|
-
)
|
|
1419
|
+
# secret not available: Check test mode and defaults
|
|
1420
|
+
is_test_mode = TB_SECRET_IN_TEST_MODE in template_execution_results
|
|
1421
|
+
if default is not None:
|
|
1422
|
+
# Use provided default value
|
|
1423
|
+
return default
|
|
1424
|
+
if is_test_mode:
|
|
1425
|
+
# In test mode without default - return placeholder
|
|
1426
|
+
return Symbol("{" + sqlescape(x) + ": String}")
|
|
1427
|
+
# Not in test mode, no secret, no default - raise error
|
|
1428
|
+
raise SQLTemplateException(
|
|
1429
|
+
f"Cannot access secret '{x}'. Check the secret exists in the Workspace and the token has the required scope."
|
|
1430
|
+
)
|
|
1442
1431
|
except Exception:
|
|
1443
1432
|
raise SQLTemplateException(
|
|
1444
1433
|
f"Cannot access secret '{x}'. Check the secret exists in the Workspace and the token has the required scope."
|
|
@@ -1522,14 +1511,12 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
|
|
|
1522
1511
|
|
|
1523
1512
|
if line:
|
|
1524
1513
|
raise SQLTemplateException(f"{message.strip()} line {line[0]}")
|
|
1525
|
-
|
|
1526
|
-
raise SQLTemplateException(f"{message.strip()}")
|
|
1514
|
+
raise SQLTemplateException(f"{message.strip()}")
|
|
1527
1515
|
except Exception as e:
|
|
1528
1516
|
if isinstance(e, SQLTemplateException):
|
|
1529
1517
|
raise e
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
raise ValueError(str(e))
|
|
1518
|
+
logging.exception(f"Error on unbound local error: {e}")
|
|
1519
|
+
raise ValueError(str(e))
|
|
1533
1520
|
except TypeError as e:
|
|
1534
1521
|
error = str(e)
|
|
1535
1522
|
if "not supported between instances of 'Placeholder' and " in str(e):
|
|
@@ -1960,29 +1947,27 @@ def get_var_data(content, node_id=None):
|
|
|
1960
1947
|
# ast.Constant nodes, so .value holds str/bytes/int/float/bool/None.
|
|
1961
1948
|
if type(x) == ast.Constant: # noqa: E721
|
|
1962
1949
|
return x.value
|
|
1963
|
-
|
|
1950
|
+
if type(x) == ast.Name: # noqa: E721
|
|
1964
1951
|
return x.id
|
|
1965
|
-
|
|
1952
|
+
if type(x) == ast.List: # noqa: E721
|
|
1966
1953
|
# List can hold different types
|
|
1967
1954
|
return _get_list_var_data(x)
|
|
1968
|
-
|
|
1955
|
+
if type(x) == ast.BinOp: # noqa: E721
|
|
1969
1956
|
# in this case there could be several variables
|
|
1970
1957
|
# if that's the case the left one is the main
|
|
1971
1958
|
r = node_to_value(x.left)
|
|
1972
1959
|
if not r:
|
|
1973
1960
|
r = node_to_value(x.right)
|
|
1974
1961
|
return r
|
|
1975
|
-
|
|
1962
|
+
if type(x) == ast.UnaryOp and type(x.operand) == ast.Constant: # noqa: E721
|
|
1976
1963
|
if type(x.op) == ast.USub: # noqa: E721
|
|
1977
1964
|
return x.operand.value * -1
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
# don't let this ruin the parsing
|
|
1985
|
-
pass
|
|
1965
|
+
return x.operand.value
|
|
1966
|
+
try:
|
|
1967
|
+
return x.id
|
|
1968
|
+
except Exception:
|
|
1969
|
+
# don't let this ruin the parsing
|
|
1970
|
+
pass
|
|
1986
1971
|
return None
|
|
1987
1972
|
|
|
1988
1973
|
def _get_list_var_data(x):
|
|
@@ -1994,7 +1979,7 @@ def get_var_data(content, node_id=None):
|
|
|
1994
1979
|
# ast.Bytes/ast.Str/ast.Num/ast.NameConstant (removed in 3.14).
|
|
1995
1980
|
if type(first_elem) == ast.Constant: # noqa: E721
|
|
1996
1981
|
return [elem.value for elem in x.elts]
|
|
1997
|
-
|
|
1982
|
+
if type(first_elem) == ast.Name: # noqa: E721
|
|
1998
1983
|
return [elem.id for elem in x.elts]
|
|
1999
1984
|
|
|
2000
1985
|
return []
|
|
@@ -2013,8 +1998,7 @@ def get_var_data(content, node_id=None):
|
|
|
2013
1998
|
# to just `ast.parse(content)` once we confirm all environments use patched Python versions.
|
|
2014
1999
|
def parse_content(content, retries=0):
|
|
2015
2000
|
try:
|
|
2016
|
-
|
|
2017
|
-
return parsed
|
|
2001
|
+
return ast.parse(content)
|
|
2018
2002
|
except Exception as e:
|
|
2019
2003
|
if "AST constructor recursion depth mismatch" not in str(e):
|
|
2020
2004
|
raise e
|
|
@@ -2042,8 +2026,7 @@ def get_var_data(content, node_id=None):
|
|
|
2042
2026
|
if type(x) == ast.Call: # noqa: E721
|
|
2043
2027
|
# Nested calls are traversed via `ast.iter_child_nodes` below.
|
|
2044
2028
|
continue
|
|
2045
|
-
|
|
2046
|
-
args.append(node_to_value(x))
|
|
2029
|
+
args.append(node_to_value(x))
|
|
2047
2030
|
|
|
2048
2031
|
kwargs = {}
|
|
2049
2032
|
for x in node.keywords:
|
|
@@ -2531,8 +2514,7 @@ def format_SQLTemplateException_message(e: SQLTemplateException, vars_and_types:
|
|
|
2531
2514
|
f"{REQUIRED_PARAM_NOT_DEFINED}. Check the parameters {join_with_different_last_separator(vars_with_default_none)}. Please provide a value or set a default value in the pipe code.",
|
|
2532
2515
|
e.documentation,
|
|
2533
2516
|
)
|
|
2534
|
-
|
|
2535
|
-
raise e
|
|
2517
|
+
raise e
|
|
2536
2518
|
|
|
2537
2519
|
|
|
2538
2520
|
def render_sql_template(
|
|
@@ -2913,12 +2895,12 @@ def render_sql_template(
|
|
|
2913
2895
|
# https://gitlab.com/tinybird/analytics/-/issues/943
|
|
2914
2896
|
if "length" in v and not v["length"]:
|
|
2915
2897
|
raise SQLTemplateException("length cannot be used as a variable name or as a function inside of a template")
|
|
2916
|
-
|
|
2898
|
+
if "missing 1 required positional argument" in str(e):
|
|
2917
2899
|
raise SQLTemplateException(
|
|
2918
2900
|
"one of the transform type functions is missing an argument",
|
|
2919
2901
|
documentation="/cli/advanced-templates.html#transform-types-functions",
|
|
2920
2902
|
)
|
|
2921
|
-
|
|
2903
|
+
if "not callable" in str(e) or "unhashable type" in str(e):
|
|
2922
2904
|
raise SQLTemplateException(
|
|
2923
2905
|
"wrong syntax, you might be using a not valid function inside a control block",
|
|
2924
2906
|
documentation="/cli/advanced-templates.html",
|
|
@@ -3063,7 +3045,7 @@ def render_template_with_secrets(
|
|
|
3063
3045
|
)
|
|
3064
3046
|
return '""'
|
|
3065
3047
|
return value
|
|
3066
|
-
|
|
3048
|
+
if default is not None:
|
|
3067
3049
|
if isinstance(default, str) and len(default) == 0:
|
|
3068
3050
|
if empty_secret_raises:
|
|
3069
3051
|
raise SQLTemplateException(
|
|
@@ -3071,10 +3053,9 @@ def render_template_with_secrets(
|
|
|
3071
3053
|
)
|
|
3072
3054
|
return '""'
|
|
3073
3055
|
return default
|
|
3074
|
-
|
|
3075
|
-
|
|
3076
|
-
|
|
3077
|
-
)
|
|
3056
|
+
raise SQLTemplateException(
|
|
3057
|
+
f"Cannot access secret '{secret_name}'. Check the secret exists in the Workspace and the token has the required scope."
|
|
3058
|
+
)
|
|
3078
3059
|
|
|
3079
3060
|
# Create the template
|
|
3080
3061
|
t = Template(content, name=name, autoescape=None)
|
tinybird/sql_template_fmt.py
CHANGED
|
@@ -33,8 +33,7 @@ class TBLine(Line):
|
|
|
33
33
|
Tinybird => This is overriden from the base Line because we want SQL inside a template to be indented
|
|
34
34
|
https://github.com/tconbeer/sqlfmt/blob/c11775b92d8a45f0e91d871b81a88a894d620bec/src/sqlfmt/line.py#L92
|
|
35
35
|
"""
|
|
36
|
-
|
|
37
|
-
return prefix
|
|
36
|
+
return INDENT * (self.depth[0] + self.depth[1])
|
|
38
37
|
|
|
39
38
|
|
|
40
39
|
def from_nodes(
|
|
@@ -106,8 +105,7 @@ def _format_jinja_node(self, node: Node, max_length: int) -> bool:
|
|
|
106
105
|
|
|
107
106
|
return tag.is_blackened
|
|
108
107
|
|
|
109
|
-
|
|
110
|
-
return False
|
|
108
|
+
return False
|
|
111
109
|
|
|
112
110
|
|
|
113
111
|
# Some monkey patching
|
|
@@ -158,8 +156,7 @@ class TinybirdNodeManager(NodeManager):
|
|
|
158
156
|
TokenType.SET_OPERATOR,
|
|
159
157
|
):
|
|
160
158
|
return " ".join(token.token.split())
|
|
161
|
-
|
|
162
|
-
return token.token
|
|
159
|
+
return token.token
|
|
163
160
|
|
|
164
161
|
|
|
165
162
|
class TinybirdDialect(ClickHouse):
|
|
@@ -230,12 +227,11 @@ class TinybirdDialect(ClickHouse):
|
|
|
230
227
|
Creates and returns an analyzer that uses the Dialect's rules for lexing
|
|
231
228
|
Custom NodeManager for Tinybird
|
|
232
229
|
"""
|
|
233
|
-
|
|
230
|
+
return Analyzer(
|
|
234
231
|
line_length=line_length,
|
|
235
232
|
rules=self.get_rules(),
|
|
236
233
|
node_manager=TinybirdNodeManager(self.case_sensitive_names, self.lower_keywords),
|
|
237
234
|
)
|
|
238
|
-
return analyzer
|
|
239
235
|
|
|
240
236
|
|
|
241
237
|
@dataclass
|
|
@@ -253,11 +249,10 @@ class TBMode(Mode):
|
|
|
253
249
|
def _calc_str(self) -> str:
|
|
254
250
|
if self.is_multiline:
|
|
255
251
|
return self.token.token + "\n"
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
return marker + " " + comment_text + "\n"
|
|
252
|
+
marker, comment_text = self._comment_parts()
|
|
253
|
+
if comment_text == "":
|
|
254
|
+
return marker + "\n"
|
|
255
|
+
return marker + " " + comment_text + "\n"
|
|
261
256
|
|
|
262
257
|
|
|
263
258
|
Comment._calc_str = property(_calc_str)
|
tinybird/sql_toolset.py
CHANGED
|
@@ -314,9 +314,9 @@ def _sql_get_used_tables_impl(
|
|
|
314
314
|
msg = str(e)
|
|
315
315
|
if "is restricted. Contact support@tinybird.co" in msg:
|
|
316
316
|
raise InvalidFunction(msg=msg) from e
|
|
317
|
-
|
|
317
|
+
if "Unknown function tb_secret" in msg:
|
|
318
318
|
raise InvalidFunction(msg="Unknown function tb_secret. Usage: {{tb_secret('secret_name')}}") from e
|
|
319
|
-
|
|
319
|
+
if "Unknown function tb_var" in msg:
|
|
320
320
|
raise InvalidFunction(msg="Unknown function tb_var. Usage: {{tb_var('var_name')}}") from e
|
|
321
321
|
raise
|
|
322
322
|
# Do not cache this fallback result: the returned sql string can contain
|
|
@@ -660,9 +660,7 @@ def replace_tables(
|
|
|
660
660
|
|
|
661
661
|
# Fix for empty database names in JOINs - remove empty backticks like ``.table_name
|
|
662
662
|
# that are generated when chquery.replace_tables processes tuples with empty database names
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
return sql
|
|
663
|
+
return sql.replace("``.", "")
|
|
666
664
|
|
|
667
665
|
|
|
668
666
|
def is_invalid_resource(
|
tinybird/syncasync.py
CHANGED
|
@@ -142,8 +142,7 @@ class Local:
|
|
|
142
142
|
storage = self._get_storage()
|
|
143
143
|
if key in storage:
|
|
144
144
|
return storage[key]
|
|
145
|
-
|
|
146
|
-
raise AttributeError(f"{self!r} object has no attribute {key!r}")
|
|
145
|
+
raise AttributeError(f"{self!r} object has no attribute {key!r}")
|
|
147
146
|
|
|
148
147
|
def __setattr__(self, key, value):
|
|
149
148
|
if key in ("_context_refs", "_thread_critical", "_thread_lock", "_attr_name"):
|
|
@@ -151,6 +150,7 @@ class Local:
|
|
|
151
150
|
with self._thread_lock:
|
|
152
151
|
storage = self._get_storage()
|
|
153
152
|
storage[key] = value
|
|
153
|
+
return None
|
|
154
154
|
|
|
155
155
|
def __delattr__(self, key):
|
|
156
156
|
with self._thread_lock:
|
|
@@ -252,13 +252,12 @@ def _iscoroutinefunction_or_partial(func: Any) -> bool:
|
|
|
252
252
|
# this to exist. Code taken from CPython.
|
|
253
253
|
if sys.version_info >= (3, 8):
|
|
254
254
|
return asyncio.iscoroutinefunction(func)
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
func = func.func
|
|
255
|
+
while inspect.ismethod(func):
|
|
256
|
+
func = func.__func__
|
|
257
|
+
while isinstance(func, functools.partial):
|
|
258
|
+
func = func.func
|
|
260
259
|
|
|
261
|
-
|
|
260
|
+
return asyncio.iscoroutinefunction(func)
|
|
262
261
|
|
|
263
262
|
|
|
264
263
|
class ThreadSensitiveContext:
|
|
@@ -660,9 +659,8 @@ class SyncToAsync:
|
|
|
660
659
|
if hasattr(asyncio, "current_task"):
|
|
661
660
|
# Python 3.7 and up
|
|
662
661
|
return asyncio.current_task()
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
return asyncio.Task.current_task()
|
|
662
|
+
# Python 3.6
|
|
663
|
+
return asyncio.Task.current_task()
|
|
666
664
|
except RuntimeError:
|
|
667
665
|
return None
|
|
668
666
|
|
tinybird/tb/__cli__.py
CHANGED
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/forward/commands'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '4.6.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '4.6.20.dev0'
|
|
8
|
+
__revision__ = 'fee8cae'
|
tinybird/tb/client.py
CHANGED
|
@@ -272,14 +272,13 @@ class TinyB:
|
|
|
272
272
|
|
|
273
273
|
def alter_tokens(self, name: str, scopes: List[str]):
|
|
274
274
|
if not scopes:
|
|
275
|
-
return
|
|
275
|
+
return None
|
|
276
276
|
scopes_url: str = "&".join([f"scope={scope}" for scope in scopes])
|
|
277
277
|
url = f"/v0/tokens/{name}"
|
|
278
278
|
if len(url + "?" + scopes_url) > TinyB.MAX_GET_LENGTH:
|
|
279
279
|
return self._req(url, method="PUT", data=scopes_url)
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
return self._req(url, method="PUT", data="")
|
|
280
|
+
url = url + "?" + scopes_url
|
|
281
|
+
return self._req(url, method="PUT", data="")
|
|
283
282
|
|
|
284
283
|
def datasources(self, branch: Optional[str] = None, attrs: Optional[str] = None) -> List[Dict[str, Any]]:
|
|
285
284
|
params = {}
|
|
@@ -300,16 +299,13 @@ class TinyB:
|
|
|
300
299
|
return self._req(f"/v0/variables/{name}")
|
|
301
300
|
|
|
302
301
|
def create_secret(self, name: str, value: str):
|
|
303
|
-
|
|
304
|
-
return response
|
|
302
|
+
return self._req("/v0/variables", method="POST", data={"name": name, "value": value})
|
|
305
303
|
|
|
306
304
|
def update_secret(self, name: str, value: str):
|
|
307
|
-
|
|
308
|
-
return response
|
|
305
|
+
return self._req(f"/v0/variables/{name}", method="PUT", data={"value": value})
|
|
309
306
|
|
|
310
307
|
def delete_secret(self, name: str):
|
|
311
|
-
|
|
312
|
-
return response
|
|
308
|
+
return self._req(f"/v0/variables/{name}", method="DELETE")
|
|
313
309
|
|
|
314
310
|
def get_connections(self, service: Optional[str] = None):
|
|
315
311
|
params = {}
|
|
@@ -537,7 +533,7 @@ class TinyB:
|
|
|
537
533
|
if projection_mode:
|
|
538
534
|
params["projection_mode"] = projection_mode
|
|
539
535
|
return self._req(f"/v1/datasources/{datasource_name}/delete", method="POST", data=params)
|
|
540
|
-
|
|
536
|
+
if dry_run:
|
|
541
537
|
params["dry_run"] = "true"
|
|
542
538
|
return self._req(f"/v0/datasources/{datasource_name}/delete", method="POST", data=params)
|
|
543
539
|
|
|
@@ -635,8 +631,7 @@ class TinyB:
|
|
|
635
631
|
node_name = node["params"]["name"] if node.get("params", None) else node["name"]
|
|
636
632
|
if datasource_name:
|
|
637
633
|
params["datasource"] = datasource_name
|
|
638
|
-
|
|
639
|
-
return response
|
|
634
|
+
return self._req(f"/v0/pipes/{pipe_name}/nodes/{node_name}/analysis?{urlencode(params)}")
|
|
640
635
|
|
|
641
636
|
def populate_node(
|
|
642
637
|
self,
|
|
@@ -654,8 +649,7 @@ class TinyB:
|
|
|
654
649
|
params.update({"populate_condition": populate_condition})
|
|
655
650
|
if on_demand_compute:
|
|
656
651
|
params.update({"on_demand_compute": "true"})
|
|
657
|
-
|
|
658
|
-
return response
|
|
652
|
+
return self._req(f"/v0/pipes/{pipe_name}/nodes/{node_name}/population?{urlencode(params)}", method="POST")
|
|
659
653
|
|
|
660
654
|
def pipes(self, branch=None, dependencies: bool = False, node_attrs=None, attrs=None) -> List[Dict[str, Any]]:
|
|
661
655
|
params = {
|
|
@@ -687,9 +681,8 @@ class TinyB:
|
|
|
687
681
|
query_string = urlencode(params)
|
|
688
682
|
if len(url + "?" + query_string) > TinyB.MAX_GET_LENGTH:
|
|
689
683
|
return self._req(f"/v0/pipes/{pipe_name_or_uid}.{format}", method="POST", data=params)
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
return self._req(url)
|
|
684
|
+
url = url + "?" + query_string
|
|
685
|
+
return self._req(url)
|
|
693
686
|
|
|
694
687
|
def pipe_create(self, pipe_name: str, sql: str):
|
|
695
688
|
return self._req(f"/v0/pipes?name={pipe_name}&sql={quote(sql, safe='')}", method="POST", data=sql.encode())
|
|
@@ -777,8 +770,7 @@ class TinyB:
|
|
|
777
770
|
|
|
778
771
|
if len(sql) > TinyB.MAX_GET_LENGTH:
|
|
779
772
|
return self._req(f"/v0/sql?{urlencode(params)}", data=sql, method="POST")
|
|
780
|
-
|
|
781
|
-
return self._req(f"/v0/sql?q={quote(sql, safe='')}&{urlencode(params)}")
|
|
773
|
+
return self._req(f"/v0/sql?q={quote(sql, safe='')}&{urlencode(params)}")
|
|
782
774
|
|
|
783
775
|
def jobs(
|
|
784
776
|
self, status: Optional[Tuple[str, ...]] = None, kind: Optional[Tuple[str, ...]] = None
|
|
@@ -1006,7 +998,7 @@ class TinyB:
|
|
|
1006
998
|
|
|
1007
999
|
def add_workspaces_to_organization(self, organization_id: str, workspace_ids: List[str]):
|
|
1008
1000
|
if not workspace_ids:
|
|
1009
|
-
return
|
|
1001
|
+
return None
|
|
1010
1002
|
return self._req(
|
|
1011
1003
|
f"/v0/organizations/{organization_id}/workspaces",
|
|
1012
1004
|
method="PUT",
|
|
@@ -1344,8 +1336,7 @@ class TinyB:
|
|
|
1344
1336
|
return next((connector for connector in result["connectors"] if connector_equals(connector, kwargs)), None)
|
|
1345
1337
|
|
|
1346
1338
|
def regions(self):
|
|
1347
|
-
|
|
1348
|
-
return regions
|
|
1339
|
+
return self._req("/v0/regions")
|
|
1349
1340
|
|
|
1350
1341
|
def datasource_query_copy(self, datasource_name: str, sql_query: str):
|
|
1351
1342
|
params = {"copy_to": datasource_name}
|
tinybird/tb/modules/cicd.py
CHANGED
|
@@ -182,6 +182,7 @@ class CICDGeneratorBase:
|
|
|
182
182
|
return FeedbackManager.warning_for_cicd_file(
|
|
183
183
|
file_name=cicd_file.file_name, warning_message=cicd_file.warning_message.format(**params)
|
|
184
184
|
)
|
|
185
|
+
return None
|
|
185
186
|
|
|
186
187
|
def is_already_generated(self, path: str) -> bool:
|
|
187
188
|
for cicd_file in self.cicd_files:
|
tinybird/tb/modules/cli.py
CHANGED
|
@@ -25,7 +25,6 @@ from tinybird.iterating.data_branch_modes import DataBranchMode
|
|
|
25
25
|
from tinybird.tb import __cli__
|
|
26
26
|
from tinybird.tb.check_pypi import CheckPypi
|
|
27
27
|
from tinybird.tb.client import (
|
|
28
|
-
AuthException,
|
|
29
28
|
AuthNoTokenException,
|
|
30
29
|
TinyB,
|
|
31
30
|
)
|
|
@@ -734,12 +733,15 @@ def cli(
|
|
|
734
733
|
# Keep config path pointing to the user project root even when using a virtual SDK project.
|
|
735
734
|
config["path"] = folder if sdk_virtual_project else str(project.path)
|
|
736
735
|
# If they have passed a token or host as parameter and it's different that record in .tinyb, refresh the workspace id
|
|
736
|
+
# Best-effort only: e.g. a still-Classic workspace makes this v1 (Forward-only)
|
|
737
|
+
# call fail with a non-auth error, and that must not block the CLI from
|
|
738
|
+
# starting up (commands like `migrate-to-forward` are meant to run on it).
|
|
737
739
|
if token or host:
|
|
738
740
|
try:
|
|
739
741
|
workspace = client.workspace_info(version="v1")
|
|
740
742
|
config["id"] = workspace.get("id", "")
|
|
741
743
|
config["name"] = workspace.get("name", "")
|
|
742
|
-
except
|
|
744
|
+
except Exception:
|
|
743
745
|
pass
|
|
744
746
|
|
|
745
747
|
ctx.ensure_object(dict)["config"] = config
|
tinybird/tb/modules/common.py
CHANGED
|
@@ -223,6 +223,7 @@ class AliasedGroup(click.Group):
|
|
|
223
223
|
cm = click.Group.get_command(self, ctx, cmd_name)
|
|
224
224
|
if cm is not None:
|
|
225
225
|
return cm
|
|
226
|
+
return None
|
|
226
227
|
|
|
227
228
|
def resolve_command(self, ctx, args):
|
|
228
229
|
# always return the command's name, not the alias
|
|
@@ -449,7 +450,7 @@ def analyze_file(filename: str, client: TinyB, format: str):
|
|
|
449
450
|
meta, data = _analyze(filename, client, format)
|
|
450
451
|
schema = meta["analysis"]["schema"]
|
|
451
452
|
schema = schema.replace(", ", ",\n ")
|
|
452
|
-
|
|
453
|
+
return f"""DESCRIPTION >
|
|
453
454
|
Generated from {filename}
|
|
454
455
|
|
|
455
456
|
SCHEMA >
|
|
@@ -459,7 +460,6 @@ ENGINE "MergeTree"
|
|
|
459
460
|
# ENGINE_SORTING_KEY "user_id, timestamp"
|
|
460
461
|
# ENGINE_TTL "timestamp + toIntervalDay(60)"
|
|
461
462
|
# Learn more at https://www.tinybird.co/docs/forward/dev-reference/datafiles/datasource-files"""
|
|
462
|
-
return content
|
|
463
463
|
|
|
464
464
|
|
|
465
465
|
def _generate_datafile(
|
|
@@ -516,6 +516,7 @@ def ask_for_region_interactively(regions):
|
|
|
516
516
|
available_options = ", ".join(map(str, range(1, len(regions) + 1)))
|
|
517
517
|
click.echo(FeedbackManager.error_region_index(host_index=region_index, available_options=available_options))
|
|
518
518
|
region_index = -1
|
|
519
|
+
return None
|
|
519
520
|
|
|
520
521
|
|
|
521
522
|
def get_region_info(ctx, region=None):
|
|
@@ -711,10 +712,8 @@ def print_branch_regression_tests_summary(client, job_id, host, response=None):
|
|
|
711
712
|
if isinstance(metric, float):
|
|
712
713
|
if is_percentage:
|
|
713
714
|
return f"{round(metric, 3):+} %"
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
else:
|
|
717
|
-
return metric
|
|
715
|
+
return f"{round(metric, 3)} seconds"
|
|
716
|
+
return metric
|
|
718
717
|
|
|
719
718
|
failed = False
|
|
720
719
|
response = client.job(job_id) if job_id else response or {"progress": []}
|
|
@@ -2087,9 +2086,7 @@ def get_gcs_svc_account_creds() -> str:
|
|
|
2087
2086
|
)
|
|
2088
2087
|
|
|
2089
2088
|
assert isinstance(creds, str)
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
return creds_without_new_lines
|
|
2089
|
+
return creds.replace("\n", "")
|
|
2093
2090
|
|
|
2094
2091
|
|
|
2095
2092
|
class DataConnectorType(str, Enum):
|
|
@@ -2478,9 +2475,8 @@ def create_workspace_branch(
|
|
|
2478
2475
|
job_response = wait_job(config.get_client(), job_id, job_url, "Environment creation")
|
|
2479
2476
|
if job_response is None:
|
|
2480
2477
|
raise CLIException(f"Empty job API response (job_id: {job_id}, job_url: {job_url})")
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
is_summary = "partitions" in response
|
|
2478
|
+
response = job_response.get("result", {})
|
|
2479
|
+
is_summary = "partitions" in response
|
|
2484
2480
|
|
|
2485
2481
|
except Exception as e:
|
|
2486
2482
|
raise CLIException(FeedbackManager.error_exception(error=str(e)))
|
tinybird/tb/modules/config.py
CHANGED
|
@@ -114,8 +114,7 @@ class CLIConfig:
|
|
|
114
114
|
def get_value_origin(self, key: str) -> ConfigValueOrigin:
|
|
115
115
|
if key in self._values:
|
|
116
116
|
return self._values[key].origin
|
|
117
|
-
|
|
118
|
-
return ConfigValueOrigin.NONE
|
|
117
|
+
return ConfigValueOrigin.NONE
|
|
119
118
|
|
|
120
119
|
def persist_to_file(self, override_with_path: Optional[str] = None) -> None:
|
|
121
120
|
if override_with_path:
|
|
@@ -781,7 +781,7 @@ def meta_to_datasource_datafile(
|
|
|
781
781
|
__value String -- Set KAFKA_STORE_RAW_VALUE to True to store the raw value of the message
|
|
782
782
|
# __headers Map(String, String) -- Set KAFKA_STORE_HEADERS to True to store the headers of the message
|
|
783
783
|
# Learn more at https://www.tinybird.co/docs/forward/get-data-in/connectors/kafka#kafka-meta-columns"""
|
|
784
|
-
|
|
784
|
+
return f"""SCHEMA >
|
|
785
785
|
{schema}{"," if schema else ""}
|
|
786
786
|
{kafka_meta_columns}
|
|
787
787
|
|
|
@@ -797,7 +797,6 @@ KAFKA_AUTO_OFFSET_RESET {kafka_auto_offset_reset}
|
|
|
797
797
|
# KAFKA_STORE_HEADERS True
|
|
798
798
|
# Learn more at https://www.tinybird.co/docs/forward/get-data-in/connectors/kafka#kafka-datasource-settings
|
|
799
799
|
"""
|
|
800
|
-
return ds_content
|
|
801
800
|
|
|
802
801
|
|
|
803
802
|
def inject_tb_secret(secret_name: str, default_value: str) -> str:
|
|
@@ -60,23 +60,21 @@ def select_schedule(schedule: Optional[str]) -> str:
|
|
|
60
60
|
|
|
61
61
|
if choice == 1:
|
|
62
62
|
return "@auto"
|
|
63
|
-
|
|
63
|
+
if choice == 2:
|
|
64
64
|
return "@once"
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
return "@auto"
|
|
65
|
+
click.echo(FeedbackManager.warning(message="Invalid option. Defaulting to @auto."))
|
|
66
|
+
return "@auto"
|
|
68
67
|
|
|
69
68
|
|
|
70
69
|
def format_file_size(size: int) -> str:
|
|
71
70
|
"""Format file size in human-readable format."""
|
|
72
71
|
if size >= 1024 * 1024 * 1024:
|
|
73
72
|
return f"{size / (1024 * 1024 * 1024):.2f} GB"
|
|
74
|
-
|
|
73
|
+
if size >= 1024 * 1024:
|
|
75
74
|
return f"{size / (1024 * 1024):.2f} MB"
|
|
76
|
-
|
|
75
|
+
if size >= 1024:
|
|
77
76
|
return f"{size / 1024:.2f} KB"
|
|
78
|
-
|
|
79
|
-
return f"{size} B"
|
|
77
|
+
return f"{size} B"
|
|
80
78
|
|
|
81
79
|
|
|
82
80
|
def select_sample_file_uri(
|
|
@@ -217,13 +215,12 @@ def get_format_from_uri(uri: str) -> str:
|
|
|
217
215
|
|
|
218
216
|
if uri_lower.endswith(".csv"):
|
|
219
217
|
return "csv"
|
|
220
|
-
|
|
218
|
+
if uri_lower.endswith((".ndjson", ".jsonl", ".json")):
|
|
221
219
|
return "ndjson"
|
|
222
|
-
|
|
220
|
+
if uri_lower.endswith(".parquet"):
|
|
223
221
|
return "parquet"
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
return "ndjson"
|
|
222
|
+
# Default to ndjson for unknown formats
|
|
223
|
+
return "ndjson"
|
|
227
224
|
|
|
228
225
|
|
|
229
226
|
def meta_to_schema(meta: list[dict[str, Any]], file_format: str) -> str:
|
|
@@ -242,9 +239,8 @@ def meta_to_schema(meta: list[dict[str, Any]], file_format: str) -> str:
|
|
|
242
239
|
if file_format == "csv":
|
|
243
240
|
# CSV files don't use JSONPath
|
|
244
241
|
return ",\n ".join([f"`{col['name']}` {col['type']}" for col in meta])
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
return ",\n ".join([f"`{col['name']}` {col['type']} `json:$.{col['name']}`" for col in meta])
|
|
242
|
+
# NDJSON and Parquet use JSONPath
|
|
243
|
+
return ",\n ".join([f"`{col['name']}` {col['type']} `json:$.{col['name']}`" for col in meta])
|
|
248
244
|
|
|
249
245
|
|
|
250
246
|
def meta_to_s3_datasource_datafile(
|
|
@@ -276,7 +272,7 @@ def meta_to_s3_datasource_datafile(
|
|
|
276
272
|
import_format if import_format is not None and import_format != "auto" else get_format_from_uri(bucket_uri)
|
|
277
273
|
)
|
|
278
274
|
schema: str = meta_to_schema(meta, file_format)
|
|
279
|
-
|
|
275
|
+
return f"""SCHEMA >
|
|
280
276
|
{schema}
|
|
281
277
|
|
|
282
278
|
ENGINE "MergeTree"
|
|
@@ -289,7 +285,6 @@ IMPORT_SCHEDULE "{import_schedule}"
|
|
|
289
285
|
IMPORT_FORMAT "{file_format}"
|
|
290
286
|
# Learn more at https://www.tinybird.co/docs/forward/get-data-in/connectors/s3#datasource-settings
|
|
291
287
|
"""
|
|
292
|
-
return ds_content
|
|
293
288
|
|
|
294
289
|
|
|
295
290
|
def connection_create_s3(
|
tinybird/tb/modules/create.py
CHANGED
|
@@ -272,8 +272,7 @@ S3_ARN {{{{ tb_secret("{role_arn_secret_name}"{default_secret}) }}}}
|
|
|
272
272
|
S3_REGION {region}
|
|
273
273
|
# Learn more at https://www.tinybird.co/docs/forward/get-data-in/connectors/s3#s3-connection-settings
|
|
274
274
|
"""
|
|
275
|
-
|
|
276
|
-
return file_path
|
|
275
|
+
return generate_connection_file(name, content, folder, skip_feedback=True)
|
|
277
276
|
|
|
278
277
|
|
|
279
278
|
def generate_dynamodb_connection_file_with_secret(
|
|
@@ -284,16 +283,14 @@ DYNAMODB_ARN {{{{ tb_secret("{role_arn_secret_name}") }}}}
|
|
|
284
283
|
DYNAMODB_REGION {region}
|
|
285
284
|
# Learn more at https://www.tinybird.co/docs/forward/get-data-in/connectors/dynamodb
|
|
286
285
|
"""
|
|
287
|
-
|
|
288
|
-
return file_path
|
|
286
|
+
return generate_connection_file(name, content, folder, skip_feedback=True)
|
|
289
287
|
|
|
290
288
|
|
|
291
289
|
def generate_gcs_connection_file_with_secrets(name: str, service: str, svc_account_creds: str, folder: str) -> Path:
|
|
292
290
|
content = f"""TYPE {service}
|
|
293
291
|
GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON {{{{ tb_secret("{svc_account_creds}") }}}}
|
|
294
292
|
"""
|
|
295
|
-
|
|
296
|
-
return file_path
|
|
293
|
+
return generate_connection_file(name, content, folder, skip_feedback=True)
|
|
297
294
|
|
|
298
295
|
|
|
299
296
|
def create_env_file(folder: str):
|
|
@@ -861,7 +861,7 @@ def process_file(
|
|
|
861
861
|
|
|
862
862
|
return resources
|
|
863
863
|
|
|
864
|
-
|
|
864
|
+
if DataFileExtensions.PIPE in filename:
|
|
865
865
|
doc = parse_pipe(filename).datafile
|
|
866
866
|
version = f"__v{doc.version}" if doc.version is not None else ""
|
|
867
867
|
name = os.path.basename(filename).split(".")[0]
|
|
@@ -958,8 +958,7 @@ def process_file(
|
|
|
958
958
|
"filtering_tags": doc.filtering_tags,
|
|
959
959
|
}
|
|
960
960
|
]
|
|
961
|
-
|
|
962
|
-
raise click.ClickException(FeedbackManager.error_file_extension(filename=filename))
|
|
961
|
+
raise click.ClickException(FeedbackManager.error_file_extension(filename=filename))
|
|
963
962
|
|
|
964
963
|
|
|
965
964
|
def sizeof_fmt(num: float, suffix: str = "b") -> str:
|
|
@@ -252,10 +252,7 @@ def is_materialized(resource: Optional[Dict[str, Any]]) -> bool:
|
|
|
252
252
|
if not resource:
|
|
253
253
|
return False
|
|
254
254
|
|
|
255
|
-
|
|
256
|
-
node.get("params", {}).get("type", None) == "materialized" for node in resource.get("nodes", []) or []
|
|
257
|
-
)
|
|
258
|
-
return is_materialized
|
|
255
|
+
return any(node.get("params", {}).get("type", None) == "materialized" for node in resource.get("nodes", []) or [])
|
|
259
256
|
|
|
260
257
|
|
|
261
258
|
def get_target_materialized_data_source_name(resource: Optional[Dict[str, Any]]) -> Optional[str]:
|
|
@@ -77,6 +77,7 @@ def diff_files(
|
|
|
77
77
|
click.echo(f"Unsupported file type: {filename}")
|
|
78
78
|
if lines:
|
|
79
79
|
return [f"{l}\n" for l in lines.split("\n")] if with_format else lines # noqa: E741
|
|
80
|
+
return None
|
|
80
81
|
|
|
81
82
|
try:
|
|
82
83
|
lines1 = parse(from_file, with_format)
|
|
@@ -86,7 +87,7 @@ def diff_files(
|
|
|
86
87
|
raise click.ClickException(FeedbackManager.error_diff_file(filename=filename))
|
|
87
88
|
|
|
88
89
|
if not lines1 or not lines2:
|
|
89
|
-
return
|
|
90
|
+
return None
|
|
90
91
|
|
|
91
92
|
diff = difflib.unified_diff(
|
|
92
93
|
lines1, lines2, fromfile=f"{Path(from_file).name} {from_file_suffix}", tofile=f"{to_file} {to_file_suffix}"
|
|
@@ -81,17 +81,16 @@ def format_engine(
|
|
|
81
81
|
file_parts.append(DATAFILE_NEW_LINE)
|
|
82
82
|
file_parts.append(DATAFILE_NEW_LINE)
|
|
83
83
|
return file_parts
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
for
|
|
90
|
-
|
|
91
|
-
file_parts.append(f"ENGINE_{arg[0].upper()} {elem or empty}")
|
|
92
|
-
file_parts.append(DATAFILE_NEW_LINE)
|
|
84
|
+
if node.get("engine", None):
|
|
85
|
+
empty = '""'
|
|
86
|
+
file_parts.append(f"ENGINE {node['engine']['type']}" if node.get("engine", {}).get("type") else empty)
|
|
87
|
+
file_parts.append(DATAFILE_NEW_LINE)
|
|
88
|
+
for arg in sorted(node["engine"].get("args", [])):
|
|
89
|
+
elem = ", ".join([x.strip() for x in arg[1].split(",")])
|
|
90
|
+
file_parts.append(f"ENGINE_{arg[0].upper()} {elem or empty}")
|
|
93
91
|
file_parts.append(DATAFILE_NEW_LINE)
|
|
94
|
-
|
|
92
|
+
file_parts.append(DATAFILE_NEW_LINE)
|
|
93
|
+
return file_parts
|
|
95
94
|
|
|
96
95
|
|
|
97
96
|
def format_sql(
|
|
@@ -89,8 +89,7 @@ def format_connection(
|
|
|
89
89
|
format_include(file_parts, doc)
|
|
90
90
|
|
|
91
91
|
result = "".join(file_parts)
|
|
92
|
-
|
|
93
|
-
return result
|
|
92
|
+
return result.rstrip("\n") + "\n"
|
|
94
93
|
|
|
95
94
|
|
|
96
95
|
def format_type(file_parts: List[str], node: Dict[str, Any]) -> List[str]:
|
|
@@ -83,8 +83,7 @@ def format_datasource(
|
|
|
83
83
|
format_shared_with(file_parts, doc)
|
|
84
84
|
format_forward_query(file_parts, doc)
|
|
85
85
|
result = "".join(file_parts)
|
|
86
|
-
|
|
87
|
-
return result
|
|
86
|
+
return result.rstrip("\n") + "\n"
|
|
88
87
|
|
|
89
88
|
|
|
90
89
|
def format_schema(file_parts: List[str], node: Dict[str, Any]) -> List[str]:
|
|
@@ -105,13 +105,12 @@ class PipeChecker(unittest.TestCase):
|
|
|
105
105
|
headers = {"Authorization": f"Bearer {self.token}"}
|
|
106
106
|
if self.http_method == "GET":
|
|
107
107
|
return requests.get(pipe_url, headers=headers, verify=not getenv_bool("TB_DISABLE_SSL_CHECKS", False))
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
)
|
|
108
|
+
return requests.post(
|
|
109
|
+
pipe_url,
|
|
110
|
+
headers=headers,
|
|
111
|
+
verify=not getenv_bool("TB_DISABLE_SSL_CHECKS", False),
|
|
112
|
+
data=self.pipe_request_params,
|
|
113
|
+
)
|
|
115
114
|
|
|
116
115
|
def _write_performance(self):
|
|
117
116
|
return ""
|
|
@@ -902,7 +902,7 @@ def process_file(
|
|
|
902
902
|
|
|
903
903
|
return resources
|
|
904
904
|
|
|
905
|
-
|
|
905
|
+
if DataFileExtensions.PIPE in filename:
|
|
906
906
|
doc, warnings = parse_pipe(filename)
|
|
907
907
|
version = f"__v{doc.version}" if doc.version is not None else ""
|
|
908
908
|
name = os.path.basename(filename).split(".")[0]
|
|
@@ -999,11 +999,10 @@ def process_file(
|
|
|
999
999
|
"filtering_tags": doc.filtering_tags,
|
|
1000
1000
|
}
|
|
1001
1001
|
]
|
|
1002
|
-
|
|
1002
|
+
if DataFileExtensions.CONNECTION in filename:
|
|
1003
1003
|
return []
|
|
1004
1004
|
|
|
1005
|
-
|
|
1006
|
-
raise click.ClickException(FeedbackManager.error_file_extension(filename=filename))
|
|
1005
|
+
raise click.ClickException(FeedbackManager.error_file_extension(filename=filename))
|
|
1007
1006
|
|
|
1008
1007
|
|
|
1009
1008
|
def sizeof_fmt(num: float, suffix: str = "b") -> str:
|
|
@@ -372,7 +372,7 @@ def datasource_append(
|
|
|
372
372
|
|
|
373
373
|
if datasource_index == 0:
|
|
374
374
|
click.echo(FeedbackManager.warning(message="Datasource type selection cancelled by user"))
|
|
375
|
-
return
|
|
375
|
+
return
|
|
376
376
|
|
|
377
377
|
try:
|
|
378
378
|
datasource_name = datasources[int(datasource_index) - 1]["name"]
|
|
@@ -403,7 +403,7 @@ def datasource_append(
|
|
|
403
403
|
|
|
404
404
|
if data_index == 0:
|
|
405
405
|
click.echo(FeedbackManager.warning(message="Data selection cancelled by user"))
|
|
406
|
-
return
|
|
406
|
+
return
|
|
407
407
|
|
|
408
408
|
try:
|
|
409
409
|
data_index = int(data_index)
|
|
@@ -463,8 +463,7 @@ def datasource_append(
|
|
|
463
463
|
if is_quarantined:
|
|
464
464
|
analyze_quarantine(datasource_name, project, client)
|
|
465
465
|
return
|
|
466
|
-
|
|
467
|
-
raise e
|
|
466
|
+
raise e
|
|
468
467
|
if job_ids:
|
|
469
468
|
_echo_v1_import_jobs_queued(job_ids, "Append")
|
|
470
469
|
else:
|
|
@@ -806,7 +805,7 @@ def datasource_delete_rows(
|
|
|
806
805
|
)
|
|
807
806
|
)
|
|
808
807
|
break
|
|
809
|
-
|
|
808
|
+
if res["status"] == "error":
|
|
810
809
|
print("\n") # noqa: T201
|
|
811
810
|
raise CLIDatasourceException(FeedbackManager.error_exception(error=res["error"]))
|
|
812
811
|
time.sleep(1)
|
|
@@ -1237,7 +1236,7 @@ ENGINE "MergeTree"
|
|
|
1237
1236
|
wizard_data["exit_reason"] = "user_cancelled_type_selection"
|
|
1238
1237
|
wizard_data["duration_seconds"] = round(time.time() - start_time, 2)
|
|
1239
1238
|
add_telemetry_event("system_info", **wizard_data)
|
|
1240
|
-
return
|
|
1239
|
+
return
|
|
1241
1240
|
|
|
1242
1241
|
try:
|
|
1243
1242
|
datasource_type = dt_keys[int(datasource_type_index) - 1]
|
|
@@ -1560,7 +1559,7 @@ ENGINE "MergeTree"
|
|
|
1560
1559
|
wizard_data["exit_reason"] = "user_cancelled_kafka_configuration"
|
|
1561
1560
|
wizard_data["duration_seconds"] = round(time.time() - start_time, 2)
|
|
1562
1561
|
add_telemetry_event("system_info", **wizard_data)
|
|
1563
|
-
return
|
|
1562
|
+
return
|
|
1564
1563
|
else:
|
|
1565
1564
|
click.echo(FeedbackManager.error(message="Invalid option. Please select 1-6."))
|
|
1566
1565
|
|
|
@@ -1680,7 +1679,7 @@ ENGINE "MergeTree"
|
|
|
1680
1679
|
wizard_data["exit_reason"] = "user_cancelled_s3_configuration"
|
|
1681
1680
|
wizard_data["duration_seconds"] = round(time.time() - start_time, 2)
|
|
1682
1681
|
add_telemetry_event("system_info", **wizard_data)
|
|
1683
|
-
return
|
|
1682
|
+
return
|
|
1684
1683
|
else:
|
|
1685
1684
|
click.echo(FeedbackManager.error(message="Invalid option. Please select 1-6."))
|
|
1686
1685
|
|
|
@@ -36,8 +36,7 @@ def api_fetch(
|
|
|
36
36
|
if r.status_code == 200:
|
|
37
37
|
logging.debug(json.dumps(r.json(), indent=2))
|
|
38
38
|
return r.json()
|
|
39
|
-
|
|
40
|
-
raise Exception(f"Request failed with status code {r.status_code}")
|
|
39
|
+
raise Exception(f"Request failed with status code {r.status_code}")
|
|
41
40
|
except Exception:
|
|
42
41
|
retries += 1
|
|
43
42
|
if retries > max_retries:
|
|
@@ -747,10 +746,9 @@ def create_deployment(
|
|
|
747
746
|
def _build_data_movement_message(kind: str, source_mv_name: Optional[str]) -> str:
|
|
748
747
|
if kind == "backfill_with_mv_queries":
|
|
749
748
|
return f"Using Materialized Pipe {source_mv_name or ''}"
|
|
750
|
-
|
|
749
|
+
if kind == "backfill_with_forward_query":
|
|
751
750
|
return "From live deployment using Forward Query"
|
|
752
|
-
|
|
753
|
-
return ""
|
|
751
|
+
return ""
|
|
754
752
|
|
|
755
753
|
|
|
756
754
|
def print_changes(result: dict, project: Project, output: Optional[str] = "human") -> None:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: tinybird
|
|
3
|
-
Version: 4.6.
|
|
3
|
+
Version: 4.6.20.dev0
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/forward/commands
|
|
6
6
|
Author: Tinybird
|
|
@@ -52,6 +52,11 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
|
|
|
52
52
|
Changelog
|
|
53
53
|
----------
|
|
54
54
|
|
|
55
|
+
4.6.18
|
|
56
|
+
********
|
|
57
|
+
|
|
58
|
+
- `Fixed` CLI commands (for example ``tb migrate-to-forward``) no longer abort on startup when the best-effort refresh of the cached workspace id/name fails for a reason other than authentication.
|
|
59
|
+
|
|
55
60
|
4.6.17
|
|
56
61
|
********
|
|
57
62
|
|
|
@@ -4,11 +4,11 @@ tinybird/feedback_manager.py,sha256=2n0s6AMbjtFsELJN9pJnG5JjhM3lUueXBgF4mZ0WiCw,
|
|
|
4
4
|
tinybird/git_settings.py,sha256=ETrcrvnhi3I9W-cGOwILXTBBNVFm5lYbeDdgTFaxT48,3898
|
|
5
5
|
tinybird/prompts.py,sha256=FNYpphDYlrXpm5AQM3yfQPbj4gd_TVnwoaoeswtx0OE,32699
|
|
6
6
|
tinybird/service_datasources.py,sha256=xte9fnT2-_y3KlXlFK5BC2gCcPNe9pjdQW_5bW99wTs,56786
|
|
7
|
-
tinybird/sql.py,sha256=
|
|
8
|
-
tinybird/sql_template.py,sha256=
|
|
9
|
-
tinybird/sql_template_fmt.py,sha256=
|
|
10
|
-
tinybird/sql_toolset.py,sha256=
|
|
11
|
-
tinybird/syncasync.py,sha256=
|
|
7
|
+
tinybird/sql.py,sha256=vela5dH34dNs8UoEk-j2FjP9B7qhgWhb2BP0opw_YTU,47359
|
|
8
|
+
tinybird/sql_template.py,sha256=1FXc85MARJeGT_uJtojDO9aryemVhQ4UuGsREXNaEYg,131708
|
|
9
|
+
tinybird/sql_template_fmt.py,sha256=UZbCNmFNdSBTC1joEu-1C11vBbyWgG_sCbYJFqSKNwo,10528
|
|
10
|
+
tinybird/sql_toolset.py,sha256=rHTPyHJLow9NMGz5gOoPdk5r7JRBULeAZ2iKM8bEip8,28297
|
|
11
|
+
tinybird/syncasync.py,sha256=VV2yn9TKc-aEr0hzQQWIwUcklWZG8AHv7fj6V5LnQSU,27718
|
|
12
12
|
tinybird/tornado_template.py,sha256=8e0ipnBQI4-RO5sFokybDNwkd1CBxGqvnnyJ5kjs5XU,42060
|
|
13
13
|
tinybird/ch_utils/constants.py,sha256=yTNizMzgYNBzUc2EV3moBfdrDIggOe9hiuAgWF7sv2c,4333
|
|
14
14
|
tinybird/ch_utils/engine.py,sha256=BhNEHeBuuORBHWde9UeApJgtac0jj285i3fDVFf08h0,37188
|
|
@@ -19,27 +19,27 @@ tinybird/datafile/parse_datasource.py,sha256=yd58HrUF4yNJXLn6OsvKGpZJpvrcjLGAeJG
|
|
|
19
19
|
tinybird/datafile/parse_pipe.py,sha256=StMwF0Y1ny94hQxgPgcyx6-Jr46gCYBuPGpHF4bFiSQ,4956
|
|
20
20
|
tinybird/iterating/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
21
|
tinybird/iterating/data_branch_modes.py,sha256=5YuDa-gr8mKwmES8Xro6TRKbQtaOtIw4GbC606Qhu3o,184
|
|
22
|
-
tinybird/tb/__cli__.py,sha256=
|
|
22
|
+
tinybird/tb/__cli__.py,sha256=6u3k4WiUJKvoQ9yuah87GnZw0ecIYEvr0iUcIytwTs4,246
|
|
23
23
|
tinybird/tb/check_pypi.py,sha256=Gp0HkHHDFMSDL6nxKlOY51z7z1Uv-2LRexNTZSHHGmM,552
|
|
24
24
|
tinybird/tb/cli.py,sha256=IjiGfNIpxSxi1odK1kMj9s8lEhx3sAUgGA263XdmyR0,1119
|
|
25
|
-
tinybird/tb/client.py,sha256=
|
|
25
|
+
tinybird/tb/client.py,sha256=u-X6p_BDE5Zq585Q7GFTXEUqYjjWau2xlhcEme6wzbo,58557
|
|
26
26
|
tinybird/tb/config.py,sha256=l6NtYvSPZDVOtt1W4G4ExXXDIf0hm6lvRnRoDMbdrsY,5429
|
|
27
27
|
tinybird/tb/modules/branch.py,sha256=U50nj2kZllCkOHQfJWQ-YQ260pvlcssgStMSovmqMiU,10157
|
|
28
28
|
tinybird/tb/modules/build.py,sha256=q7-aVM7q1tApiax_y6vYSCZq4cJ9ATSAm9D-elpNp7I,6543
|
|
29
29
|
tinybird/tb/modules/build_common.py,sha256=Ks_fUbS376x6KaSihUl8MMvAk4pXYIMZauVL_SOpVEA,25533
|
|
30
|
-
tinybird/tb/modules/cicd.py,sha256=
|
|
31
|
-
tinybird/tb/modules/cli.py,sha256=
|
|
32
|
-
tinybird/tb/modules/common.py,sha256=
|
|
33
|
-
tinybird/tb/modules/config.py,sha256=
|
|
30
|
+
tinybird/tb/modules/cicd.py,sha256=xKa7XBQ1wgkYeG9lBLJQcwM7P18cY0c_QeF-KFO9_HI,8287
|
|
31
|
+
tinybird/tb/modules/cli.py,sha256=6leKHypUjieVS9DbEXizFmUZq2lbJFlBDXYyvy_v8G4,44230
|
|
32
|
+
tinybird/tb/modules/common.py,sha256=KMPEKBFLtaD-b4-5DsDZe80R-2TG6porw8ChdKmkc-Y,94214
|
|
33
|
+
tinybird/tb/modules/config.py,sha256=U13HYGCILZjtWeP-LG1g63GDGR0CbqT0QE_cBZeNeOs,11157
|
|
34
34
|
tinybird/tb/modules/connection.py,sha256=HwHn0YgwqcKEYCLr2OqDzJzAUFd65Wv8MZiSUw_TLII,18452
|
|
35
35
|
tinybird/tb/modules/connection_dynamodb.py,sha256=vaB28d6t8nOd6kcUQGMUkl3IZin5DYSBN2TBPm2968Q,9490
|
|
36
|
-
tinybird/tb/modules/connection_kafka.py,sha256=
|
|
37
|
-
tinybird/tb/modules/connection_s3.py,sha256=
|
|
36
|
+
tinybird/tb/modules/connection_kafka.py,sha256=AwGRrHqkV8mb5xSsFEEZ1HuamOPRukFRr5HsUJLfnbE,33502
|
|
37
|
+
tinybird/tb/modules/connection_s3.py,sha256=PeXZ4NHKhvAz0pUpXLPc7V1A8s9fllbyZJdyUFIC5Pw,16393
|
|
38
38
|
tinybird/tb/modules/copy.py,sha256=Apzjxiyinp6KmgamypPKEe3BAnpxG7MwYcSYkHiG8sA,6033
|
|
39
|
-
tinybird/tb/modules/create.py,sha256=
|
|
40
|
-
tinybird/tb/modules/datasource.py,sha256=
|
|
39
|
+
tinybird/tb/modules/create.py,sha256=kPeDZb60cfEyA8nIv_8JiMRAaZp4_03YsJfx4gQAgjs,22869
|
|
40
|
+
tinybird/tb/modules/datasource.py,sha256=6QGnjy5RDggyXq3GJ3ejeNo0Rx8itsn6zGvLKmz-_ek,79999
|
|
41
41
|
tinybird/tb/modules/deployment.py,sha256=vB4QSMsN4-DH-yDfSESQ5KKXgzC1t1fdCsqJ9lh8ej0,20956
|
|
42
|
-
tinybird/tb/modules/deployment_common.py,sha256=
|
|
42
|
+
tinybird/tb/modules/deployment_common.py,sha256=vU3MsZu4FGvgx9P2i1ulPElhc6LlwA1c_KiNb1nsEBk,36137
|
|
43
43
|
tinybird/tb/modules/deprecations.py,sha256=XXekrzPO9v12F1ToQDUGzLYJJ2wrEUlGKOkLCSdfHiM,4935
|
|
44
44
|
tinybird/tb/modules/endpoint.py,sha256=yRTh6rQFJ98LA0hSC8rPD3EcDaJj41gk9oCsgcZPu_c,12112
|
|
45
45
|
tinybird/tb/modules/exceptions.py,sha256=_1BHy0OixEkeF0fOCu1_1Pj8pJS4BNfUyucqCViJGTw,5958
|
|
@@ -79,18 +79,18 @@ tinybird/tb/modules/ts_project.py,sha256=XeDN-3gthT7GZYIFn0l-y5RJaofB_2T-I8gq6dL
|
|
|
79
79
|
tinybird/tb/modules/watch.py,sha256=HTZwqlGWDLJPBxBh04J0tAZDP_6BhqLupHBDx4uZDYo,5218
|
|
80
80
|
tinybird/tb/modules/workspace.py,sha256=7shWyyZD9IT32EeP0aEgsZGk0U1ZdT6hWnVfctRoIJk,8939
|
|
81
81
|
tinybird/tb/modules/workspace_members.py,sha256=8oQLTczEh9cIdD3iF-N3SJuqLtKdF-_g-YzeVaqKGP0,9486
|
|
82
|
-
tinybird/tb/modules/datafile/build.py,sha256=
|
|
82
|
+
tinybird/tb/modules/datafile/build.py,sha256=RzP8vB4yMZmqVNtcHnc40RDdaZhYhFz7oebauf1Y9mQ,40887
|
|
83
83
|
tinybird/tb/modules/datafile/build_common.py,sha256=2yNdxe49IMA9wNvl25NemY2Iaz8L66snjOdT64dm1is,4511
|
|
84
84
|
tinybird/tb/modules/datafile/build_datasource.py,sha256=keAx2lchYoxEDXQfTBtAe94Oyr4zfw3kWMumBFNtYso,16914
|
|
85
|
-
tinybird/tb/modules/datafile/build_pipe.py,sha256=
|
|
86
|
-
tinybird/tb/modules/datafile/diff.py,sha256=
|
|
85
|
+
tinybird/tb/modules/datafile/build_pipe.py,sha256=4e8bZWcJBw4WdP4sUAfRN8BU9Iu9383jZNZbNPGyOSU,11217
|
|
86
|
+
tinybird/tb/modules/datafile/diff.py,sha256=gxaYQ7SNNxUSQa9Bv92dahymSzx6oOaoGuhNM8_x72g,6709
|
|
87
87
|
tinybird/tb/modules/datafile/fixture.py,sha256=6s2LiVPxwCP9hLCW4TD7tU2AYo0iT_QvgJHJj-KY3E8,1063
|
|
88
|
-
tinybird/tb/modules/datafile/format_common.py,sha256=
|
|
89
|
-
tinybird/tb/modules/datafile/format_connection.py,sha256=
|
|
90
|
-
tinybird/tb/modules/datafile/format_datasource.py,sha256=
|
|
91
|
-
tinybird/tb/modules/datafile/format_pipe.py,sha256=
|
|
92
|
-
tinybird/tb/modules/datafile/pipe_checker.py,sha256=
|
|
93
|
-
tinybird/tb/modules/datafile/playground.py,sha256=
|
|
88
|
+
tinybird/tb/modules/datafile/format_common.py,sha256=G7ux7dHRioYNi5RFLUOy0NIaxlEHEmAoo1nO3hgijDk,4100
|
|
89
|
+
tinybird/tb/modules/datafile/format_connection.py,sha256=9opv2kxizA_05VbBAWGBH10Oyql21XnE3IDB2DoQvCs,5069
|
|
90
|
+
tinybird/tb/modules/datafile/format_datasource.py,sha256=_w5Y4tgl3Sejf0T8146F_owC1p5tFxXoTaSP5ggT36A,5305
|
|
91
|
+
tinybird/tb/modules/datafile/format_pipe.py,sha256=Rs_2P55vz8r0zg3dI0pNa-9azufuiU5tjORlyTqWLr8,7704
|
|
92
|
+
tinybird/tb/modules/datafile/pipe_checker.py,sha256=jpHfaql5sjYPKb9jSYMwV0pWSVncTZIokijKZAgWweY,24929
|
|
93
|
+
tinybird/tb/modules/datafile/playground.py,sha256=o1jnAAN72OVK_gpiDez73EtZc4uvL2n9j8yCZbShfvo,40948
|
|
94
94
|
tinybird/tb/modules/datafile/pull.py,sha256=MBLT9qO51Bz2BQwj2h1jTwlOVokFKZyfbxehRQOcdeQ,6441
|
|
95
95
|
tinybird/tb/modules/tinyunit/tinyunit.py,sha256=ZhKSwk5uIRggxDAPaqYZrcMMI0hLN1qGNIf-eRoJJWc,11055
|
|
96
96
|
tinybird/tb_cli_modules/cicd.py,sha256=i2Mw8AbmEVNBcEPYdio7liy3PGqh1ezVFZ0OmJ9ww5o,13809
|
|
@@ -99,8 +99,8 @@ tinybird/tb_cli_modules/config.py,sha256=lukiJeihI09blGeeS0FWeqVbKWhqzZmyoaJDIL0
|
|
|
99
99
|
tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
|
|
100
100
|
tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
101
101
|
tinybird/tb_cli_modules/telemetry.py,sha256=W098H6jmS4kpE7hN3tadaREBTf7oMocel-lkKWN0pU8,10466
|
|
102
|
-
tinybird-4.6.
|
|
103
|
-
tinybird-4.6.
|
|
104
|
-
tinybird-4.6.
|
|
105
|
-
tinybird-4.6.
|
|
106
|
-
tinybird-4.6.
|
|
102
|
+
tinybird-4.6.20.dev0.dist-info/METADATA,sha256=pYYRtFxp5t3_BE37SADxgxjkFnwfmE0EWJIQfferHi4,17670
|
|
103
|
+
tinybird-4.6.20.dev0.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
|
|
104
|
+
tinybird-4.6.20.dev0.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
|
|
105
|
+
tinybird-4.6.20.dev0.dist-info/top_level.txt,sha256=ZIQJTPCzMqnfDzM_hEGZrJqDSEcKnIK_49T86DGWpyQ,78
|
|
106
|
+
tinybird-4.6.20.dev0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|