tinybird-cli 6.5.6.dev0__py3-none-any.whl → 6.5.8.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tinybird/__cli__.py +2 -2
- tinybird/ch_utils/constants.py +1 -0
- tinybird/ch_utils/engine.py +90 -54
- tinybird/client.py +80 -89
- tinybird/config.py +0 -4
- tinybird/datafile_common.py +72 -45
- tinybird/datatypes.py +34 -12
- tinybird/feedback_manager.py +5 -0
- tinybird/git_settings.py +1 -3
- tinybird/sql.py +3 -10
- tinybird/sql_template.py +128 -98
- tinybird/sql_template_fmt.py +10 -15
- tinybird/sql_toolset.py +63 -11
- tinybird/syncasync.py +9 -54
- tinybird/tb_cli_modules/branch.py +4 -25
- tinybird/tb_cli_modules/cicd.py +2 -0
- tinybird/tb_cli_modules/cli.py +5 -9
- tinybird/tb_cli_modules/common.py +33 -87
- tinybird/tb_cli_modules/config.py +1 -2
- tinybird/tb_cli_modules/datasource.py +15 -55
- tinybird/tb_cli_modules/pipe.py +49 -34
- tinybird/tb_cli_modules/tinyunit/tinyunit.py +4 -4
- tinybird/tb_cli_modules/token.py +1 -2
- tinybird/tb_cli_modules/workspace.py +11 -13
- tinybird/tornado_template.py +227 -83
- tinybird/utils/__init__.py +0 -0
- tinybird/utils/bools.py +19 -0
- {tinybird_cli-6.5.6.dev0.dist-info → tinybird_cli-6.5.8.dev0.dist-info}/METADATA +15 -1
- tinybird_cli-6.5.8.dev0.dist-info/RECORD +45 -0
- tinybird_cli-6.5.6.dev0.dist-info/RECORD +0 -43
- {tinybird_cli-6.5.6.dev0.dist-info → tinybird_cli-6.5.8.dev0.dist-info}/WHEEL +0 -0
- {tinybird_cli-6.5.6.dev0.dist-info → tinybird_cli-6.5.8.dev0.dist-info}/entry_points.txt +0 -0
- {tinybird_cli-6.5.6.dev0.dist-info → tinybird_cli-6.5.8.dev0.dist-info}/top_level.txt +0 -0
tinybird/__cli__.py
CHANGED
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/cli'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '6.5.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '6.5.8.dev0'
|
|
8
|
+
__revision__ = '57adbc63'
|
tinybird/ch_utils/constants.py
CHANGED
tinybird/ch_utils/engine.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import functools
|
|
1
2
|
import logging
|
|
2
3
|
import re
|
|
3
4
|
from collections import defaultdict
|
|
@@ -25,6 +26,9 @@ DEFAULT_EMPTY_PARAMETERS = ["ttl", "partition_key", "sorting_key"]
|
|
|
25
26
|
# * `N` is a number
|
|
26
27
|
SIMPLE_TTL_DEFINITION = re.compile(r"""^([a-zA-Z0-9_\-\.\(\)\ \/\*]*) \+ (toInterval[a-zA-Z]*\([0-9]+\))$""")
|
|
27
28
|
|
|
29
|
+
# ClickHouse 25.8 quotes the engine name `Null`, as `null` is a keyword.
|
|
30
|
+
_BACKQUOTED_ENGINE_NAME = re.compile(r"^`(\w+)`")
|
|
31
|
+
|
|
28
32
|
|
|
29
33
|
class TableDetails:
|
|
30
34
|
"""
|
|
@@ -78,6 +82,16 @@ class TableDetails:
|
|
|
78
82
|
|
|
79
83
|
@property
|
|
80
84
|
def engine_full(self) -> Optional[str]:
|
|
85
|
+
"""
|
|
86
|
+
>>> TableDetails({"engine_full": "`Null`", "engine": "Null"}).engine_full
|
|
87
|
+
'Null'
|
|
88
|
+
>>> TableDetails({"engine_full": "`Null`()", "engine": "Null"}).engine_full
|
|
89
|
+
'Null()'
|
|
90
|
+
>>> TableDetails({"engine_full": "Null", "engine": "Null"}).to_json()
|
|
91
|
+
{'engine_full': 'Null', 'engine': 'Null'}
|
|
92
|
+
>>> TableDetails({"engine_full": "`Null`", "engine": "Null"}).to_json()
|
|
93
|
+
{'engine_full': 'Null', 'engine': 'Null'}
|
|
94
|
+
"""
|
|
81
95
|
_engine_full: Optional[str] = self.details.get("engine_full", None)
|
|
82
96
|
if not _engine_full:
|
|
83
97
|
return None
|
|
@@ -86,7 +100,7 @@ class TableDetails:
|
|
|
86
100
|
# We cannot remove index_granularity = 8192 blindly because it might be followed by other settings
|
|
87
101
|
if settings and settings.strip().lower() == "index_granularity = 8192":
|
|
88
102
|
_engine_full = _engine_full.replace(" SETTINGS index_granularity = 8192", "")
|
|
89
|
-
return engine_replicated_to_local(_engine_full)
|
|
103
|
+
return engine_replicated_to_local(_BACKQUOTED_ENGINE_NAME.sub(r"\1", _engine_full))
|
|
90
104
|
|
|
91
105
|
@property
|
|
92
106
|
def original_engine_full(self) -> Optional[str]:
|
|
@@ -111,8 +125,7 @@ class TableDetails:
|
|
|
111
125
|
|
|
112
126
|
@property
|
|
113
127
|
def version(self):
|
|
114
|
-
|
|
115
|
-
return _version
|
|
128
|
+
return self.details.get("version", None)
|
|
116
129
|
|
|
117
130
|
def is_replicated(self) -> bool:
|
|
118
131
|
engine: Optional[str] = self.details.get("engine", None)
|
|
@@ -183,6 +196,7 @@ class TableDetails:
|
|
|
183
196
|
settings = self.details.get("settings", None)
|
|
184
197
|
if settings and settings.strip().lower() != "index_granularity = 8192":
|
|
185
198
|
return settings
|
|
199
|
+
return None
|
|
186
200
|
|
|
187
201
|
@property
|
|
188
202
|
def ttl(self):
|
|
@@ -190,23 +204,19 @@ class TableDetails:
|
|
|
190
204
|
|
|
191
205
|
@property
|
|
192
206
|
def ver(self):
|
|
193
|
-
|
|
194
|
-
return _ver
|
|
207
|
+
return self.details.get("ver", None)
|
|
195
208
|
|
|
196
209
|
@property
|
|
197
210
|
def is_deleted(self):
|
|
198
|
-
|
|
199
|
-
return _is_deleted
|
|
211
|
+
return self.details.get("is_deleted", None)
|
|
200
212
|
|
|
201
213
|
@property
|
|
202
214
|
def columns(self):
|
|
203
|
-
|
|
204
|
-
return _columns
|
|
215
|
+
return self.details.get("columns", None)
|
|
205
216
|
|
|
206
217
|
@property
|
|
207
218
|
def sign(self):
|
|
208
|
-
|
|
209
|
-
return _sign
|
|
219
|
+
return self.details.get("sign", None)
|
|
210
220
|
|
|
211
221
|
@property
|
|
212
222
|
def statistics(self) -> Dict[str, Any]:
|
|
@@ -444,38 +454,70 @@ def get_engine_config(engine: str):
|
|
|
444
454
|
)
|
|
445
455
|
|
|
446
456
|
|
|
447
|
-
def
|
|
448
|
-
|
|
457
|
+
def validate_engine_config(
|
|
458
|
+
engine: str, args: dict, schema: Optional[str] = None, columns: Optional[List[Dict[str, Any]]] = None
|
|
459
|
+
) -> None:
|
|
460
|
+
if schema is not None and columns is not None:
|
|
461
|
+
raise ValueError("You can not use 'schema' and 'columns' at the same time")
|
|
462
|
+
_, (params, options) = get_engine_config(engine)
|
|
463
|
+
schema_columns = parse_table_structure(schema) if schema is not None else (columns or [])
|
|
464
|
+
engine_settings = {key.replace("engine_", ""): value for key, value in args.items()}
|
|
465
|
+
|
|
466
|
+
for arg in engine_settings:
|
|
467
|
+
if not hasattr(TableDetails, arg):
|
|
468
|
+
raise ValueError(f"engine_{arg} is not a valid option")
|
|
469
|
+
|
|
449
470
|
for p in params:
|
|
450
|
-
if p.required and p.name not in
|
|
471
|
+
if p.required and p.name not in engine_settings:
|
|
451
472
|
raise ValueError(f"Missing required parameter '{p.name}'")
|
|
452
|
-
param_value =
|
|
473
|
+
param_value = engine_settings.get(p.name) or p.default_value
|
|
453
474
|
if not param_value:
|
|
454
475
|
continue
|
|
455
476
|
if p.is_valid:
|
|
456
477
|
check_is_valid(
|
|
457
|
-
valid_check=p.is_valid,
|
|
478
|
+
valid_check=p.is_valid,
|
|
479
|
+
check_type="parameter",
|
|
480
|
+
columns=schema_columns,
|
|
481
|
+
tb_param=p.tb_param,
|
|
482
|
+
value=param_value,
|
|
458
483
|
)
|
|
484
|
+
|
|
485
|
+
for o in options:
|
|
486
|
+
if o.required and o.name not in engine_settings:
|
|
487
|
+
raise ValueError(f"Missing required option '{o.name}'")
|
|
488
|
+
option_value = engine_settings.get(o.name) or o.default_value
|
|
489
|
+
if o.is_valid:
|
|
490
|
+
check_is_valid(
|
|
491
|
+
valid_check=o.is_valid,
|
|
492
|
+
check_type="option",
|
|
493
|
+
columns=schema_columns,
|
|
494
|
+
tb_param=o.tb_param,
|
|
495
|
+
value=option_value,
|
|
496
|
+
)
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def engine_params(params: List[EngineParam], args: Dict) -> List[str]:
|
|
500
|
+
params_values = []
|
|
501
|
+
for p in params:
|
|
502
|
+
param_value = args.get(p.name, None) or p.default_value
|
|
503
|
+
if not param_value:
|
|
504
|
+
continue
|
|
459
505
|
params_values.append(param_value)
|
|
460
506
|
return params_values
|
|
461
507
|
|
|
462
508
|
|
|
463
|
-
def engine_options(
|
|
509
|
+
def engine_options(options: List[EngineOption], args: Dict) -> List[str]:
|
|
464
510
|
options_values = []
|
|
465
511
|
engine_settings = ""
|
|
466
512
|
|
|
467
513
|
for o in options:
|
|
468
|
-
if o.required and o.name not in args:
|
|
469
|
-
raise ValueError(f"Missing required option '{o.name}'")
|
|
470
514
|
option_value = args.get(o.name) or o.default_value
|
|
471
|
-
if o.is_valid:
|
|
472
|
-
check_is_valid(
|
|
473
|
-
valid_check=o.is_valid, check_type="option", columns=columns, tb_param=o.tb_param, value=option_value
|
|
474
|
-
)
|
|
475
|
-
|
|
476
515
|
if option_value:
|
|
477
516
|
if o.sql.lower() == "settings":
|
|
478
517
|
engine_settings = f"{o.sql} {option_value}"
|
|
518
|
+
# Conditional TTLs fail if we wrap the value in parentheses
|
|
519
|
+
elif o.sql.lower() == "ttl":
|
|
520
|
+
options_values.append(f"{o.sql} {option_value}")
|
|
479
521
|
else:
|
|
480
522
|
options_values.append(f"{o.sql} ({option_value})")
|
|
481
523
|
|
|
@@ -491,7 +533,7 @@ def check_is_valid(
|
|
|
491
533
|
columns: List[Dict[str, Any]],
|
|
492
534
|
tb_param: str,
|
|
493
535
|
value: str,
|
|
494
|
-
):
|
|
536
|
+
) -> None:
|
|
495
537
|
"""
|
|
496
538
|
>>> check_is_valid(sorting_key_is_valid, 'option', ['column-name'], 'sorting_key', 'date')
|
|
497
539
|
|
|
@@ -508,15 +550,13 @@ def check_is_valid(
|
|
|
508
550
|
raise ValueError(f"Invalid value '{value}' for {check_type} '{tb_param}', reason: {e}")
|
|
509
551
|
|
|
510
552
|
|
|
511
|
-
def build_engine(
|
|
512
|
-
engine
|
|
513
|
-
):
|
|
514
|
-
return f"{engine}({', '.join(engine_params(columns, params, args))}) {' '.join(engine_options(columns, options, args))}".strip()
|
|
553
|
+
def build_engine(engine: str, params: List[EngineParam], options: List[EngineOption], args: Dict) -> str:
|
|
554
|
+
return f"{engine}({', '.join(engine_params(params, args))}) {' '.join(engine_options(options, args))}".strip()
|
|
515
555
|
|
|
516
556
|
|
|
517
557
|
def engine_full_from_dict(
|
|
518
558
|
engine: str, args: dict, schema: Optional[str] = None, columns: Optional[List[Dict[str, Any]]] = None
|
|
519
|
-
):
|
|
559
|
+
) -> str:
|
|
520
560
|
"""
|
|
521
561
|
>>> schema = ''
|
|
522
562
|
>>> engine_full_from_dict('wadus', {}, schema=schema)
|
|
@@ -536,7 +576,9 @@ def engine_full_from_dict(
|
|
|
536
576
|
>>> engine_full_from_dict('MergeTree', {'sorting_key': 'local_date, cod_store'}, schema=schema)
|
|
537
577
|
'MergeTree() ORDER BY (local_date, cod_store)'
|
|
538
578
|
>>> engine_full_from_dict('MergeTree', {'partition_key': 'toDate(timestamp)', 'sorting_key': 'local_date, cod_store', 'settings': 'index_granularity = 32, index_granularity_bytes = 2048', 'ttl': 'toDate(local_date) + INTERVAL 1 DAY'}, schema=schema)
|
|
539
|
-
'MergeTree() PARTITION BY (toDate(timestamp)) ORDER BY (local_date, cod_store) TTL
|
|
579
|
+
'MergeTree() PARTITION BY (toDate(timestamp)) ORDER BY (local_date, cod_store) TTL toDate(local_date) + INTERVAL 1 DAY SETTINGS index_granularity = 32, index_granularity_bytes = 2048'
|
|
580
|
+
>>> engine_full_from_dict('MergeTree', {'sorting_key': 'timestamp', 'ttl': 'timestamp + INTERVAL 1 DAY DELETE WHERE is_deleted = 1'}, schema=schema)
|
|
581
|
+
'MergeTree() ORDER BY (timestamp) TTL timestamp + INTERVAL 1 DAY DELETE WHERE is_deleted = 1'
|
|
540
582
|
|
|
541
583
|
>>> schema = ''
|
|
542
584
|
>>> engine_full_from_dict('CollapsingMergeTree', {'sign': 'sign_column'}, schema=schema)
|
|
@@ -605,20 +647,10 @@ def engine_full_from_dict(
|
|
|
605
647
|
'ReplacingMergeTree(updated_at) PARTITION BY (tuple()) ORDER BY (project_id) SETTINGS index_granularity = 32'
|
|
606
648
|
"""
|
|
607
649
|
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
engine_config = get_engine_config(engine)
|
|
611
|
-
name, (params, options) = engine_config
|
|
612
|
-
if columns is None and schema is not None:
|
|
613
|
-
columns = parse_table_structure(schema)
|
|
614
|
-
|
|
650
|
+
validate_engine_config(engine, args, schema=schema, columns=columns)
|
|
651
|
+
name, (params, options) = get_engine_config(engine)
|
|
615
652
|
engine_settings = {key.replace("engine_", ""): value for key, value in args.items()}
|
|
616
|
-
|
|
617
|
-
for arg in engine_settings:
|
|
618
|
-
if not hasattr(TableDetails, arg):
|
|
619
|
-
raise ValueError(f"engine_{arg} is not a valid option")
|
|
620
|
-
|
|
621
|
-
return build_engine(name, columns, params, options, engine_settings)
|
|
653
|
+
return build_engine(name, params, options, engine_settings)
|
|
622
654
|
|
|
623
655
|
|
|
624
656
|
def engine_params_from_engine_full(engine_full: str) -> Dict[str, Any]:
|
|
@@ -694,8 +726,7 @@ def ttl_from_engine(engine: str) -> Optional[str]:
|
|
|
694
726
|
return None
|
|
695
727
|
settings_array = engine.split(" SETTINGS ")
|
|
696
728
|
settings = " SETTINGS " + settings_array[1] if len(settings_array) > 1 else None
|
|
697
|
-
|
|
698
|
-
return ttl
|
|
729
|
+
return ttl_array[1][: -(len(settings))] if settings else ttl_array[1]
|
|
699
730
|
|
|
700
731
|
|
|
701
732
|
def ttl_condition_from_engine_full(engine_full: Optional[str]) -> Optional[str]:
|
|
@@ -759,17 +790,22 @@ def ttl_condition_from_engine_full(engine_full: Optional[str]) -> Optional[str]:
|
|
|
759
790
|
return None
|
|
760
791
|
|
|
761
792
|
|
|
793
|
+
@functools.lru_cache(maxsize=1024)
|
|
794
|
+
def _index_lines(create_table_query_expr: str) -> tuple[str, ...]:
|
|
795
|
+
from tinybird.sql_toolset import format_sql
|
|
796
|
+
|
|
797
|
+
return tuple(
|
|
798
|
+
line.strip() for line in format_sql(create_table_query_expr).splitlines() if line.strip().startswith("INDEX")
|
|
799
|
+
)
|
|
800
|
+
|
|
801
|
+
|
|
762
802
|
def _parse_indexes(create_table_query_expr: str) -> List[TableIndex]:
|
|
763
|
-
|
|
803
|
+
# format_sql is CPU-heavy and runs on the event loop. ClickHouse writes the
|
|
804
|
+
# INDEX keyword in uppercase in create_table_query.
|
|
805
|
+
if "INDEX" not in create_table_query_expr:
|
|
764
806
|
return []
|
|
765
807
|
try:
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
indexes = [
|
|
769
|
-
line.strip()
|
|
770
|
-
for line in format_sql(create_table_query_expr).splitlines()
|
|
771
|
-
if line.strip().startswith("INDEX")
|
|
772
|
-
]
|
|
808
|
+
indexes = list(_index_lines(create_table_query_expr))
|
|
773
809
|
except ModuleNotFoundError:
|
|
774
810
|
# this is not needed from CLI
|
|
775
811
|
return []
|
tinybird/client.py
CHANGED
|
@@ -47,14 +47,6 @@ class TimeoutException(Exception):
|
|
|
47
47
|
pass
|
|
48
48
|
|
|
49
49
|
|
|
50
|
-
class ReachRetryLimit(Exception):
|
|
51
|
-
pass
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
class ConnectorNothingToLoad(Exception):
|
|
55
|
-
pass
|
|
56
|
-
|
|
57
|
-
|
|
58
50
|
class JobException(Exception):
|
|
59
51
|
pass
|
|
60
52
|
|
|
@@ -266,14 +258,13 @@ class TinyB:
|
|
|
266
258
|
|
|
267
259
|
async def alter_tokens(self, name: str, scopes: List[str]):
|
|
268
260
|
if not scopes:
|
|
269
|
-
return
|
|
261
|
+
return None
|
|
270
262
|
scopes_url: str = "&".join([f"scope={scope}" for scope in scopes])
|
|
271
263
|
url = f"/v0/tokens/{name}"
|
|
272
264
|
if len(url + "?" + scopes_url) > TinyB.MAX_GET_LENGTH:
|
|
273
265
|
return await self._req(url, method="PUT", data=scopes_url)
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
return await self._req(url, method="PUT", data="")
|
|
266
|
+
url = url + "?" + scopes_url
|
|
267
|
+
return await self._req(url, method="PUT", data="")
|
|
277
268
|
|
|
278
269
|
async def datasources(self, branch: Optional[str] = None, used_by: bool = False) -> List[Dict[str, Any]]:
|
|
279
270
|
params = {}
|
|
@@ -294,16 +285,13 @@ class TinyB:
|
|
|
294
285
|
return await self._req(f"/v0/variables/{name}")
|
|
295
286
|
|
|
296
287
|
async def create_secret(self, name: str, value: str):
|
|
297
|
-
|
|
298
|
-
return response
|
|
288
|
+
return await self._req("/v0/variables", method="POST", data={"name": name, "value": value})
|
|
299
289
|
|
|
300
290
|
async def update_secret(self, name: str, value: str):
|
|
301
|
-
|
|
302
|
-
return response
|
|
291
|
+
return await self._req(f"/v0/variables/{name}", method="PUT", data={"value": value})
|
|
303
292
|
|
|
304
293
|
async def delete_secret(self, name: str):
|
|
305
|
-
|
|
306
|
-
return response
|
|
294
|
+
return await self._req(f"/v0/variables/{name}", method="DELETE")
|
|
307
295
|
|
|
308
296
|
async def get_connections(self, service: Optional[str] = None):
|
|
309
297
|
params = {}
|
|
@@ -411,7 +399,17 @@ class TinyB:
|
|
|
411
399
|
sql_condition: Optional[str] = None,
|
|
412
400
|
format: str = "csv",
|
|
413
401
|
replace_options: Optional[Set[str]] = None,
|
|
414
|
-
):
|
|
402
|
+
) -> Dict[str, Any]:
|
|
403
|
+
if mode in ("append", "replace"):
|
|
404
|
+
return await self.datasource_append_url(
|
|
405
|
+
table_name,
|
|
406
|
+
url,
|
|
407
|
+
mode=mode,
|
|
408
|
+
sql_condition=sql_condition,
|
|
409
|
+
format=format,
|
|
410
|
+
replace_options=replace_options,
|
|
411
|
+
)
|
|
412
|
+
|
|
415
413
|
params = {"name": table_name, "url": url, "mode": mode, "debug": "blocks_block_log", "format": format}
|
|
416
414
|
|
|
417
415
|
if sql_condition:
|
|
@@ -428,6 +426,33 @@ class TinyB:
|
|
|
428
426
|
|
|
429
427
|
return await self.wait_for_job(res["id"], status_callback, backoff_multiplier=1.5, maximum_backoff_seconds=20)
|
|
430
428
|
|
|
429
|
+
async def datasource_append_url(
|
|
430
|
+
self,
|
|
431
|
+
datasource_name: str,
|
|
432
|
+
url: str,
|
|
433
|
+
mode: str = "append",
|
|
434
|
+
sql_condition: Optional[str] = None,
|
|
435
|
+
format: str = "csv",
|
|
436
|
+
replace_options: Optional[Set[str]] = None,
|
|
437
|
+
) -> Dict[str, Any]:
|
|
438
|
+
params = {"format": format, "url": url}
|
|
439
|
+
if sql_condition:
|
|
440
|
+
params["replace_condition"] = sql_condition
|
|
441
|
+
if replace_options:
|
|
442
|
+
for option in replace_options:
|
|
443
|
+
params[option] = "true"
|
|
444
|
+
|
|
445
|
+
res = await self._req(
|
|
446
|
+
f"/v1/datasources/{quote(datasource_name, safe='')}/{mode}?{urlencode(params, safe='')}",
|
|
447
|
+
method="POST",
|
|
448
|
+
data=b"",
|
|
449
|
+
)
|
|
450
|
+
if not isinstance(res, dict):
|
|
451
|
+
raise RuntimeError("We couldn't confirm that your import started. Please try again.")
|
|
452
|
+
if "error" in res:
|
|
453
|
+
raise Exception(res["error"])
|
|
454
|
+
return res
|
|
455
|
+
|
|
431
456
|
async def datasource_delete(self, datasource_name: str, force: bool = False, dry_run: bool = False):
|
|
432
457
|
params = {"force": "true" if force else "false", "dry_run": "true" if dry_run else "false"}
|
|
433
458
|
return await self._req(f"/v0/datasources/{datasource_name}?{urlencode(params)}", method="DELETE")
|
|
@@ -437,58 +462,37 @@ class TinyB:
|
|
|
437
462
|
datasource_name: str,
|
|
438
463
|
file: Union[str, Path],
|
|
439
464
|
mode: str = "append",
|
|
440
|
-
status_callback=None,
|
|
465
|
+
status_callback=None, # noqa: ARG002 - kept for v0 caller compatibility
|
|
441
466
|
sql_condition: Optional[str] = None,
|
|
442
467
|
format: str = "csv",
|
|
443
468
|
replace_options: Optional[Set[str]] = None,
|
|
444
|
-
use_v1: bool = False,
|
|
445
469
|
):
|
|
446
|
-
|
|
447
|
-
|
|
470
|
+
# V1 returns the import job immediately, so waiting and progress reporting
|
|
471
|
+
# belong to the caller.
|
|
472
|
+
async with aiofiles.open(file, "rb") as content:
|
|
473
|
+
file_content = await content.read()
|
|
474
|
+
v1_params = {"format": format}
|
|
448
475
|
if sql_condition:
|
|
449
|
-
|
|
476
|
+
v1_params["replace_condition"] = sql_condition
|
|
450
477
|
if replace_options:
|
|
451
478
|
for option in list(replace_options):
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
if replace_options:
|
|
462
|
-
for option in list(replace_options):
|
|
463
|
-
v1_params[option] = "true"
|
|
464
|
-
content_types = {
|
|
465
|
-
"csv": "text/csv",
|
|
466
|
-
"ndjson": "application/x-ndjson",
|
|
467
|
-
"parquet": "application/vnd.apache.parquet",
|
|
468
|
-
}
|
|
469
|
-
headers = {"Content-Type": content_types[format]}
|
|
470
|
-
if str(file).endswith(".gz"):
|
|
471
|
-
headers["Content-Encoding"] = "gzip"
|
|
472
|
-
return await self._req(
|
|
473
|
-
f"/v1/datasources/{quote(datasource_name, safe='')}/{mode}?{urlencode(v1_params, safe='')}",
|
|
474
|
-
data=file_content,
|
|
475
|
-
headers=headers,
|
|
476
|
-
method="POST",
|
|
477
|
-
)
|
|
478
|
-
|
|
479
|
-
if format == "csv":
|
|
480
|
-
files = {"csv": ("csv", file_content)}
|
|
481
|
-
else:
|
|
482
|
-
files = {"ndjson": ("ndjson", file_content)}
|
|
483
|
-
|
|
479
|
+
v1_params[option] = "true"
|
|
480
|
+
content_types = {
|
|
481
|
+
"csv": "text/csv",
|
|
482
|
+
"ndjson": "application/x-ndjson",
|
|
483
|
+
"parquet": "application/vnd.apache.parquet",
|
|
484
|
+
}
|
|
485
|
+
headers = {"Content-Type": content_types[format]}
|
|
486
|
+
if str(file).endswith(".gz"):
|
|
487
|
+
headers["Content-Encoding"] = "gzip"
|
|
484
488
|
res = await self._req(
|
|
485
|
-
f"
|
|
486
|
-
|
|
489
|
+
f"/v1/datasources/{quote(datasource_name, safe='')}/{mode}?{urlencode(v1_params, safe='')}",
|
|
490
|
+
data=file_content,
|
|
491
|
+
headers=headers,
|
|
487
492
|
method="POST",
|
|
488
493
|
)
|
|
489
|
-
if
|
|
490
|
-
|
|
491
|
-
|
|
494
|
+
if not isinstance(res, dict):
|
|
495
|
+
raise RuntimeError("We couldn't confirm that your import started. Please try again.")
|
|
492
496
|
return res
|
|
493
497
|
|
|
494
498
|
async def datasource_truncate(self, datasource_name: str):
|
|
@@ -598,8 +602,7 @@ class TinyB:
|
|
|
598
602
|
node_name = node["params"]["name"] if node.get("params", None) else node["name"]
|
|
599
603
|
if datasource_name:
|
|
600
604
|
params["datasource"] = datasource_name
|
|
601
|
-
|
|
602
|
-
return response
|
|
605
|
+
return await self._req(f"/v0/pipes/{pipe_name}/nodes/{node_name}/analysis?{urlencode(params)}")
|
|
603
606
|
|
|
604
607
|
async def populate_node(
|
|
605
608
|
self,
|
|
@@ -621,10 +624,7 @@ class TinyB:
|
|
|
621
624
|
params.update({"populate_condition": populate_condition})
|
|
622
625
|
if on_demand_compute:
|
|
623
626
|
params.update({"on_demand_compute": "true"})
|
|
624
|
-
|
|
625
|
-
f"/v0/pipes/{pipe_name}/nodes/{node_name}/population?{urlencode(params)}", method="POST"
|
|
626
|
-
)
|
|
627
|
-
return response
|
|
627
|
+
return await self._req(f"/v0/pipes/{pipe_name}/nodes/{node_name}/population?{urlencode(params)}", method="POST")
|
|
628
628
|
|
|
629
629
|
async def pipes(self, branch=None, dependencies: bool = False, node_attrs=None, attrs=None) -> List[Dict[str, Any]]:
|
|
630
630
|
params = {
|
|
@@ -656,9 +656,8 @@ class TinyB:
|
|
|
656
656
|
query_string = urlencode(params)
|
|
657
657
|
if len(url + "?" + query_string) > TinyB.MAX_GET_LENGTH:
|
|
658
658
|
return await self._req(f"/v0/pipes/{pipe_name_or_uid}.{format}", method="POST", data=params)
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
return await self._req(url)
|
|
659
|
+
url = url + "?" + query_string
|
|
660
|
+
return await self._req(url)
|
|
662
661
|
|
|
663
662
|
async def pipe_create(self, pipe_name: str, sql: str):
|
|
664
663
|
return await self._req(
|
|
@@ -684,6 +683,7 @@ class TinyB:
|
|
|
684
683
|
target_datasource: Optional[str] = None,
|
|
685
684
|
schedule_cron: Optional[str] = None,
|
|
686
685
|
mode: Optional[str] = None,
|
|
686
|
+
on_demand_compute: Optional[bool] = None,
|
|
687
687
|
):
|
|
688
688
|
data = {"schedule_cron": schedule_cron}
|
|
689
689
|
|
|
@@ -693,6 +693,9 @@ class TinyB:
|
|
|
693
693
|
if mode:
|
|
694
694
|
data["mode"] = mode
|
|
695
695
|
|
|
696
|
+
if on_demand_compute is not None:
|
|
697
|
+
data["on_demand_compute"] = "true" if on_demand_compute else "false"
|
|
698
|
+
|
|
696
699
|
return await self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/copy", method="PUT", data=data)
|
|
697
700
|
|
|
698
701
|
async def pipe_remove_copy(self, pipe_name_or_id: str, node_id: str):
|
|
@@ -747,8 +750,7 @@ class TinyB:
|
|
|
747
750
|
|
|
748
751
|
if len(sql) > TinyB.MAX_GET_LENGTH:
|
|
749
752
|
return await self._req(f"/v0/sql?{urlencode(params)}", data=sql, method="POST")
|
|
750
|
-
|
|
751
|
-
return await self._req(f"/v0/sql?q={quote(sql, safe='')}&{urlencode(params)}")
|
|
753
|
+
return await self._req(f"/v0/sql?q={quote(sql, safe='')}&{urlencode(params)}")
|
|
752
754
|
|
|
753
755
|
async def jobs(self, status=None):
|
|
754
756
|
jobs = (await self._req("/v0/jobs"))["jobs"]
|
|
@@ -849,7 +851,6 @@ class TinyB:
|
|
|
849
851
|
assert_time_increase_percentage: Optional[float] = None,
|
|
850
852
|
assert_bytes_read_increase_percentage: Optional[float] = None,
|
|
851
853
|
assert_max_time: Optional[float] = None,
|
|
852
|
-
run_in_main: Optional[bool] = False,
|
|
853
854
|
):
|
|
854
855
|
test: Dict[str, Any] = {
|
|
855
856
|
test_type: {
|
|
@@ -879,20 +880,12 @@ class TinyB:
|
|
|
879
880
|
]
|
|
880
881
|
|
|
881
882
|
data = json.dumps(regression_commands)
|
|
882
|
-
|
|
883
|
-
url = f"/v0/environments/{branch_id}/regression/main"
|
|
884
|
-
else:
|
|
885
|
-
url = f"/v0/environments/{branch_id}/regression"
|
|
883
|
+
url = f"/v0/environments/{branch_id}/regression"
|
|
886
884
|
return await self._req(url, method="POST", data=data, headers={"Content-Type": "application/json"})
|
|
887
885
|
|
|
888
|
-
async def branch_regression_tests_file(
|
|
889
|
-
self, branch_id: str, regression_commands: List[Dict[str, Any]], run_in_main: Optional[bool] = False
|
|
890
|
-
):
|
|
886
|
+
async def branch_regression_tests_file(self, branch_id: str, regression_commands: List[Dict[str, Any]]):
|
|
891
887
|
data = json.dumps(regression_commands)
|
|
892
|
-
|
|
893
|
-
url = f"/v0/environments/{branch_id}/regression/main"
|
|
894
|
-
else:
|
|
895
|
-
url = f"/v0/environments/{branch_id}/regression"
|
|
888
|
+
url = f"/v0/environments/{branch_id}/regression"
|
|
896
889
|
return await self._req(url, method="POST", data=data, headers={"Content-Type": "application/json"})
|
|
897
890
|
|
|
898
891
|
async def delete_workspace(self, id: str, hard_delete_confirmation: Optional[str], version: str = "v0"):
|
|
@@ -943,7 +936,7 @@ class TinyB:
|
|
|
943
936
|
|
|
944
937
|
async def add_workspaces_to_organization(self, organization_id: str, workspace_ids: List[str]):
|
|
945
938
|
if not workspace_ids:
|
|
946
|
-
return
|
|
939
|
+
return None
|
|
947
940
|
return await self._req(
|
|
948
941
|
f"/v0/organizations/{organization_id}/workspaces",
|
|
949
942
|
method="PUT",
|
|
@@ -1126,8 +1119,7 @@ class TinyB:
|
|
|
1126
1119
|
from tinybird.sql_toolset import format_sql
|
|
1127
1120
|
|
|
1128
1121
|
return format_sql(sql)
|
|
1129
|
-
|
|
1130
|
-
return await self._sql_get_format_remote(sql, with_clickhouse_format)
|
|
1122
|
+
return await self._sql_get_format_remote(sql, with_clickhouse_format)
|
|
1131
1123
|
except ModuleNotFoundError:
|
|
1132
1124
|
return await self._sql_get_format_remote(sql, with_clickhouse_format)
|
|
1133
1125
|
|
|
@@ -1189,8 +1181,7 @@ class TinyB:
|
|
|
1189
1181
|
return next((connector for connector in result["connectors"] if connector_equals(connector, kwargs)), None)
|
|
1190
1182
|
|
|
1191
1183
|
async def regions(self):
|
|
1192
|
-
|
|
1193
|
-
return regions
|
|
1184
|
+
return await self._req("/v0/regions")
|
|
1194
1185
|
|
|
1195
1186
|
async def datasource_query_copy(self, datasource_name: str, sql_query: str):
|
|
1196
1187
|
params = {"copy_to": datasource_name}
|
tinybird/config.py
CHANGED
|
@@ -49,11 +49,7 @@ LEGACY_HOSTS = {
|
|
|
49
49
|
"https://ui.europe-west2.gcp.tinybird.co": "https://cloud.tinybird.co/gcp/europe-west2",
|
|
50
50
|
"https://ui.ap-east.aws.tinybird.co": "https://app.tinybird.co/aws/ap-east-1",
|
|
51
51
|
"https://ui.ap-east-1.aws.tinybird.co": "https://app.tinybird.co/aws/ap-east-1",
|
|
52
|
-
"https://ui.split.tinybird.co": "https://app.tinybird.co/aws/split-us-east",
|
|
53
|
-
"https://ui.split.us-west-2.aws.tinybird.co": "https://app.tinybird.co/aws/split-us-west-2",
|
|
54
52
|
"https://ui.northamerica-northeast2.gcp.tinybird.co": "https://app.tinybird.co/gcp/northamerica-northeast2",
|
|
55
|
-
"https://api.split.tinybird.co": "https://app.tinybird.co/aws/split-us-east",
|
|
56
|
-
"https://api.split.us-west-2.aws.tinybird.co": "https://app.tinybird.co/aws/split-us-west-2",
|
|
57
53
|
"https://ui.wadus1.gcp.tinybird.co": "https://app.wadus.tinybird.co/gcp/wadus1",
|
|
58
54
|
"https://ui.wadus2.gcp.tinybird.co": "https://app.wadus.tinybird.co/gcp/wadus2",
|
|
59
55
|
"https://ui.wadus3.gcp.tinybird.co": "https://app.wadus.tinybird.co/gcp/wadus3",
|