tinybird 0.0.1.dev68__py3-none-any.whl → 0.0.1.dev70__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/ch_utils/engine.py +2 -4
- tinybird/context.py +0 -1
- tinybird/sql_template.py +1 -3
- tinybird/sql_toolset.py +3 -3
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/modules/auth.py +1 -1
- tinybird/tb/modules/build.py +12 -1
- tinybird/tb/modules/cli.py +15 -6
- tinybird/tb/modules/common.py +9 -9
- tinybird/tb/modules/datafile/build_common.py +1 -1
- tinybird/tb/modules/datafile/build_datasource.py +1 -1
- tinybird/tb/modules/datafile/common.py +18 -1
- tinybird/tb/modules/datafile/fixture.py +7 -0
- tinybird/tb/modules/datafile/pipe_checker.py +1 -1
- tinybird/tb/modules/datasource.py +65 -2
- tinybird/tb/modules/deployment.py +31 -5
- tinybird/tb/modules/endpoint.py +1 -1
- tinybird/tb/modules/fmt.py +1 -1
- tinybird/tb/modules/materialization.py +5 -5
- tinybird/tb/modules/mock.py +26 -19
- tinybird/tb/modules/pipe.py +1 -1
- tinybird/tb/modules/project.py +3 -3
- tinybird/tb/modules/shell.py +13 -21
- tinybird/tb/modules/test.py +1 -1
- tinybird/tb/modules/token.py +4 -4
- tinybird/tb/modules/watch.py +1 -1
- tinybird/tb/modules/workspace.py +6 -6
- tinybird/tb/modules/workspace_members.py +6 -6
- tinybird/tb_cli_modules/common.py +2 -2
- tinybird/tornado_template.py +2 -1
- tinybird-0.0.1.dev70.dist-info/METADATA +73 -0
- {tinybird-0.0.1.dev68.dist-info → tinybird-0.0.1.dev70.dist-info}/RECORD +35 -35
- {tinybird-0.0.1.dev68.dist-info → tinybird-0.0.1.dev70.dist-info}/WHEEL +1 -1
- tinybird-0.0.1.dev68.dist-info/METADATA +0 -64
- {tinybird-0.0.1.dev68.dist-info → tinybird-0.0.1.dev70.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev68.dist-info → tinybird-0.0.1.dev70.dist-info}/top_level.txt +0 -0
tinybird/ch_utils/engine.py
CHANGED
|
@@ -456,7 +456,7 @@ ENABLED_ENGINES = [
|
|
|
456
456
|
MERGETREE_OPTIONS,
|
|
457
457
|
),
|
|
458
458
|
# AggregatingMergeTree()
|
|
459
|
-
engine_config("AggregatingMergeTree", options=
|
|
459
|
+
engine_config("AggregatingMergeTree", options=MERGETREE_OPTIONS),
|
|
460
460
|
# CollapsingMergeTree(sign)
|
|
461
461
|
engine_config(
|
|
462
462
|
"CollapsingMergeTree",
|
|
@@ -631,9 +631,7 @@ def engine_full_from_dict(
|
|
|
631
631
|
|
|
632
632
|
>>> schema = 'sign_column Int8'
|
|
633
633
|
>>> engine_full_from_dict('AggregatingMergeTree', {}, schema=schema)
|
|
634
|
-
|
|
635
|
-
...
|
|
636
|
-
ValueError: Missing required option 'sorting_key'
|
|
634
|
+
'AggregatingMergeTree() ORDER BY (tuple())'
|
|
637
635
|
|
|
638
636
|
>>> columns=[]
|
|
639
637
|
>>> columns.append({'name': 'key_column', 'type': 'Int8', 'codec': None, 'default_value': None, 'nullable': False, 'normalized_name': 'key_column'})
|
tinybird/context.py
CHANGED
|
@@ -20,5 +20,4 @@ engine: ContextVar[str] = ContextVar("engine")
|
|
|
20
20
|
wait_parameter: ContextVar[bool] = ContextVar("wait_parameter")
|
|
21
21
|
api_host: ContextVar[str] = ContextVar("api_host")
|
|
22
22
|
ff_split_to_array_escape: ContextVar[bool] = ContextVar("ff_split_to_array_escape")
|
|
23
|
-
ff_preprocess_parameters_circuit_breaker: ContextVar[bool] = ContextVar("ff_preprocess_parameters_circuit_breaker")
|
|
24
23
|
ff_column_json_backticks_circuit_breaker: ContextVar[bool] = ContextVar("ff_column_json_backticks_circuit_breaker")
|
tinybird/sql_template.py
CHANGED
|
@@ -14,7 +14,6 @@ from tornado.util import ObjectDict, exec_in, unicode_type
|
|
|
14
14
|
|
|
15
15
|
from tinybird.context import (
|
|
16
16
|
ff_column_json_backticks_circuit_breaker,
|
|
17
|
-
ff_preprocess_parameters_circuit_breaker,
|
|
18
17
|
ff_split_to_array_escape,
|
|
19
18
|
)
|
|
20
19
|
|
|
@@ -2244,14 +2243,13 @@ def render_sql_template(
|
|
|
2244
2243
|
tinybird.sql_template.SQLTemplateException: Template Syntax Error: Required parameter is not defined. Check the parameters test. Please provide a value or set a default value in the pipe code.
|
|
2245
2244
|
"""
|
|
2246
2245
|
escape_split_to_array = ff_split_to_array_escape.get(False)
|
|
2247
|
-
bypass_preprocess_variables = ff_preprocess_parameters_circuit_breaker.get(False)
|
|
2248
2246
|
|
|
2249
2247
|
t, template_variables, variable_warnings = get_template_and_variables(
|
|
2250
2248
|
sql, name, escape_arrays=escape_split_to_array
|
|
2251
2249
|
)
|
|
2252
2250
|
template_variables_with_types = get_var_names_and_types_cached(t)
|
|
2253
2251
|
|
|
2254
|
-
if
|
|
2252
|
+
if variables is not None:
|
|
2255
2253
|
processed_variables = preprocess_variables(variables, template_variables_with_types)
|
|
2256
2254
|
variables.update(processed_variables)
|
|
2257
2255
|
|
tinybird/sql_toolset.py
CHANGED
|
@@ -3,7 +3,7 @@ import logging
|
|
|
3
3
|
from collections import defaultdict
|
|
4
4
|
from datetime import datetime
|
|
5
5
|
from functools import lru_cache
|
|
6
|
-
from typing import FrozenSet, List, Optional, Set, Tuple
|
|
6
|
+
from typing import FrozenSet, List, Optional, Set, Tuple, Union
|
|
7
7
|
|
|
8
8
|
from chtoolset import query as chquery
|
|
9
9
|
from toposort import toposort
|
|
@@ -172,7 +172,7 @@ def tables_or_sql(replacement: dict, table_functions=False) -> set:
|
|
|
172
172
|
return {replacement}
|
|
173
173
|
|
|
174
174
|
|
|
175
|
-
def _separate_as_tuple_if_contains_database_and_table(definition: str) -> str
|
|
175
|
+
def _separate_as_tuple_if_contains_database_and_table(definition: str) -> Union[str, Tuple[str, str]]:
|
|
176
176
|
if "." in definition:
|
|
177
177
|
database_and_table_separated = definition.split(".")
|
|
178
178
|
return database_and_table_separated[0], database_and_table_separated[1]
|
|
@@ -255,7 +255,7 @@ def replace_tables(
|
|
|
255
255
|
function_allow_list=function_allow_list,
|
|
256
256
|
)
|
|
257
257
|
seen_tables = set()
|
|
258
|
-
table: Tuple[str, str]
|
|
258
|
+
table: Union[Tuple[str, str], Tuple[str, str, str]]
|
|
259
259
|
if function_allow_list is None:
|
|
260
260
|
_enabled_table_functions = ENABLED_TABLE_FUNCTIONS
|
|
261
261
|
else:
|
tinybird/tb/__cli__.py
CHANGED
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '0.0.1.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '0.0.1.dev70'
|
|
8
|
+
__revision__ = 'f8064d4'
|
tinybird/tb/modules/auth.py
CHANGED
|
@@ -30,7 +30,7 @@ from tinybird.tb.modules.regions import Region
|
|
|
30
30
|
@click.option("--token", help="Use auth token, defaults to TB_TOKEN envvar, then to the .tinyb file")
|
|
31
31
|
@click.option(
|
|
32
32
|
"--host",
|
|
33
|
-
help="Set custom host if it's different than https://api.tinybird.co. Check https://www.tinybird.co/docs/api-reference
|
|
33
|
+
help="Set custom host if it's different than https://api.tinybird.co. Check https://www.tinybird.co/docs/api-reference#regions-and-endpoints for the available list of regions",
|
|
34
34
|
)
|
|
35
35
|
@click.option(
|
|
36
36
|
"--region", envvar="TB_REGION", help="Set region. Run 'tb auth ls' to show available regions. Overrides host."
|
tinybird/tb/modules/build.py
CHANGED
|
@@ -15,7 +15,7 @@ from tinybird.client import TinyB
|
|
|
15
15
|
from tinybird.tb.modules.cli import cli
|
|
16
16
|
from tinybird.tb.modules.common import push_data
|
|
17
17
|
from tinybird.tb.modules.datafile.build import folder_build
|
|
18
|
-
from tinybird.tb.modules.datafile.fixture import get_fixture_dir
|
|
18
|
+
from tinybird.tb.modules.datafile.fixture import get_fixture_dir, persist_fixture
|
|
19
19
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
20
20
|
from tinybird.tb.modules.project import Project
|
|
21
21
|
from tinybird.tb.modules.shell import Shell, print_table_formatted
|
|
@@ -210,6 +210,8 @@ def process(
|
|
|
210
210
|
build_failed = False
|
|
211
211
|
if file_changed and file_changed.endswith(".ndjson"):
|
|
212
212
|
rebuild_fixture(project, tb_client, file_changed)
|
|
213
|
+
elif file_changed and file_changed.endswith(".sql"):
|
|
214
|
+
rebuild_fixture_sql(project, tb_client, file_changed)
|
|
213
215
|
else:
|
|
214
216
|
try:
|
|
215
217
|
build_project(project, tb_client, file_changed)
|
|
@@ -247,3 +249,12 @@ def run_watch(
|
|
|
247
249
|
)
|
|
248
250
|
watcher_thread.start()
|
|
249
251
|
shell.run()
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def rebuild_fixture_sql(project: Project, tb_client: TinyB, sql_file: str) -> None:
|
|
255
|
+
sql_path = Path(sql_file)
|
|
256
|
+
datasource_name = sql_path.stem
|
|
257
|
+
sql = sql_path.read_text()
|
|
258
|
+
result = asyncio.run(tb_client.query(f"{sql} FORMAT JSON"))
|
|
259
|
+
data = result.get("data", [])
|
|
260
|
+
persist_fixture(datasource_name, data, project.folder)
|
tinybird/tb/modules/cli.py
CHANGED
|
@@ -127,6 +127,7 @@ async def cli(
|
|
|
127
127
|
ctx.ensure_object(dict)["client"] = client
|
|
128
128
|
|
|
129
129
|
ctx.ensure_object(dict)["project"] = project
|
|
130
|
+
ctx.ensure_object(dict)["env"] = get_target_env(cloud, build)
|
|
130
131
|
|
|
131
132
|
|
|
132
133
|
@cli.command(hidden=True)
|
|
@@ -147,12 +148,12 @@ async def pull(ctx: Context, force: bool, fmt: bool) -> None:
|
|
|
147
148
|
@click.option("--no-deps", is_flag=True, default=False, help="Print only data sources with no pipes using them")
|
|
148
149
|
@click.option("--match", default=None, help="Retrieve any resource matching the pattern")
|
|
149
150
|
@click.option("--pipe", default=None, help="Retrieve any resource used by pipe")
|
|
150
|
-
@click.option("--datasource", default=None, help="Retrieve resources depending on this
|
|
151
|
+
@click.option("--datasource", default=None, help="Retrieve resources depending on this data source")
|
|
151
152
|
@click.option(
|
|
152
153
|
"--check-for-partial-replace",
|
|
153
154
|
is_flag=True,
|
|
154
155
|
default=False,
|
|
155
|
-
help="Retrieve dependant
|
|
156
|
+
help="Retrieve dependant data sources that will have their data replaced if a partial replace is executed in the data source selected",
|
|
156
157
|
)
|
|
157
158
|
@click.option("--recursive", is_flag=True, default=False, help="Calculate recursive dependencies")
|
|
158
159
|
@click.pass_context
|
|
@@ -186,7 +187,7 @@ async def dependencies(
|
|
|
186
187
|
|
|
187
188
|
@cli.command(
|
|
188
189
|
name="diff",
|
|
189
|
-
short_help="
|
|
190
|
+
short_help="Diff local datafiles to the corresponding remote files in the workspace. For the case of .datasource files it just diffs VERSION and SCHEMA, since ENGINE, KAFKA or other metadata is considered immutable.",
|
|
190
191
|
)
|
|
191
192
|
@click.argument("filename", type=click.Path(exists=True), nargs=-1, required=False)
|
|
192
193
|
@click.option(
|
|
@@ -203,7 +204,7 @@ async def dependencies(
|
|
|
203
204
|
"--main",
|
|
204
205
|
is_flag=True,
|
|
205
206
|
default=False,
|
|
206
|
-
help="
|
|
207
|
+
help="Diff local datafiles to the corresponding remote files in the main workspace. Only works when authenticated on a Branch.",
|
|
207
208
|
hidden=True,
|
|
208
209
|
)
|
|
209
210
|
@click.pass_context
|
|
@@ -255,7 +256,7 @@ async def diff(
|
|
|
255
256
|
@cli.command()
|
|
256
257
|
@click.argument("query", required=False)
|
|
257
258
|
@click.option("--rows_limit", default=100, help="Max number of rows retrieved")
|
|
258
|
-
@click.option("--pipeline", default=None, help="The name of the
|
|
259
|
+
@click.option("--pipeline", default=None, help="The name of the pipe to run the SQL Query")
|
|
259
260
|
@click.option("--pipe", default=None, help="The path to the .pipe file to run the SQL Query of a specific NODE")
|
|
260
261
|
@click.option("--node", default=None, help="The NODE name")
|
|
261
262
|
@click.option(
|
|
@@ -393,7 +394,7 @@ async def create_ctx_client(ctx: Context, config: Dict[str, Any], cloud: bool, b
|
|
|
393
394
|
|
|
394
395
|
commands_always_cloud = ["pull"]
|
|
395
396
|
commands_always_build = ["build", "test", "dev"]
|
|
396
|
-
commands_always_local = ["create"
|
|
397
|
+
commands_always_local = ["create"]
|
|
397
398
|
if (
|
|
398
399
|
(cloud or command in commands_always_cloud)
|
|
399
400
|
and command not in commands_always_build
|
|
@@ -407,3 +408,11 @@ async def create_ctx_client(ctx: Context, config: Dict[str, Any], cloud: bool, b
|
|
|
407
408
|
if not build and command not in commands_always_local and command not in commands_always_build:
|
|
408
409
|
click.echo(FeedbackManager.gray(message="Running against Tinybird Local\n"))
|
|
409
410
|
return await get_tinybird_local_client(config, build=build)
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def get_target_env(cloud: bool, build: bool) -> str:
|
|
414
|
+
if cloud:
|
|
415
|
+
return "cloud"
|
|
416
|
+
if build:
|
|
417
|
+
return "build"
|
|
418
|
+
return "local"
|
tinybird/tb/modules/common.py
CHANGED
|
@@ -571,7 +571,7 @@ def region_from_host(region_name_or_host, regions):
|
|
|
571
571
|
|
|
572
572
|
def ask_for_user_token(action: str, ui_host: str) -> str:
|
|
573
573
|
return click.prompt(
|
|
574
|
-
f'\nUse the token called "user token"
|
|
574
|
+
f'\nUse the token called "user token" to {action}. Copy it from {ui_host}/tokens and paste it here',
|
|
575
575
|
hide_input=True,
|
|
576
576
|
show_default=False,
|
|
577
577
|
default=None,
|
|
@@ -591,13 +591,13 @@ async def check_user_token(ctx: Context, token: str):
|
|
|
591
591
|
if not is_authenticated.get("is_valid", False):
|
|
592
592
|
raise CLIWorkspaceException(
|
|
593
593
|
FeedbackManager.error_exception(
|
|
594
|
-
error='Invalid token.
|
|
594
|
+
error='Invalid token. Make sure you are using the "user token" instead of the "admin your@email" token.'
|
|
595
595
|
)
|
|
596
596
|
)
|
|
597
597
|
if is_authenticated.get("is_valid") and not is_authenticated.get("is_user", False):
|
|
598
598
|
raise CLIWorkspaceException(
|
|
599
599
|
FeedbackManager.error_exception(
|
|
600
|
-
error='Invalid user authentication.
|
|
600
|
+
error='Invalid user authentication. Make sure you are using the "user token" instead of the "admin your@email" token.'
|
|
601
601
|
)
|
|
602
602
|
)
|
|
603
603
|
|
|
@@ -614,13 +614,13 @@ async def check_user_token_with_client(client: TinyB, token: str):
|
|
|
614
614
|
if not is_authenticated.get("is_valid", False):
|
|
615
615
|
raise CLIWorkspaceException(
|
|
616
616
|
FeedbackManager.error_exception(
|
|
617
|
-
error='Invalid token.
|
|
617
|
+
error='Invalid token. Make sure you are using the "user token" instead of the "admin your@email" token.'
|
|
618
618
|
)
|
|
619
619
|
)
|
|
620
620
|
if is_authenticated.get("is_valid") and not is_authenticated.get("is_user", False):
|
|
621
621
|
raise CLIWorkspaceException(
|
|
622
622
|
FeedbackManager.error_exception(
|
|
623
|
-
error='Invalid user authentication.
|
|
623
|
+
error='Invalid user authentication. Make sure you are using the "user token" instead of the "admin your@email" token.'
|
|
624
624
|
)
|
|
625
625
|
)
|
|
626
626
|
|
|
@@ -1153,17 +1153,17 @@ def validate_kafka_bootstrap_servers(host_and_port):
|
|
|
1153
1153
|
|
|
1154
1154
|
def validate_kafka_key(s):
|
|
1155
1155
|
if not isinstance(s, str):
|
|
1156
|
-
raise CLIException("Key format is not correct, it
|
|
1156
|
+
raise CLIException("Key format is not correct, it must be a string")
|
|
1157
1157
|
|
|
1158
1158
|
|
|
1159
1159
|
def validate_kafka_secret(s):
|
|
1160
1160
|
if not isinstance(s, str):
|
|
1161
|
-
raise CLIException("Password format is not correct, it
|
|
1161
|
+
raise CLIException("Password format is not correct, it must be a string")
|
|
1162
1162
|
|
|
1163
1163
|
|
|
1164
1164
|
def validate_string_connector_param(param, s):
|
|
1165
1165
|
if not isinstance(s, str):
|
|
1166
|
-
raise CLIConnectionException(param + " format is not correct, it
|
|
1166
|
+
raise CLIConnectionException(param + " format is not correct, it must be a string")
|
|
1167
1167
|
|
|
1168
1168
|
|
|
1169
1169
|
async def validate_connection_name(client, connection_name, service):
|
|
@@ -1434,7 +1434,7 @@ async def try_update_config_with_remote(
|
|
|
1434
1434
|
def ask_for_admin_token_interactively(ui_host: str, default_token: Optional[str]) -> str:
|
|
1435
1435
|
return (
|
|
1436
1436
|
click.prompt(
|
|
1437
|
-
f'\nCopy the "admin your@email" token from {ui_host}/tokens and paste it here {"OR press enter to use the token from .tinyb file" if default_token else ""}',
|
|
1437
|
+
f'\nCopy the "admin your@email" token from {ui_host}/tokens and paste it here {"OR press enter to use the token from the .tinyb file" if default_token else ""}',
|
|
1438
1438
|
hide_input=True,
|
|
1439
1439
|
show_default=False,
|
|
1440
1440
|
default=default_token,
|
|
@@ -94,7 +94,7 @@ async def update_tags_in_resource(rs: Dict[str, Any], resource_type: str, client
|
|
|
94
94
|
resource_name = persisted_ds.get("name", "")
|
|
95
95
|
except DoesNotExistException:
|
|
96
96
|
click.echo(
|
|
97
|
-
FeedbackManager.error_tag_generic("Could not get the latest
|
|
97
|
+
FeedbackManager.error_tag_generic("Could not get the latest data source info for updating its tags.")
|
|
98
98
|
)
|
|
99
99
|
elif resource_type == "pipe":
|
|
100
100
|
pipe_name = rs["name"]
|
|
@@ -60,7 +60,7 @@ async def new_ds(
|
|
|
60
60
|
|
|
61
61
|
if engine_param.lower() == "join":
|
|
62
62
|
deprecation_notice = FeedbackManager.warning_deprecated(
|
|
63
|
-
warning="Data
|
|
63
|
+
warning="Data sources with Join engine are deprecated and will be removed in the next major release of tinybird-cli. Use MergeTree instead."
|
|
64
64
|
)
|
|
65
65
|
click.echo(deprecation_notice)
|
|
66
66
|
|
|
@@ -20,6 +20,7 @@ from string import Template
|
|
|
20
20
|
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, cast
|
|
21
21
|
|
|
22
22
|
import click
|
|
23
|
+
from croniter import croniter
|
|
23
24
|
from mypy_extensions import KwArg, VarArg
|
|
24
25
|
|
|
25
26
|
from tinybird.ch_utils.engine import ENABLED_ENGINES
|
|
@@ -200,6 +201,20 @@ class Datafile:
|
|
|
200
201
|
def set_kind(self, kind: DatafileKind):
|
|
201
202
|
self.kind = kind
|
|
202
203
|
|
|
204
|
+
def validate_copy_node(self, node: Dict[str, Any]):
|
|
205
|
+
if "target_datasource" not in node:
|
|
206
|
+
raise DatafileValidationError("COPY node missing target datasource")
|
|
207
|
+
# copy mode must be append or replace
|
|
208
|
+
if node.get("mode") and node["mode"] not in ["append", "replace"]:
|
|
209
|
+
raise DatafileValidationError("COPY node mode must be append or replace")
|
|
210
|
+
# copy schedule must be @on-demand or a cron-expression
|
|
211
|
+
if (
|
|
212
|
+
node.get("copy_schedule")
|
|
213
|
+
and node["copy_schedule"] != ON_DEMAND
|
|
214
|
+
and not croniter.is_valid(node["copy_schedule"])
|
|
215
|
+
):
|
|
216
|
+
raise DatafileValidationError("COPY node schedule must be @on-demand or a valid cron expression.")
|
|
217
|
+
|
|
203
218
|
def validate(self):
|
|
204
219
|
if self.kind == DatafileKind.pipe:
|
|
205
220
|
# TODO(eclbg):
|
|
@@ -208,7 +223,7 @@ class Datafile:
|
|
|
208
223
|
# [x] Materialized nodes have target datasource
|
|
209
224
|
# [x] Only one materialized node
|
|
210
225
|
# [x] Only one node of any specific type
|
|
211
|
-
#
|
|
226
|
+
# (rbarbadillo): there's a HUGE amount of validations in api_pipes.py, we should somehow merge them
|
|
212
227
|
for node in self.nodes:
|
|
213
228
|
if "sql" not in node:
|
|
214
229
|
raise DatafileValidationError(f"SQL missing for node {repr(node['name'])}")
|
|
@@ -220,6 +235,8 @@ class Datafile:
|
|
|
220
235
|
raise DatafileValidationError("Multiple non-standard nodes in pipe. There can only be one")
|
|
221
236
|
if node.get("type", "").lower() == PipeNodeTypes.MATERIALIZED and "datasource" not in node:
|
|
222
237
|
raise DatafileValidationError(f"Materialized node {repr(node['name'])} missing target datasource")
|
|
238
|
+
if node.get("type", "").lower() == PipeNodeTypes.COPY:
|
|
239
|
+
self.validate_copy_node(node)
|
|
223
240
|
elif self.kind == DatafileKind.datasource:
|
|
224
241
|
# TODO(eclbg):
|
|
225
242
|
# [x] Just one node
|
|
@@ -8,6 +8,13 @@ def get_fixture_dir(folder: str) -> Path:
|
|
|
8
8
|
return Path(folder) / "fixtures"
|
|
9
9
|
|
|
10
10
|
|
|
11
|
+
def persist_fixture_sql(fixture_name: str, sql: str, folder: str) -> Path:
|
|
12
|
+
fixture_dir = get_fixture_dir(folder)
|
|
13
|
+
fixture_file = fixture_dir / f"{fixture_name}.sql"
|
|
14
|
+
fixture_file.write_text(sql)
|
|
15
|
+
return fixture_file
|
|
16
|
+
|
|
17
|
+
|
|
11
18
|
def persist_fixture(fixture_name: str, data: Union[List[Dict[str, Any]], str], folder: str, format="ndjson") -> Path:
|
|
12
19
|
fixture_dir = get_fixture_dir(folder)
|
|
13
20
|
fixture_file = fixture_dir / f"{fixture_name}.{format}"
|
|
@@ -412,7 +412,7 @@ class PipeCheckerRunner:
|
|
|
412
412
|
)
|
|
413
413
|
|
|
414
414
|
result = PipeCheckerTextTestResult(
|
|
415
|
-
self.checker_stream_result_class(sys.stdout),
|
|
415
|
+
self.checker_stream_result_class(sys.stdout),
|
|
416
416
|
descriptions=True,
|
|
417
417
|
verbosity=2,
|
|
418
418
|
custom_output=custom_output,
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import asyncio
|
|
7
7
|
import json
|
|
8
|
+
import os
|
|
8
9
|
import re
|
|
9
10
|
from typing import Optional
|
|
10
11
|
|
|
@@ -23,8 +24,10 @@ from tinybird.tb.modules.common import (
|
|
|
23
24
|
push_data,
|
|
24
25
|
)
|
|
25
26
|
from tinybird.tb.modules.datafile.common import get_name_version
|
|
27
|
+
from tinybird.tb.modules.datafile.fixture import persist_fixture
|
|
26
28
|
from tinybird.tb.modules.exceptions import CLIDatasourceException
|
|
27
29
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
30
|
+
from tinybird.tb.modules.project import Project
|
|
28
31
|
|
|
29
32
|
|
|
30
33
|
@cli.group()
|
|
@@ -34,7 +37,7 @@ def datasource(ctx):
|
|
|
34
37
|
|
|
35
38
|
|
|
36
39
|
@datasource.command(name="ls")
|
|
37
|
-
@click.option("--match", default=None, help="Retrieve any resources matching the pattern.
|
|
40
|
+
@click.option("--match", default=None, help="Retrieve any resources matching the pattern. For example, --match _test")
|
|
38
41
|
@click.option(
|
|
39
42
|
"--format",
|
|
40
43
|
"format_",
|
|
@@ -116,7 +119,7 @@ async def datasource_append(
|
|
|
116
119
|
concurrency: int,
|
|
117
120
|
):
|
|
118
121
|
"""
|
|
119
|
-
Appends data to an existing
|
|
122
|
+
Appends data to an existing data source from URL, local file or a connector
|
|
120
123
|
|
|
121
124
|
- Load from URL `tb datasource append [datasource_name] https://url_to_csv`
|
|
122
125
|
|
|
@@ -391,3 +394,63 @@ async def datasource_data(ctx: Context, datasource: str, limit: int):
|
|
|
391
394
|
echo_safe_humanfriendly_tables_format_smart_table(
|
|
392
395
|
data=[d.values() for d in res["data"]], column_names=res["data"][0].keys()
|
|
393
396
|
)
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
@datasource.command(name="export")
|
|
400
|
+
@click.argument("datasource")
|
|
401
|
+
@click.option(
|
|
402
|
+
"--format",
|
|
403
|
+
"format_",
|
|
404
|
+
type=click.Choice(["csv", "ndjson"], case_sensitive=False),
|
|
405
|
+
default="ndjson",
|
|
406
|
+
help="Output format (csv or ndjson)",
|
|
407
|
+
)
|
|
408
|
+
@click.option("--rows", type=int, default=100, help="Number of rows to export (default: 100)")
|
|
409
|
+
@click.option("--where", type=str, default=None, help="Condition to filter data")
|
|
410
|
+
@click.option("--target", type=str, help="Target file path (default: datasource_name.{format})")
|
|
411
|
+
@click.pass_context
|
|
412
|
+
@coro
|
|
413
|
+
async def datasource_export(
|
|
414
|
+
ctx: Context,
|
|
415
|
+
datasource: str,
|
|
416
|
+
format_: str,
|
|
417
|
+
rows: int,
|
|
418
|
+
where: Optional[str],
|
|
419
|
+
target: Optional[str],
|
|
420
|
+
):
|
|
421
|
+
"""Export data from a datasource to a file in CSV or NDJSON format
|
|
422
|
+
|
|
423
|
+
Example usage:
|
|
424
|
+
- Export all rows as CSV: tb datasource export my_datasource
|
|
425
|
+
- Export 1000 rows as NDJSON: tb datasource export my_datasource --format ndjson --rows 1000
|
|
426
|
+
- Export to specific file: tb datasource export my_datasource --output ./data/export.csv
|
|
427
|
+
"""
|
|
428
|
+
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
429
|
+
project: Project = ctx.ensure_object(dict)["project"]
|
|
430
|
+
|
|
431
|
+
# Determine output filename if not provided
|
|
432
|
+
if not target:
|
|
433
|
+
target = f"{datasource}.{format_}"
|
|
434
|
+
|
|
435
|
+
# Build query with optional row limit
|
|
436
|
+
query = f"SELECT * FROM {datasource} WHERE {where or 1} LIMIT {rows}"
|
|
437
|
+
|
|
438
|
+
click.echo(FeedbackManager.highlight(message=f"\n» Exporting {datasource} to {target}"))
|
|
439
|
+
|
|
440
|
+
try:
|
|
441
|
+
if format_ == "csv":
|
|
442
|
+
query += " FORMAT CSVWithNames"
|
|
443
|
+
else:
|
|
444
|
+
query += " FORMAT JSONEachRow"
|
|
445
|
+
|
|
446
|
+
res = await client.query(query)
|
|
447
|
+
|
|
448
|
+
fixture_path = persist_fixture(datasource, res, project.folder)
|
|
449
|
+
file_size = os.path.getsize(fixture_path)
|
|
450
|
+
|
|
451
|
+
click.echo(
|
|
452
|
+
FeedbackManager.success(message=f"✓ Exported data to {target} ({humanfriendly.format_size(file_size)})")
|
|
453
|
+
)
|
|
454
|
+
|
|
455
|
+
except Exception as e:
|
|
456
|
+
raise CLIDatasourceException(FeedbackManager.error(message=str(e)))
|
|
@@ -44,12 +44,15 @@ def promote_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
|
|
|
44
44
|
if candidate_deployment.get("live"):
|
|
45
45
|
click.echo(FeedbackManager.error(message="Candidate deployment is already live"))
|
|
46
46
|
else:
|
|
47
|
-
click.echo(FeedbackManager.success(message="
|
|
47
|
+
click.echo(FeedbackManager.success(message="Setting candidate deployment as live"))
|
|
48
48
|
|
|
49
49
|
TINYBIRD_API_URL = f"{host}/v1/deployments/{candidate_deployment.get('id')}/set-live"
|
|
50
50
|
r = requests.post(TINYBIRD_API_URL, headers=headers)
|
|
51
51
|
result = r.json()
|
|
52
52
|
logging.debug(json.dumps(result, indent=2))
|
|
53
|
+
if result.get("error"):
|
|
54
|
+
click.echo(FeedbackManager.error(message=result.get("error")))
|
|
55
|
+
sys.exit(1)
|
|
53
56
|
|
|
54
57
|
click.echo(FeedbackManager.success(message="Removing old deployment"))
|
|
55
58
|
|
|
@@ -57,6 +60,9 @@ def promote_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
|
|
|
57
60
|
r = requests.delete(TINYBIRD_API_URL, headers=headers)
|
|
58
61
|
result = r.json()
|
|
59
62
|
logging.debug(json.dumps(result, indent=2))
|
|
63
|
+
if result.get("error"):
|
|
64
|
+
click.echo(FeedbackManager.error(message=result.get("error")))
|
|
65
|
+
sys.exit(1)
|
|
60
66
|
|
|
61
67
|
click.echo(FeedbackManager.success(message="Deployment promotion successfully started"))
|
|
62
68
|
|
|
@@ -110,6 +116,9 @@ def rollback_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
|
|
|
110
116
|
r = requests.post(TINYBIRD_API_URL, headers=headers)
|
|
111
117
|
result = r.json()
|
|
112
118
|
logging.debug(json.dumps(result, indent=2))
|
|
119
|
+
if result.get("error"):
|
|
120
|
+
click.echo(FeedbackManager.error(message=result.get("error")))
|
|
121
|
+
sys.exit(1)
|
|
113
122
|
|
|
114
123
|
click.echo(FeedbackManager.success(message="Removing current deployment"))
|
|
115
124
|
|
|
@@ -117,6 +126,9 @@ def rollback_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
|
|
|
117
126
|
r = requests.delete(TINYBIRD_API_URL, headers=headers)
|
|
118
127
|
result = r.json()
|
|
119
128
|
logging.debug(json.dumps(result, indent=2))
|
|
129
|
+
if result.get("error"):
|
|
130
|
+
click.echo(FeedbackManager.error(message=result.get("error")))
|
|
131
|
+
sys.exit(1)
|
|
120
132
|
|
|
121
133
|
click.echo(FeedbackManager.success(message="Deployment rollback successfully started"))
|
|
122
134
|
|
|
@@ -254,13 +266,13 @@ def deployment_rollback(ctx: click.Context, wait: bool) -> None:
|
|
|
254
266
|
@click.option(
|
|
255
267
|
"--wait/--no-wait",
|
|
256
268
|
is_flag=True,
|
|
257
|
-
default=
|
|
269
|
+
default=True,
|
|
258
270
|
help="Wait for deploy to finish. Disabled by default.",
|
|
259
271
|
)
|
|
260
272
|
@click.option(
|
|
261
273
|
"--auto/--no-auto",
|
|
262
274
|
is_flag=True,
|
|
263
|
-
default=
|
|
275
|
+
default=True,
|
|
264
276
|
help="Auto-promote the deployment. Only works if --wait is enabled. Disabled by default.",
|
|
265
277
|
)
|
|
266
278
|
@click.option(
|
|
@@ -299,6 +311,7 @@ def create_deployment(
|
|
|
299
311
|
}
|
|
300
312
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
301
313
|
client = ctx.ensure_object(dict)["client"]
|
|
314
|
+
config = ctx.ensure_object(dict)["config"]
|
|
302
315
|
TINYBIRD_API_URL = f"{client.host}/v1/deploy"
|
|
303
316
|
TINYBIRD_API_KEY = client.token
|
|
304
317
|
|
|
@@ -345,8 +358,21 @@ def create_deployment(
|
|
|
345
358
|
deploy_result = result.get("result")
|
|
346
359
|
if deploy_result == "success":
|
|
347
360
|
print_changes(result, project)
|
|
348
|
-
|
|
349
|
-
|
|
361
|
+
deployment = result.get("deployment", {})
|
|
362
|
+
# We show the url in the case of region is public
|
|
363
|
+
if client.host == "https://api.europe-west2.gcp.tinybird.co":
|
|
364
|
+
click.echo(
|
|
365
|
+
FeedbackManager.gray(message="Deployment URL: ")
|
|
366
|
+
+ FeedbackManager.info(
|
|
367
|
+
message=f"https://cloud.tinybird.co/gcp/europe-west2/{config.get('name')}/deployments/{deployment.get('id')}"
|
|
368
|
+
)
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
if wait:
|
|
372
|
+
click.echo(FeedbackManager.info(message="\n✓ Deployment submitted successfully"))
|
|
373
|
+
else:
|
|
374
|
+
click.echo(FeedbackManager.success(message="\n✓ Deployment submitted successfully"))
|
|
375
|
+
|
|
350
376
|
feedback = deployment.get("feedback", [])
|
|
351
377
|
for f in feedback:
|
|
352
378
|
click.echo(
|
tinybird/tb/modules/endpoint.py
CHANGED
|
@@ -29,7 +29,7 @@ def endpoint(ctx):
|
|
|
29
29
|
|
|
30
30
|
|
|
31
31
|
@endpoint.command(name="ls")
|
|
32
|
-
@click.option("--match", default=None, help="Retrieve any
|
|
32
|
+
@click.option("--match", default=None, help="Retrieve any resource matching the pattern. For example, --match _test")
|
|
33
33
|
@click.option(
|
|
34
34
|
"--format",
|
|
35
35
|
"format_",
|
tinybird/tb/modules/fmt.py
CHANGED
|
@@ -58,7 +58,7 @@ async def fmt(
|
|
|
58
58
|
elif (".pipe" in extensions) or (".incl" in extensions):
|
|
59
59
|
result = await format_pipe(filename, line_length, skip_eval=True)
|
|
60
60
|
else:
|
|
61
|
-
click.echo("Unsupported file type. Supported files types are: .pipe
|
|
61
|
+
click.echo("Unsupported file type. Supported files types are: .pipe and .datasource")
|
|
62
62
|
return None
|
|
63
63
|
|
|
64
64
|
if diff:
|
|
@@ -23,7 +23,7 @@ def materialization(ctx):
|
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
@materialization.command(name="ls")
|
|
26
|
-
@click.option("--match", default=None, help="Retrieve any resourcing matching the pattern.
|
|
26
|
+
@click.option("--match", default=None, help="Retrieve any resourcing matching the pattern. For example, --match _test")
|
|
27
27
|
@click.option(
|
|
28
28
|
"--format",
|
|
29
29
|
"format_",
|
|
@@ -79,16 +79,16 @@ async def materialization_ls(ctx: click.Context, match: str, format_: str):
|
|
|
79
79
|
"--sql-condition",
|
|
80
80
|
type=str,
|
|
81
81
|
default=None,
|
|
82
|
-
help="Populate with a SQL condition to be applied to the trigger
|
|
82
|
+
help="Populate with a SQL condition to be applied to the trigger data source of the materialized view. For instance, `--sql-condition='date == toYYYYMM(now())'` it'll populate taking all the rows from the trigger data source which `date` is the current month. Use it together with --populate. --sql-condition is not taken into account if the --subset param is present. Including in the ``sql_condition`` any column present in the data source ``engine_sorting_key`` will make the populate job process less data.",
|
|
83
83
|
)
|
|
84
84
|
@click.option(
|
|
85
|
-
"--truncate", is_flag=True, default=False, help="Truncates the materialized
|
|
85
|
+
"--truncate", is_flag=True, default=False, help="Truncates the materialized data source before populating it."
|
|
86
86
|
)
|
|
87
87
|
@click.option(
|
|
88
88
|
"--unlink-on-populate-error",
|
|
89
89
|
is_flag=True,
|
|
90
90
|
default=False,
|
|
91
|
-
help="If the populate job fails the
|
|
91
|
+
help="If the populate job fails the materialized view is unlinked and new data won't be ingested in the materialized view. First time a populate job fails, the materialized view is always unlinked.",
|
|
92
92
|
)
|
|
93
93
|
@click.option(
|
|
94
94
|
"--wait",
|
|
@@ -107,7 +107,7 @@ async def pipe_populate(
|
|
|
107
107
|
unlink_on_populate_error: bool,
|
|
108
108
|
wait: bool,
|
|
109
109
|
):
|
|
110
|
-
"""Populate the result of a Materialized Node into the target
|
|
110
|
+
"""Populate the result of a Materialized Node into the target materialized view"""
|
|
111
111
|
cl = create_tb_client(ctx)
|
|
112
112
|
|
|
113
113
|
pipe = await cl.pipe(pipe_name)
|