tinybird-cli 5.8.0.dev1__tar.gz → 5.8.0.dev3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/PKG-INFO +11 -1
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/__cli__.py +2 -2
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/client.py +18 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/datafile.py +33 -18
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/feedback_manager.py +14 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/sql_template.py +12 -3
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli.py +1 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/cli.py +0 -84
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/datasource.py +62 -0
- tinybird-cli-5.8.0.dev3/tinybird/tb_cli_modules/fmt.py +90 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird_cli.egg-info/PKG-INFO +11 -1
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird_cli.egg-info/SOURCES.txt +1 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/setup.cfg +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/ch_utils/constants.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/ch_utils/engine.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/check_pypi.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/config.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/connectors.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/context.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/datatypes.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/git_settings.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/sql.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/sql_template_fmt.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/sql_toolset.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/syncasync.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/auth.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/branch.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/cicd.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/common.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/config.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/connection.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/exceptions.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/job.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/pipe.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/regions.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/tag.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/telemetry.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/test.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/token.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/workspace.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/workspace_members.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tornado_template.py +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird_cli.egg-info/dependency_links.txt +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird_cli.egg-info/entry_points.txt +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird_cli.egg-info/requires.txt +0 -0
- {tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird_cli.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: tinybird-cli
|
|
3
|
-
Version: 5.8.0.
|
|
3
|
+
Version: 5.8.0.dev3
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/cli/introduction.html
|
|
6
6
|
Author: Tinybird
|
|
@@ -18,6 +18,16 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
|
|
|
18
18
|
Changelog
|
|
19
19
|
----------
|
|
20
20
|
|
|
21
|
+
5.8.0.dev3
|
|
22
|
+
**********
|
|
23
|
+
|
|
24
|
+
- `Added` `tb datasource scheduling` commands to manage the scheduling of a Data Source
|
|
25
|
+
|
|
26
|
+
5.8.0.dev2
|
|
27
|
+
***********
|
|
28
|
+
|
|
29
|
+
- `Added` support to `TAGS` in `tb fmt`.
|
|
30
|
+
|
|
21
31
|
5.8.0.dev1
|
|
22
32
|
***********
|
|
23
33
|
|
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '5.8.0.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '5.8.0.dev3'
|
|
8
|
+
__revision__ = '43074f3'
|
|
@@ -474,6 +474,24 @@ class TinyB(object):
|
|
|
474
474
|
async def datasource_sync(self, datasource_id: str):
|
|
475
475
|
return await self._req(f"/v0/datasources/{datasource_id}/scheduling/runs", method="POST", data="")
|
|
476
476
|
|
|
477
|
+
async def datasource_scheduling_state(self, datasource_id: str):
|
|
478
|
+
response = await self._req(f"/v0/datasources/{datasource_id}/scheduling/state", method="GET")
|
|
479
|
+
return response["state"]
|
|
480
|
+
|
|
481
|
+
async def datasource_scheduling_pause(self, datasource_id: str):
|
|
482
|
+
return await self._req(
|
|
483
|
+
f"/v0/datasources/{datasource_id}/scheduling/state",
|
|
484
|
+
method="PUT",
|
|
485
|
+
data='{"state": "paused"}',
|
|
486
|
+
)
|
|
487
|
+
|
|
488
|
+
async def datasource_scheduling_resume(self, datasource_id: str):
|
|
489
|
+
return await self._req(
|
|
490
|
+
f"/v0/datasources/{datasource_id}/scheduling/state",
|
|
491
|
+
method="PUT",
|
|
492
|
+
data='{"state": "running"}',
|
|
493
|
+
)
|
|
494
|
+
|
|
477
495
|
async def datasource_exchange(self, datasource_a: str, datasource_b: str):
|
|
478
496
|
payload = {"datasource_a": datasource_a, "datasource_b": datasource_b}
|
|
479
497
|
return await self._req("/v0/datasources/exchange", method="POST", data=payload)
|
|
@@ -904,30 +904,31 @@ def eval_var(s: str, skip: bool = False) -> str:
|
|
|
904
904
|
return Template(s).safe_substitute(os.environ)
|
|
905
905
|
|
|
906
906
|
|
|
907
|
-
def parse_tags(tags: str) -> Tuple[
|
|
907
|
+
def parse_tags(tags: str) -> Tuple[str, List[str]]:
|
|
908
908
|
"""
|
|
909
909
|
Parses a string of tags into:
|
|
910
|
-
- kv_tags: a
|
|
911
|
-
|
|
910
|
+
- kv_tags: a string of key-value tags: the previous tags we have for operational purposes. It
|
|
911
|
+
has the format key=value&key2=value2 (with_staging=true&with_last_date=true)
|
|
912
912
|
- filtering_tags: a list of tags that are used for filtering.
|
|
913
913
|
|
|
914
914
|
Example: "with_staging=true&with_last_date=true,billing,stats" ->
|
|
915
915
|
kv_tags = {"with_staging": "true", "with_last_date": "true"}
|
|
916
916
|
filtering_tags = ["billing", "stats"]
|
|
917
917
|
"""
|
|
918
|
-
kv_tags =
|
|
918
|
+
kv_tags = []
|
|
919
919
|
filtering_tags = []
|
|
920
920
|
|
|
921
921
|
entries = tags.split(",")
|
|
922
922
|
for entry in entries:
|
|
923
923
|
trimmed_entry = entry.strip()
|
|
924
924
|
if "=" in trimmed_entry:
|
|
925
|
-
|
|
926
|
-
kv_tags.update(the_tags)
|
|
925
|
+
kv_tags.append(trimmed_entry)
|
|
927
926
|
else:
|
|
928
927
|
filtering_tags.append(trimmed_entry)
|
|
929
928
|
|
|
930
|
-
|
|
929
|
+
all_kv_tags = "&".join(kv_tags)
|
|
930
|
+
|
|
931
|
+
return all_kv_tags, filtering_tags
|
|
931
932
|
|
|
932
933
|
|
|
933
934
|
def parse(
|
|
@@ -1128,14 +1129,19 @@ def parse(
|
|
|
1128
1129
|
return _f
|
|
1129
1130
|
|
|
1130
1131
|
def tags(*args: str, **kwargs: Any) -> None:
|
|
1132
|
+
raw_tags = _unquote((" ".join(args)).strip())
|
|
1133
|
+
operational_tags, filtering_tags = parse_tags(raw_tags)
|
|
1134
|
+
|
|
1131
1135
|
# Pipe nodes or Data Sources
|
|
1132
|
-
if parser_state.current_node:
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1136
|
+
if parser_state.current_node and operational_tags:
|
|
1137
|
+
operational_tags_args = (operational_tags,)
|
|
1138
|
+
assign_node_var("tags")(*operational_tags_args, **kwargs)
|
|
1139
|
+
|
|
1140
|
+
if filtering_tags:
|
|
1141
|
+
if doc.filtering_tags is None:
|
|
1142
|
+
doc.filtering_tags = filtering_tags
|
|
1143
|
+
else:
|
|
1144
|
+
doc.filtering_tags += filtering_tags
|
|
1139
1145
|
|
|
1140
1146
|
cmds = {
|
|
1141
1147
|
"from": assign("from"),
|
|
@@ -1552,9 +1558,8 @@ async def process_file(
|
|
|
1552
1558
|
del params["format"]
|
|
1553
1559
|
|
|
1554
1560
|
if "tags" in node:
|
|
1555
|
-
tags,
|
|
1561
|
+
tags = {k: v[0] for k, v in urllib.parse.parse_qs(node["tags"]).items()}
|
|
1556
1562
|
params.update(tags)
|
|
1557
|
-
doc.filtering_tags = filtering_tags
|
|
1558
1563
|
|
|
1559
1564
|
resources: List[Dict[str, Any]] = []
|
|
1560
1565
|
|
|
@@ -1654,7 +1659,7 @@ async def process_file(
|
|
|
1654
1659
|
sql = re.sub("([\t \\n']+|^)" + old + "([\t \\n'\\)]+|$)", "\\1" + new + "\\2", sql)
|
|
1655
1660
|
|
|
1656
1661
|
if "tags" in node:
|
|
1657
|
-
tags,
|
|
1662
|
+
tags = {k: v[0] for k, v in urllib.parse.parse_qs(node["tags"]).items()}
|
|
1658
1663
|
params.update(tags)
|
|
1659
1664
|
|
|
1660
1665
|
nodes.append(
|
|
@@ -4817,19 +4822,20 @@ async def format_datasource(
|
|
|
4817
4822
|
if for_deploy_diff:
|
|
4818
4823
|
format_description(file_parts, doc)
|
|
4819
4824
|
format_tokens(file_parts, doc)
|
|
4825
|
+
format_tags(file_parts, doc)
|
|
4820
4826
|
format_schema(file_parts, doc.nodes[0])
|
|
4821
4827
|
format_indices(file_parts, doc.nodes[0])
|
|
4822
4828
|
await format_engine(file_parts, doc.nodes[0], only_ttl=True if not for_deploy_diff else False, client=client)
|
|
4823
4829
|
if for_deploy_diff:
|
|
4824
4830
|
format_import_settings(file_parts, doc.nodes[0])
|
|
4825
4831
|
format_shared_with(file_parts, doc)
|
|
4826
|
-
|
|
4827
4832
|
else:
|
|
4828
4833
|
format_sources(file_parts, doc)
|
|
4829
4834
|
format_maintainer(file_parts, doc)
|
|
4830
4835
|
format_version(file_parts, doc)
|
|
4831
4836
|
format_description(file_parts, doc)
|
|
4832
4837
|
format_tokens(file_parts, doc)
|
|
4838
|
+
format_tags(file_parts, doc)
|
|
4833
4839
|
format_schema(file_parts, doc.nodes[0])
|
|
4834
4840
|
format_indices(file_parts, doc.nodes[0])
|
|
4835
4841
|
await format_engine(file_parts, doc.nodes[0])
|
|
@@ -4905,6 +4911,14 @@ def format_shared_with(file_parts: List[str], doc: Datafile) -> List[str]:
|
|
|
4905
4911
|
return file_parts
|
|
4906
4912
|
|
|
4907
4913
|
|
|
4914
|
+
def format_tags(file_parts: List[str], doc: Datafile) -> List[str]:
|
|
4915
|
+
if doc.filtering_tags:
|
|
4916
|
+
file_parts.append(f'TAGS {", ".join(doc.filtering_tags)}')
|
|
4917
|
+
file_parts.append(DATAFILE_NEW_LINE)
|
|
4918
|
+
file_parts.append(DATAFILE_NEW_LINE)
|
|
4919
|
+
return file_parts
|
|
4920
|
+
|
|
4921
|
+
|
|
4908
4922
|
async def format_engine(
|
|
4909
4923
|
file_parts: List[str], node: Dict[str, Any], only_ttl: bool = False, client: Optional[TinyB] = None
|
|
4910
4924
|
) -> List[str]:
|
|
@@ -5041,6 +5055,7 @@ async def format_pipe(
|
|
|
5041
5055
|
format_version(file_parts, doc)
|
|
5042
5056
|
format_description(file_parts, doc)
|
|
5043
5057
|
format_tokens(file_parts, doc)
|
|
5058
|
+
format_tags(file_parts, doc)
|
|
5044
5059
|
if doc.includes and not unroll_includes:
|
|
5045
5060
|
for k in doc.includes:
|
|
5046
5061
|
# We filter only the include files as we currently have 2 items for each include
|
|
@@ -157,6 +157,15 @@ class FeedbackManager:
|
|
|
157
157
|
error_creating_copy_job = error_message("Failed creating copy job: {error}")
|
|
158
158
|
error_pausing_copy_pipe = error_message("Failed pausing copy pipe: {error}")
|
|
159
159
|
error_resuming_copy_pipe = error_message("Failed resuming copy pipe: {error}")
|
|
160
|
+
error_pausing_datasource_scheduling = error_message(
|
|
161
|
+
"Failed pausing scheduling for Data Source '{datasource}': {error}"
|
|
162
|
+
)
|
|
163
|
+
error_resuming_datasource_scheduling = error_message(
|
|
164
|
+
"Failed resuming scheduling for Data Source '{datasource}': {error}"
|
|
165
|
+
)
|
|
166
|
+
error_datasource_scheduling_state = error_message(
|
|
167
|
+
"Failed requesting scheduling state for Data Source '{datasource}': {error}"
|
|
168
|
+
)
|
|
160
169
|
error_creating_pipe = error_message("Failed creating pipe {error}")
|
|
161
170
|
error_creating_sink_job = error_message("Failed creating sink job: {error}")
|
|
162
171
|
error_running_on_demand_sink_job = error_message("Failed running on-demand sink job: {error}")
|
|
@@ -681,6 +690,9 @@ Ready? """
|
|
|
681
690
|
info_datasource_title = print_message("** {title}", bcolors.BOLD)
|
|
682
691
|
info_datasource_row = info_message("{row}")
|
|
683
692
|
info_datasource_delete_rows_job_url = info_message("** Delete rows job url {url}")
|
|
693
|
+
info_datasource_scheduling_state = info_message("** Scheduling state for Data Source '{datasource}': {state}")
|
|
694
|
+
info_datasource_scheduling_pause = info_message("** Pausing scheduling...")
|
|
695
|
+
info_datasource_scheduling_resume = info_message("** Resuming scheduling...")
|
|
684
696
|
info_pipes = info_message("** Pipes:")
|
|
685
697
|
info_pipe_name = info_message("** - {pipe}")
|
|
686
698
|
info_using_node = print_message("** Using last node {node} as endpoint")
|
|
@@ -933,6 +945,8 @@ Ready? """
|
|
|
933
945
|
success_datasource_unshared = success_message(
|
|
934
946
|
"** The Data Source {datasource} has been correctly unshared from {workspace}"
|
|
935
947
|
)
|
|
948
|
+
success_datasource_scheduling_resumed = success_message("""** Scheduling resumed for Data Source '{datasource}'""")
|
|
949
|
+
success_datasource_scheduling_paused = success_message("""** Scheduling paused for Data Source '{datasource}'""")
|
|
936
950
|
success_connection_created = success_message("** Connection {id} created successfully!")
|
|
937
951
|
|
|
938
952
|
# TODO: Update the message when the .env feature is implemented
|
|
@@ -386,14 +386,14 @@ def array_type(types): # noqa: C901
|
|
|
386
386
|
for i, t in enumerate(list_values):
|
|
387
387
|
if _type in testers:
|
|
388
388
|
if testers[_type](str(t)):
|
|
389
|
-
values.append(expression_wrapper(types[_type](t),
|
|
389
|
+
values.append(expression_wrapper(types[_type](t), str(t)))
|
|
390
390
|
else:
|
|
391
391
|
raise SQLTemplateException(
|
|
392
392
|
f"Error validating {x}[{i}]({t}) to type {_type}",
|
|
393
393
|
documentation="/cli/advanced-templates.html",
|
|
394
394
|
)
|
|
395
395
|
else:
|
|
396
|
-
values.append(expression_wrapper(types.get(_type, lambda x: x)(t),
|
|
396
|
+
values.append(expression_wrapper(types.get(_type, lambda x: x)(t), str(t)))
|
|
397
397
|
return Expression(f"[{','.join(map(str, values))}]")
|
|
398
398
|
except AttributeError as e:
|
|
399
399
|
logging.warning(f"AttributeError on Array: {e}")
|
|
@@ -1314,8 +1314,9 @@ def expression_wrapper(x, name, escape_arrays: bool = False):
|
|
|
1314
1314
|
elif isinstance(x, Comment):
|
|
1315
1315
|
return "-- {x} \n"
|
|
1316
1316
|
if x is None:
|
|
1317
|
+
truncated_name = name[:20] + "..." if len(name) > 20 else name
|
|
1317
1318
|
raise SQLTemplateException(
|
|
1318
|
-
f'expression "{
|
|
1319
|
+
f'expression "{truncated_name}" evaluated to null', documentation="/cli/advanced-templates.html"
|
|
1319
1320
|
)
|
|
1320
1321
|
if isinstance(x, list) and escape_arrays:
|
|
1321
1322
|
logging.warning(f"expression_wrapper -> list :{x}:")
|
|
@@ -1992,6 +1993,14 @@ def render_sql_template(
|
|
|
1992
1993
|
Traceback (most recent call last):
|
|
1993
1994
|
...
|
|
1994
1995
|
tinybird.sql_template.SQLTemplateException: Template Syntax Error: expression "test" evaluated to null
|
|
1996
|
+
>>> render_sql_template("SELECT {{testisasuperlongthingandwedontwanttoreturnthefullthing}}", {'token':'testing'})
|
|
1997
|
+
Traceback (most recent call last):
|
|
1998
|
+
...
|
|
1999
|
+
tinybird.sql_template.SQLTemplateException: Template Syntax Error: expression "testisasuperlongthin..." evaluated to null
|
|
2000
|
+
>>> render_sql_template("SELECT {{ Array(embedding, 'Float32') }}", {'token':'testing', 'embedding': '1,2,3,4, null'})
|
|
2001
|
+
Traceback (most recent call last):
|
|
2002
|
+
...
|
|
2003
|
+
tinybird.sql_template.SQLTemplateException: Template Syntax Error: Error validating 1,2,3,4, null[4]( null) to type Float32
|
|
1995
2004
|
>>> render_sql_template('{% if test %}SELECT 1{% else %} select 2 {% end %}')
|
|
1996
2005
|
(' select 2 ', {}, [])
|
|
1997
2006
|
>>> render_sql_template('{% if Int32(test, 1) %}SELECT 1{% else %} select 2 {% end %}')
|
|
@@ -10,6 +10,7 @@ import tinybird.tb_cli_modules.cli
|
|
|
10
10
|
import tinybird.tb_cli_modules.common
|
|
11
11
|
import tinybird.tb_cli_modules.connection
|
|
12
12
|
import tinybird.tb_cli_modules.datasource
|
|
13
|
+
import tinybird.tb_cli_modules.fmt
|
|
13
14
|
import tinybird.tb_cli_modules.job
|
|
14
15
|
import tinybird.tb_cli_modules.pipe
|
|
15
16
|
import tinybird.tb_cli_modules.tag
|
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
# - If it makes sense and only when strictly necessary, you can create utility functions in this file.
|
|
4
4
|
# - But please, **do not** interleave utility functions and command definitions.
|
|
5
5
|
|
|
6
|
-
import difflib
|
|
7
6
|
import json
|
|
8
7
|
import logging
|
|
9
8
|
import os
|
|
@@ -36,21 +35,16 @@ from tinybird.datafile import (
|
|
|
36
35
|
Datafile,
|
|
37
36
|
ParseException,
|
|
38
37
|
build_graph,
|
|
39
|
-
color_diff,
|
|
40
38
|
create_release,
|
|
41
39
|
diff_command,
|
|
42
40
|
folder_pull,
|
|
43
41
|
folder_push,
|
|
44
|
-
format_datasource,
|
|
45
|
-
format_pipe,
|
|
46
42
|
get_project_filenames,
|
|
47
43
|
get_resource_versions,
|
|
48
44
|
has_internal_datafiles,
|
|
49
|
-
is_file_a_datasource,
|
|
50
45
|
parse_datasource,
|
|
51
46
|
parse_pipe,
|
|
52
47
|
parse_token,
|
|
53
|
-
peek,
|
|
54
48
|
wait_job,
|
|
55
49
|
)
|
|
56
50
|
from tinybird.feedback_manager import FeedbackManager
|
|
@@ -753,84 +747,6 @@ async def dependencies(
|
|
|
753
747
|
raise CLIException(FeedbackManager.error_partial_replace_cant_be_executed(datasource=datasource))
|
|
754
748
|
|
|
755
749
|
|
|
756
|
-
@cli.command()
|
|
757
|
-
@click.argument("filenames", type=click.Path(exists=True), nargs=-1, required=True)
|
|
758
|
-
@click.option(
|
|
759
|
-
"--line-length",
|
|
760
|
-
is_flag=False,
|
|
761
|
-
default=100,
|
|
762
|
-
help="A number indicating the maximum characters per line in the node SQL, lines will be splitted based on the SQL syntax and the number of characters passed as a parameter",
|
|
763
|
-
)
|
|
764
|
-
@click.option("--dry-run", is_flag=True, default=False, help="Don't ask to override the local file")
|
|
765
|
-
@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation to overwrite the local file")
|
|
766
|
-
@click.option(
|
|
767
|
-
"--diff",
|
|
768
|
-
is_flag=True,
|
|
769
|
-
default=False,
|
|
770
|
-
help="Formats local file, prints the diff and exits 1 if different, 0 if equal",
|
|
771
|
-
)
|
|
772
|
-
@click.pass_context
|
|
773
|
-
@coro
|
|
774
|
-
async def fmt(
|
|
775
|
-
ctx: Context, filenames: List[str], line_length: int, dry_run: bool, yes: bool, diff: bool
|
|
776
|
-
) -> Optional[str]:
|
|
777
|
-
"""
|
|
778
|
-
Formats a .datasource, .pipe or .incl file
|
|
779
|
-
|
|
780
|
-
This command removes comments starting with # from the file, use DESCRIPTION instead.
|
|
781
|
-
|
|
782
|
-
The format command tries to parse the datafile so syntax errors might rise.
|
|
783
|
-
|
|
784
|
-
.incl files must contain a NODE definition
|
|
785
|
-
"""
|
|
786
|
-
|
|
787
|
-
result = ""
|
|
788
|
-
failed = []
|
|
789
|
-
for filename in filenames:
|
|
790
|
-
if not diff:
|
|
791
|
-
click.echo(filename)
|
|
792
|
-
extensions = Path(filename).suffixes
|
|
793
|
-
if is_file_a_datasource(filename):
|
|
794
|
-
result = await format_datasource(filename, skip_eval=True)
|
|
795
|
-
elif (".pipe" in extensions) or (".incl" in extensions):
|
|
796
|
-
result = await format_pipe(filename, line_length, skip_eval=True)
|
|
797
|
-
else:
|
|
798
|
-
click.echo("Unsupported file type. Supported files types are: .pipe, .incl and .datasource")
|
|
799
|
-
return None
|
|
800
|
-
|
|
801
|
-
if diff:
|
|
802
|
-
result = result.rstrip("\n")
|
|
803
|
-
lines_fmt = [f"{line}\n" for line in result.split("\n")]
|
|
804
|
-
with open(filename, "r") as file:
|
|
805
|
-
lines_file = file.readlines()
|
|
806
|
-
diff_result = difflib.unified_diff(
|
|
807
|
-
lines_file, lines_fmt, fromfile=f"{Path(filename).name} local", tofile="fmt datafile"
|
|
808
|
-
)
|
|
809
|
-
diff_result = color_diff(diff_result)
|
|
810
|
-
not_empty, diff_lines = peek(diff_result)
|
|
811
|
-
if not_empty:
|
|
812
|
-
sys.stdout.writelines(diff_lines)
|
|
813
|
-
failed.append(filename)
|
|
814
|
-
click.echo("")
|
|
815
|
-
else:
|
|
816
|
-
click.echo(result)
|
|
817
|
-
if dry_run:
|
|
818
|
-
return None
|
|
819
|
-
|
|
820
|
-
if yes or click.confirm(FeedbackManager.prompt_override_local_file(name=filename)):
|
|
821
|
-
with open(f"{filename}", "w") as file:
|
|
822
|
-
file.write(result)
|
|
823
|
-
|
|
824
|
-
click.echo(FeedbackManager.success_generated_local_file(file=filename))
|
|
825
|
-
|
|
826
|
-
if len(failed):
|
|
827
|
-
click.echo(FeedbackManager.error_failed_to_format_files(number=len(failed)))
|
|
828
|
-
for f in failed:
|
|
829
|
-
click.echo(f"tb fmt {f} --yes")
|
|
830
|
-
sys.exit(1)
|
|
831
|
-
return result
|
|
832
|
-
|
|
833
|
-
|
|
834
750
|
@cli.command(
|
|
835
751
|
name="diff",
|
|
836
752
|
short_help="Diffs local datafiles to the corresponding remote files in the workspace. For the case of .datasource files it just diffs VERSION and SCHEMA, since ENGINE, KAFKA or other metadata is considered immutable.",
|
|
@@ -805,3 +805,65 @@ async def datasource_copy_from_main(
|
|
|
805
805
|
if wait:
|
|
806
806
|
base_msg = "Copy from Main Workspace" if sql_from_main else f"Copy from {sql}"
|
|
807
807
|
await wait_job(client, job_id, job_url, f"{base_msg} to {datasource_name}")
|
|
808
|
+
|
|
809
|
+
|
|
810
|
+
@datasource.group(name="scheduling")
|
|
811
|
+
@click.pass_context
|
|
812
|
+
def datasource_scheduling(ctx: Context) -> None:
|
|
813
|
+
"""Data Source scheduling commands."""
|
|
814
|
+
|
|
815
|
+
|
|
816
|
+
@datasource_scheduling.command(name="state")
|
|
817
|
+
@click.argument("datasource_name")
|
|
818
|
+
@click.pass_context
|
|
819
|
+
@coro
|
|
820
|
+
async def datasource_scheduling_state(ctx: Context, datasource_name: str) -> None:
|
|
821
|
+
"""Get the scheduling state of a Data Source."""
|
|
822
|
+
client: TinyB = ctx.obj["client"]
|
|
823
|
+
try:
|
|
824
|
+
state = await client.datasource_scheduling_state(datasource_name)
|
|
825
|
+
click.echo(FeedbackManager.info_datasource_scheduling_state(datasource=datasource_name, state=state))
|
|
826
|
+
except Exception as e:
|
|
827
|
+
raise CLIDatasourceException(
|
|
828
|
+
FeedbackManager.error_datasource_scheduling_state(datasource=datasource_name, error=e)
|
|
829
|
+
)
|
|
830
|
+
|
|
831
|
+
|
|
832
|
+
@datasource_scheduling.command(name="pause")
|
|
833
|
+
@click.argument("datasource_name")
|
|
834
|
+
@click.pass_context
|
|
835
|
+
@coro
|
|
836
|
+
async def datasource_scheduling_pause(ctx: Context, datasource_name: str) -> None:
|
|
837
|
+
"""Pause the scheduling of a Data Source."""
|
|
838
|
+
|
|
839
|
+
click.echo(FeedbackManager.info_datasource_scheduling_pause())
|
|
840
|
+
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
841
|
+
|
|
842
|
+
try:
|
|
843
|
+
await client.datasource_scheduling_pause(datasource_name)
|
|
844
|
+
click.echo(FeedbackManager.success_datasource_scheduling_paused(datasource=datasource_name))
|
|
845
|
+
|
|
846
|
+
except Exception as e:
|
|
847
|
+
raise CLIDatasourceException(
|
|
848
|
+
FeedbackManager.error_pausing_datasource_scheduling(datasource=datasource_name, error=e)
|
|
849
|
+
)
|
|
850
|
+
|
|
851
|
+
|
|
852
|
+
@datasource_scheduling.command(name="resume")
|
|
853
|
+
@click.argument("datasource_name")
|
|
854
|
+
@click.pass_context
|
|
855
|
+
@coro
|
|
856
|
+
async def datasource_scheduling_resume(ctx: Context, datasource_name: str) -> None:
|
|
857
|
+
"""Resume the scheduling of a Data Source."""
|
|
858
|
+
|
|
859
|
+
click.echo(FeedbackManager.info_datasource_scheduling_resume())
|
|
860
|
+
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
861
|
+
|
|
862
|
+
try:
|
|
863
|
+
await client.datasource_scheduling_resume(datasource_name)
|
|
864
|
+
click.echo(FeedbackManager.success_datasource_scheduling_resumed(datasource=datasource_name))
|
|
865
|
+
|
|
866
|
+
except Exception as e:
|
|
867
|
+
raise CLIDatasourceException(
|
|
868
|
+
FeedbackManager.error_resuming_datasource_scheduling(datasource=datasource_name, error=e)
|
|
869
|
+
)
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import difflib
|
|
2
|
+
import sys
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import List, Optional
|
|
5
|
+
|
|
6
|
+
import click
|
|
7
|
+
from click import Context
|
|
8
|
+
|
|
9
|
+
from tinybird.datafile import color_diff, format_datasource, format_pipe, is_file_a_datasource, peek
|
|
10
|
+
from tinybird.feedback_manager import FeedbackManager
|
|
11
|
+
from tinybird.tb_cli_modules.cli import cli
|
|
12
|
+
from tinybird.tb_cli_modules.common import coro
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@cli.command()
|
|
16
|
+
@click.argument("filenames", type=click.Path(exists=True), nargs=-1, required=True)
|
|
17
|
+
@click.option(
|
|
18
|
+
"--line-length",
|
|
19
|
+
is_flag=False,
|
|
20
|
+
default=100,
|
|
21
|
+
help="A number indicating the maximum characters per line in the node SQL, lines will be splitted based on the SQL syntax and the number of characters passed as a parameter",
|
|
22
|
+
)
|
|
23
|
+
@click.option("--dry-run", is_flag=True, default=False, help="Don't ask to override the local file")
|
|
24
|
+
@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation to overwrite the local file")
|
|
25
|
+
@click.option(
|
|
26
|
+
"--diff",
|
|
27
|
+
is_flag=True,
|
|
28
|
+
default=False,
|
|
29
|
+
help="Formats local file, prints the diff and exits 1 if different, 0 if equal",
|
|
30
|
+
)
|
|
31
|
+
@click.pass_context
|
|
32
|
+
@coro
|
|
33
|
+
async def fmt(
|
|
34
|
+
ctx: Context, filenames: List[str], line_length: int, dry_run: bool, yes: bool, diff: bool
|
|
35
|
+
) -> Optional[str]:
|
|
36
|
+
"""
|
|
37
|
+
Formats a .datasource, .pipe or .incl file
|
|
38
|
+
|
|
39
|
+
This command removes comments starting with # from the file, use DESCRIPTION instead.
|
|
40
|
+
|
|
41
|
+
The format command tries to parse the datafile so syntax errors might rise.
|
|
42
|
+
|
|
43
|
+
.incl files must contain a NODE definition
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
result = ""
|
|
47
|
+
failed = []
|
|
48
|
+
for filename in filenames:
|
|
49
|
+
if not diff:
|
|
50
|
+
click.echo(filename)
|
|
51
|
+
extensions = Path(filename).suffixes
|
|
52
|
+
if is_file_a_datasource(filename):
|
|
53
|
+
result = await format_datasource(filename, skip_eval=True)
|
|
54
|
+
elif (".pipe" in extensions) or (".incl" in extensions):
|
|
55
|
+
result = await format_pipe(filename, line_length, skip_eval=True)
|
|
56
|
+
else:
|
|
57
|
+
click.echo("Unsupported file type. Supported files types are: .pipe, .incl and .datasource")
|
|
58
|
+
return None
|
|
59
|
+
|
|
60
|
+
if diff:
|
|
61
|
+
result = result.rstrip("\n")
|
|
62
|
+
lines_fmt = [f"{line}\n" for line in result.split("\n")]
|
|
63
|
+
with open(filename, "r") as file:
|
|
64
|
+
lines_file = file.readlines()
|
|
65
|
+
diff_result = difflib.unified_diff(
|
|
66
|
+
lines_file, lines_fmt, fromfile=f"{Path(filename).name} local", tofile="fmt datafile"
|
|
67
|
+
)
|
|
68
|
+
diff_result = color_diff(diff_result)
|
|
69
|
+
not_empty, diff_lines = peek(diff_result)
|
|
70
|
+
if not_empty:
|
|
71
|
+
sys.stdout.writelines(diff_lines)
|
|
72
|
+
failed.append(filename)
|
|
73
|
+
click.echo("")
|
|
74
|
+
else:
|
|
75
|
+
click.echo(result)
|
|
76
|
+
if dry_run:
|
|
77
|
+
return None
|
|
78
|
+
|
|
79
|
+
if yes or click.confirm(FeedbackManager.prompt_override_local_file(name=filename)):
|
|
80
|
+
with open(f"{filename}", "w") as file:
|
|
81
|
+
file.write(result)
|
|
82
|
+
|
|
83
|
+
click.echo(FeedbackManager.success_generated_local_file(file=filename))
|
|
84
|
+
|
|
85
|
+
if len(failed):
|
|
86
|
+
click.echo(FeedbackManager.error_failed_to_format_files(number=len(failed)))
|
|
87
|
+
for f in failed:
|
|
88
|
+
click.echo(f"tb fmt {f} --yes")
|
|
89
|
+
sys.exit(1)
|
|
90
|
+
return result
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: tinybird-cli
|
|
3
|
-
Version: 5.8.0.
|
|
3
|
+
Version: 5.8.0.dev3
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/cli/introduction.html
|
|
6
6
|
Author: Tinybird
|
|
@@ -18,6 +18,16 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
|
|
|
18
18
|
Changelog
|
|
19
19
|
----------
|
|
20
20
|
|
|
21
|
+
5.8.0.dev3
|
|
22
|
+
**********
|
|
23
|
+
|
|
24
|
+
- `Added` `tb datasource scheduling` commands to manage the scheduling of a Data Source
|
|
25
|
+
|
|
26
|
+
5.8.0.dev2
|
|
27
|
+
***********
|
|
28
|
+
|
|
29
|
+
- `Added` support to `TAGS` in `tb fmt`.
|
|
30
|
+
|
|
21
31
|
5.8.0.dev1
|
|
22
32
|
***********
|
|
23
33
|
|
|
@@ -26,6 +26,7 @@ tinybird/tb_cli_modules/config.py
|
|
|
26
26
|
tinybird/tb_cli_modules/connection.py
|
|
27
27
|
tinybird/tb_cli_modules/datasource.py
|
|
28
28
|
tinybird/tb_cli_modules/exceptions.py
|
|
29
|
+
tinybird/tb_cli_modules/fmt.py
|
|
29
30
|
tinybird/tb_cli_modules/job.py
|
|
30
31
|
tinybird/tb_cli_modules/pipe.py
|
|
31
32
|
tinybird/tb_cli_modules/regions.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/tinyunit/tinyunit.py
RENAMED
|
File without changes
|
{tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird/tb_cli_modules/workspace_members.py
RENAMED
|
File without changes
|
|
File without changes
|
{tinybird-cli-5.8.0.dev1 → tinybird-cli-5.8.0.dev3}/tinybird_cli.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|