tinybird-cli 5.0.0.dev5__tar.gz → 5.0.1.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/PKG-INFO +12 -29
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/__cli__.py +2 -2
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/ch_utils/constants.py +1 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/client.py +28 -17
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/datafile.py +29 -1
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/feedback_manager.py +3 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/sql_toolset.py +69 -16
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/pipe.py +7 -2
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird_cli.egg-info/PKG-INFO +12 -29
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/setup.cfg +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/ch_utils/engine.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/check_pypi.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/config.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/connectors.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/context.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/datatypes.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/git_settings.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/sql.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/sql_template.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/sql_template_fmt.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/syncasync.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/auth.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/branch.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/cicd.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/cli.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/common.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/config.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/connection.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/datasource.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/exceptions.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/job.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/regions.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/telemetry.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/test.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/token.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/workspace.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/workspace_members.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tornado_template.py +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird_cli.egg-info/SOURCES.txt +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird_cli.egg-info/dependency_links.txt +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird_cli.egg-info/entry_points.txt +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird_cli.egg-info/requires.txt +0 -0
- {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird_cli.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: tinybird-cli
|
|
3
|
-
Version: 5.0.
|
|
3
|
+
Version: 5.0.1.dev1
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/cli/introduction.html
|
|
6
6
|
Author: Tinybird
|
|
@@ -18,42 +18,25 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
|
|
|
18
18
|
Changelog
|
|
19
19
|
----------
|
|
20
20
|
|
|
21
|
-
5.
|
|
22
|
-
|
|
21
|
+
5.1.0.dev1
|
|
22
|
+
***********
|
|
23
23
|
|
|
24
|
-
- `Added`
|
|
24
|
+
- `Added` COPY pipes now support `MODE replace`
|
|
25
25
|
|
|
26
|
-
5.0.0
|
|
27
|
-
|
|
26
|
+
5.0.0
|
|
27
|
+
******
|
|
28
28
|
|
|
29
|
-
- `
|
|
29
|
+
- `Breaking change` Make `insertion_date` column explicit. This column is no longer inferred, it must be present in the Data Source file.
|
|
30
30
|
|
|
31
|
-
|
|
32
|
-
************
|
|
31
|
+
Detailed info at https://www.tinybird.co/docs/changelog/2024-06-17-insertion_date-deprecation
|
|
33
32
|
|
|
33
|
+
- `Added` parameter to `tb pipe regression-test` --relative-change
|
|
34
|
+
- `Added` `tb push` displays warnings when using a reserved parameter in a pipe.
|
|
35
|
+
- `Added` --role parameter to `tb workspace members add`
|
|
34
36
|
- `Changed` Point region hosts to new `app.tinybird.co` domain
|
|
35
|
-
|
|
36
|
-
5.0.0.dev2
|
|
37
|
-
************
|
|
38
|
-
|
|
37
|
+
- `Improved` help message of `tb workspace members set-role` now displays the 3 valid roles: viewer|guest|admin.
|
|
39
38
|
- `Improved` syntax error messages when parsing endpoints.
|
|
40
39
|
|
|
41
|
-
5.0.0.dev1
|
|
42
|
-
************
|
|
43
|
-
|
|
44
|
-
- `Added` `tb push` displays warnings when using a reserved parameter in a pipe.
|
|
45
|
-
|
|
46
|
-
5.0.0.dev0
|
|
47
|
-
************
|
|
48
|
-
|
|
49
|
-
- `Changed` Make `insertion_date` column explicit. This column is no longer inferred, it must be present in the Data Source file.
|
|
50
|
-
|
|
51
|
-
4.1.2.dev0
|
|
52
|
-
************
|
|
53
|
-
|
|
54
|
-
- `Added` parameter to `tb pipe regression-test` --relative-change
|
|
55
|
-
|
|
56
|
-
|
|
57
40
|
4.1.1
|
|
58
41
|
************
|
|
59
42
|
|
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '5.0.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '5.0.1.dev1'
|
|
8
|
+
__revision__ = '3d0cfe4'
|
|
@@ -4,6 +4,7 @@ LIVE_WS_NAME = "live"
|
|
|
4
4
|
SNAPSHOT_WS_NAME = "snapshot"
|
|
5
5
|
|
|
6
6
|
ENABLED_TABLE_FUNCTIONS = {"generateRandom", "null", "numbers", "numbers_mt", "values", "zeros", "zeros_mt"}
|
|
7
|
+
COPY_ENABLED_TABLE_FUNCTIONS = frozenset(["postgresql"])
|
|
7
8
|
|
|
8
9
|
ENABLED_SYSTEM_TABLES = {
|
|
9
10
|
"functions",
|
|
@@ -11,6 +11,7 @@ import requests.adapters
|
|
|
11
11
|
from requests import Response
|
|
12
12
|
from urllib3 import Retry
|
|
13
13
|
|
|
14
|
+
from tinybird.ch_utils.constants import COPY_ENABLED_TABLE_FUNCTIONS
|
|
14
15
|
from tinybird.syncasync import sync_to_async
|
|
15
16
|
from tinybird.tb_cli_modules.regions import fill_with_public_regions
|
|
16
17
|
from tinybird.tb_cli_modules.telemetry import add_telemetry_event
|
|
@@ -470,6 +471,8 @@ class TinyB(object):
|
|
|
470
471
|
self, pipe_name: str, node: Dict[str, Any], dry_run: str = "false", datasource_name: Optional[str] = None
|
|
471
472
|
):
|
|
472
473
|
params = {**{"include_datafile": "true", "dry_run": dry_run}, **node.get("params", node)}
|
|
474
|
+
if "mode" in params:
|
|
475
|
+
params.pop("mode")
|
|
473
476
|
node_name = node["params"]["name"] if node.get("params", None) else node["name"]
|
|
474
477
|
if datasource_name:
|
|
475
478
|
params["datasource"] = datasource_name
|
|
@@ -549,34 +552,33 @@ class TinyB(object):
|
|
|
549
552
|
async def pipe_remove_endpoint(self, pipe_name_or_uid: str, published_node_uid: str):
|
|
550
553
|
return await self._req(f"/v0/pipes/{pipe_name_or_uid}/nodes/{published_node_uid}/endpoint", method="DELETE")
|
|
551
554
|
|
|
552
|
-
async def pipe_create_copy(
|
|
553
|
-
self, pipe_name_or_id: str, node_id: str, target_datasource: str, schedule_cron: Optional[str] = None
|
|
554
|
-
):
|
|
555
|
-
data = {"target_datasource": target_datasource}
|
|
556
|
-
if schedule_cron:
|
|
557
|
-
data["schedule_cron"] = schedule_cron
|
|
558
|
-
|
|
559
|
-
return await self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/copy", method="POST", data=data)
|
|
560
|
-
|
|
561
555
|
async def pipe_update_copy(
|
|
562
556
|
self,
|
|
563
557
|
pipe_name_or_id: str,
|
|
564
558
|
node_id: str,
|
|
565
559
|
target_datasource: Optional[str] = None,
|
|
566
560
|
schedule_cron: Optional[str] = None,
|
|
561
|
+
mode: Optional[str] = None,
|
|
567
562
|
):
|
|
568
563
|
data = {"schedule_cron": schedule_cron}
|
|
569
564
|
|
|
570
565
|
if target_datasource:
|
|
571
566
|
data["target_datasource"] = target_datasource
|
|
572
567
|
|
|
568
|
+
if mode:
|
|
569
|
+
data["mode"] = mode
|
|
570
|
+
|
|
573
571
|
return await self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/copy", method="PUT", data=data)
|
|
574
572
|
|
|
575
573
|
async def pipe_remove_copy(self, pipe_name_or_id: str, node_id: str):
|
|
576
574
|
return await self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/copy", method="DELETE")
|
|
577
575
|
|
|
578
|
-
async def pipe_run_copy(
|
|
576
|
+
async def pipe_run_copy(
|
|
577
|
+
self, pipe_name_or_id: str, params: Optional[Dict[str, str]] = None, mode: Optional[str] = None
|
|
578
|
+
):
|
|
579
579
|
params = {**params} if params else {}
|
|
580
|
+
if mode:
|
|
581
|
+
params["_mode"] = mode
|
|
580
582
|
return await self._req(f"/v0/pipes/{pipe_name_or_id}/copy?{urlencode(params)}", method="POST")
|
|
581
583
|
|
|
582
584
|
async def pipe_resume_copy(self, pipe_name_or_id: str):
|
|
@@ -1035,23 +1037,32 @@ class TinyB(object):
|
|
|
1035
1037
|
return result["q"]
|
|
1036
1038
|
|
|
1037
1039
|
@staticmethod
|
|
1038
|
-
def _sql_get_used_tables_local(sql: str, raising: bool = False) -> List[str]:
|
|
1040
|
+
def _sql_get_used_tables_local(sql: str, raising: bool = False, is_copy: Optional[bool] = False) -> List[str]:
|
|
1039
1041
|
from tinybird.sql_toolset import sql_get_used_tables
|
|
1040
1042
|
|
|
1041
|
-
tables = sql_get_used_tables(
|
|
1043
|
+
tables = sql_get_used_tables(
|
|
1044
|
+
sql, raising, table_functions=False, function_allow_list=COPY_ENABLED_TABLE_FUNCTIONS if is_copy else None
|
|
1045
|
+
)
|
|
1042
1046
|
return [t[1] if t[0] == "" else f"{t[0]}.{t[1]}" for t in tables]
|
|
1043
1047
|
|
|
1044
|
-
async def _sql_get_used_tables_remote(
|
|
1045
|
-
|
|
1048
|
+
async def _sql_get_used_tables_remote(
|
|
1049
|
+
self, sql: str, raising: bool = False, is_copy: Optional[bool] = False
|
|
1050
|
+
) -> List[str]:
|
|
1051
|
+
params = {
|
|
1052
|
+
"q": sql,
|
|
1053
|
+
"raising": "true" if raising else "false",
|
|
1054
|
+
"table_functions": "false",
|
|
1055
|
+
"is_copy": "true" if is_copy else "false",
|
|
1056
|
+
}
|
|
1046
1057
|
result = await self._req("/v0/sql_tables", data=params, method="POST")
|
|
1047
1058
|
return [t[1] if t[0] == "" else f"{t[0]}.{t[1]}" for t in result["tables"]]
|
|
1048
1059
|
|
|
1049
1060
|
# Get used tables from a query. Does not include table functions
|
|
1050
|
-
async def sql_get_used_tables(self, sql: str, raising: bool = False) -> List[str]:
|
|
1061
|
+
async def sql_get_used_tables(self, sql: str, raising: bool = False, is_copy: Optional[bool] = False) -> List[str]:
|
|
1051
1062
|
try:
|
|
1052
|
-
return self._sql_get_used_tables_local(sql, raising)
|
|
1063
|
+
return self._sql_get_used_tables_local(sql, raising, is_copy)
|
|
1053
1064
|
except ModuleNotFoundError:
|
|
1054
|
-
return await self._sql_get_used_tables_remote(sql, raising)
|
|
1065
|
+
return await self._sql_get_used_tables_remote(sql, raising, is_copy)
|
|
1055
1066
|
|
|
1056
1067
|
@staticmethod
|
|
1057
1068
|
def _replace_tables_local(q: str, replacements):
|
|
@@ -118,6 +118,18 @@ class DataFileExtensions:
|
|
|
118
118
|
class CopyParameters:
|
|
119
119
|
TARGET_DATASOURCE = "target_datasource"
|
|
120
120
|
COPY_SCHEDULE = "copy_schedule"
|
|
121
|
+
COPY_MODE = "copy_mode"
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class CopyModes:
|
|
125
|
+
APPEND = "append"
|
|
126
|
+
REPLACE = "replace"
|
|
127
|
+
|
|
128
|
+
valid_modes = (APPEND, REPLACE)
|
|
129
|
+
|
|
130
|
+
@staticmethod
|
|
131
|
+
def is_valid(node_mode):
|
|
132
|
+
return node_mode.lower() in CopyModes.valid_modes
|
|
121
133
|
|
|
122
134
|
|
|
123
135
|
PREVIEW_CONNECTOR_SERVICES = ["s3", "s3_iamrole", "gcs"]
|
|
@@ -1076,6 +1088,8 @@ def parse(
|
|
|
1076
1088
|
"tags": assign_node_var("tags"),
|
|
1077
1089
|
"target_datasource": assign_node_var("target_datasource"),
|
|
1078
1090
|
"copy_schedule": assign_node_var(CopyParameters.COPY_SCHEDULE),
|
|
1091
|
+
"copy_mode": assign_node_var("mode"),
|
|
1092
|
+
"mode": assign_node_var("mode"),
|
|
1079
1093
|
"resource": assign_node_var("resource"),
|
|
1080
1094
|
"filter": assign_node_var("filter"),
|
|
1081
1095
|
"token": add_token,
|
|
@@ -1476,6 +1490,7 @@ async def process_file(
|
|
|
1476
1490
|
deps = []
|
|
1477
1491
|
nodes: List[Dict[str, Any]] = []
|
|
1478
1492
|
|
|
1493
|
+
is_copy = any([node for node in doc.nodes if node.get("type", "standard").lower() == PipeNodeTypes.COPY])
|
|
1479
1494
|
for node in doc.nodes:
|
|
1480
1495
|
sql = node["sql"]
|
|
1481
1496
|
node_type = node.get("type", "standard").lower()
|
|
@@ -1485,6 +1500,7 @@ async def process_file(
|
|
|
1485
1500
|
"description": node.get("description", ""),
|
|
1486
1501
|
"target_datasource": node.get("target_datasource", None),
|
|
1487
1502
|
"copy_schedule": node.get(CopyParameters.COPY_SCHEDULE, None),
|
|
1503
|
+
"mode": node.get("mode", CopyModes.APPEND),
|
|
1488
1504
|
}
|
|
1489
1505
|
|
|
1490
1506
|
is_export_node = ExportReplacements.is_export_node(node)
|
|
@@ -1504,7 +1520,7 @@ async def process_file(
|
|
|
1504
1520
|
sql_rendered = sql
|
|
1505
1521
|
|
|
1506
1522
|
try:
|
|
1507
|
-
dependencies = await tb_client.sql_get_used_tables(sql_rendered, raising=True)
|
|
1523
|
+
dependencies = await tb_client.sql_get_used_tables(sql_rendered, raising=True, is_copy=is_copy)
|
|
1508
1524
|
deps += [t for t in dependencies if t not in [n["name"] for n in doc.nodes]]
|
|
1509
1525
|
|
|
1510
1526
|
except Exception as e:
|
|
@@ -2482,6 +2498,12 @@ async def check_copy_pipe(pipe, copy_node, tb_client: TinyB):
|
|
|
2482
2498
|
if not is_valid_cron:
|
|
2483
2499
|
raise CLIPipeException(FeedbackManager.error_creating_copy_pipe_invalid_cron(schedule_cron=schedule_cron))
|
|
2484
2500
|
|
|
2501
|
+
mode = copy_node["params"].get("mode", CopyModes.APPEND)
|
|
2502
|
+
is_valid_mode = CopyModes.is_valid(mode)
|
|
2503
|
+
|
|
2504
|
+
if not is_valid_mode:
|
|
2505
|
+
raise CLIPipeException(FeedbackManager.error_creating_copy_pipe_invalid_mode(mode=mode))
|
|
2506
|
+
|
|
2485
2507
|
if not pipe:
|
|
2486
2508
|
return
|
|
2487
2509
|
|
|
@@ -2763,6 +2785,7 @@ async def new_pipe(
|
|
|
2763
2785
|
try:
|
|
2764
2786
|
target_datasource = copy_node.get(CopyParameters.TARGET_DATASOURCE, None)
|
|
2765
2787
|
schedule_cron = copy_node.get(CopyParameters.COPY_SCHEDULE, None)
|
|
2788
|
+
mode = copy_node.get("mode", CopyModes.APPEND)
|
|
2766
2789
|
schedule_cron = None if schedule_cron == ON_DEMAND else schedule_cron
|
|
2767
2790
|
current_target_datasource_id = data["copy_target_datasource"]
|
|
2768
2791
|
target_datasource_response = await tb_client.get_datasource(target_datasource)
|
|
@@ -2776,6 +2799,7 @@ async def new_pipe(
|
|
|
2776
2799
|
current_schedule_cron = current_schedule.get("cron", None) if current_schedule else None
|
|
2777
2800
|
schedule_cron_should_be_removed = current_schedule_cron and not schedule_cron
|
|
2778
2801
|
copy_params["schedule_cron"] = "None" if schedule_cron_should_be_removed else schedule_cron
|
|
2802
|
+
copy_params["mode"] = mode
|
|
2779
2803
|
await tb_client.pipe_update_copy(**copy_params)
|
|
2780
2804
|
except Exception as e:
|
|
2781
2805
|
raise Exception(
|
|
@@ -4627,6 +4651,10 @@ async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[
|
|
|
4627
4651
|
file_parts.append(node_type_upper)
|
|
4628
4652
|
file_parts.append(DATAFILE_NEW_LINE)
|
|
4629
4653
|
file_parts.append(f'TARGET_DATASOURCE {node["target_datasource"]}')
|
|
4654
|
+
if node.get("mode"):
|
|
4655
|
+
file_parts.append(DATAFILE_NEW_LINE)
|
|
4656
|
+
file_parts.append(f'COPY_MODE {node.get("mode")}')
|
|
4657
|
+
|
|
4630
4658
|
if CopyParameters.COPY_SCHEDULE in node and node[CopyParameters.COPY_SCHEDULE]:
|
|
4631
4659
|
is_ondemand = node[CopyParameters.COPY_SCHEDULE].lower() == ON_DEMAND
|
|
4632
4660
|
file_parts.append(DATAFILE_NEW_LINE)
|
|
@@ -126,6 +126,9 @@ class FeedbackManager:
|
|
|
126
126
|
error_creating_copy_pipe_invalid_cron = error_message(
|
|
127
127
|
"Cannot create Copy pipe. Invalid cron expression: '{schedule_cron}'"
|
|
128
128
|
)
|
|
129
|
+
error_creating_copy_pipe_invalid_mode = error_message(
|
|
130
|
+
"Cannot create Copy pipe. Invalid MODE expression: '{mode}'. Valid modes are: 'append', 'replace'"
|
|
131
|
+
)
|
|
129
132
|
error_creating_sink_pipe_invalid_cron = error_message(
|
|
130
133
|
"Cannot create Sink Pipe. Invalid cron expression: '{schedule_cron}'"
|
|
131
134
|
)
|
|
@@ -3,20 +3,26 @@ import logging
|
|
|
3
3
|
from collections import defaultdict
|
|
4
4
|
from datetime import datetime
|
|
5
5
|
from functools import lru_cache
|
|
6
|
-
from typing import Any, List, Optional, Set, Tuple
|
|
6
|
+
from typing import Any, FrozenSet, List, Optional, Set, Tuple
|
|
7
7
|
|
|
8
8
|
from chtoolset import query as chquery
|
|
9
9
|
from toposort import toposort
|
|
10
10
|
|
|
11
|
-
from tinybird.ch_utils.constants import ENABLED_TABLE_FUNCTIONS
|
|
11
|
+
from tinybird.ch_utils.constants import COPY_ENABLED_TABLE_FUNCTIONS, ENABLED_TABLE_FUNCTIONS
|
|
12
12
|
|
|
13
13
|
VALID_REMOTE = "VALID_REMOTE"
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
class InvalidFunction(ValueError):
|
|
17
17
|
def __init__(self, msg: str = "", table_function_names: str = ""):
|
|
18
|
+
if any([fn for fn in COPY_ENABLED_TABLE_FUNCTIONS if fn in msg]):
|
|
19
|
+
msg = msg.replace("is restricted", "is restricted to Copy Pipes")
|
|
20
|
+
|
|
18
21
|
if table_function_names:
|
|
19
|
-
|
|
22
|
+
if "postgresql" in table_function_names:
|
|
23
|
+
self.msg = "The postgresql table function is only allowed in Copy Pipes"
|
|
24
|
+
else:
|
|
25
|
+
self.msg = f"The query uses disabled table functions: '{table_function_names}'"
|
|
20
26
|
else:
|
|
21
27
|
self.msg = msg
|
|
22
28
|
super().__init__(self.msg)
|
|
@@ -58,7 +64,11 @@ def format_where_for_mutation_command(where_clause: str) -> str:
|
|
|
58
64
|
|
|
59
65
|
@lru_cache(maxsize=2**13)
|
|
60
66
|
def sql_get_used_tables_cached(
|
|
61
|
-
sql: str,
|
|
67
|
+
sql: str,
|
|
68
|
+
raising: bool = False,
|
|
69
|
+
default_database: str = "",
|
|
70
|
+
table_functions: bool = True,
|
|
71
|
+
function_allow_list: Optional[FrozenSet[str]] = None,
|
|
62
72
|
) -> List[Tuple[str, str, str]]:
|
|
63
73
|
"""
|
|
64
74
|
>>> sql_get_used_tables("SELECT 1 FROM the_table")
|
|
@@ -79,7 +89,11 @@ def sql_get_used_tables_cached(
|
|
|
79
89
|
[('d_d3926a', 't_976af08ec4b547419e729c63e754b17b', '')]
|
|
80
90
|
"""
|
|
81
91
|
try:
|
|
82
|
-
|
|
92
|
+
_function_allow_list = list() if function_allow_list is None else list(function_allow_list)
|
|
93
|
+
|
|
94
|
+
tables: List[Tuple[str, str, str]] = chquery.tables(
|
|
95
|
+
sql, default_database=default_database, function_allow_list=_function_allow_list
|
|
96
|
+
)
|
|
83
97
|
if not table_functions:
|
|
84
98
|
return [(t[0], t[1], "") for t in tables if t[0] or t[1]]
|
|
85
99
|
return tables
|
|
@@ -93,10 +107,17 @@ def sql_get_used_tables_cached(
|
|
|
93
107
|
|
|
94
108
|
|
|
95
109
|
def sql_get_used_tables(
|
|
96
|
-
sql: str,
|
|
110
|
+
sql: str,
|
|
111
|
+
raising: bool = False,
|
|
112
|
+
default_database: str = "",
|
|
113
|
+
table_functions: bool = True,
|
|
114
|
+
function_allow_list: Optional[FrozenSet[str]] = None,
|
|
97
115
|
) -> List[Tuple[str, str, str]]:
|
|
98
|
-
|
|
99
|
-
|
|
116
|
+
hashable_list = frozenset() if function_allow_list is None else function_allow_list
|
|
117
|
+
|
|
118
|
+
return copy.copy(
|
|
119
|
+
sql_get_used_tables_cached(sql, raising, default_database, table_functions, function_allow_list=hashable_list)
|
|
120
|
+
)
|
|
100
121
|
|
|
101
122
|
|
|
102
123
|
class ReplacementsDict(dict):
|
|
@@ -150,9 +171,18 @@ def replace_tables_chquery_cached(
|
|
|
150
171
|
default_database: str = "",
|
|
151
172
|
output_one_line: bool = False,
|
|
152
173
|
timestamp: Optional[datetime] = None,
|
|
174
|
+
function_allow_list: Optional[FrozenSet[str]] = None,
|
|
153
175
|
) -> str:
|
|
154
176
|
replacements = dict(sorted_replacements) if sorted_replacements else {}
|
|
155
|
-
|
|
177
|
+
_function_allow_list = list() if function_allow_list is None else list(function_allow_list)
|
|
178
|
+
|
|
179
|
+
return chquery.replace_tables(
|
|
180
|
+
sql,
|
|
181
|
+
replacements,
|
|
182
|
+
default_database=default_database,
|
|
183
|
+
one_line=output_one_line,
|
|
184
|
+
function_allow_list=_function_allow_list,
|
|
185
|
+
)
|
|
156
186
|
|
|
157
187
|
|
|
158
188
|
def replace_tables(
|
|
@@ -164,15 +194,19 @@ def replace_tables(
|
|
|
164
194
|
valid_tables: Optional[Set[Tuple[str, str]]] = None,
|
|
165
195
|
output_one_line: bool = False,
|
|
166
196
|
timestamp: Optional[datetime] = None,
|
|
197
|
+
function_allow_list: Optional[FrozenSet[str]] = None,
|
|
167
198
|
) -> str:
|
|
168
199
|
"""
|
|
169
200
|
Given a query and a list of table replacements, returns the query after applying the table replacements.
|
|
170
201
|
It takes into account dependencies between replacement subqueries (if any)
|
|
171
202
|
It also validates the sql to verify it's valid and doesn't use unknown or prohibited functions
|
|
172
203
|
"""
|
|
204
|
+
hashable_list = frozenset() if function_allow_list is None else function_allow_list
|
|
173
205
|
if not replacements:
|
|
174
206
|
# Always call replace_tables to do validation and formatting
|
|
175
|
-
return replace_tables_chquery_cached(
|
|
207
|
+
return replace_tables_chquery_cached(
|
|
208
|
+
sql, None, output_one_line=output_one_line, timestamp=timestamp, function_allow_list=hashable_list
|
|
209
|
+
)
|
|
176
210
|
|
|
177
211
|
_replaced_with = set()
|
|
178
212
|
_replacements = ReplacementsDict()
|
|
@@ -182,14 +216,24 @@ def replace_tables(
|
|
|
182
216
|
_replaced_with.add(r)
|
|
183
217
|
|
|
184
218
|
deps: defaultdict = defaultdict(set)
|
|
185
|
-
_tables = sql_get_used_tables(
|
|
219
|
+
_tables = sql_get_used_tables(
|
|
220
|
+
sql,
|
|
221
|
+
default_database=default_database,
|
|
222
|
+
raising=True,
|
|
223
|
+
table_functions=check_functions,
|
|
224
|
+
function_allow_list=function_allow_list,
|
|
225
|
+
)
|
|
186
226
|
seen_tables = set()
|
|
187
227
|
table: Tuple[str, str] | Tuple[str, str, str]
|
|
228
|
+
if function_allow_list is None:
|
|
229
|
+
_enabled_table_functions = ENABLED_TABLE_FUNCTIONS
|
|
230
|
+
else:
|
|
231
|
+
_enabled_table_functions = ENABLED_TABLE_FUNCTIONS.union(set(function_allow_list))
|
|
188
232
|
while _tables:
|
|
189
233
|
table = _tables.pop()
|
|
190
234
|
if len(table) == 3:
|
|
191
235
|
first_table, second_table, last_table = table
|
|
192
|
-
if last_table and last_table not in
|
|
236
|
+
if last_table and last_table not in _enabled_table_functions:
|
|
193
237
|
raise InvalidFunction(table_function_names=last_table)
|
|
194
238
|
if first_table or second_table:
|
|
195
239
|
table = (first_table, second_table)
|
|
@@ -204,7 +248,7 @@ def replace_tables(
|
|
|
204
248
|
if len(dependent_table) == 3:
|
|
205
249
|
if (
|
|
206
250
|
dependent_table[2]
|
|
207
|
-
and dependent_table[2] not in
|
|
251
|
+
and dependent_table[2] not in _enabled_table_functions
|
|
208
252
|
and not (dependent_table[2] in ["cluster"] and replacement[0] == VALID_REMOTE)
|
|
209
253
|
):
|
|
210
254
|
raise InvalidFunction(table_function_names=dependent_table[2])
|
|
@@ -219,7 +263,9 @@ def replace_tables(
|
|
|
219
263
|
deps_sorted = list(reversed(list(toposort(deps))))
|
|
220
264
|
|
|
221
265
|
if not deps_sorted:
|
|
222
|
-
return replace_tables_chquery_cached(
|
|
266
|
+
return replace_tables_chquery_cached(
|
|
267
|
+
sql, None, output_one_line=output_one_line, timestamp=timestamp, function_allow_list=hashable_list
|
|
268
|
+
)
|
|
223
269
|
|
|
224
270
|
for current_deps in deps_sorted:
|
|
225
271
|
current_replacements = {}
|
|
@@ -251,10 +297,17 @@ def replace_tables(
|
|
|
251
297
|
# We need to transform the dictionary into something cacheable, so a sorted tuple of tuples it is
|
|
252
298
|
r = tuple(sorted([(k, v) for k, v in current_replacements.items()]))
|
|
253
299
|
sql = replace_tables_chquery_cached(
|
|
254
|
-
sql,
|
|
300
|
+
sql,
|
|
301
|
+
r,
|
|
302
|
+
default_database=default_database,
|
|
303
|
+
output_one_line=output_one_line,
|
|
304
|
+
timestamp=timestamp,
|
|
305
|
+
function_allow_list=hashable_list,
|
|
255
306
|
)
|
|
256
307
|
else:
|
|
257
|
-
sql = replace_tables_chquery_cached(
|
|
308
|
+
sql = replace_tables_chquery_cached(
|
|
309
|
+
sql, None, output_one_line=output_one_line, timestamp=timestamp, function_allow_list=hashable_list
|
|
310
|
+
)
|
|
258
311
|
|
|
259
312
|
return sql
|
|
260
313
|
|
|
@@ -604,6 +604,9 @@ async def regression_test(
|
|
|
604
604
|
@pipe_copy.command(name="run", short_help="Run an on-demand copy job")
|
|
605
605
|
@click.argument("pipe_name_or_id")
|
|
606
606
|
@click.option("--wait", is_flag=True, default=False, help="Wait for the copy job to finish")
|
|
607
|
+
@click.option(
|
|
608
|
+
"--mode", type=click.Choice(["append", "replace"], case_sensitive=True), default=None, help="Copy strategy"
|
|
609
|
+
)
|
|
607
610
|
@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
|
|
608
611
|
@click.option(
|
|
609
612
|
"--param",
|
|
@@ -615,7 +618,9 @@ async def regression_test(
|
|
|
615
618
|
)
|
|
616
619
|
@click.pass_context
|
|
617
620
|
@coro
|
|
618
|
-
async def pipe_copy_run(
|
|
621
|
+
async def pipe_copy_run(
|
|
622
|
+
ctx: click.Context, pipe_name_or_id: str, wait: bool, mode: str, yes: bool, param: Optional[Tuple[str]]
|
|
623
|
+
):
|
|
619
624
|
"""Run an on-demand copy job"""
|
|
620
625
|
|
|
621
626
|
params = dict(key_value.split("=") for key_value in param) if param else {}
|
|
@@ -625,7 +630,7 @@ async def pipe_copy_run(ctx: click.Context, pipe_name_or_id: str, wait: bool, ye
|
|
|
625
630
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
626
631
|
|
|
627
632
|
try:
|
|
628
|
-
response = await client.pipe_run_copy(pipe_name_or_id, params)
|
|
633
|
+
response = await client.pipe_run_copy(pipe_name_or_id, params, mode)
|
|
629
634
|
|
|
630
635
|
job_id = response["job"]["job_id"]
|
|
631
636
|
job_url = response["job"]["job_url"]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: tinybird-cli
|
|
3
|
-
Version: 5.0.
|
|
3
|
+
Version: 5.0.1.dev1
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/cli/introduction.html
|
|
6
6
|
Author: Tinybird
|
|
@@ -18,42 +18,25 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
|
|
|
18
18
|
Changelog
|
|
19
19
|
----------
|
|
20
20
|
|
|
21
|
-
5.
|
|
22
|
-
|
|
21
|
+
5.1.0.dev1
|
|
22
|
+
***********
|
|
23
23
|
|
|
24
|
-
- `Added`
|
|
24
|
+
- `Added` COPY pipes now support `MODE replace`
|
|
25
25
|
|
|
26
|
-
5.0.0
|
|
27
|
-
|
|
26
|
+
5.0.0
|
|
27
|
+
******
|
|
28
28
|
|
|
29
|
-
- `
|
|
29
|
+
- `Breaking change` Make `insertion_date` column explicit. This column is no longer inferred, it must be present in the Data Source file.
|
|
30
30
|
|
|
31
|
-
|
|
32
|
-
************
|
|
31
|
+
Detailed info at https://www.tinybird.co/docs/changelog/2024-06-17-insertion_date-deprecation
|
|
33
32
|
|
|
33
|
+
- `Added` parameter to `tb pipe regression-test` --relative-change
|
|
34
|
+
- `Added` `tb push` displays warnings when using a reserved parameter in a pipe.
|
|
35
|
+
- `Added` --role parameter to `tb workspace members add`
|
|
34
36
|
- `Changed` Point region hosts to new `app.tinybird.co` domain
|
|
35
|
-
|
|
36
|
-
5.0.0.dev2
|
|
37
|
-
************
|
|
38
|
-
|
|
37
|
+
- `Improved` help message of `tb workspace members set-role` now displays the 3 valid roles: viewer|guest|admin.
|
|
39
38
|
- `Improved` syntax error messages when parsing endpoints.
|
|
40
39
|
|
|
41
|
-
5.0.0.dev1
|
|
42
|
-
************
|
|
43
|
-
|
|
44
|
-
- `Added` `tb push` displays warnings when using a reserved parameter in a pipe.
|
|
45
|
-
|
|
46
|
-
5.0.0.dev0
|
|
47
|
-
************
|
|
48
|
-
|
|
49
|
-
- `Changed` Make `insertion_date` column explicit. This column is no longer inferred, it must be present in the Data Source file.
|
|
50
|
-
|
|
51
|
-
4.1.2.dev0
|
|
52
|
-
************
|
|
53
|
-
|
|
54
|
-
- `Added` parameter to `tb pipe regression-test` --relative-change
|
|
55
|
-
|
|
56
|
-
|
|
57
40
|
4.1.1
|
|
58
41
|
************
|
|
59
42
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/tinyunit/tinyunit.py
RENAMED
|
File without changes
|
{tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/workspace_members.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird_cli.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|