tinybird-cli 5.0.0.dev5__tar.gz → 5.0.1.dev1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/PKG-INFO +12 -29
  2. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/__cli__.py +2 -2
  3. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/ch_utils/constants.py +1 -0
  4. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/client.py +28 -17
  5. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/datafile.py +29 -1
  6. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/feedback_manager.py +3 -0
  7. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/sql_toolset.py +69 -16
  8. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/pipe.py +7 -2
  9. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird_cli.egg-info/PKG-INFO +12 -29
  10. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/setup.cfg +0 -0
  11. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/ch_utils/engine.py +0 -0
  12. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/check_pypi.py +0 -0
  13. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/config.py +0 -0
  14. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/connectors.py +0 -0
  15. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/context.py +0 -0
  16. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/datatypes.py +0 -0
  17. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/git_settings.py +0 -0
  18. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/sql.py +0 -0
  19. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/sql_template.py +0 -0
  20. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/sql_template_fmt.py +0 -0
  21. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/syncasync.py +0 -0
  22. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli.py +0 -0
  23. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/auth.py +0 -0
  24. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/branch.py +0 -0
  25. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/cicd.py +0 -0
  26. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/cli.py +0 -0
  27. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/common.py +0 -0
  28. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/config.py +0 -0
  29. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/connection.py +0 -0
  30. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/datasource.py +0 -0
  31. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/exceptions.py +0 -0
  32. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/job.py +0 -0
  33. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/regions.py +0 -0
  34. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/telemetry.py +0 -0
  35. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/test.py +0 -0
  36. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -0
  37. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py +0 -0
  38. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/token.py +0 -0
  39. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/workspace.py +0 -0
  40. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/workspace_members.py +0 -0
  41. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird/tornado_template.py +0 -0
  42. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird_cli.egg-info/SOURCES.txt +0 -0
  43. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird_cli.egg-info/dependency_links.txt +0 -0
  44. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird_cli.egg-info/entry_points.txt +0 -0
  45. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird_cli.egg-info/requires.txt +0 -0
  46. {tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev1}/tinybird_cli.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: tinybird-cli
3
- Version: 5.0.0.dev5
3
+ Version: 5.0.1.dev1
4
4
  Summary: Tinybird Command Line Tool
5
5
  Home-page: https://www.tinybird.co/docs/cli/introduction.html
6
6
  Author: Tinybird
@@ -18,42 +18,25 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
18
18
  Changelog
19
19
  ----------
20
20
 
21
- 5.0.0.dev5
22
- ************
21
+ 5.1.0.dev1
22
+ ***********
23
23
 
24
- - `Added` parameter to `tb workspace members add` --role
24
+ - `Added` COPY pipes now support `MODE replace`
25
25
 
26
- 5.0.0.dev4
27
- ************
26
+ 5.0.0
27
+ ******
28
28
 
29
- - `Improved` help message of `tb workspace members set-role` now displays the 3 valid roles: viewer|guest|admin.
29
+ - `Breaking change` Make `insertion_date` column explicit. This column is no longer inferred, it must be present in the Data Source file.
30
30
 
31
- 5.0.0.dev3
32
- ************
31
+ Detailed info at https://www.tinybird.co/docs/changelog/2024-06-17-insertion_date-deprecation
33
32
 
33
+ - `Added` parameter to `tb pipe regression-test` --relative-change
34
+ - `Added` `tb push` displays warnings when using a reserved parameter in a pipe.
35
+ - `Added` --role parameter to `tb workspace members add`
34
36
  - `Changed` Point region hosts to new `app.tinybird.co` domain
35
-
36
- 5.0.0.dev2
37
- ************
38
-
37
+ - `Improved` help message of `tb workspace members set-role` now displays the 3 valid roles: viewer|guest|admin.
39
38
  - `Improved` syntax error messages when parsing endpoints.
40
39
 
41
- 5.0.0.dev1
42
- ************
43
-
44
- - `Added` `tb push` displays warnings when using a reserved parameter in a pipe.
45
-
46
- 5.0.0.dev0
47
- ************
48
-
49
- - `Changed` Make `insertion_date` column explicit. This column is no longer inferred, it must be present in the Data Source file.
50
-
51
- 4.1.2.dev0
52
- ************
53
-
54
- - `Added` parameter to `tb pipe regression-test` --relative-change
55
-
56
-
57
40
  4.1.1
58
41
  ************
59
42
 
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
4
4
  __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
5
5
  __author__ = 'Tinybird'
6
6
  __author_email__ = 'support@tinybird.co'
7
- __version__ = '5.0.0.dev5'
8
- __revision__ = 'e2df9f1'
7
+ __version__ = '5.0.1.dev1'
8
+ __revision__ = '3d0cfe4'
@@ -4,6 +4,7 @@ LIVE_WS_NAME = "live"
4
4
  SNAPSHOT_WS_NAME = "snapshot"
5
5
 
6
6
  ENABLED_TABLE_FUNCTIONS = {"generateRandom", "null", "numbers", "numbers_mt", "values", "zeros", "zeros_mt"}
7
+ COPY_ENABLED_TABLE_FUNCTIONS = frozenset(["postgresql"])
7
8
 
8
9
  ENABLED_SYSTEM_TABLES = {
9
10
  "functions",
@@ -11,6 +11,7 @@ import requests.adapters
11
11
  from requests import Response
12
12
  from urllib3 import Retry
13
13
 
14
+ from tinybird.ch_utils.constants import COPY_ENABLED_TABLE_FUNCTIONS
14
15
  from tinybird.syncasync import sync_to_async
15
16
  from tinybird.tb_cli_modules.regions import fill_with_public_regions
16
17
  from tinybird.tb_cli_modules.telemetry import add_telemetry_event
@@ -470,6 +471,8 @@ class TinyB(object):
470
471
  self, pipe_name: str, node: Dict[str, Any], dry_run: str = "false", datasource_name: Optional[str] = None
471
472
  ):
472
473
  params = {**{"include_datafile": "true", "dry_run": dry_run}, **node.get("params", node)}
474
+ if "mode" in params:
475
+ params.pop("mode")
473
476
  node_name = node["params"]["name"] if node.get("params", None) else node["name"]
474
477
  if datasource_name:
475
478
  params["datasource"] = datasource_name
@@ -549,34 +552,33 @@ class TinyB(object):
549
552
  async def pipe_remove_endpoint(self, pipe_name_or_uid: str, published_node_uid: str):
550
553
  return await self._req(f"/v0/pipes/{pipe_name_or_uid}/nodes/{published_node_uid}/endpoint", method="DELETE")
551
554
 
552
- async def pipe_create_copy(
553
- self, pipe_name_or_id: str, node_id: str, target_datasource: str, schedule_cron: Optional[str] = None
554
- ):
555
- data = {"target_datasource": target_datasource}
556
- if schedule_cron:
557
- data["schedule_cron"] = schedule_cron
558
-
559
- return await self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/copy", method="POST", data=data)
560
-
561
555
  async def pipe_update_copy(
562
556
  self,
563
557
  pipe_name_or_id: str,
564
558
  node_id: str,
565
559
  target_datasource: Optional[str] = None,
566
560
  schedule_cron: Optional[str] = None,
561
+ mode: Optional[str] = None,
567
562
  ):
568
563
  data = {"schedule_cron": schedule_cron}
569
564
 
570
565
  if target_datasource:
571
566
  data["target_datasource"] = target_datasource
572
567
 
568
+ if mode:
569
+ data["mode"] = mode
570
+
573
571
  return await self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/copy", method="PUT", data=data)
574
572
 
575
573
  async def pipe_remove_copy(self, pipe_name_or_id: str, node_id: str):
576
574
  return await self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/copy", method="DELETE")
577
575
 
578
- async def pipe_run_copy(self, pipe_name_or_id: str, params: Optional[Dict[str, str]] = None):
576
+ async def pipe_run_copy(
577
+ self, pipe_name_or_id: str, params: Optional[Dict[str, str]] = None, mode: Optional[str] = None
578
+ ):
579
579
  params = {**params} if params else {}
580
+ if mode:
581
+ params["_mode"] = mode
580
582
  return await self._req(f"/v0/pipes/{pipe_name_or_id}/copy?{urlencode(params)}", method="POST")
581
583
 
582
584
  async def pipe_resume_copy(self, pipe_name_or_id: str):
@@ -1035,23 +1037,32 @@ class TinyB(object):
1035
1037
  return result["q"]
1036
1038
 
1037
1039
  @staticmethod
1038
- def _sql_get_used_tables_local(sql: str, raising: bool = False) -> List[str]:
1040
+ def _sql_get_used_tables_local(sql: str, raising: bool = False, is_copy: Optional[bool] = False) -> List[str]:
1039
1041
  from tinybird.sql_toolset import sql_get_used_tables
1040
1042
 
1041
- tables = sql_get_used_tables(sql, raising, table_functions=False)
1043
+ tables = sql_get_used_tables(
1044
+ sql, raising, table_functions=False, function_allow_list=COPY_ENABLED_TABLE_FUNCTIONS if is_copy else None
1045
+ )
1042
1046
  return [t[1] if t[0] == "" else f"{t[0]}.{t[1]}" for t in tables]
1043
1047
 
1044
- async def _sql_get_used_tables_remote(self, sql: str, raising: bool = False) -> List[str]:
1045
- params = {"q": sql, "raising": "true" if raising else "false", "table_functions": "false"}
1048
+ async def _sql_get_used_tables_remote(
1049
+ self, sql: str, raising: bool = False, is_copy: Optional[bool] = False
1050
+ ) -> List[str]:
1051
+ params = {
1052
+ "q": sql,
1053
+ "raising": "true" if raising else "false",
1054
+ "table_functions": "false",
1055
+ "is_copy": "true" if is_copy else "false",
1056
+ }
1046
1057
  result = await self._req("/v0/sql_tables", data=params, method="POST")
1047
1058
  return [t[1] if t[0] == "" else f"{t[0]}.{t[1]}" for t in result["tables"]]
1048
1059
 
1049
1060
  # Get used tables from a query. Does not include table functions
1050
- async def sql_get_used_tables(self, sql: str, raising: bool = False) -> List[str]:
1061
+ async def sql_get_used_tables(self, sql: str, raising: bool = False, is_copy: Optional[bool] = False) -> List[str]:
1051
1062
  try:
1052
- return self._sql_get_used_tables_local(sql, raising)
1063
+ return self._sql_get_used_tables_local(sql, raising, is_copy)
1053
1064
  except ModuleNotFoundError:
1054
- return await self._sql_get_used_tables_remote(sql, raising)
1065
+ return await self._sql_get_used_tables_remote(sql, raising, is_copy)
1055
1066
 
1056
1067
  @staticmethod
1057
1068
  def _replace_tables_local(q: str, replacements):
@@ -118,6 +118,18 @@ class DataFileExtensions:
118
118
  class CopyParameters:
119
119
  TARGET_DATASOURCE = "target_datasource"
120
120
  COPY_SCHEDULE = "copy_schedule"
121
+ COPY_MODE = "copy_mode"
122
+
123
+
124
+ class CopyModes:
125
+ APPEND = "append"
126
+ REPLACE = "replace"
127
+
128
+ valid_modes = (APPEND, REPLACE)
129
+
130
+ @staticmethod
131
+ def is_valid(node_mode):
132
+ return node_mode.lower() in CopyModes.valid_modes
121
133
 
122
134
 
123
135
  PREVIEW_CONNECTOR_SERVICES = ["s3", "s3_iamrole", "gcs"]
@@ -1076,6 +1088,8 @@ def parse(
1076
1088
  "tags": assign_node_var("tags"),
1077
1089
  "target_datasource": assign_node_var("target_datasource"),
1078
1090
  "copy_schedule": assign_node_var(CopyParameters.COPY_SCHEDULE),
1091
+ "copy_mode": assign_node_var("mode"),
1092
+ "mode": assign_node_var("mode"),
1079
1093
  "resource": assign_node_var("resource"),
1080
1094
  "filter": assign_node_var("filter"),
1081
1095
  "token": add_token,
@@ -1476,6 +1490,7 @@ async def process_file(
1476
1490
  deps = []
1477
1491
  nodes: List[Dict[str, Any]] = []
1478
1492
 
1493
+ is_copy = any([node for node in doc.nodes if node.get("type", "standard").lower() == PipeNodeTypes.COPY])
1479
1494
  for node in doc.nodes:
1480
1495
  sql = node["sql"]
1481
1496
  node_type = node.get("type", "standard").lower()
@@ -1485,6 +1500,7 @@ async def process_file(
1485
1500
  "description": node.get("description", ""),
1486
1501
  "target_datasource": node.get("target_datasource", None),
1487
1502
  "copy_schedule": node.get(CopyParameters.COPY_SCHEDULE, None),
1503
+ "mode": node.get("mode", CopyModes.APPEND),
1488
1504
  }
1489
1505
 
1490
1506
  is_export_node = ExportReplacements.is_export_node(node)
@@ -1504,7 +1520,7 @@ async def process_file(
1504
1520
  sql_rendered = sql
1505
1521
 
1506
1522
  try:
1507
- dependencies = await tb_client.sql_get_used_tables(sql_rendered, raising=True)
1523
+ dependencies = await tb_client.sql_get_used_tables(sql_rendered, raising=True, is_copy=is_copy)
1508
1524
  deps += [t for t in dependencies if t not in [n["name"] for n in doc.nodes]]
1509
1525
 
1510
1526
  except Exception as e:
@@ -2482,6 +2498,12 @@ async def check_copy_pipe(pipe, copy_node, tb_client: TinyB):
2482
2498
  if not is_valid_cron:
2483
2499
  raise CLIPipeException(FeedbackManager.error_creating_copy_pipe_invalid_cron(schedule_cron=schedule_cron))
2484
2500
 
2501
+ mode = copy_node["params"].get("mode", CopyModes.APPEND)
2502
+ is_valid_mode = CopyModes.is_valid(mode)
2503
+
2504
+ if not is_valid_mode:
2505
+ raise CLIPipeException(FeedbackManager.error_creating_copy_pipe_invalid_mode(mode=mode))
2506
+
2485
2507
  if not pipe:
2486
2508
  return
2487
2509
 
@@ -2763,6 +2785,7 @@ async def new_pipe(
2763
2785
  try:
2764
2786
  target_datasource = copy_node.get(CopyParameters.TARGET_DATASOURCE, None)
2765
2787
  schedule_cron = copy_node.get(CopyParameters.COPY_SCHEDULE, None)
2788
+ mode = copy_node.get("mode", CopyModes.APPEND)
2766
2789
  schedule_cron = None if schedule_cron == ON_DEMAND else schedule_cron
2767
2790
  current_target_datasource_id = data["copy_target_datasource"]
2768
2791
  target_datasource_response = await tb_client.get_datasource(target_datasource)
@@ -2776,6 +2799,7 @@ async def new_pipe(
2776
2799
  current_schedule_cron = current_schedule.get("cron", None) if current_schedule else None
2777
2800
  schedule_cron_should_be_removed = current_schedule_cron and not schedule_cron
2778
2801
  copy_params["schedule_cron"] = "None" if schedule_cron_should_be_removed else schedule_cron
2802
+ copy_params["mode"] = mode
2779
2803
  await tb_client.pipe_update_copy(**copy_params)
2780
2804
  except Exception as e:
2781
2805
  raise Exception(
@@ -4627,6 +4651,10 @@ async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[
4627
4651
  file_parts.append(node_type_upper)
4628
4652
  file_parts.append(DATAFILE_NEW_LINE)
4629
4653
  file_parts.append(f'TARGET_DATASOURCE {node["target_datasource"]}')
4654
+ if node.get("mode"):
4655
+ file_parts.append(DATAFILE_NEW_LINE)
4656
+ file_parts.append(f'COPY_MODE {node.get("mode")}')
4657
+
4630
4658
  if CopyParameters.COPY_SCHEDULE in node and node[CopyParameters.COPY_SCHEDULE]:
4631
4659
  is_ondemand = node[CopyParameters.COPY_SCHEDULE].lower() == ON_DEMAND
4632
4660
  file_parts.append(DATAFILE_NEW_LINE)
@@ -126,6 +126,9 @@ class FeedbackManager:
126
126
  error_creating_copy_pipe_invalid_cron = error_message(
127
127
  "Cannot create Copy pipe. Invalid cron expression: '{schedule_cron}'"
128
128
  )
129
+ error_creating_copy_pipe_invalid_mode = error_message(
130
+ "Cannot create Copy pipe. Invalid MODE expression: '{mode}'. Valid modes are: 'append', 'replace'"
131
+ )
129
132
  error_creating_sink_pipe_invalid_cron = error_message(
130
133
  "Cannot create Sink Pipe. Invalid cron expression: '{schedule_cron}'"
131
134
  )
@@ -3,20 +3,26 @@ import logging
3
3
  from collections import defaultdict
4
4
  from datetime import datetime
5
5
  from functools import lru_cache
6
- from typing import Any, List, Optional, Set, Tuple
6
+ from typing import Any, FrozenSet, List, Optional, Set, Tuple
7
7
 
8
8
  from chtoolset import query as chquery
9
9
  from toposort import toposort
10
10
 
11
- from tinybird.ch_utils.constants import ENABLED_TABLE_FUNCTIONS
11
+ from tinybird.ch_utils.constants import COPY_ENABLED_TABLE_FUNCTIONS, ENABLED_TABLE_FUNCTIONS
12
12
 
13
13
  VALID_REMOTE = "VALID_REMOTE"
14
14
 
15
15
 
16
16
  class InvalidFunction(ValueError):
17
17
  def __init__(self, msg: str = "", table_function_names: str = ""):
18
+ if any([fn for fn in COPY_ENABLED_TABLE_FUNCTIONS if fn in msg]):
19
+ msg = msg.replace("is restricted", "is restricted to Copy Pipes")
20
+
18
21
  if table_function_names:
19
- self.msg = f"The query uses disabled table functions: '{table_function_names}'"
22
+ if "postgresql" in table_function_names:
23
+ self.msg = "The postgresql table function is only allowed in Copy Pipes"
24
+ else:
25
+ self.msg = f"The query uses disabled table functions: '{table_function_names}'"
20
26
  else:
21
27
  self.msg = msg
22
28
  super().__init__(self.msg)
@@ -58,7 +64,11 @@ def format_where_for_mutation_command(where_clause: str) -> str:
58
64
 
59
65
  @lru_cache(maxsize=2**13)
60
66
  def sql_get_used_tables_cached(
61
- sql: str, raising: bool = False, default_database: str = "", table_functions: bool = True
67
+ sql: str,
68
+ raising: bool = False,
69
+ default_database: str = "",
70
+ table_functions: bool = True,
71
+ function_allow_list: Optional[FrozenSet[str]] = None,
62
72
  ) -> List[Tuple[str, str, str]]:
63
73
  """
64
74
  >>> sql_get_used_tables("SELECT 1 FROM the_table")
@@ -79,7 +89,11 @@ def sql_get_used_tables_cached(
79
89
  [('d_d3926a', 't_976af08ec4b547419e729c63e754b17b', '')]
80
90
  """
81
91
  try:
82
- tables: List[Tuple[str, str, str]] = chquery.tables(sql, default_database=default_database)
92
+ _function_allow_list = list() if function_allow_list is None else list(function_allow_list)
93
+
94
+ tables: List[Tuple[str, str, str]] = chquery.tables(
95
+ sql, default_database=default_database, function_allow_list=_function_allow_list
96
+ )
83
97
  if not table_functions:
84
98
  return [(t[0], t[1], "") for t in tables if t[0] or t[1]]
85
99
  return tables
@@ -93,10 +107,17 @@ def sql_get_used_tables_cached(
93
107
 
94
108
 
95
109
  def sql_get_used_tables(
96
- sql: str, raising: bool = False, default_database: str = "", table_functions: bool = True
110
+ sql: str,
111
+ raising: bool = False,
112
+ default_database: str = "",
113
+ table_functions: bool = True,
114
+ function_allow_list: Optional[FrozenSet[str]] = None,
97
115
  ) -> List[Tuple[str, str, str]]:
98
- # We return a copy here since the callers might (and replace_tables does) edit the resulting object
99
- return copy.copy(sql_get_used_tables_cached(sql, raising, default_database, table_functions))
116
+ hashable_list = frozenset() if function_allow_list is None else function_allow_list
117
+
118
+ return copy.copy(
119
+ sql_get_used_tables_cached(sql, raising, default_database, table_functions, function_allow_list=hashable_list)
120
+ )
100
121
 
101
122
 
102
123
  class ReplacementsDict(dict):
@@ -150,9 +171,18 @@ def replace_tables_chquery_cached(
150
171
  default_database: str = "",
151
172
  output_one_line: bool = False,
152
173
  timestamp: Optional[datetime] = None,
174
+ function_allow_list: Optional[FrozenSet[str]] = None,
153
175
  ) -> str:
154
176
  replacements = dict(sorted_replacements) if sorted_replacements else {}
155
- return chquery.replace_tables(sql, replacements, default_database=default_database, one_line=output_one_line)
177
+ _function_allow_list = list() if function_allow_list is None else list(function_allow_list)
178
+
179
+ return chquery.replace_tables(
180
+ sql,
181
+ replacements,
182
+ default_database=default_database,
183
+ one_line=output_one_line,
184
+ function_allow_list=_function_allow_list,
185
+ )
156
186
 
157
187
 
158
188
  def replace_tables(
@@ -164,15 +194,19 @@ def replace_tables(
164
194
  valid_tables: Optional[Set[Tuple[str, str]]] = None,
165
195
  output_one_line: bool = False,
166
196
  timestamp: Optional[datetime] = None,
197
+ function_allow_list: Optional[FrozenSet[str]] = None,
167
198
  ) -> str:
168
199
  """
169
200
  Given a query and a list of table replacements, returns the query after applying the table replacements.
170
201
  It takes into account dependencies between replacement subqueries (if any)
171
202
  It also validates the sql to verify it's valid and doesn't use unknown or prohibited functions
172
203
  """
204
+ hashable_list = frozenset() if function_allow_list is None else function_allow_list
173
205
  if not replacements:
174
206
  # Always call replace_tables to do validation and formatting
175
- return replace_tables_chquery_cached(sql, None, output_one_line=output_one_line, timestamp=timestamp)
207
+ return replace_tables_chquery_cached(
208
+ sql, None, output_one_line=output_one_line, timestamp=timestamp, function_allow_list=hashable_list
209
+ )
176
210
 
177
211
  _replaced_with = set()
178
212
  _replacements = ReplacementsDict()
@@ -182,14 +216,24 @@ def replace_tables(
182
216
  _replaced_with.add(r)
183
217
 
184
218
  deps: defaultdict = defaultdict(set)
185
- _tables = sql_get_used_tables(sql, default_database=default_database, raising=True, table_functions=check_functions)
219
+ _tables = sql_get_used_tables(
220
+ sql,
221
+ default_database=default_database,
222
+ raising=True,
223
+ table_functions=check_functions,
224
+ function_allow_list=function_allow_list,
225
+ )
186
226
  seen_tables = set()
187
227
  table: Tuple[str, str] | Tuple[str, str, str]
228
+ if function_allow_list is None:
229
+ _enabled_table_functions = ENABLED_TABLE_FUNCTIONS
230
+ else:
231
+ _enabled_table_functions = ENABLED_TABLE_FUNCTIONS.union(set(function_allow_list))
188
232
  while _tables:
189
233
  table = _tables.pop()
190
234
  if len(table) == 3:
191
235
  first_table, second_table, last_table = table
192
- if last_table and last_table not in ENABLED_TABLE_FUNCTIONS:
236
+ if last_table and last_table not in _enabled_table_functions:
193
237
  raise InvalidFunction(table_function_names=last_table)
194
238
  if first_table or second_table:
195
239
  table = (first_table, second_table)
@@ -204,7 +248,7 @@ def replace_tables(
204
248
  if len(dependent_table) == 3:
205
249
  if (
206
250
  dependent_table[2]
207
- and dependent_table[2] not in ENABLED_TABLE_FUNCTIONS
251
+ and dependent_table[2] not in _enabled_table_functions
208
252
  and not (dependent_table[2] in ["cluster"] and replacement[0] == VALID_REMOTE)
209
253
  ):
210
254
  raise InvalidFunction(table_function_names=dependent_table[2])
@@ -219,7 +263,9 @@ def replace_tables(
219
263
  deps_sorted = list(reversed(list(toposort(deps))))
220
264
 
221
265
  if not deps_sorted:
222
- return replace_tables_chquery_cached(sql, None, output_one_line=output_one_line, timestamp=timestamp)
266
+ return replace_tables_chquery_cached(
267
+ sql, None, output_one_line=output_one_line, timestamp=timestamp, function_allow_list=hashable_list
268
+ )
223
269
 
224
270
  for current_deps in deps_sorted:
225
271
  current_replacements = {}
@@ -251,10 +297,17 @@ def replace_tables(
251
297
  # We need to transform the dictionary into something cacheable, so a sorted tuple of tuples it is
252
298
  r = tuple(sorted([(k, v) for k, v in current_replacements.items()]))
253
299
  sql = replace_tables_chquery_cached(
254
- sql, r, default_database=default_database, output_one_line=output_one_line, timestamp=timestamp
300
+ sql,
301
+ r,
302
+ default_database=default_database,
303
+ output_one_line=output_one_line,
304
+ timestamp=timestamp,
305
+ function_allow_list=hashable_list,
255
306
  )
256
307
  else:
257
- sql = replace_tables_chquery_cached(sql, None, output_one_line=output_one_line, timestamp=timestamp)
308
+ sql = replace_tables_chquery_cached(
309
+ sql, None, output_one_line=output_one_line, timestamp=timestamp, function_allow_list=hashable_list
310
+ )
258
311
 
259
312
  return sql
260
313
 
@@ -604,6 +604,9 @@ async def regression_test(
604
604
  @pipe_copy.command(name="run", short_help="Run an on-demand copy job")
605
605
  @click.argument("pipe_name_or_id")
606
606
  @click.option("--wait", is_flag=True, default=False, help="Wait for the copy job to finish")
607
+ @click.option(
608
+ "--mode", type=click.Choice(["append", "replace"], case_sensitive=True), default=None, help="Copy strategy"
609
+ )
607
610
  @click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
608
611
  @click.option(
609
612
  "--param",
@@ -615,7 +618,9 @@ async def regression_test(
615
618
  )
616
619
  @click.pass_context
617
620
  @coro
618
- async def pipe_copy_run(ctx: click.Context, pipe_name_or_id: str, wait: bool, yes: bool, param: Optional[Tuple[str]]):
621
+ async def pipe_copy_run(
622
+ ctx: click.Context, pipe_name_or_id: str, wait: bool, mode: str, yes: bool, param: Optional[Tuple[str]]
623
+ ):
619
624
  """Run an on-demand copy job"""
620
625
 
621
626
  params = dict(key_value.split("=") for key_value in param) if param else {}
@@ -625,7 +630,7 @@ async def pipe_copy_run(ctx: click.Context, pipe_name_or_id: str, wait: bool, ye
625
630
  client: TinyB = ctx.ensure_object(dict)["client"]
626
631
 
627
632
  try:
628
- response = await client.pipe_run_copy(pipe_name_or_id, params)
633
+ response = await client.pipe_run_copy(pipe_name_or_id, params, mode)
629
634
 
630
635
  job_id = response["job"]["job_id"]
631
636
  job_url = response["job"]["job_url"]
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: tinybird-cli
3
- Version: 5.0.0.dev5
3
+ Version: 5.0.1.dev1
4
4
  Summary: Tinybird Command Line Tool
5
5
  Home-page: https://www.tinybird.co/docs/cli/introduction.html
6
6
  Author: Tinybird
@@ -18,42 +18,25 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
18
18
  Changelog
19
19
  ----------
20
20
 
21
- 5.0.0.dev5
22
- ************
21
+ 5.1.0.dev1
22
+ ***********
23
23
 
24
- - `Added` parameter to `tb workspace members add` --role
24
+ - `Added` COPY pipes now support `MODE replace`
25
25
 
26
- 5.0.0.dev4
27
- ************
26
+ 5.0.0
27
+ ******
28
28
 
29
- - `Improved` help message of `tb workspace members set-role` now displays the 3 valid roles: viewer|guest|admin.
29
+ - `Breaking change` Make `insertion_date` column explicit. This column is no longer inferred, it must be present in the Data Source file.
30
30
 
31
- 5.0.0.dev3
32
- ************
31
+ Detailed info at https://www.tinybird.co/docs/changelog/2024-06-17-insertion_date-deprecation
33
32
 
33
+ - `Added` parameter to `tb pipe regression-test` --relative-change
34
+ - `Added` `tb push` displays warnings when using a reserved parameter in a pipe.
35
+ - `Added` --role parameter to `tb workspace members add`
34
36
  - `Changed` Point region hosts to new `app.tinybird.co` domain
35
-
36
- 5.0.0.dev2
37
- ************
38
-
37
+ - `Improved` help message of `tb workspace members set-role` now displays the 3 valid roles: viewer|guest|admin.
39
38
  - `Improved` syntax error messages when parsing endpoints.
40
39
 
41
- 5.0.0.dev1
42
- ************
43
-
44
- - `Added` `tb push` displays warnings when using a reserved parameter in a pipe.
45
-
46
- 5.0.0.dev0
47
- ************
48
-
49
- - `Changed` Make `insertion_date` column explicit. This column is no longer inferred, it must be present in the Data Source file.
50
-
51
- 4.1.2.dev0
52
- ************
53
-
54
- - `Added` parameter to `tb pipe regression-test` --relative-change
55
-
56
-
57
40
  4.1.1
58
41
  ************
59
42