tinybird-cli 5.0.0.dev5__tar.gz → 5.0.1.dev0__tar.gz

{tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.0.0.dev5
+Version: 5.0.1.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,42 +18,20 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
-5.0.0.dev5
-************
-
-- `Added` parameter to `tb workspace members add` --role
-
-5.0.0.dev4
-************
+5.0.0
+******
 
-- `Improved` help message of `tb workspace members set-role` now displays the 3 valid roles: viewer|guest|admin.
+- `Breaking change` Make `insertion_date` column explicit. This column is no longer inferred, it must be present in the Data Source file.
 
-5.0.0.dev3
-************
+  Detailed info at https://www.tinybird.co/docs/changelog/2024-06-17-insertion_date-deprecation
 
+- `Added` parameter to `tb pipe regression-test` --relative-change
+- `Added` `tb push` displays warnings when using a reserved parameter in a pipe.
+- `Added` --role parameter to `tb workspace members add`
 - `Changed` Point region hosts to new `app.tinybird.co` domain
-
-5.0.0.dev2
-************
-
+- `Improved` help message of `tb workspace members set-role` now displays the 3 valid roles: viewer|guest|admin.
 - `Improved` syntax error messages when parsing endpoints.
 
-5.0.0.dev1
-************
-
-- `Added` `tb push` displays warnings when using a reserved parameter in a pipe.
-
-5.0.0.dev0
-************
-
-- `Changed` Make `insertion_date` column explicit. This column is no longer inferred, it must be present in the Data Source file.
-
-4.1.2.dev0
-************
-
-- `Added` parameter to `tb pipe regression-test` --relative-change
-
-
 4.1.1
 ************
 

{tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev0}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '5.0.0.dev5'
-__revision__ = 'e2df9f1'
+__version__ = '5.0.1.dev0'
+__revision__ = '2af0ad5'

{tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev0}/tinybird/ch_utils/constants.py

@@ -4,6 +4,7 @@ LIVE_WS_NAME = "live"
 SNAPSHOT_WS_NAME = "snapshot"
 
 ENABLED_TABLE_FUNCTIONS = {"generateRandom", "null", "numbers", "numbers_mt", "values", "zeros", "zeros_mt"}
+COPY_ENABLED_TABLE_FUNCTIONS = frozenset(["postgresql"])
 
 ENABLED_SYSTEM_TABLES = {
     "functions",

{tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev0}/tinybird/client.py

@@ -11,6 +11,7 @@ import requests.adapters
 from requests import Response
 from urllib3 import Retry
 
+from tinybird.ch_utils.constants import COPY_ENABLED_TABLE_FUNCTIONS
 from tinybird.syncasync import sync_to_async
 from tinybird.tb_cli_modules.regions import fill_with_public_regions
 from tinybird.tb_cli_modules.telemetry import add_telemetry_event
@@ -1035,23 +1036,32 @@ class TinyB(object):
         return result["q"]
 
     @staticmethod
-    def _sql_get_used_tables_local(sql: str, raising: bool = False) -> List[str]:
+    def _sql_get_used_tables_local(sql: str, raising: bool = False, is_copy: Optional[bool] = False) -> List[str]:
         from tinybird.sql_toolset import sql_get_used_tables
 
-        tables = sql_get_used_tables(sql, raising, table_functions=False)
+        tables = sql_get_used_tables(
+            sql, raising, table_functions=False, function_allow_list=COPY_ENABLED_TABLE_FUNCTIONS if is_copy else None
+        )
         return [t[1] if t[0] == "" else f"{t[0]}.{t[1]}" for t in tables]
 
-    async def _sql_get_used_tables_remote(self, sql: str, raising: bool = False) -> List[str]:
-        params = {"q": sql, "raising": "true" if raising else "false", "table_functions": "false"}
+    async def _sql_get_used_tables_remote(
+        self, sql: str, raising: bool = False, is_copy: Optional[bool] = False
+    ) -> List[str]:
+        params = {
+            "q": sql,
+            "raising": "true" if raising else "false",
+            "table_functions": "false",
+            "is_copy": "true" if is_copy else "false",
+        }
         result = await self._req("/v0/sql_tables", data=params, method="POST")
         return [t[1] if t[0] == "" else f"{t[0]}.{t[1]}" for t in result["tables"]]
 
     # Get used tables from a query. Does not include table functions
-    async def sql_get_used_tables(self, sql: str, raising: bool = False) -> List[str]:
+    async def sql_get_used_tables(self, sql: str, raising: bool = False, is_copy: Optional[bool] = False) -> List[str]:
         try:
-            return self._sql_get_used_tables_local(sql, raising)
+            return self._sql_get_used_tables_local(sql, raising, is_copy)
         except ModuleNotFoundError:
-            return await self._sql_get_used_tables_remote(sql, raising)
+            return await self._sql_get_used_tables_remote(sql, raising, is_copy)
 
     @staticmethod
     def _replace_tables_local(q: str, replacements):

{tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev0}/tinybird/datafile.py

@@ -1476,6 +1476,7 @@ async def process_file(
         deps = []
         nodes: List[Dict[str, Any]] = []
 
+        is_copy = any([node for node in doc.nodes if node.get("type", "standard").lower() == PipeNodeTypes.COPY])
         for node in doc.nodes:
             sql = node["sql"]
             node_type = node.get("type", "standard").lower()
@@ -1504,7 +1505,7 @@ async def process_file(
                 sql_rendered = sql
 
             try:
-                dependencies = await tb_client.sql_get_used_tables(sql_rendered, raising=True)
+                dependencies = await tb_client.sql_get_used_tables(sql_rendered, raising=True, is_copy=is_copy)
                 deps += [t for t in dependencies if t not in [n["name"] for n in doc.nodes]]
 
             except Exception as e:

{tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev0}/tinybird/sql_toolset.py

@@ -3,20 +3,26 @@ import logging
 from collections import defaultdict
 from datetime import datetime
 from functools import lru_cache
-from typing import Any, List, Optional, Set, Tuple
+from typing import Any, FrozenSet, List, Optional, Set, Tuple
 
 from chtoolset import query as chquery
 from toposort import toposort
 
-from tinybird.ch_utils.constants import ENABLED_TABLE_FUNCTIONS
+from tinybird.ch_utils.constants import COPY_ENABLED_TABLE_FUNCTIONS, ENABLED_TABLE_FUNCTIONS
 
 VALID_REMOTE = "VALID_REMOTE"
 
 
 class InvalidFunction(ValueError):
     def __init__(self, msg: str = "", table_function_names: str = ""):
+        if any([fn for fn in COPY_ENABLED_TABLE_FUNCTIONS if fn in msg]):
+            msg = msg.replace("is restricted", "is restricted to Copy Pipes")
+
         if table_function_names:
-            self.msg = f"The query uses disabled table functions: '{table_function_names}'"
+            if "postgresql" in table_function_names:
+                self.msg = "The postgresql table function is only allowed in Copy Pipes"
+            else:
+                self.msg = f"The query uses disabled table functions: '{table_function_names}'"
         else:
             self.msg = msg
         super().__init__(self.msg)
@@ -58,7 +64,11 @@ def format_where_for_mutation_command(where_clause: str) -> str:
 
 @lru_cache(maxsize=2**13)
 def sql_get_used_tables_cached(
-    sql: str, raising: bool = False, default_database: str = "", table_functions: bool = True
+    sql: str,
+    raising: bool = False,
+    default_database: str = "",
+    table_functions: bool = True,
+    function_allow_list: Optional[FrozenSet[str]] = None,
 ) -> List[Tuple[str, str, str]]:
     """
     >>> sql_get_used_tables("SELECT 1 FROM the_table")
@@ -79,7 +89,11 @@ def sql_get_used_tables_cached(
     [('d_d3926a', 't_976af08ec4b547419e729c63e754b17b', '')]
     """
     try:
-        tables: List[Tuple[str, str, str]] = chquery.tables(sql, default_database=default_database)
+        _function_allow_list = list() if function_allow_list is None else list(function_allow_list)
+
+        tables: List[Tuple[str, str, str]] = chquery.tables(
+            sql, default_database=default_database, function_allow_list=_function_allow_list
+        )
         if not table_functions:
             return [(t[0], t[1], "") for t in tables if t[0] or t[1]]
         return tables
@@ -93,10 +107,17 @@ def sql_get_used_tables_cached(
 
 
 def sql_get_used_tables(
-    sql: str, raising: bool = False, default_database: str = "", table_functions: bool = True
+    sql: str,
+    raising: bool = False,
+    default_database: str = "",
+    table_functions: bool = True,
+    function_allow_list: Optional[FrozenSet[str]] = None,
 ) -> List[Tuple[str, str, str]]:
-    # We return a copy here since the callers might (and replace_tables does) edit the resulting object
-    return copy.copy(sql_get_used_tables_cached(sql, raising, default_database, table_functions))
+    hashable_list = frozenset() if function_allow_list is None else function_allow_list
+
+    return copy.copy(
+        sql_get_used_tables_cached(sql, raising, default_database, table_functions, function_allow_list=hashable_list)
+    )
 
 
 class ReplacementsDict(dict):
@@ -150,9 +171,18 @@ def replace_tables_chquery_cached(
     default_database: str = "",
     output_one_line: bool = False,
     timestamp: Optional[datetime] = None,
+    function_allow_list: Optional[FrozenSet[str]] = None,
 ) -> str:
     replacements = dict(sorted_replacements) if sorted_replacements else {}
-    return chquery.replace_tables(sql, replacements, default_database=default_database, one_line=output_one_line)
+    _function_allow_list = list() if function_allow_list is None else list(function_allow_list)
+
+    return chquery.replace_tables(
+        sql,
+        replacements,
+        default_database=default_database,
+        one_line=output_one_line,
+        function_allow_list=_function_allow_list,
+    )
 
 
 def replace_tables(
@@ -164,15 +194,19 @@ def replace_tables(
     valid_tables: Optional[Set[Tuple[str, str]]] = None,
     output_one_line: bool = False,
     timestamp: Optional[datetime] = None,
+    function_allow_list: Optional[FrozenSet[str]] = None,
 ) -> str:
     """
     Given a query and a list of table replacements, returns the query after applying the table replacements.
     It takes into account dependencies between replacement subqueries (if any)
     It also validates the sql to verify it's valid and doesn't use unknown or prohibited functions
     """
+    hashable_list = frozenset() if function_allow_list is None else function_allow_list
     if not replacements:
         # Always call replace_tables to do validation and formatting
-        return replace_tables_chquery_cached(sql, None, output_one_line=output_one_line, timestamp=timestamp)
+        return replace_tables_chquery_cached(
+            sql, None, output_one_line=output_one_line, timestamp=timestamp, function_allow_list=hashable_list
+        )
 
     _replaced_with = set()
     _replacements = ReplacementsDict()
@@ -182,14 +216,24 @@ def replace_tables(
         _replaced_with.add(r)
 
     deps: defaultdict = defaultdict(set)
-    _tables = sql_get_used_tables(sql, default_database=default_database, raising=True, table_functions=check_functions)
+    _tables = sql_get_used_tables(
+        sql,
+        default_database=default_database,
+        raising=True,
+        table_functions=check_functions,
+        function_allow_list=function_allow_list,
+    )
     seen_tables = set()
     table: Tuple[str, str] | Tuple[str, str, str]
+    if function_allow_list is None:
+        _enabled_table_functions = ENABLED_TABLE_FUNCTIONS
+    else:
+        _enabled_table_functions = ENABLED_TABLE_FUNCTIONS.union(set(function_allow_list))
     while _tables:
         table = _tables.pop()
         if len(table) == 3:
             first_table, second_table, last_table = table
-            if last_table and last_table not in ENABLED_TABLE_FUNCTIONS:
+            if last_table and last_table not in _enabled_table_functions:
                 raise InvalidFunction(table_function_names=last_table)
             if first_table or second_table:
                 table = (first_table, second_table)
@@ -204,7 +248,7 @@ def replace_tables(
                 if len(dependent_table) == 3:
                     if (
                         dependent_table[2]
-                        and dependent_table[2] not in ENABLED_TABLE_FUNCTIONS
+                        and dependent_table[2] not in _enabled_table_functions
                         and not (dependent_table[2] in ["cluster"] and replacement[0] == VALID_REMOTE)
                     ):
                         raise InvalidFunction(table_function_names=dependent_table[2])
@@ -219,7 +263,9 @@ def replace_tables(
     deps_sorted = list(reversed(list(toposort(deps))))
 
     if not deps_sorted:
-        return replace_tables_chquery_cached(sql, None, output_one_line=output_one_line, timestamp=timestamp)
+        return replace_tables_chquery_cached(
+            sql, None, output_one_line=output_one_line, timestamp=timestamp, function_allow_list=hashable_list
+        )
 
     for current_deps in deps_sorted:
         current_replacements = {}
@@ -251,10 +297,17 @@ def replace_tables(
             # We need to transform the dictionary into something cacheable, so a sorted tuple of tuples it is
             r = tuple(sorted([(k, v) for k, v in current_replacements.items()]))
             sql = replace_tables_chquery_cached(
-                sql, r, default_database=default_database, output_one_line=output_one_line, timestamp=timestamp
+                sql,
+                r,
+                default_database=default_database,
+                output_one_line=output_one_line,
+                timestamp=timestamp,
+                function_allow_list=hashable_list,
             )
         else:
-            sql = replace_tables_chquery_cached(sql, None, output_one_line=output_one_line, timestamp=timestamp)
+            sql = replace_tables_chquery_cached(
+                sql, None, output_one_line=output_one_line, timestamp=timestamp, function_allow_list=hashable_list
+            )
 
     return sql
 

{tinybird-cli-5.0.0.dev5 → tinybird-cli-5.0.1.dev0}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.0.0.dev5
+Version: 5.0.1.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,42 +18,20 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
-5.0.0.dev5
-************
-
-- `Added` parameter to `tb workspace members add` --role
-
-5.0.0.dev4
-************
+5.0.0
+******
 
-- `Improved` help message of `tb workspace members set-role` now displays the 3 valid roles: viewer|guest|admin.
+- `Breaking change` Make `insertion_date` column explicit. This column is no longer inferred, it must be present in the Data Source file.
 
-5.0.0.dev3
-************
+  Detailed info at https://www.tinybird.co/docs/changelog/2024-06-17-insertion_date-deprecation
 
+- `Added` parameter to `tb pipe regression-test` --relative-change
+- `Added` `tb push` displays warnings when using a reserved parameter in a pipe.
+- `Added` --role parameter to `tb workspace members add`
 - `Changed` Point region hosts to new `app.tinybird.co` domain
-
-5.0.0.dev2
-************
-
+- `Improved` help message of `tb workspace members set-role` now displays the 3 valid roles: viewer|guest|admin.
 - `Improved` syntax error messages when parsing endpoints.
 
-5.0.0.dev1
-************
-
-- `Added` `tb push` displays warnings when using a reserved parameter in a pipe.
-
-5.0.0.dev0
-************
-
-- `Changed` Make `insertion_date` column explicit. This column is no longer inferred, it must be present in the Data Source file.
-
-4.1.2.dev0
-************
-
-- `Added` parameter to `tb pipe regression-test` --relative-change
-
-
 4.1.1
 ************