tinybird-cli 5.1.1.dev0__tar.gz → 5.1.1.dev1__tar.gz

{tinybird-cli-5.1.1.dev0 → tinybird-cli-5.1.1.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.1.1.dev0
+Version: 5.1.1.dev1
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,6 +18,11 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.1.1.dev1
+***********
+
+- `Added` internal release
+
 5.1.0
 ******
 

{tinybird-cli-5.1.1.dev0 → tinybird-cli-5.1.1.dev1}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '5.1.1.dev0'
-__revision__ = '1b678aa'
+__version__ = '5.1.1.dev1'
+__revision__ = '2922d6f'

{tinybird-cli-5.1.1.dev0 → tinybird-cli-5.1.1.dev1}/tinybird/sql_template.py

@@ -7,7 +7,7 @@ from datetime import datetime
 from functools import lru_cache
 from io import StringIO
 from json import loads
-from typing import Any, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 from tornado import escape
 from tornado.util import ObjectDict, exec_in, unicode_type
@@ -17,6 +17,32 @@ from tinybird.context import ff_preprocess_parameters_circuit_breaker, ff_split_
 from .datatypes import testers
 from .tornado_template import VALID_CUSTOM_FUNCTION_NAMES, SecurityException, Template
 
+TB_SECRET_IN_TEST_MODE = "tb_secret_dont_raise"
+TB_SECRET_PREFIX = "tb_secret_"
+CH_PARAM_PREFIX = "param_"
+
+
+def secret_template_key(secret_name: str) -> str:
+    return f"{TB_SECRET_PREFIX}{secret_name}"
+
+
+def is_secret_template_key(key: str) -> bool:
+    return key.startswith(TB_SECRET_PREFIX)
+
+
+class TemplateExecutionResults(dict):
+    def __init__(self, *args, **kwargs):
+        super(TemplateExecutionResults, self).__init__(*args, **kwargs)
+
+    def subdict(self, key: str) -> Dict[str, Any]:
+        return {k: v for k, v in self.items() if k.startswith(key)}
+
+    def get_params(self) -> Dict[str, Any]:
+        return self.subdict(CH_PARAM_PREFIX)
+
+    def get_params_keys(self) -> List[str]:
+        return [key.split(CH_PARAM_PREFIX)[-1] for key in list(self.get_params().keys())]
+
 
 class SQLTemplateCustomError(Exception):
     def __init__(self, err, code=400):
@@ -1333,10 +1359,29 @@ for p in DEFAULT_PARAM_NAMES:  # we handle these in an specific manner
 error_vars = ["error", "custom_error"]
 
 
-def generate(self, **kwargs) -> Tuple[str, dict]:
+def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
     """Generate this template with the given arguments."""
     namespace = {}
-    template_execution_results = {}
+    template_execution_results = TemplateExecutionResults()
+    for key in kwargs:
+        if is_secret_template_key(key):
+            template_execution_results[key] = kwargs[key]
+
+    def set_tb_secret(x):
+        try:
+            key = secret_template_key(x)
+            if key in template_execution_results:
+                val = template_execution_results[key]
+                template_execution_results[f"{CH_PARAM_PREFIX}{x}"] = val
+                return Symbol("{" + sqlescape(x) + ": String}")
+            else:
+                is_test_mode = TB_SECRET_IN_TEST_MODE in template_execution_results
+                if is_test_mode:
+                    return Symbol("{" + sqlescape(x) + ": String}")
+                else:
+                    raise SQLTemplateException(f"Secret '{x}' does not exist in Workspace")
+        except Exception:
+            raise SQLTemplateException(f"Secret '{x}' does not exist in Workspace")
 
     def set_max_threads(x):
         try:
@@ -1374,6 +1419,7 @@ def generate(self, **kwargs) -> Tuple[str, dict]:
             "__name__": self.name.replace(".", "_"),
             "__loader__": ObjectDict(get_source=lambda name: self.code),
             "max_threads": set_max_threads,
+            "tb_secret": set_tb_secret,
             "backend_hint": set_backend_hint,
             "cache_ttl": set_cache_ttl,
             "activate": set_activate,
@@ -1856,9 +1902,10 @@ def preprocess_variables(variables: dict, t: Template):
 def render_sql_template(
     sql: str,
     variables: Optional[dict] = None,
+    secrets: Optional[dict] = None,
     test_mode: bool = False,
     name: Optional[str] = None,
-) -> Tuple[str, dict, list]:
+) -> Tuple[str, TemplateExecutionResults, list]:
     """
     >>> render_sql_template("select * from table where f = {{Float32(foo)}}", { 'foo': -1 })
     ("select * from table where f = toFloat32('-1.0')", {}, [])
@@ -2058,6 +2105,20 @@ def render_sql_template(
     tinybird.tornado_template.UnClosedIfError: Missing {% end %} block for if at line 1
     >>> render_sql_template("select * from table where str = {{pipeline}}", { 'pipeline': 'test' })
     ("select * from table where str = 'test'", {}, ['pipeline'])
+    >>> render_sql_template("select * from table where str = {{tb_secret('test')}}", secrets = { 'tb_secret_test': '1234' })
+    ('select * from table where str = {test: String}', {'param_test': '1234'}, [])
+    >>> render_sql_template("select * from table where str = {{tb_secret('test')}}", variables = { 'test': '1234' })
+    Traceback (most recent call last):
+    ...
+    tinybird.sql_template.SQLTemplateException: Template Syntax Error: Secret 'test' does not exist in Workspace
+    >>> render_sql_template("select * from table where str = {{tb_secret('test')}}", test_mode=True)
+    ('select * from table where str = {test: String}', {}, [])
+    >>> render_sql_template("select * from table where str = {{tb_secret('test')}}", secrets = { 'tb_secret_test': '1234' }, test_mode=True)
+    ('select * from table where str = {test: String}', {'param_test': '1234'}, [])
+    >>> render_sql_template("select * from table where str = {{tb_secret('test')}}", secrets = { 'tb_secret_test2': '1234' })
+    Traceback (most recent call last):
+    ...
+    tinybird.sql_template.SQLTemplateException: Template Syntax Error: Secret 'test' does not exist in Workspace
     """
     escape_split_to_array = ff_split_to_array_escape.get(False)
     bypass_preprocess_variables = ff_preprocess_parameters_circuit_breaker.get(False)
@@ -2076,10 +2137,17 @@ def render_sql_template(
             return Comment("error launched")
 
         v: dict = {x["name"]: Placeholder(x["name"], x["line"]) for x in template_variables}
+        is_tb_secret = any([s for s in template_variables if s["name"] == "tb_secret"])
 
         if variables:
             v.update(variables)
 
+        if secrets:
+            v.update(secrets)
+
+        if is_tb_secret:
+            v.update({TB_SECRET_IN_TEST_MODE: None})
+
         v.update(type_fns_check)
         v.update(
             {
@@ -2093,10 +2161,17 @@ def render_sql_template(
         v = {x["name"]: None for x in template_variables}
         if variables:
             v.update(variables)
+
+        if secrets:
+            v.update(secrets)
         v.update(type_fns)
 
     try:
-        return *generate(t, **v), variable_warnings
+        sql, template_execution_results = generate(t, **v)
+        for key in list(template_execution_results.keys()):
+            if is_secret_template_key(key) and key in template_execution_results:
+                del template_execution_results[key]
+        return sql, template_execution_results, variable_warnings
     except NameError as e:
         raise SQLTemplateException(e, documentation="/cli/advanced-templates.html#defined")
     except SQLTemplateException:

{tinybird-cli-5.1.1.dev0 → tinybird-cli-5.1.1.dev1}/tinybird/sql_toolset.py

@@ -14,15 +14,15 @@ VALID_REMOTE = "VALID_REMOTE"
 
 
 class InvalidFunction(ValueError):
-    def __init__(self, msg: str = "", table_function_names: str = ""):
+    def __init__(self, msg: str = "", table_function_name: str = ""):
         if any([fn for fn in COPY_ENABLED_TABLE_FUNCTIONS if fn in msg]):
             msg = msg.replace("is restricted", "is restricted to Copy Pipes")
 
-        if table_function_names:
-            if "postgresql" in table_function_names:
-                self.msg = "The postgresql table function is only allowed in Copy Pipes"
+        if table_function_name:
+            if table_function_name in COPY_ENABLED_TABLE_FUNCTIONS:
+                self.msg = f"The {table_function_name} table function is only allowed in Copy Pipes"
             else:
-                self.msg = f"The query uses disabled table functions: '{table_function_names}'"
+                self.msg = f"The query uses disabled table functions: '{table_function_name}'"
         else:
             self.msg = msg
         super().__init__(self.msg)
@@ -102,6 +102,8 @@ def sql_get_used_tables_cached(
             msg = str(e)
             if "is restricted. Contact support@tinybird.co" in msg:
                 raise InvalidFunction(msg=msg) from e
+            elif "Unknown function tb_secret" in msg:
+                raise InvalidFunction(msg="Unknown function tb_secret. Usage: {{tb_secret('secret_name')}}") from e
             raise
         return [(default_database, sql, "")]
 
@@ -234,7 +236,7 @@ def replace_tables(
         if len(table) == 3:
             first_table, second_table, last_table = table
             if last_table and last_table not in _enabled_table_functions:
-                raise InvalidFunction(table_function_names=last_table)
+                raise InvalidFunction(table_function_name=last_table)
             if first_table or second_table:
                 table = (first_table, second_table)
             else:
@@ -251,7 +253,7 @@ def replace_tables(
                         and dependent_table[2] not in _enabled_table_functions
                         and not (dependent_table[2] in ["cluster"] and replacement[0] == VALID_REMOTE)
                     ):
-                        raise InvalidFunction(table_function_names=dependent_table[2])
+                        raise InvalidFunction(table_function_name=dependent_table[2])
                     if dependent_table[0] or dependent_table[1]:
                         dependent_table = (dependent_table[0], dependent_table[1])
                     else:

{tinybird-cli-5.1.1.dev0 → tinybird-cli-5.1.1.dev1}/tinybird/tornado_template.py

@@ -1050,6 +1050,7 @@ VALID_FUNCTION_NAMES = {
     "activate",
     "sql_unescape",
     "max_threads",
+    "tb_secret",
     "Int",
     "table",
     "TABLE",

{tinybird-cli-5.1.1.dev0 → tinybird-cli-5.1.1.dev1}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.1.1.dev0
+Version: 5.1.1.dev1
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,6 +18,11 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.1.1.dev1
+***********
+
+- `Added` internal release
+
 5.1.0
 ******