tinybird-cli 5.1.1.dev0__tar.gz → 5.1.1.dev2__tar.gz

{tinybird-cli-5.1.1.dev0 → tinybird-cli-5.1.1.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.1.1.dev0
+Version: 5.1.1.dev2
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,6 +18,16 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.1.1.dev2
+************
+
+- `Added` `tb pipe unlink` command
+
+5.1.1.dev1
+***********
+
+- `Added` internal release
+
 5.1.0
 ******
 

{tinybird-cli-5.1.1.dev0 → tinybird-cli-5.1.1.dev2}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '5.1.1.dev0'
-__revision__ = '1b678aa'
+__version__ = '5.1.1.dev2'
+__revision__ = '1417339'

{tinybird-cli-5.1.1.dev0 → tinybird-cli-5.1.1.dev2}/tinybird/feedback_manager.py

@@ -357,6 +357,7 @@ class FeedbackManager:
     error_number_of_fixed_params_and_resources_mismatch = error_message(
         "The number of --fixed-params options must not exceed the number of --scope and --resource options."
     )
+    error_unlinking_pipe_not_linked = error_message("** {pipe} is not linked (MV, Copy, or Sink).")
 
     info_incl_relative_path = info_message("** Relative path {path} does not exist, skipping.")
     info_ignoring_incl_file = info_message(
@@ -670,7 +671,9 @@ Ready? """
     info_materialized_datasource_used = info_message(
         "** Materialized pipe '{pipe}' using the Data Source '{datasource}'"
     )
-    info_materialized_unlinking_pipe = info_message("** Unlinking materialized pipe {pipe}")
+    info_unlinking_materialized_pipe = info_message("** Unlinking materialized pipe {pipe}")
+    info_unlinking_copy_pipe = info_message("** Unlinking copy pipe {pipe}")
+    info_unlinking_sink_pipe = info_message("** Unlinking sink pipe {pipe}")
     info_materialized_unlinking_pipe_not_found = info_message("** {pipe} not found")
     info_materialized_dry_unlinking_pipe = info_message("** [DRY RUN] Unlinking materialized pipe {pipe}")
     info_copy_datasource_created = info_message("** Copy pipe '{pipe}' created the Data Source '{datasource}'")
@@ -839,6 +842,7 @@ Ready? """
 ** Pipe URL: {host}/v0/pipes/{pipe}
 """
     )
+    success_pipe_unlinked = success_message("""** Pipe {pipe} unlinked""")
     success_node_copy = success_message(
         """** Node set to be used to copy data!
 ** Pipe URL: {host}/v0/pipes/{pipe}

{tinybird-cli-5.1.1.dev0 → tinybird-cli-5.1.1.dev2}/tinybird/sql_template.py

@@ -17,6 +17,36 @@ from tinybird.context import ff_preprocess_parameters_circuit_breaker, ff_split_
 from .datatypes import testers
 from .tornado_template import VALID_CUSTOM_FUNCTION_NAMES, SecurityException, Template
 
+TB_SECRET_IN_TEST_MODE = "tb_secret_dont_raise"
+TB_SECRET_PREFIX = "tb_secret_"
+CH_PARAM_PREFIX = "param_"
+
+
+def secret_template_key(secret_name: str) -> str:
+    return f"{TB_SECRET_PREFIX}{secret_name}"
+
+
+def is_secret_template_key(key: str) -> bool:
+    return key.startswith(TB_SECRET_PREFIX)
+
+
+class TemplateExecutionResults(dict):
+    def __init__(self, *args, **kwargs):
+        super(TemplateExecutionResults, self).__init__(*args, **kwargs)
+        self.template_params = set()
+        self.ch_params = set()
+
+    def add_template_param(self, param: str):
+        self.template_params.add(param)
+
+    def add_ch_param(self, param: str):
+        self.ch_params.add(param)
+
+    def update_all(self, other: "TemplateExecutionResults"):
+        self.update(other)
+        self.ch_params.update(other.ch_params)
+        self.template_params.update(other.template_params)
+
 
 class SQLTemplateCustomError(Exception):
     def __init__(self, err, code=400):
@@ -1333,10 +1363,30 @@ for p in DEFAULT_PARAM_NAMES:  # we handle these in an specific manner
 error_vars = ["error", "custom_error"]
 
 
-def generate(self, **kwargs) -> Tuple[str, dict]:
+def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
     """Generate this template with the given arguments."""
     namespace = {}
-    template_execution_results = {}
+    template_execution_results = TemplateExecutionResults()
+    for key in kwargs.get("tb_secrets", []):
+        if is_secret_template_key(key):
+            template_execution_results.add_template_param(key)
+    if TB_SECRET_IN_TEST_MODE in kwargs:
+        template_execution_results[TB_SECRET_IN_TEST_MODE] = None
+
+    def set_tb_secret(x):
+        try:
+            key = secret_template_key(x)
+            if key in template_execution_results.template_params:
+                template_execution_results.add_ch_param(x)
+                return Symbol("{" + sqlescape(x) + ": String}")
+            else:
+                is_test_mode = TB_SECRET_IN_TEST_MODE in template_execution_results
+                if is_test_mode:
+                    return Symbol("{" + sqlescape(x) + ": String}")
+                else:
+                    raise SQLTemplateException(f"Secret '{x}' does not exist in Workspace")
+        except Exception:
+            raise SQLTemplateException(f"Secret '{x}' does not exist in Workspace")
 
     def set_max_threads(x):
         try:
@@ -1374,6 +1424,7 @@ def generate(self, **kwargs) -> Tuple[str, dict]:
             "__name__": self.name.replace(".", "_"),
             "__loader__": ObjectDict(get_source=lambda name: self.code),
             "max_threads": set_max_threads,
+            "tb_secret": set_tb_secret,
             "backend_hint": set_backend_hint,
             "cache_ttl": set_cache_ttl,
             "activate": set_activate,
@@ -1856,9 +1907,10 @@ def preprocess_variables(variables: dict, t: Template):
 def render_sql_template(
     sql: str,
     variables: Optional[dict] = None,
+    secrets: Optional[List[str]] = None,
     test_mode: bool = False,
     name: Optional[str] = None,
-) -> Tuple[str, dict, list]:
+) -> Tuple[str, TemplateExecutionResults, list]:
     """
     >>> render_sql_template("select * from table where f = {{Float32(foo)}}", { 'foo': -1 })
     ("select * from table where f = toFloat32('-1.0')", {}, [])
@@ -2058,6 +2110,20 @@ def render_sql_template(
     tinybird.tornado_template.UnClosedIfError: Missing {% end %} block for if at line 1
     >>> render_sql_template("select * from table where str = {{pipeline}}", { 'pipeline': 'test' })
     ("select * from table where str = 'test'", {}, ['pipeline'])
+    >>> render_sql_template("select * from table where str = {{tb_secret('test')}}", secrets = [ 'tb_secret_test' ])
+    ('select * from table where str = {test: String}', {}, [])
+    >>> render_sql_template("select * from table where str = {{tb_secret('test')}}", variables = { 'test': '1234' })
+    Traceback (most recent call last):
+    ...
+    tinybird.sql_template.SQLTemplateException: Template Syntax Error: Secret 'test' does not exist in Workspace
+    >>> render_sql_template("select * from table where str = {{tb_secret('test')}}", test_mode=True)
+    ('select * from table where str = {test: String}', {}, [])
+    >>> render_sql_template("select * from table where str = {{tb_secret('test')}}", secrets = [ 'tb_secret_test' ], test_mode=True)
+    ('select * from table where str = {test: String}', {}, [])
+    >>> render_sql_template("select * from table where str = {{tb_secret('test')}}", secrets = [ 'tb_secret_test2' ])
+    Traceback (most recent call last):
+    ...
+    tinybird.sql_template.SQLTemplateException: Template Syntax Error: Secret 'test' does not exist in Workspace
     """
     escape_split_to_array = ff_split_to_array_escape.get(False)
     bypass_preprocess_variables = ff_preprocess_parameters_circuit_breaker.get(False)
@@ -2076,10 +2142,17 @@ def render_sql_template(
             return Comment("error launched")
 
         v: dict = {x["name"]: Placeholder(x["name"], x["line"]) for x in template_variables}
+        is_tb_secret = any([s for s in template_variables if s["name"] == "tb_secret"])
 
         if variables:
             v.update(variables)
 
+        if secrets:
+            v.update({"tb_secrets": secrets})
+
+        if is_tb_secret:
+            v.update({TB_SECRET_IN_TEST_MODE: None})
+
         v.update(type_fns_check)
         v.update(
             {
@@ -2093,10 +2166,19 @@ def render_sql_template(
         v = {x["name"]: None for x in template_variables}
         if variables:
             v.update(variables)
+
+        if secrets:
+            v.update({"tb_secrets": secrets})
         v.update(type_fns)
 
     try:
-        return *generate(t, **v), variable_warnings
+        sql, template_execution_results = generate(t, **v)
+        try:
+            if TB_SECRET_IN_TEST_MODE in template_execution_results:
+                del template_execution_results[TB_SECRET_IN_TEST_MODE]
+        except Exception:
+            pass
+        return sql, template_execution_results, variable_warnings
     except NameError as e:
         raise SQLTemplateException(e, documentation="/cli/advanced-templates.html#defined")
     except SQLTemplateException:

{tinybird-cli-5.1.1.dev0 → tinybird-cli-5.1.1.dev2}/tinybird/sql_toolset.py

@@ -14,15 +14,15 @@ VALID_REMOTE = "VALID_REMOTE"
 
 
 class InvalidFunction(ValueError):
-    def __init__(self, msg: str = "", table_function_names: str = ""):
+    def __init__(self, msg: str = "", table_function_name: str = ""):
         if any([fn for fn in COPY_ENABLED_TABLE_FUNCTIONS if fn in msg]):
             msg = msg.replace("is restricted", "is restricted to Copy Pipes")
 
-        if table_function_names:
-            if "postgresql" in table_function_names:
-                self.msg = "The postgresql table function is only allowed in Copy Pipes"
+        if table_function_name:
+            if table_function_name in COPY_ENABLED_TABLE_FUNCTIONS:
+                self.msg = f"The {table_function_name} table function is only allowed in Copy Pipes"
             else:
-                self.msg = f"The query uses disabled table functions: '{table_function_names}'"
+                self.msg = f"The query uses disabled table functions: '{table_function_name}'"
         else:
             self.msg = msg
         super().__init__(self.msg)
@@ -102,6 +102,8 @@ def sql_get_used_tables_cached(
             msg = str(e)
             if "is restricted. Contact support@tinybird.co" in msg:
                 raise InvalidFunction(msg=msg) from e
+            elif "Unknown function tb_secret" in msg:
+                raise InvalidFunction(msg="Unknown function tb_secret. Usage: {{tb_secret('secret_name')}}") from e
             raise
         return [(default_database, sql, "")]
 
@@ -234,7 +236,7 @@ def replace_tables(
         if len(table) == 3:
             first_table, second_table, last_table = table
             if last_table and last_table not in _enabled_table_functions:
-                raise InvalidFunction(table_function_names=last_table)
+                raise InvalidFunction(table_function_name=last_table)
             if first_table or second_table:
                 table = (first_table, second_table)
             else:
@@ -251,7 +253,7 @@ def replace_tables(
                         and dependent_table[2] not in _enabled_table_functions
                         and not (dependent_table[2] in ["cluster"] and replacement[0] == VALID_REMOTE)
                     ):
-                        raise InvalidFunction(table_function_names=dependent_table[2])
+                        raise InvalidFunction(table_function_name=dependent_table[2])
                     if dependent_table[0] or dependent_table[1]:
                         dependent_table = (dependent_table[0], dependent_table[1])
                     else:

{tinybird-cli-5.1.1.dev0 → tinybird-cli-5.1.1.dev2}/tinybird/tb_cli_modules/pipe.py

@@ -16,7 +16,7 @@ from click import Context
 import tinybird.context as context
 from tinybird.client import DoesNotExistException, TinyB
 from tinybird.config import DEFAULT_API_HOST, FeatureFlags
-from tinybird.datafile import folder_push, get_name_version, process_file, wait_job
+from tinybird.datafile import PipeTypes, folder_push, get_name_version, process_file, wait_job
 from tinybird.feedback_manager import FeedbackManager
 from tinybird.tb_cli_modules.branch import warn_if_in_live
 from tinybird.tb_cli_modules.cli import cli
@@ -267,6 +267,74 @@ async def pipe_populate(
         await wait_job(cl, job_id, job_url, "Populating")
 
 
+@pipe.command(name="unlink")
+@click.argument("pipe_name_or_id")
+@click.argument("node_uid", default=None, required=False)
+@click.pass_context
+@coro
+async def pipe_unlink_output_node(
+    ctx: click.Context,
+    pipe_name_or_id: str,
+    node_uid: Optional[str] = None,
+):
+    """Unlink the output of a pipe. Works for Materialized Views, Copy Pipes, and Sinks."""
+    client: TinyB = ctx.ensure_object(dict)["client"]
+
+    try:
+        pipe = await client.pipe(pipe_name_or_id)
+
+        if pipe["type"] not in [PipeTypes.MATERIALIZED, PipeTypes.COPY, PipeTypes.DATA_SINK]:
+            raise CLIPipeException(FeedbackManager.error_unlinking_pipe_not_linked(pipe=pipe_name_or_id))
+
+        if pipe["type"] == PipeTypes.MATERIALIZED:
+            click.echo(FeedbackManager.info_unlinking_materialized_pipe(pipe=pipe["name"]))
+
+            if not node_uid:
+                for node in pipe["nodes"]:
+                    if "materialized" in node and node["materialized"] is not None:
+                        node_uid = node["id"]
+                        break
+
+            if not node_uid:
+                raise CLIPipeException(FeedbackManager.error_unlinking_pipe_not_linked(pipe=pipe_name_or_id))
+            else:
+                await client.pipe_unlink_materialized(pipe["name"], node_uid)
+                click.echo(FeedbackManager.success_pipe_unlinked(pipe=pipe["name"]))
+
+        if pipe["type"] == PipeTypes.COPY:
+            click.echo(FeedbackManager.info_unlinking_copy_pipe(pipe=pipe["name"]))
+
+            if not node_uid:
+                for node in pipe["nodes"]:
+                    if node["node_type"] == "copy":
+                        node_uid = node["id"]
+                        break
+
+            if not node_uid:
+                raise CLIPipeException(FeedbackManager.error_unlinking_pipe_not_linked(pipe=pipe_name_or_id))
+            else:
+                await client.pipe_remove_copy(pipe["name"], node_uid)
+                click.echo(FeedbackManager.success_pipe_unlinked(pipe=pipe["name"]))
+
+        if pipe["type"] == PipeTypes.DATA_SINK:
+            click.echo(FeedbackManager.info_unlinking_sink_pipe(pipe=pipe["name"]))
+
+            if not node_uid:
+                for node in pipe["nodes"]:
+                    if node["node_type"] == "sink":
+                        node_uid = node["id"]
+                        break
+
+            if not node_uid:
+                raise CLIPipeException(FeedbackManager.error_unlinking_pipe_not_linked(pipe=pipe_name_or_id))
+            else:
+                await client.pipe_remove_sink(pipe["name"], node_uid)
+                click.echo(FeedbackManager.success_pipe_unlinked(pipe=pipe["name"]))
+
+    except Exception as e:
+        raise CLIPipeException(FeedbackManager.error_exception(error=e))
+
+
 @pipe.command(name="append")
 @click.argument("pipe_name_or_uid")
 @click.argument("sql")

{tinybird-cli-5.1.1.dev0 → tinybird-cli-5.1.1.dev2}/tinybird/tb_cli_modules/workspace.py

@@ -151,7 +151,7 @@ async def clear_workspace(ctx: Context, yes: bool, dry_run: bool) -> None:
                             break
 
                     if node_id:
-                        click.echo(FeedbackManager.info_materialized_unlinking_pipe(pipe=pipe["name"]))
+                        click.echo(FeedbackManager.info_unlinking_materialized_pipe(pipe=pipe["name"]))
                         try:
                             await client.pipe_unlink_materialized(pipe["name"], node_id)
                         except DoesNotExistException:

{tinybird-cli-5.1.1.dev0 → tinybird-cli-5.1.1.dev2}/tinybird/tornado_template.py

@@ -1050,6 +1050,7 @@ VALID_FUNCTION_NAMES = {
     "activate",
     "sql_unescape",
     "max_threads",
+    "tb_secret",
     "Int",
     "table",
     "TABLE",

{tinybird-cli-5.1.1.dev0 → tinybird-cli-5.1.1.dev2}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.1.1.dev0
+Version: 5.1.1.dev2
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,6 +18,16 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.1.1.dev2
+************
+
+- `Added` `tb pipe unlink` command
+
+5.1.1.dev1
+***********
+
+- `Added` internal release
+
 5.1.0
 ******