tinybird-cli 5.1.1.dev1__tar.gz → 5.1.1.dev3__tar.gz

{tinybird-cli-5.1.1.dev1 → tinybird-cli-5.1.1.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.1.1.dev1
+Version: 5.1.1.dev3
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,6 +18,16 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.1.1.dev3
+***********
+
+- `Improved` No need for `--node` param in `tb pipe populate`
+
+5.1.1.dev2
+************
+
+- `Added` `tb pipe unlink` command
+
 5.1.1.dev1
 ***********
 

{tinybird-cli-5.1.1.dev1 → tinybird-cli-5.1.1.dev3}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '5.1.1.dev1'
-__revision__ = '2922d6f'
+__version__ = '5.1.1.dev3'
+__revision__ = '7908f0f'

{tinybird-cli-5.1.1.dev1 → tinybird-cli-5.1.1.dev3}/tinybird/feedback_manager.py

@@ -357,6 +357,12 @@ class FeedbackManager:
     error_number_of_fixed_params_and_resources_mismatch = error_message(
         "The number of --fixed-params options must not exceed the number of --scope and --resource options."
     )
+    error_pipe_not_materialized = error_message("Pipe {pipe} is not materialized.")
+    error_populate_no_materialized_in_pipe = error_message("No materialized nodes in pipe {pipe}. Use --node param.")
+    error_populate_several_materialized_in_pipe = error_message(
+        "Several materialized nodes in pipe {pipe}. Use --node param."
+    )
+    error_unlinking_pipe_not_linked = error_message("** {pipe} is not linked (MV, Copy, or Sink).")
 
     info_incl_relative_path = info_message("** Relative path {path} does not exist, skipping.")
     info_ignoring_incl_file = info_message(
@@ -670,7 +676,9 @@ Ready? """
     info_materialized_datasource_used = info_message(
         "** Materialized pipe '{pipe}' using the Data Source '{datasource}'"
     )
-    info_materialized_unlinking_pipe = info_message("** Unlinking materialized pipe {pipe}")
+    info_unlinking_materialized_pipe = info_message("** Unlinking materialized pipe {pipe}")
+    info_unlinking_copy_pipe = info_message("** Unlinking copy pipe {pipe}")
+    info_unlinking_sink_pipe = info_message("** Unlinking sink pipe {pipe}")
     info_materialized_unlinking_pipe_not_found = info_message("** {pipe} not found")
     info_materialized_dry_unlinking_pipe = info_message("** [DRY RUN] Unlinking materialized pipe {pipe}")
     info_copy_datasource_created = info_message("** Copy pipe '{pipe}' created the Data Source '{datasource}'")
@@ -839,6 +847,7 @@ Ready? """
 ** Pipe URL: {host}/v0/pipes/{pipe}
 """
     )
+    success_pipe_unlinked = success_message("""** Pipe {pipe} unlinked""")
     success_node_copy = success_message(
         """** Node set to be used to copy data!
 ** Pipe URL: {host}/v0/pipes/{pipe}

{tinybird-cli-5.1.1.dev1 → tinybird-cli-5.1.1.dev3}/tinybird/sql_template.py

@@ -7,7 +7,7 @@ from datetime import datetime
 from functools import lru_cache
 from io import StringIO
 from json import loads
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, List, Optional, Tuple, Union
 
 from tornado import escape
 from tornado.util import ObjectDict, exec_in, unicode_type
@@ -33,15 +33,19 @@ def is_secret_template_key(key: str) -> bool:
 class TemplateExecutionResults(dict):
     def __init__(self, *args, **kwargs):
         super(TemplateExecutionResults, self).__init__(*args, **kwargs)
+        self.template_params = set()
+        self.ch_params = set()
 
-    def subdict(self, key: str) -> Dict[str, Any]:
-        return {k: v for k, v in self.items() if k.startswith(key)}
+    def add_template_param(self, param: str):
+        self.template_params.add(param)
 
-    def get_params(self) -> Dict[str, Any]:
-        return self.subdict(CH_PARAM_PREFIX)
+    def add_ch_param(self, param: str):
+        self.ch_params.add(param)
 
-    def get_params_keys(self) -> List[str]:
-        return [key.split(CH_PARAM_PREFIX)[-1] for key in list(self.get_params().keys())]
+    def update_all(self, other: "TemplateExecutionResults"):
+        self.update(other)
+        self.ch_params.update(other.ch_params)
+        self.template_params.update(other.template_params)
 
 
 class SQLTemplateCustomError(Exception):
@@ -1363,25 +1367,30 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
     """Generate this template with the given arguments."""
     namespace = {}
     template_execution_results = TemplateExecutionResults()
-    for key in kwargs:
+    for key in kwargs.get("tb_secrets", []):
         if is_secret_template_key(key):
-            template_execution_results[key] = kwargs[key]
+            template_execution_results.add_template_param(key)
+    if TB_SECRET_IN_TEST_MODE in kwargs:
+        template_execution_results[TB_SECRET_IN_TEST_MODE] = None
 
     def set_tb_secret(x):
         try:
             key = secret_template_key(x)
-            if key in template_execution_results:
-                val = template_execution_results[key]
-                template_execution_results[f"{CH_PARAM_PREFIX}{x}"] = val
+            if key in template_execution_results.template_params:
+                template_execution_results.add_ch_param(x)
                 return Symbol("{" + sqlescape(x) + ": String}")
             else:
                 is_test_mode = TB_SECRET_IN_TEST_MODE in template_execution_results
                 if is_test_mode:
                     return Symbol("{" + sqlescape(x) + ": String}")
                 else:
-                    raise SQLTemplateException(f"Secret '{x}' does not exist in Workspace")
+                    raise SQLTemplateException(
+                        f"Cannot access secret '{x}'. Check the secret exists in the Workspace and the token has the required scope."
+                    )
         except Exception:
-            raise SQLTemplateException(f"Secret '{x}' does not exist in Workspace")
+            raise SQLTemplateException(
+                f"Cannot access secret '{x}'. Check the secret exists in the Workspace and the token has the required scope."
+            )
 
     def set_max_threads(x):
         try:
@@ -1902,7 +1911,7 @@ def preprocess_variables(variables: dict, t: Template):
 def render_sql_template(
     sql: str,
     variables: Optional[dict] = None,
-    secrets: Optional[dict] = None,
+    secrets: Optional[List[str]] = None,
     test_mode: bool = False,
     name: Optional[str] = None,
 ) -> Tuple[str, TemplateExecutionResults, list]:
@@ -2105,20 +2114,20 @@ def render_sql_template(
     tinybird.tornado_template.UnClosedIfError: Missing {% end %} block for if at line 1
     >>> render_sql_template("select * from table where str = {{pipeline}}", { 'pipeline': 'test' })
     ("select * from table where str = 'test'", {}, ['pipeline'])
-    >>> render_sql_template("select * from table where str = {{tb_secret('test')}}", secrets = { 'tb_secret_test': '1234' })
-    ('select * from table where str = {test: String}', {'param_test': '1234'}, [])
+    >>> render_sql_template("select * from table where str = {{tb_secret('test')}}", secrets = [ 'tb_secret_test' ])
+    ('select * from table where str = {test: String}', {}, [])
     >>> render_sql_template("select * from table where str = {{tb_secret('test')}}", variables = { 'test': '1234' })
     Traceback (most recent call last):
     ...
-    tinybird.sql_template.SQLTemplateException: Template Syntax Error: Secret 'test' does not exist in Workspace
+    tinybird.sql_template.SQLTemplateException: Template Syntax Error: Cannot access secret 'test'. Check the secret exists in the Workspace and the token has the required scope.
     >>> render_sql_template("select * from table where str = {{tb_secret('test')}}", test_mode=True)
     ('select * from table where str = {test: String}', {}, [])
-    >>> render_sql_template("select * from table where str = {{tb_secret('test')}}", secrets = { 'tb_secret_test': '1234' }, test_mode=True)
-    ('select * from table where str = {test: String}', {'param_test': '1234'}, [])
-    >>> render_sql_template("select * from table where str = {{tb_secret('test')}}", secrets = { 'tb_secret_test2': '1234' })
+    >>> render_sql_template("select * from table where str = {{tb_secret('test')}}", secrets = [ 'tb_secret_test' ], test_mode=True)
+    ('select * from table where str = {test: String}', {}, [])
+    >>> render_sql_template("select * from table where str = {{tb_secret('test')}}", secrets = [ 'tb_secret_test2' ])
     Traceback (most recent call last):
     ...
-    tinybird.sql_template.SQLTemplateException: Template Syntax Error: Secret 'test' does not exist in Workspace
+    tinybird.sql_template.SQLTemplateException: Template Syntax Error: Cannot access secret 'test'. Check the secret exists in the Workspace and the token has the required scope.
     """
     escape_split_to_array = ff_split_to_array_escape.get(False)
     bypass_preprocess_variables = ff_preprocess_parameters_circuit_breaker.get(False)
@@ -2143,7 +2152,7 @@ def render_sql_template(
             v.update(variables)
 
         if secrets:
-            v.update(secrets)
+            v.update({"tb_secrets": secrets})
 
         if is_tb_secret:
             v.update({TB_SECRET_IN_TEST_MODE: None})
@@ -2163,14 +2172,16 @@ def render_sql_template(
             v.update(variables)
 
         if secrets:
-            v.update(secrets)
+            v.update({"tb_secrets": secrets})
         v.update(type_fns)
 
     try:
         sql, template_execution_results = generate(t, **v)
-        for key in list(template_execution_results.keys()):
-            if is_secret_template_key(key) and key in template_execution_results:
-                del template_execution_results[key]
+        try:
+            if TB_SECRET_IN_TEST_MODE in template_execution_results:
+                del template_execution_results[TB_SECRET_IN_TEST_MODE]
+        except Exception:
+            pass
         return sql, template_execution_results, variable_warnings
     except NameError as e:
         raise SQLTemplateException(e, documentation="/cli/advanced-templates.html#defined")

{tinybird-cli-5.1.1.dev1 → tinybird-cli-5.1.1.dev3}/tinybird/tb_cli_modules/pipe.py

@@ -16,7 +16,7 @@ from click import Context
 import tinybird.context as context
 from tinybird.client import DoesNotExistException, TinyB
 from tinybird.config import DEFAULT_API_HOST, FeatureFlags
-from tinybird.datafile import folder_push, get_name_version, process_file, wait_job
+from tinybird.datafile import PipeTypes, folder_push, get_name_version, process_file, wait_job
 from tinybird.feedback_manager import FeedbackManager
 from tinybird.tb_cli_modules.branch import warn_if_in_live
 from tinybird.tb_cli_modules.cli import cli
@@ -212,7 +212,7 @@ async def pipe_ls(ctx: Context, match: str, format_: str):
 
 @pipe.command(name="populate")
 @click.argument("pipe_name")
-@click.option("--node", type=str, help="Name of the materialized node.", required=True)
+@click.option("--node", type=str, help="Name of the materialized node.", default=None, required=False)
 @click.option(
     "--sql-condition",
     type=str,
@@ -247,6 +247,23 @@ async def pipe_populate(
 ):
     """Populate the result of a Materialized Node into the target Materialized View"""
     cl = create_tb_client(ctx)
+
+    pipe = await cl.pipe(pipe_name)
+
+    if pipe["type"] != PipeTypes.MATERIALIZED:
+        raise CLIPipeException(FeedbackManager.error_pipe_not_materialized(pipe=pipe_name))
+
+    if not node:
+        materialized_ids = [pipe_node["id"] for pipe_node in pipe["nodes"] if pipe_node.get("materialized") is not None]
+
+        if not materialized_ids:
+            raise CLIPipeException(FeedbackManager.error_populate_no_materialized_in_pipe(pipe=pipe_name))
+
+        elif len(materialized_ids) > 1:
+            raise CLIPipeException(FeedbackManager.error_populate_several_materialized_in_pipe(pipe=pipe_name))
+
+        node = materialized_ids[0]
+
     response = await cl.populate_node(
         pipe_name,
         node,
@@ -267,6 +284,74 @@ async def pipe_populate(
         await wait_job(cl, job_id, job_url, "Populating")
 
 
+@pipe.command(name="unlink")
+@click.argument("pipe_name_or_id")
+@click.argument("node_uid", default=None, required=False)
+@click.pass_context
+@coro
+async def pipe_unlink_output_node(
+    ctx: click.Context,
+    pipe_name_or_id: str,
+    node_uid: Optional[str] = None,
+):
+    """Unlink the output of a pipe. Works for Materialized Views, Copy Pipes, and Sinks."""
+    client: TinyB = ctx.ensure_object(dict)["client"]
+
+    try:
+        pipe = await client.pipe(pipe_name_or_id)
+
+        if pipe["type"] not in [PipeTypes.MATERIALIZED, PipeTypes.COPY, PipeTypes.DATA_SINK]:
+            raise CLIPipeException(FeedbackManager.error_unlinking_pipe_not_linked(pipe=pipe_name_or_id))
+
+        if pipe["type"] == PipeTypes.MATERIALIZED:
+            click.echo(FeedbackManager.info_unlinking_materialized_pipe(pipe=pipe["name"]))
+
+            if not node_uid:
+                for node in pipe["nodes"]:
+                    if "materialized" in node and node["materialized"] is not None:
+                        node_uid = node["id"]
+                        break
+
+            if not node_uid:
+                raise CLIPipeException(FeedbackManager.error_unlinking_pipe_not_linked(pipe=pipe_name_or_id))
+            else:
+                await client.pipe_unlink_materialized(pipe["name"], node_uid)
+                click.echo(FeedbackManager.success_pipe_unlinked(pipe=pipe["name"]))
+
+        if pipe["type"] == PipeTypes.COPY:
+            click.echo(FeedbackManager.info_unlinking_copy_pipe(pipe=pipe["name"]))
+
+            if not node_uid:
+                for node in pipe["nodes"]:
+                    if node["node_type"] == "copy":
+                        node_uid = node["id"]
+                        break
+
+            if not node_uid:
+                raise CLIPipeException(FeedbackManager.error_unlinking_pipe_not_linked(pipe=pipe_name_or_id))
+            else:
+                await client.pipe_remove_copy(pipe["name"], node_uid)
+                click.echo(FeedbackManager.success_pipe_unlinked(pipe=pipe["name"]))
+
+        if pipe["type"] == PipeTypes.DATA_SINK:
+            click.echo(FeedbackManager.info_unlinking_sink_pipe(pipe=pipe["name"]))
+
+            if not node_uid:
+                for node in pipe["nodes"]:
+                    if node["node_type"] == "sink":
+                        node_uid = node["id"]
+                        break
+
+            if not node_uid:
+                raise CLIPipeException(FeedbackManager.error_unlinking_pipe_not_linked(pipe=pipe_name_or_id))
+            else:
+                await client.pipe_remove_sink(pipe["name"], node_uid)
+                click.echo(FeedbackManager.success_pipe_unlinked(pipe=pipe["name"]))
+
+    except Exception as e:
+        raise CLIPipeException(FeedbackManager.error_exception(error=e))
+
+
 @pipe.command(name="append")
 @click.argument("pipe_name_or_uid")
 @click.argument("sql")

{tinybird-cli-5.1.1.dev1 → tinybird-cli-5.1.1.dev3}/tinybird/tb_cli_modules/workspace.py

@@ -151,7 +151,7 @@ async def clear_workspace(ctx: Context, yes: bool, dry_run: bool) -> None:
                             break
 
                     if node_id:
-                        click.echo(FeedbackManager.info_materialized_unlinking_pipe(pipe=pipe["name"]))
+                        click.echo(FeedbackManager.info_unlinking_materialized_pipe(pipe=pipe["name"]))
                         try:
                             await client.pipe_unlink_materialized(pipe["name"], node_id)
                         except DoesNotExistException:

{tinybird-cli-5.1.1.dev1 → tinybird-cli-5.1.1.dev3}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.1.1.dev1
+Version: 5.1.1.dev3
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,6 +18,16 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.1.1.dev3
+***********
+
+- `Improved` No need for `--node` param in `tb pipe populate`
+
+5.1.1.dev2
+************
+
+- `Added` `tb pipe unlink` command
+
 5.1.1.dev1
 ***********