tinybird-cli 5.7.1.dev0__tar.gz → 5.8.0.dev1__tar.gz

{tinybird-cli-5.7.1.dev0 → tinybird-cli-5.8.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.7.1.dev0
+Version: 5.8.0.dev1
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,6 +18,16 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.8.0.dev1
+***********
+
+- `Added` new `tb tag` command.
+
+5.8.0.dev0
+***********
+
+- `Added` support to `TAGS` in `tb pull` and `tb push`. Allows tagging resources for filtering in the UI.
+
 5.7.0
 **********
 

{tinybird-cli-5.7.1.dev0 → tinybird-cli-5.8.0.dev1}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '5.7.1.dev0'
-__revision__ = '2608aa4'
+__version__ = '5.8.0.dev1'
+__revision__ = 'a3056db'

{tinybird-cli-5.7.1.dev0 → tinybird-cli-5.8.0.dev1}/tinybird/client.py

@@ -103,12 +103,20 @@ class TinyB(object):
         self.semver = semver
 
     async def _req(
-        self, endpoint: str, data=None, files=None, method: str = "GET", retries: int = LIMIT_RETRIES, **kwargs
+        self,
+        endpoint: str,
+        data=None,
+        files=None,
+        method: str = "GET",
+        retries: int = LIMIT_RETRIES,
+        use_token: Optional[str] = None,
+        **kwargs,
     ):  # noqa: C901
         url = f"{self.host.strip('/')}/{endpoint.strip('/')}"
 
-        if self.token:
-            url += ("&" if "?" in endpoint else "?") + "token=" + self.token
+        token_to_use = use_token if use_token else self.token
+        if token_to_use:
+            url += ("&" if "?" in endpoint else "?") + "token=" + token_to_use
         if self.version:
             url += ("&" if "?" in url else "?") + "cli_version=" + quote(self.version)
         if self.semver:
@@ -162,7 +170,7 @@ class TinyB(object):
 
         if response.status_code == 403:
             error = parse_error_response(response)
-            if not self.token:
+            if not token_to_use:
                 raise AuthNoTokenException(f"Forbidden: {error}")
             raise AuthException(f"Forbidden: {error}")
         if response.status_code == 204 or response.status_code == 205:
@@ -1210,3 +1218,47 @@ class TinyB(object):
 
     async def check_auth_login(self) -> Dict[str, Any]:
         return await self._req("/v0/auth")
+
+    async def get_all_tags(self, token: Optional[str] = None) -> Dict[str, Any]:
+        return await self._req("/v0/tags", use_token=token)
+
+    async def create_tag_with_resource(
+        self, name: str, resource_id: str, resource_name: str, resource_type: str, token: Optional[str] = None
+    ):
+        return await self._req(
+            "/v0/tags",
+            method="POST",
+            headers={"Content-Type": "application/json"},
+            data=json.dumps(
+                {
+                    "name": name,
+                    "resources": [{"id": resource_id, "name": resource_name, "type": resource_type}],
+                }
+            ),
+            use_token=token,
+        )
+
+    async def create_tag(self, name: str, token: Optional[str] = None):
+        return await self._req(
+            "/v0/tags",
+            method="POST",
+            headers={"Content-Type": "application/json"},
+            data=json.dumps({"name": name}),
+            use_token=token,
+        )
+
+    async def update_tag(self, name: str, resources: List[Dict[str, Any]], token: Optional[str] = None):
+        await self._req(
+            f"/v0/tags/{name}",
+            method="PUT",
+            headers={"Content-Type": "application/json"},
+            data=json.dumps(
+                {
+                    "resources": resources,
+                }
+            ),
+            use_token=token,
+        )
+
+    async def delete_tag(self, name: str, token: Optional[str] = None):
+        await self._req(f"/v0/tags/{name}", method="DELETE", use_token=token)

{tinybird-cli-5.7.1.dev0 → tinybird-cli-5.8.0.dev1}/tinybird/datafile.py

@@ -760,6 +760,7 @@ class Datafile:
         self.includes: Dict[str, Any] = {}
         self.shared_with: List[str] = []
         self.warnings: List[str] = []
+        self.filtering_tags: Optional[List[str]] = None
 
     def validate(self) -> None:
         for x in self.nodes:
@@ -903,6 +904,32 @@ def eval_var(s: str, skip: bool = False) -> str:
     return Template(s).safe_substitute(os.environ)
 
 
+def parse_tags(tags: str) -> Tuple[Dict[str, str], List[str]]:
+    """
+    Parses a string of tags into:
+    - kv_tags: a dictionary of key-value tags: the previous tags we have for operational purposes. They
+        have the format key=value&key2=value2 (with_staging=true&with_last_date=true)
+    - filtering_tags: a list of tags that are used for filtering.
+
+    Example: "with_staging=true&with_last_date=true,billing,stats" ->
+        kv_tags = {"with_staging": "true", "with_last_date": "true"}
+        filtering_tags = ["billing", "stats"]
+    """
+    kv_tags = {}
+    filtering_tags = []
+
+    entries = tags.split(",")
+    for entry in entries:
+        trimmed_entry = entry.strip()
+        if "=" in trimmed_entry:
+            the_tags = {k: v[0] for k, v in urllib.parse.parse_qs(trimmed_entry).items()}
+            kv_tags.update(the_tags)
+        else:
+            filtering_tags.append(trimmed_entry)
+
+    return kv_tags, filtering_tags
+
+
 def parse(
     s: str,
     default_node: Optional[str] = None,
@@ -1100,6 +1127,16 @@ def parse(
 
         return _f
 
+    def tags(*args: str, **kwargs: Any) -> None:
+        # Pipe nodes or Data Sources
+        if parser_state.current_node:
+            assign_node_var("tags")(*args, **kwargs)
+        else:
+            # Pipes, at pipe level
+            raw_tags = _unquote((" ".join(args)).strip())
+            _, filtering_tags = parse_tags(raw_tags)
+            doc.filtering_tags = filtering_tags
+
     cmds = {
         "from": assign("from"),
         "source": sources,
@@ -1120,7 +1157,7 @@ def parse(
         "description": description,
         "type": assign_node_var("type"),
         "datasource": assign_node_var("datasource"),
-        "tags": assign_node_var("tags"),
+        "tags": tags,
         "target_datasource": assign_node_var("target_datasource"),
         "copy_schedule": assign_node_var(CopyParameters.COPY_SCHEDULE),
         "copy_mode": assign_node_var("mode"),
@@ -1515,8 +1552,9 @@ async def process_file(
             del params["format"]
 
         if "tags" in node:
-            tags = {k: v[0] for k, v in urllib.parse.parse_qs(node["tags"]).items()}
+            tags, filtering_tags = parse_tags(node["tags"])
             params.update(tags)
+            doc.filtering_tags = filtering_tags
 
         resources: List[Dict[str, Any]] = []
 
@@ -1530,6 +1568,7 @@ async def process_file(
                 "deps": deps,
                 "tokens": doc.tokens,
                 "shared_with": doc.shared_with,
+                "filtering_tags": doc.filtering_tags,
             }
         )
 
@@ -1615,7 +1654,7 @@ async def process_file(
                 sql = re.sub("([\t \\n']+|^)" + old + "([\t \\n'\\)]+|$)", "\\1" + new + "\\2", sql)
 
             if "tags" in node:
-                tags = {k: v[0] for k, v in urllib.parse.parse_qs(node["tags"]).items()}
+                tags, _ = parse_tags(node["tags"])  # Only interested in the "old" tags
                 params.update(tags)
 
             nodes.append(
@@ -1638,6 +1677,7 @@ async def process_file(
                 "tokens": doc.tokens,
                 "description": description,
                 "warnings": doc.warnings,
+                "filtering_tags": doc.filtering_tags,
             }
         ]
     elif DataFileExtensions.TOKEN in filename:
@@ -2646,6 +2686,121 @@ def show_materialized_view_warnings(warnings):
         )
 
 
+async def update_tags(resource_id: str, resource_name: str, resource_type: str, tags: List[str], tb_client: TinyB):
+    def get_tags_for_resource(all_tags: dict, resource_id: str, resource_name: str) -> List[str]:
+        tag_names = []
+
+        for tag in all_tags.get("tags", []):
+            for resource in tag.get("resources", []):
+                if resource.get("id") == resource_id or resource.get("name") == resource_name:
+                    tag_names.append(tag.get("name"))
+                    break  # No need to check other resources in this tag
+
+        return tag_names
+
+    def get_tag(all_tags: dict, tag_name: str) -> Optional[dict]:
+        for tag in all_tags.get("tags", []):
+            if tag.get("name") == tag_name:
+                return tag
+        return None
+
+    def compare_tags(current_tags: List[str], new_tags: List[str]) -> Tuple[List[str], List[str]]:
+        tags_to_add = list(set(new_tags) - set(current_tags))
+        tags_to_remove = list(set(current_tags) - set(new_tags))
+        return tags_to_add, tags_to_remove
+
+    token_from_main = await get_token_from_main_branch(tb_client)
+
+    try:
+        all_tags = await tb_client.get_all_tags(token=token_from_main)
+    except Exception as e:
+        raise Exception(FeedbackManager.error_getting_tags(error=str(e)))
+
+    # Get all tags of that resource
+    current_tags = get_tags_for_resource(all_tags, resource_id, resource_name)
+
+    # Get the tags to add and remove
+    tags_to_add, tags_to_remove = compare_tags(current_tags, tags)
+
+    # Tags to add
+    for tag_name in tags_to_add:
+        tag = get_tag(all_tags, tag_name)
+
+        if not tag:
+            # Create new tag
+            try:
+                await tb_client.create_tag_with_resource(
+                    name=tag_name,
+                    resource_id=resource_id,
+                    resource_name=resource_name,
+                    resource_type=resource_type,
+                    token=token_from_main,
+                )
+            except Exception as e:
+                raise Exception(FeedbackManager.error_creating_tag(error=str(e)))
+        else:
+            # Update tag with new resource
+            resources = tag.get("resources", [])
+            resources.append({"id": resource_id, "name": resource_name, "type": resource_type})
+            try:
+                await tb_client.update_tag(tag.get("name", tag_name), resources, token=token_from_main)
+            except Exception as e:
+                raise Exception(FeedbackManager.error_updating_tag(error=str(e)))
+
+    # Tags to delete
+    for tag_name in tags_to_remove:
+        tag = get_tag(all_tags, tag_name)
+
+        if tag:
+            resources = tag.get("resources", [])
+            resources = [resource for resource in resources if resource.get("name") != resource_name]
+            try:
+                await tb_client.update_tag(tag.get("name", tag_name), resources, token=token_from_main)
+            except Exception as e:
+                raise Exception(FeedbackManager.error_updating_tag(error=str(e)))
+
+
+async def update_tags_in_resource(rs: Dict[str, Any], resource_type: str, client: TinyB):
+    filtering_tags = rs.get("filtering_tags", [])
+
+    if not filtering_tags:
+        return
+
+    resource_id = ""
+    resource_name = ""
+
+    if resource_type == "datasource":
+        ds_name = rs["params"]["name"]
+        try:
+            persisted_ds = await client.get_datasource(ds_name)
+            resource_id = persisted_ds.get("id", "")
+            resource_name = persisted_ds.get("name", "")
+        except DoesNotExistException:
+            click.echo(
+                FeedbackManager.error_tag_generic("Could not get the latest Data Source info for updating its tags.")
+            )
+    elif resource_type == "pipe":
+        pipe_name = rs["name"]
+        try:
+            persisted_pipe = await client.pipe(pipe_name)
+            resource_id = persisted_pipe.get("id", "")
+            resource_name = persisted_pipe.get("name", "")
+        except DoesNotExistException:
+            click.echo(FeedbackManager.error_tag_generic("Could not get the latest Pipe info for updating its tags."))
+
+    if resource_id and resource_name:
+        try:
+            await update_tags(
+                resource_id=resource_id,
+                resource_name=resource_name,
+                resource_type=resource_type,
+                tags=filtering_tags,
+                tb_client=client,
+            )
+        except Exception as e:
+            click.echo(FeedbackManager.error_tag_generic(error=str(e)))
+
+
 async def get_token_from_main_branch(branch_tb_client: TinyB) -> Optional[str]:
     token_from_main_branch = None
     current_workspace = await branch_tb_client.workspace_info()
@@ -3289,32 +3444,33 @@ async def new_ds(
 
     if alter_response and make_changes:
         # alter operation finished
-        return
-    # removed replacing by default. When a datasource is removed data is
-    # removed and all the references needs to be updated
-    if (
-        os.getenv("TB_I_KNOW_WHAT_I_AM_DOING")
-        and click.prompt(FeedbackManager.info_ask_for_datasource_confirmation()) == ds_name
-    ):  # TODO move to CLI
-        try:
-            await client.datasource_delete(ds_name)
-            click.echo(FeedbackManager.success_delete_datasource(datasource=ds_name))
-        except Exception:
-            raise click.ClickException(FeedbackManager.error_removing_datasource(datasource=ds_name))
-        return
+        pass
     else:
-        if alter_error_message:
-            raise click.ClickException(
-                FeedbackManager.error_datasource_already_exists_and_alter_failed(
-                    datasource=ds_name, alter_error_message=alter_error_message
-                )
-            )
-        if promote_error_message:
-            raise click.ClickException(
-                FeedbackManager.error_promoting_datasource(datasource=ds_name, error=promote_error_message)
-            )
+        # removed replacing by default. When a datasource is removed data is
+        # removed and all the references needs to be updated
+        if (
+            os.getenv("TB_I_KNOW_WHAT_I_AM_DOING")
+            and click.prompt(FeedbackManager.info_ask_for_datasource_confirmation()) == ds_name
+        ):  # TODO move to CLI
+            try:
+                await client.datasource_delete(ds_name)
+                click.echo(FeedbackManager.success_delete_datasource(datasource=ds_name))
+            except Exception:
+                raise click.ClickException(FeedbackManager.error_removing_datasource(datasource=ds_name))
+            return
         else:
-            click.echo(FeedbackManager.warning_datasource_already_exists(datasource=ds_name))
+            if alter_error_message:
+                raise click.ClickException(
+                    FeedbackManager.error_datasource_already_exists_and_alter_failed(
+                        datasource=ds_name, alter_error_message=alter_error_message
+                    )
+                )
+            if promote_error_message:
+                raise click.ClickException(
+                    FeedbackManager.error_promoting_datasource(datasource=ds_name, error=promote_error_message)
+                )
+            else:
+                click.echo(FeedbackManager.warning_datasource_already_exists(datasource=ds_name))
 
 
 async def new_token(token: Dict[str, Any], client: TinyB, force: bool = False):
@@ -3400,6 +3556,7 @@ async def exec_file(
             fork_downstream=fork_downstream,
             fork=fork,
         )
+        await update_tags_in_resource(r, "pipe", tb_client)
     elif r["resource"] == "datasources":
         await new_ds(
             r,
@@ -3412,6 +3569,7 @@ async def exec_file(
             fork=fork,
             git_release=git_release,
         )
+        await update_tags_in_resource(r, "datasource", tb_client)
     elif r["resource"] == "tokens":
         await new_token(r, tb_client, force)
     else:

{tinybird-cli-5.7.1.dev0 → tinybird-cli-5.8.0.dev1}/tinybird/feedback_manager.py

@@ -392,6 +392,11 @@ class FeedbackManager:
         "Several materialized nodes in pipe {pipe}. Use --node param."
     )
     error_unlinking_pipe_not_linked = error_message("** {pipe} is not linked (MV, Copy, or Sink).")
+    error_getting_tags = error_message("Error getting tags: {error}")
+    error_creating_tag = error_message("Error creating new tag: {error}")
+    error_updating_tag = error_message("Error updating tag: {error}")
+    error_tag_generic = error_message("There was an issue updating tags. {error}")
+    error_tag_not_found = error_message("Tag {tag_name} not found.")
 
     info_incl_relative_path = info_message("** Relative path {path} does not exist, skipping.")
     info_ignoring_incl_file = info_message(
@@ -606,6 +611,12 @@ Ready? """
     warning_pipe_restricted_params = warning_message(
         "** The parameter names {words} and '{last_word}' are reserved words. Please, choose another name or the pipe will not work as expected."
     )
+    warning_tag_remove_no_resources = prompt_message(
+        "Tag {tag_name} is not used by any resource. Do you want to remove it?"
+    )
+    warning_tag_remove = prompt_message(
+        "Tag {tag_name} is used by {resources_len} resources. Do you want to remove it?"
+    )
 
     info_fixtures_branch = info_message("** Data Fixtures are only pushed to Branches")
     info_materialize_push_datasource_exists = warning_message("** Data Source {name} already exists")
@@ -797,6 +808,8 @@ Ready? """
     info_release_rollback = info_message(
         "** The following resources IDs are present in the {semver} Release and will be restored:"
     )
+    info_tag_list = info_message("** Tags:")
+    info_tag_resources = info_message("** Resources tagged by {tag_name}:")
     warning_no_release = warning_message(
         "** Warning: Workspace does not have Releases, run `tb init --git` to activate them."
     )
@@ -981,5 +994,7 @@ Ready? """
     success_delete_token = success_message("** Token '{token}' removed successfully")
     success_refresh_token = success_message("** Token '{token}' refreshed successfully")
     success_copy_token = success_message("** Token '{token}' copied to clipboard")
+    success_tag_created = success_message("** Tag '{tag_name}' created!")
+    success_tag_removed = success_message("** Tag '{tag_name}' removed!")
 
     debug_running_file = print_message("** Running {file}", bcolors.CGREY)

{tinybird-cli-5.7.1.dev0 → tinybird-cli-5.8.0.dev1}/tinybird/tb_cli.py

@@ -12,6 +12,7 @@ import tinybird.tb_cli_modules.connection
 import tinybird.tb_cli_modules.datasource
 import tinybird.tb_cli_modules.job
 import tinybird.tb_cli_modules.pipe
+import tinybird.tb_cli_modules.tag
 import tinybird.tb_cli_modules.test
 import tinybird.tb_cli_modules.token
 import tinybird.tb_cli_modules.workspace

tinybird-cli-5.8.0.dev1/tinybird/tb_cli_modules/tag.py

@@ -0,0 +1,116 @@
+from typing import Optional
+
+import click
+from click import Context
+
+from tinybird.feedback_manager import FeedbackManager
+from tinybird.tb_cli_modules.cli import cli
+from tinybird.tb_cli_modules.common import (
+    coro,
+    echo_safe_humanfriendly_tables_format_smart_table,
+    get_current_main_workspace,
+)
+
+
+@cli.group()
+@click.pass_context
+def tag(ctx: Context) -> None:
+    """Tag commands"""
+
+
+@tag.command(name="ls")
+@click.argument("tag_name", required=False)
+@click.pass_context
+@coro
+async def tag_ls(ctx: Context, tag_name: Optional[str]) -> None:
+    """List all the tags of the current Workspace or the resources associated to a specific tag."""
+
+    client = ctx.ensure_object(dict)["client"]
+    config = ctx.ensure_object(dict)["config"]
+    main_workspace = await get_current_main_workspace(client, config)
+    token = main_workspace.get("token") if main_workspace else None
+
+    response = await client.get_all_tags(token=token)
+
+    if tag_name:
+        the_tag = [tag for tag in response["tags"] if tag["name"] == tag_name]
+
+        columns = ["name", "id", "type"]
+        table = []
+
+        if len(the_tag) > 0:
+            for resource in the_tag[0]["resources"]:
+                table.append([resource["name"], resource["id"], resource["type"]])
+
+        click.echo(FeedbackManager.info_tag_resources(tag_name=tag_name))
+        echo_safe_humanfriendly_tables_format_smart_table(table, column_names=columns)
+        return
+
+    columns = ["tag", "resources"]
+    table = []
+
+    for tag in response["tags"]:
+        unique_resources = []
+        for resource in tag["resources"]:
+            if resource.get("name", "") not in unique_resources:
+                unique_resources.append(resource)  # Reducing by name in case there are duplicates.
+        table.append([tag["name"], len(unique_resources)])
+
+    click.echo(FeedbackManager.info_tag_list())
+    echo_safe_humanfriendly_tables_format_smart_table(table, column_names=columns)
+
+
+@tag.command(name="create")
+@click.argument("tag_name")
+@click.pass_context
+@coro
+async def tag_create(ctx: Context, tag_name: str) -> None:
+    """Create a tag in the current Workspace."""
+
+    client = ctx.ensure_object(dict)["client"]
+    config = ctx.ensure_object(dict)["config"]
+    main_workspace = await get_current_main_workspace(client, config)
+    token = main_workspace.get("token") if main_workspace else None
+
+    await client.create_tag(name=tag_name, token=token)
+
+    click.echo(FeedbackManager.success_tag_created(tag_name=tag_name))
+
+
+@tag.command(name="rm")
+@click.argument("tag_name")
+@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation to delete the tag.")
+@click.pass_context
+@coro
+async def tag_rm(ctx: Context, tag_name: str, yes: bool) -> None:
+    """Remove a tag from the current Workspace."""
+
+    client = ctx.ensure_object(dict)["client"]
+    config = ctx.ensure_object(dict)["config"]
+    main_workspace = await get_current_main_workspace(client, config)
+    token = main_workspace.get("token") if main_workspace else None
+
+    remove_tag = True
+
+    if not yes:
+        all_tags = await client.get_all_tags(token=token)
+        the_tag = [tag for tag in all_tags["tags"] if tag["name"] == tag_name]
+        if len(the_tag) > 0:
+            unique_resources = []
+            for resource in the_tag[0]["resources"]:
+                if resource.get("name", "") not in unique_resources:
+                    unique_resources.append(resource)  # Reducing by name in case there are duplicates.
+
+            if len(unique_resources) > 0:
+                remove_tag = click.confirm(
+                    FeedbackManager.warning_tag_remove(tag_name=tag_name, resources_len=len(unique_resources))
+                )
+            else:
+                remove_tag = click.confirm(FeedbackManager.warning_tag_remove_no_resources(tag_name=tag_name))
+        else:
+            remove_tag = False
+            click.echo(FeedbackManager.error_tag_not_found(tag_name=tag_name))
+
+    if remove_tag:
+        await client.delete_tag(tag_name, token=token)
+        click.echo(FeedbackManager.success_tag_removed(tag_name=tag_name))

{tinybird-cli-5.7.1.dev0 → tinybird-cli-5.8.0.dev1}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.7.1.dev0
+Version: 5.8.0.dev1
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,6 +18,16 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.8.0.dev1
+***********
+
+- `Added` new `tb tag` command.
+
+5.8.0.dev0
+***********
+
+- `Added` support to `TAGS` in `tb pull` and `tb push`. Allows tagging resources for filtering in the UI.
+
 5.7.0
 **********
 

{tinybird-cli-5.7.1.dev0 → tinybird-cli-5.8.0.dev1}/tinybird_cli.egg-info/SOURCES.txt

@@ -29,6 +29,7 @@ tinybird/tb_cli_modules/exceptions.py
 tinybird/tb_cli_modules/job.py
 tinybird/tb_cli_modules/pipe.py
 tinybird/tb_cli_modules/regions.py
+tinybird/tb_cli_modules/tag.py
 tinybird/tb_cli_modules/telemetry.py
 tinybird/tb_cli_modules/test.py
 tinybird/tb_cli_modules/token.py