tinybird-cli 5.7.1.dev0__tar.gz → 5.8.0.dev0__tar.gz

{tinybird-cli-5.7.1.dev0 → tinybird-cli-5.8.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.7.1.dev0
+Version: 5.8.0.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,6 +18,11 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.8.0.dev0
+***********
+
+- `Added` support to `TAGS` in `tb pull` and `tb push`. Allows tagging resources for filtering in the UI.
+
 5.7.0
 **********
 

{tinybird-cli-5.7.1.dev0 → tinybird-cli-5.8.0.dev0}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '5.7.1.dev0'
-__revision__ = '2608aa4'
+__version__ = '5.8.0.dev0'
+__revision__ = 'ea7486c'

{tinybird-cli-5.7.1.dev0 → tinybird-cli-5.8.0.dev0}/tinybird/client.py

@@ -1210,3 +1210,31 @@ class TinyB(object):
 
     async def check_auth_login(self) -> Dict[str, Any]:
         return await self._req("/v0/auth")
+
+    async def get_all_tags(self) -> Dict[str, Any]:
+        return await self._req("/v0/tags")
+
+    async def create_tag_with_resource(self, name: str, resourceId: str, resourceName: str, resource_type: str):
+        return await self._req(
+            "/v0/tags",
+            method="POST",
+            headers={"Content-Type": "application/json"},
+            data=json.dumps(
+                {
+                    "name": name,
+                    "resources": [{"id": resourceId, "name": resourceName, "type": resource_type}],
+                }
+            ),
+        )
+
+    async def update_tag(self, name: str, resources: List[Dict[str, Any]]):
+        await self._req(
+            f"/v0/tags/{name}",
+            method="PUT",
+            headers={"Content-Type": "application/json"},
+            data=json.dumps(
+                {
+                    "resources": resources,
+                }
+            ),
+        )

{tinybird-cli-5.7.1.dev0 → tinybird-cli-5.8.0.dev0}/tinybird/datafile.py

@@ -760,6 +760,7 @@ class Datafile:
         self.includes: Dict[str, Any] = {}
         self.shared_with: List[str] = []
         self.warnings: List[str] = []
+        self.filtering_tags: Optional[List[str]] = None
 
     def validate(self) -> None:
         for x in self.nodes:
@@ -903,6 +904,32 @@ def eval_var(s: str, skip: bool = False) -> str:
     return Template(s).safe_substitute(os.environ)
 
 
+def parse_tags(tags: str) -> Tuple[Dict[str, str], List[str]]:
+    """
+    Parses a string of tags into:
+    - kv_tags: a dictionary of key-value tags: the previous tags we have for operational purposes. They
+        have the format key=value&key2=value2 (with_staging=true&with_last_date=true)
+    - filtering_tags: a list of tags that are used for filtering.
+
+    Example: "with_staging=true&with_last_date=true,billing,stats" ->
+        kv_tags = {"with_staging": "true", "with_last_date": "true"}
+        filtering_tags = ["billing", "stats"]
+    """
+    kv_tags = {}
+    filtering_tags = []
+
+    entries = tags.split(",")
+    for entry in entries:
+        trimmed_entry = entry.strip()
+        if "=" in trimmed_entry:
+            the_tags = {k: v[0] for k, v in urllib.parse.parse_qs(trimmed_entry).items()}
+            kv_tags.update(the_tags)
+        else:
+            filtering_tags.append(trimmed_entry)
+
+    return kv_tags, filtering_tags
+
+
 def parse(
     s: str,
     default_node: Optional[str] = None,
@@ -1100,6 +1127,16 @@ def parse(
 
         return _f
 
+    def tags(*args: str, **kwargs: Any) -> None:
+        # Pipe nodes or Data Sources
+        if parser_state.current_node:
+            assign_node_var("tags")(*args, **kwargs)
+        else:
+            # Pipes, at pipe level
+            raw_tags = _unquote((" ".join(args)).strip())
+            _, filtering_tags = parse_tags(raw_tags)
+            doc.filtering_tags = filtering_tags
+
     cmds = {
         "from": assign("from"),
         "source": sources,
@@ -1120,7 +1157,7 @@ def parse(
         "description": description,
         "type": assign_node_var("type"),
         "datasource": assign_node_var("datasource"),
-        "tags": assign_node_var("tags"),
+        "tags": tags,
         "target_datasource": assign_node_var("target_datasource"),
         "copy_schedule": assign_node_var(CopyParameters.COPY_SCHEDULE),
         "copy_mode": assign_node_var("mode"),
@@ -1515,8 +1552,9 @@ async def process_file(
             del params["format"]
 
         if "tags" in node:
-            tags = {k: v[0] for k, v in urllib.parse.parse_qs(node["tags"]).items()}
+            tags, filtering_tags = parse_tags(node["tags"])
             params.update(tags)
+            doc.filtering_tags = filtering_tags
 
         resources: List[Dict[str, Any]] = []
 
@@ -1530,6 +1568,7 @@ async def process_file(
                 "deps": deps,
                 "tokens": doc.tokens,
                 "shared_with": doc.shared_with,
+                "filtering_tags": doc.filtering_tags,
             }
         )
 
@@ -1615,7 +1654,7 @@ async def process_file(
                 sql = re.sub("([\t \\n']+|^)" + old + "([\t \\n'\\)]+|$)", "\\1" + new + "\\2", sql)
 
             if "tags" in node:
-                tags = {k: v[0] for k, v in urllib.parse.parse_qs(node["tags"]).items()}
+                tags, _ = parse_tags(node["tags"])  # Only interested in the "old" tags
                 params.update(tags)
 
             nodes.append(
@@ -1638,6 +1677,7 @@ async def process_file(
                 "tokens": doc.tokens,
                 "description": description,
                 "warnings": doc.warnings,
+                "filtering_tags": doc.filtering_tags,
             }
         ]
     elif DataFileExtensions.TOKEN in filename:
@@ -2646,6 +2686,107 @@ def show_materialized_view_warnings(warnings):
         )
 
 
+async def update_tags(resourceId: str, resourceName: str, resource_type: str, tags: List[str], tb_client: TinyB):
+    def get_tags_for_resource(all_tags: dict, resource_id: str, resource_name: str) -> List[str]:
+        tag_names = []
+
+        for tag in all_tags.get("tags", []):
+            for resource in tag.get("resources", []):
+                if resource.get("id") == resource_id or resource.get("name") == resource_name:
+                    tag_names.append(tag.get("name"))
+                    break  # No need to check other resources in this tag
+
+        return tag_names
+
+    def get_tag(all_tags: dict, tag_name: str) -> Optional[dict]:
+        for tag in all_tags.get("tags", []):
+            if tag.get("name") == tag_name:
+                return tag
+        return None
+
+    def compare_tags(current_tags: List[str], new_tags: List[str]) -> Tuple[List[str], List[str]]:
+        tags_to_add = list(set(new_tags) - set(current_tags))
+        tags_to_remove = list(set(current_tags) - set(new_tags))
+        return tags_to_add, tags_to_remove
+
+    try:
+        all_tags = await tb_client.get_all_tags()
+    except Exception as e:
+        raise Exception(FeedbackManager.error_getting_tags(error=str(e)))
+
+    # Get all tags of that resource
+    current_tags = get_tags_for_resource(all_tags, resourceId, resourceName)
+
+    # Get the tags to add and remove
+    tags_to_add, tags_to_remove = compare_tags(current_tags, tags)
+
+    # Tags to add
+    for tag_name in tags_to_add:
+        tag = get_tag(all_tags, tag_name)
+
+        if not tag:
+            # Create new tag
+            try:
+                await tb_client.create_tag_with_resource(tag_name, resourceId, resourceName, resource_type)
+            except Exception as e:
+                raise Exception(FeedbackManager.error_creating_tag(error=str(e)))
+        else:
+            # Update tag with new resource
+            resources = tag.get("resources", [])
+            resources.append({"id": resourceId, "name": resourceName, "type": resource_type})
+            try:
+                await tb_client.update_tag(tag.get("name", tag_name), resources)
+            except Exception as e:
+                raise Exception(FeedbackManager.error_updating_tag(error=str(e)))
+
+    # Tags to delete
+    for tag_name in tags_to_remove:
+        tag = get_tag(all_tags, tag_name)
+
+        if tag:
+            resources = tag.get("resources", [])
+            resources = [resource for resource in resources if resource.get("name") != resourceName]
+            try:
+                await tb_client.update_tag(tag.get("name", tag_name), resources)
+            except Exception as e:
+                raise Exception(FeedbackManager.error_updating_tag(error=str(e)))
+
+
+async def update_tags_in_resource(rs: Dict[str, Any], resource_type: str, client: TinyB):
+    filtering_tags = rs.get("filtering_tags", [])
+
+    if not filtering_tags:
+        return
+
+    resource_id = ""
+    resource_name = ""
+
+    if resource_type == "datasource":
+        ds_name = rs["params"]["name"]
+        try:
+            persisted_ds = await client.get_datasource(ds_name)
+            resource_id = persisted_ds.get("id", "")
+            resource_name = persisted_ds.get("name", "")
+        except DoesNotExistException:
+            click.echo(
+                FeedbackManager.error_tag_generic("Could not get the latest Data Source info for updating its tags.")
+            )
+    elif resource_type == "pipe":
+        pipe_name = rs["name"]
+        try:
+            persisted_pipe = await client.pipe(pipe_name)
+            resource_id = persisted_pipe.get("id", "")
+            resource_name = persisted_pipe.get("name", "")
+        except DoesNotExistException:
+            click.echo(FeedbackManager.error_tag_generic("Could not get the latest Pipe info for updating its tags."))
+
+    if resource_id and resource_name:
+        try:
+            await update_tags(resource_id, resource_name, resource_type, filtering_tags, client)
+        except Exception as e:
+            click.echo(FeedbackManager.error_tag_generic(error=str(e)))
+
+
 async def get_token_from_main_branch(branch_tb_client: TinyB) -> Optional[str]:
     token_from_main_branch = None
     current_workspace = await branch_tb_client.workspace_info()
@@ -3289,32 +3430,33 @@ async def new_ds(
 
     if alter_response and make_changes:
         # alter operation finished
-        return
-    # removed replacing by default. When a datasource is removed data is
-    # removed and all the references needs to be updated
-    if (
-        os.getenv("TB_I_KNOW_WHAT_I_AM_DOING")
-        and click.prompt(FeedbackManager.info_ask_for_datasource_confirmation()) == ds_name
-    ):  # TODO move to CLI
-        try:
-            await client.datasource_delete(ds_name)
-            click.echo(FeedbackManager.success_delete_datasource(datasource=ds_name))
-        except Exception:
-            raise click.ClickException(FeedbackManager.error_removing_datasource(datasource=ds_name))
-        return
+        pass
     else:
-        if alter_error_message:
-            raise click.ClickException(
-                FeedbackManager.error_datasource_already_exists_and_alter_failed(
-                    datasource=ds_name, alter_error_message=alter_error_message
-                )
-            )
-        if promote_error_message:
-            raise click.ClickException(
-                FeedbackManager.error_promoting_datasource(datasource=ds_name, error=promote_error_message)
-            )
+        # removed replacing by default. When a datasource is removed data is
+        # removed and all the references needs to be updated
+        if (
+            os.getenv("TB_I_KNOW_WHAT_I_AM_DOING")
+            and click.prompt(FeedbackManager.info_ask_for_datasource_confirmation()) == ds_name
+        ):  # TODO move to CLI
+            try:
+                await client.datasource_delete(ds_name)
+                click.echo(FeedbackManager.success_delete_datasource(datasource=ds_name))
+            except Exception:
+                raise click.ClickException(FeedbackManager.error_removing_datasource(datasource=ds_name))
+            return
         else:
-            click.echo(FeedbackManager.warning_datasource_already_exists(datasource=ds_name))
+            if alter_error_message:
+                raise click.ClickException(
+                    FeedbackManager.error_datasource_already_exists_and_alter_failed(
+                        datasource=ds_name, alter_error_message=alter_error_message
+                    )
+                )
+            if promote_error_message:
+                raise click.ClickException(
+                    FeedbackManager.error_promoting_datasource(datasource=ds_name, error=promote_error_message)
+                )
+            else:
+                click.echo(FeedbackManager.warning_datasource_already_exists(datasource=ds_name))
 
 
 async def new_token(token: Dict[str, Any], client: TinyB, force: bool = False):
@@ -3400,6 +3542,7 @@ async def exec_file(
             fork_downstream=fork_downstream,
             fork=fork,
         )
+        await update_tags_in_resource(r, "pipe", tb_client)
     elif r["resource"] == "datasources":
         await new_ds(
             r,
@@ -3412,6 +3555,7 @@ async def exec_file(
             fork=fork,
             git_release=git_release,
         )
+        await update_tags_in_resource(r, "datasource", tb_client)
     elif r["resource"] == "tokens":
         await new_token(r, tb_client, force)
     else:

{tinybird-cli-5.7.1.dev0 → tinybird-cli-5.8.0.dev0}/tinybird/feedback_manager.py

@@ -392,6 +392,10 @@ class FeedbackManager:
         "Several materialized nodes in pipe {pipe}. Use --node param."
     )
     error_unlinking_pipe_not_linked = error_message("** {pipe} is not linked (MV, Copy, or Sink).")
+    error_getting_tags = error_message("Error getting tags: {error}")
+    error_creating_tag = error_message("Error creating new tag: {error}")
+    error_updating_tag = error_message("Error updating tag: {error}")
+    error_tag_generic = error_message("There was an issue updating tags. {error}")
 
     info_incl_relative_path = info_message("** Relative path {path} does not exist, skipping.")
     info_ignoring_incl_file = info_message(

{tinybird-cli-5.7.1.dev0 → tinybird-cli-5.8.0.dev0}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.7.1.dev0
+Version: 5.8.0.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,6 +18,11 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.8.0.dev0
+***********
+
+- `Added` support to `TAGS` in `tb pull` and `tb push`. Allows tagging resources for filtering in the UI.
+
 5.7.0
 **********