tinybird-cli 5.7.0__tar.gz → 5.8.0__tar.gz

{tinybird-cli-5.7.0 → tinybird-cli-5.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.7.0
+Version: 5.8.0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,6 +18,14 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.8.0
+**********
+
+- `Added` `tb datasource scheduling` commands to manage the scheduling of a Data Source
+- `Added` `tb tag` commands to manage tags from the CLI.
+- `Added` support to `TAGS` in `tb fmt`.
+- `Added` support to `TAGS` in `tb pull` and `tb push`. Allows tagging resources for filtering in the UI.
+
 5.7.0
 **********
 

{tinybird-cli-5.7.0 → tinybird-cli-5.8.0}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '5.7.0'
-__revision__ = '2608aa4'
+__version__ = '5.8.0'
+__revision__ = 'cee7088'

{tinybird-cli-5.7.0 → tinybird-cli-5.8.0}/tinybird/client.py

@@ -103,12 +103,20 @@ class TinyB(object):
         self.semver = semver
 
     async def _req(
-        self, endpoint: str, data=None, files=None, method: str = "GET", retries: int = LIMIT_RETRIES, **kwargs
+        self,
+        endpoint: str,
+        data=None,
+        files=None,
+        method: str = "GET",
+        retries: int = LIMIT_RETRIES,
+        use_token: Optional[str] = None,
+        **kwargs,
     ):  # noqa: C901
         url = f"{self.host.strip('/')}/{endpoint.strip('/')}"
 
-        if self.token:
-            url += ("&" if "?" in endpoint else "?") + "token=" + self.token
+        token_to_use = use_token if use_token else self.token
+        if token_to_use:
+            url += ("&" if "?" in endpoint else "?") + "token=" + token_to_use
         if self.version:
             url += ("&" if "?" in url else "?") + "cli_version=" + quote(self.version)
         if self.semver:
@@ -162,7 +170,7 @@ class TinyB(object):
 
         if response.status_code == 403:
             error = parse_error_response(response)
-            if not self.token:
+            if not token_to_use:
                 raise AuthNoTokenException(f"Forbidden: {error}")
             raise AuthException(f"Forbidden: {error}")
         if response.status_code == 204 or response.status_code == 205:
@@ -466,6 +474,24 @@ class TinyB(object):
     async def datasource_sync(self, datasource_id: str):
         return await self._req(f"/v0/datasources/{datasource_id}/scheduling/runs", method="POST", data="")
 
+    async def datasource_scheduling_state(self, datasource_id: str):
+        response = await self._req(f"/v0/datasources/{datasource_id}/scheduling/state", method="GET")
+        return response["state"]
+
+    async def datasource_scheduling_pause(self, datasource_id: str):
+        return await self._req(
+            f"/v0/datasources/{datasource_id}/scheduling/state",
+            method="PUT",
+            data='{"state": "paused"}',
+        )
+
+    async def datasource_scheduling_resume(self, datasource_id: str):
+        return await self._req(
+            f"/v0/datasources/{datasource_id}/scheduling/state",
+            method="PUT",
+            data='{"state": "running"}',
+        )
+
     async def datasource_exchange(self, datasource_a: str, datasource_b: str):
         payload = {"datasource_a": datasource_a, "datasource_b": datasource_b}
         return await self._req("/v0/datasources/exchange", method="POST", data=payload)
@@ -1210,3 +1236,47 @@ class TinyB(object):
 
     async def check_auth_login(self) -> Dict[str, Any]:
         return await self._req("/v0/auth")
+
+    async def get_all_tags(self, token: Optional[str] = None) -> Dict[str, Any]:
+        return await self._req("/v0/tags", use_token=token)
+
+    async def create_tag_with_resource(
+        self, name: str, resource_id: str, resource_name: str, resource_type: str, token: Optional[str] = None
+    ):
+        return await self._req(
+            "/v0/tags",
+            method="POST",
+            headers={"Content-Type": "application/json"},
+            data=json.dumps(
+                {
+                    "name": name,
+                    "resources": [{"id": resource_id, "name": resource_name, "type": resource_type}],
+                }
+            ),
+            use_token=token,
+        )
+
+    async def create_tag(self, name: str, token: Optional[str] = None):
+        return await self._req(
+            "/v0/tags",
+            method="POST",
+            headers={"Content-Type": "application/json"},
+            data=json.dumps({"name": name}),
+            use_token=token,
+        )
+
+    async def update_tag(self, name: str, resources: List[Dict[str, Any]], token: Optional[str] = None):
+        await self._req(
+            f"/v0/tags/{name}",
+            method="PUT",
+            headers={"Content-Type": "application/json"},
+            data=json.dumps(
+                {
+                    "resources": resources,
+                }
+            ),
+            use_token=token,
+        )
+
+    async def delete_tag(self, name: str, token: Optional[str] = None):
+        await self._req(f"/v0/tags/{name}", method="DELETE", use_token=token)

{tinybird-cli-5.7.0 → tinybird-cli-5.8.0}/tinybird/datafile.py

@@ -760,6 +760,7 @@ class Datafile:
         self.includes: Dict[str, Any] = {}
         self.shared_with: List[str] = []
         self.warnings: List[str] = []
+        self.filtering_tags: Optional[List[str]] = None
 
     def validate(self) -> None:
         for x in self.nodes:
@@ -903,6 +904,33 @@ def eval_var(s: str, skip: bool = False) -> str:
     return Template(s).safe_substitute(os.environ)
 
 
+def parse_tags(tags: str) -> Tuple[str, List[str]]:
+    """
+    Parses a string of tags into:
+    - kv_tags: a string of key-value tags: the previous tags we have for operational purposes. It
+        has the format key=value&key2=value2 (with_staging=true&with_last_date=true)
+    - filtering_tags: a list of tags that are used for filtering.
+
+    Example: "with_staging=true&with_last_date=true,billing,stats" ->
+        kv_tags = {"with_staging": "true", "with_last_date": "true"}
+        filtering_tags = ["billing", "stats"]
+    """
+    kv_tags = []
+    filtering_tags = []
+
+    entries = tags.split(",")
+    for entry in entries:
+        trimmed_entry = entry.strip()
+        if "=" in trimmed_entry:
+            kv_tags.append(trimmed_entry)
+        else:
+            filtering_tags.append(trimmed_entry)
+
+    all_kv_tags = "&".join(kv_tags)
+
+    return all_kv_tags, filtering_tags
+
+
 def parse(
     s: str,
     default_node: Optional[str] = None,
@@ -1100,6 +1128,21 @@ def parse(
 
         return _f
 
+    def tags(*args: str, **kwargs: Any) -> None:
+        raw_tags = _unquote((" ".join(args)).strip())
+        operational_tags, filtering_tags = parse_tags(raw_tags)
+
+        # Pipe nodes or Data Sources
+        if parser_state.current_node and operational_tags:
+            operational_tags_args = (operational_tags,)
+            assign_node_var("tags")(*operational_tags_args, **kwargs)
+
+        if filtering_tags:
+            if doc.filtering_tags is None:
+                doc.filtering_tags = filtering_tags
+            else:
+                doc.filtering_tags += filtering_tags
+
     cmds = {
         "from": assign("from"),
         "source": sources,
@@ -1120,7 +1163,7 @@ def parse(
         "description": description,
         "type": assign_node_var("type"),
         "datasource": assign_node_var("datasource"),
-        "tags": assign_node_var("tags"),
+        "tags": tags,
         "target_datasource": assign_node_var("target_datasource"),
         "copy_schedule": assign_node_var(CopyParameters.COPY_SCHEDULE),
         "copy_mode": assign_node_var("mode"),
@@ -1530,6 +1573,7 @@ async def process_file(
                 "deps": deps,
                 "tokens": doc.tokens,
                 "shared_with": doc.shared_with,
+                "filtering_tags": doc.filtering_tags,
             }
         )
 
@@ -1638,6 +1682,7 @@ async def process_file(
                 "tokens": doc.tokens,
                 "description": description,
                 "warnings": doc.warnings,
+                "filtering_tags": doc.filtering_tags,
             }
         ]
     elif DataFileExtensions.TOKEN in filename:
@@ -2646,6 +2691,121 @@ def show_materialized_view_warnings(warnings):
         )
 
 
+async def update_tags(resource_id: str, resource_name: str, resource_type: str, tags: List[str], tb_client: TinyB):
+    def get_tags_for_resource(all_tags: dict, resource_id: str, resource_name: str) -> List[str]:
+        tag_names = []
+
+        for tag in all_tags.get("tags", []):
+            for resource in tag.get("resources", []):
+                if resource.get("id") == resource_id or resource.get("name") == resource_name:
+                    tag_names.append(tag.get("name"))
+                    break  # No need to check other resources in this tag
+
+        return tag_names
+
+    def get_tag(all_tags: dict, tag_name: str) -> Optional[dict]:
+        for tag in all_tags.get("tags", []):
+            if tag.get("name") == tag_name:
+                return tag
+        return None
+
+    def compare_tags(current_tags: List[str], new_tags: List[str]) -> Tuple[List[str], List[str]]:
+        tags_to_add = list(set(new_tags) - set(current_tags))
+        tags_to_remove = list(set(current_tags) - set(new_tags))
+        return tags_to_add, tags_to_remove
+
+    token_from_main = await get_token_from_main_branch(tb_client)
+
+    try:
+        all_tags = await tb_client.get_all_tags(token=token_from_main)
+    except Exception as e:
+        raise Exception(FeedbackManager.error_getting_tags(error=str(e)))
+
+    # Get all tags of that resource
+    current_tags = get_tags_for_resource(all_tags, resource_id, resource_name)
+
+    # Get the tags to add and remove
+    tags_to_add, tags_to_remove = compare_tags(current_tags, tags)
+
+    # Tags to add
+    for tag_name in tags_to_add:
+        tag = get_tag(all_tags, tag_name)
+
+        if not tag:
+            # Create new tag
+            try:
+                await tb_client.create_tag_with_resource(
+                    name=tag_name,
+                    resource_id=resource_id,
+                    resource_name=resource_name,
+                    resource_type=resource_type,
+                    token=token_from_main,
+                )
+            except Exception as e:
+                raise Exception(FeedbackManager.error_creating_tag(error=str(e)))
+        else:
+            # Update tag with new resource
+            resources = tag.get("resources", [])
+            resources.append({"id": resource_id, "name": resource_name, "type": resource_type})
+            try:
+                await tb_client.update_tag(tag.get("name", tag_name), resources, token=token_from_main)
+            except Exception as e:
+                raise Exception(FeedbackManager.error_updating_tag(error=str(e)))
+
+    # Tags to delete
+    for tag_name in tags_to_remove:
+        tag = get_tag(all_tags, tag_name)
+
+        if tag:
+            resources = tag.get("resources", [])
+            resources = [resource for resource in resources if resource.get("name") != resource_name]
+            try:
+                await tb_client.update_tag(tag.get("name", tag_name), resources, token=token_from_main)
+            except Exception as e:
+                raise Exception(FeedbackManager.error_updating_tag(error=str(e)))
+
+
+async def update_tags_in_resource(rs: Dict[str, Any], resource_type: str, client: TinyB):
+    filtering_tags = rs.get("filtering_tags", [])
+
+    if not filtering_tags:
+        return
+
+    resource_id = ""
+    resource_name = ""
+
+    if resource_type == "datasource":
+        ds_name = rs["params"]["name"]
+        try:
+            persisted_ds = await client.get_datasource(ds_name)
+            resource_id = persisted_ds.get("id", "")
+            resource_name = persisted_ds.get("name", "")
+        except DoesNotExistException:
+            click.echo(
+                FeedbackManager.error_tag_generic("Could not get the latest Data Source info for updating its tags.")
+            )
+    elif resource_type == "pipe":
+        pipe_name = rs["name"]
+        try:
+            persisted_pipe = await client.pipe(pipe_name)
+            resource_id = persisted_pipe.get("id", "")
+            resource_name = persisted_pipe.get("name", "")
+        except DoesNotExistException:
+            click.echo(FeedbackManager.error_tag_generic("Could not get the latest Pipe info for updating its tags."))
+
+    if resource_id and resource_name:
+        try:
+            await update_tags(
+                resource_id=resource_id,
+                resource_name=resource_name,
+                resource_type=resource_type,
+                tags=filtering_tags,
+                tb_client=client,
+            )
+        except Exception as e:
+            click.echo(FeedbackManager.error_tag_generic(error=str(e)))
+
+
 async def get_token_from_main_branch(branch_tb_client: TinyB) -> Optional[str]:
     token_from_main_branch = None
     current_workspace = await branch_tb_client.workspace_info()
@@ -3289,32 +3449,33 @@ async def new_ds(
 
     if alter_response and make_changes:
         # alter operation finished
-        return
-    # removed replacing by default. When a datasource is removed data is
-    # removed and all the references needs to be updated
-    if (
-        os.getenv("TB_I_KNOW_WHAT_I_AM_DOING")
-        and click.prompt(FeedbackManager.info_ask_for_datasource_confirmation()) == ds_name
-    ):  # TODO move to CLI
-        try:
-            await client.datasource_delete(ds_name)
-            click.echo(FeedbackManager.success_delete_datasource(datasource=ds_name))
-        except Exception:
-            raise click.ClickException(FeedbackManager.error_removing_datasource(datasource=ds_name))
-        return
+        pass
     else:
-        if alter_error_message:
-            raise click.ClickException(
-                FeedbackManager.error_datasource_already_exists_and_alter_failed(
-                    datasource=ds_name, alter_error_message=alter_error_message
-                )
-            )
-        if promote_error_message:
-            raise click.ClickException(
-                FeedbackManager.error_promoting_datasource(datasource=ds_name, error=promote_error_message)
-            )
+        # removed replacing by default. When a datasource is removed data is
+        # removed and all the references needs to be updated
+        if (
+            os.getenv("TB_I_KNOW_WHAT_I_AM_DOING")
+            and click.prompt(FeedbackManager.info_ask_for_datasource_confirmation()) == ds_name
+        ):  # TODO move to CLI
+            try:
+                await client.datasource_delete(ds_name)
+                click.echo(FeedbackManager.success_delete_datasource(datasource=ds_name))
+            except Exception:
+                raise click.ClickException(FeedbackManager.error_removing_datasource(datasource=ds_name))
+            return
         else:
-            click.echo(FeedbackManager.warning_datasource_already_exists(datasource=ds_name))
+            if alter_error_message:
+                raise click.ClickException(
+                    FeedbackManager.error_datasource_already_exists_and_alter_failed(
+                        datasource=ds_name, alter_error_message=alter_error_message
+                    )
+                )
+            if promote_error_message:
+                raise click.ClickException(
+                    FeedbackManager.error_promoting_datasource(datasource=ds_name, error=promote_error_message)
+                )
+            else:
+                click.echo(FeedbackManager.warning_datasource_already_exists(datasource=ds_name))
 
 
 async def new_token(token: Dict[str, Any], client: TinyB, force: bool = False):
@@ -3400,6 +3561,7 @@ async def exec_file(
             fork_downstream=fork_downstream,
             fork=fork,
         )
+        await update_tags_in_resource(r, "pipe", tb_client)
     elif r["resource"] == "datasources":
         await new_ds(
             r,
@@ -3412,6 +3574,7 @@ async def exec_file(
             fork=fork,
             git_release=git_release,
         )
+        await update_tags_in_resource(r, "datasource", tb_client)
     elif r["resource"] == "tokens":
         await new_token(r, tb_client, force)
     else:
@@ -4659,19 +4822,20 @@ async def format_datasource(
         if for_deploy_diff:
             format_description(file_parts, doc)
         format_tokens(file_parts, doc)
+        format_tags(file_parts, doc)
         format_schema(file_parts, doc.nodes[0])
         format_indices(file_parts, doc.nodes[0])
         await format_engine(file_parts, doc.nodes[0], only_ttl=True if not for_deploy_diff else False, client=client)
         if for_deploy_diff:
             format_import_settings(file_parts, doc.nodes[0])
         format_shared_with(file_parts, doc)
-
     else:
         format_sources(file_parts, doc)
         format_maintainer(file_parts, doc)
         format_version(file_parts, doc)
         format_description(file_parts, doc)
         format_tokens(file_parts, doc)
+        format_tags(file_parts, doc)
         format_schema(file_parts, doc.nodes[0])
         format_indices(file_parts, doc.nodes[0])
         await format_engine(file_parts, doc.nodes[0])
@@ -4747,6 +4911,14 @@ def format_shared_with(file_parts: List[str], doc: Datafile) -> List[str]:
     return file_parts
 
 
+def format_tags(file_parts: List[str], doc: Datafile) -> List[str]:
+    if doc.filtering_tags:
+        file_parts.append(f'TAGS {", ".join(doc.filtering_tags)}')
+        file_parts.append(DATAFILE_NEW_LINE)
+        file_parts.append(DATAFILE_NEW_LINE)
+    return file_parts
+
+
 async def format_engine(
     file_parts: List[str], node: Dict[str, Any], only_ttl: bool = False, client: Optional[TinyB] = None
 ) -> List[str]:
@@ -4883,6 +5055,7 @@ async def format_pipe(
     format_version(file_parts, doc)
     format_description(file_parts, doc)
     format_tokens(file_parts, doc)
+    format_tags(file_parts, doc)
     if doc.includes and not unroll_includes:
         for k in doc.includes:
             # We filter only the include files as we currently have 2 items for each include

{tinybird-cli-5.7.0 → tinybird-cli-5.8.0}/tinybird/feedback_manager.py

@@ -157,6 +157,15 @@ class FeedbackManager:
     error_creating_copy_job = error_message("Failed creating copy job: {error}")
     error_pausing_copy_pipe = error_message("Failed pausing copy pipe: {error}")
     error_resuming_copy_pipe = error_message("Failed resuming copy pipe: {error}")
+    error_pausing_datasource_scheduling = error_message(
+        "Failed pausing scheduling for Data Source '{datasource}': {error}"
+    )
+    error_resuming_datasource_scheduling = error_message(
+        "Failed resuming scheduling for Data Source '{datasource}': {error}"
+    )
+    error_datasource_scheduling_state = error_message(
+        "Failed requesting scheduling state for Data Source '{datasource}': {error}"
+    )
     error_creating_pipe = error_message("Failed creating pipe {error}")
     error_creating_sink_job = error_message("Failed creating sink job: {error}")
     error_running_on_demand_sink_job = error_message("Failed running on-demand sink job: {error}")
@@ -392,6 +401,11 @@ class FeedbackManager:
         "Several materialized nodes in pipe {pipe}. Use --node param."
     )
     error_unlinking_pipe_not_linked = error_message("** {pipe} is not linked (MV, Copy, or Sink).")
+    error_getting_tags = error_message("Error getting tags: {error}")
+    error_creating_tag = error_message("Error creating new tag: {error}")
+    error_updating_tag = error_message("Error updating tag: {error}")
+    error_tag_generic = error_message("There was an issue updating tags. {error}")
+    error_tag_not_found = error_message("Tag {tag_name} not found.")
 
     info_incl_relative_path = info_message("** Relative path {path} does not exist, skipping.")
     info_ignoring_incl_file = info_message(
@@ -606,6 +620,12 @@ Ready? """
     warning_pipe_restricted_params = warning_message(
         "** The parameter names {words} and '{last_word}' are reserved words. Please, choose another name or the pipe will not work as expected."
     )
+    warning_tag_remove_no_resources = prompt_message(
+        "Tag {tag_name} is not used by any resource. Do you want to remove it?"
+    )
+    warning_tag_remove = prompt_message(
+        "Tag {tag_name} is used by {resources_len} resources. Do you want to remove it?"
+    )
 
     info_fixtures_branch = info_message("** Data Fixtures are only pushed to Branches")
     info_materialize_push_datasource_exists = warning_message("** Data Source {name} already exists")
@@ -670,6 +690,9 @@ Ready? """
     info_datasource_title = print_message("** {title}", bcolors.BOLD)
     info_datasource_row = info_message("{row}")
     info_datasource_delete_rows_job_url = info_message("** Delete rows job url {url}")
+    info_datasource_scheduling_state = info_message("** Scheduling state for Data Source '{datasource}': {state}")
+    info_datasource_scheduling_pause = info_message("** Pausing scheduling...")
+    info_datasource_scheduling_resume = info_message("** Resuming scheduling...")
     info_pipes = info_message("** Pipes:")
     info_pipe_name = info_message("** - {pipe}")
     info_using_node = print_message("** Using last node {node} as endpoint")
@@ -797,6 +820,8 @@ Ready? """
     info_release_rollback = info_message(
         "** The following resources IDs are present in the {semver} Release and will be restored:"
     )
+    info_tag_list = info_message("** Tags:")
+    info_tag_resources = info_message("** Resources tagged by {tag_name}:")
     warning_no_release = warning_message(
         "** Warning: Workspace does not have Releases, run `tb init --git` to activate them."
     )
@@ -920,6 +945,8 @@ Ready? """
     success_datasource_unshared = success_message(
         "** The Data Source {datasource} has been correctly unshared from {workspace}"
     )
+    success_datasource_scheduling_resumed = success_message("""** Scheduling resumed for Data Source '{datasource}'""")
+    success_datasource_scheduling_paused = success_message("""** Scheduling paused for Data Source '{datasource}'""")
     success_connection_created = success_message("** Connection {id} created successfully!")
 
     # TODO: Update the message when the .env feature is implemented
@@ -981,5 +1008,7 @@ Ready? """
     success_delete_token = success_message("** Token '{token}' removed successfully")
     success_refresh_token = success_message("** Token '{token}' refreshed successfully")
     success_copy_token = success_message("** Token '{token}' copied to clipboard")
+    success_tag_created = success_message("** Tag '{tag_name}' created!")
+    success_tag_removed = success_message("** Tag '{tag_name}' removed!")
 
     debug_running_file = print_message("** Running {file}", bcolors.CGREY)

{tinybird-cli-5.7.0 → tinybird-cli-5.8.0}/tinybird/sql_template.py

@@ -386,14 +386,14 @@ def array_type(types):  # noqa: C901
             for i, t in enumerate(list_values):
                 if _type in testers:
                     if testers[_type](str(t)):
-                        values.append(expression_wrapper(types[_type](t), f"{x}[{i}]"))
+                        values.append(expression_wrapper(types[_type](t), str(t)))
                     else:
                         raise SQLTemplateException(
                             f"Error validating {x}[{i}]({t}) to type {_type}",
                             documentation="/cli/advanced-templates.html",
                         )
                 else:
-                    values.append(expression_wrapper(types.get(_type, lambda x: x)(t), f"{x}[{i}]"))
+                    values.append(expression_wrapper(types.get(_type, lambda x: x)(t), str(t)))
             return Expression(f"[{','.join(map(str, values))}]")
         except AttributeError as e:
             logging.warning(f"AttributeError on Array: {e}")
@@ -1314,8 +1314,9 @@ def expression_wrapper(x, name, escape_arrays: bool = False):
     elif isinstance(x, Comment):
         return "-- {x} \n"
     if x is None:
+        truncated_name = name[:20] + "..." if len(name) > 20 else name
         raise SQLTemplateException(
-            f'expression "{name}" evaluated to null', documentation="/cli/advanced-templates.html"
+            f'expression "{truncated_name}" evaluated to null', documentation="/cli/advanced-templates.html"
         )
     if isinstance(x, list) and escape_arrays:
         logging.warning(f"expression_wrapper -> list :{x}:")
@@ -1992,6 +1993,14 @@ def render_sql_template(
     Traceback (most recent call last):
     ...
     tinybird.sql_template.SQLTemplateException: Template Syntax Error: expression "test" evaluated to null
+    >>> render_sql_template("SELECT {{testisasuperlongthingandwedontwanttoreturnthefullthing}}", {'token':'testing'})
+    Traceback (most recent call last):
+    ...
+    tinybird.sql_template.SQLTemplateException: Template Syntax Error: expression "testisasuperlongthin..." evaluated to null
+    >>> render_sql_template("SELECT {{ Array(embedding, 'Float32') }}", {'token':'testing', 'embedding': '1,2,3,4, null'})
+    Traceback (most recent call last):
+    ...
+    tinybird.sql_template.SQLTemplateException: Template Syntax Error: Error validating 1,2,3,4, null[4]( null) to type Float32
     >>> render_sql_template('{% if test %}SELECT 1{% else %} select 2 {% end %}')
     (' select 2 ', {}, [])
     >>> render_sql_template('{% if Int32(test, 1) %}SELECT 1{% else %} select 2 {% end %}')

{tinybird-cli-5.7.0 → tinybird-cli-5.8.0}/tinybird/sql_toolset.py

@@ -293,7 +293,7 @@ def replace_tables(
                     and is_invalid_resource(r, database, default_database, _replaced_with, valid_tables)
                 ):
                     logging.info(
-                        "Resource not found in replace_tables: %s",
+                        "Resource not found in replace_tables in sql_toolset: %s",
                         {
                             "r": r,
                             "default_database": default_database,

{tinybird-cli-5.7.0 → tinybird-cli-5.8.0}/tinybird/tb_cli.py

@@ -10,8 +10,10 @@ import tinybird.tb_cli_modules.cli
 import tinybird.tb_cli_modules.common
 import tinybird.tb_cli_modules.connection
 import tinybird.tb_cli_modules.datasource
+import tinybird.tb_cli_modules.fmt
 import tinybird.tb_cli_modules.job
 import tinybird.tb_cli_modules.pipe
+import tinybird.tb_cli_modules.tag
 import tinybird.tb_cli_modules.test
 import tinybird.tb_cli_modules.token
 import tinybird.tb_cli_modules.workspace

{tinybird-cli-5.7.0 → tinybird-cli-5.8.0}/tinybird/tb_cli_modules/cli.py

@@ -3,7 +3,6 @@
 # - If it makes sense and only when strictly necessary, you can create utility functions in this file.
 # - But please, **do not** interleave utility functions and command definitions.
 
-import difflib
 import json
 import logging
 import os
@@ -36,21 +35,16 @@ from tinybird.datafile import (
     Datafile,
     ParseException,
     build_graph,
-    color_diff,
     create_release,
     diff_command,
     folder_pull,
     folder_push,
-    format_datasource,
-    format_pipe,
     get_project_filenames,
     get_resource_versions,
     has_internal_datafiles,
-    is_file_a_datasource,
     parse_datasource,
     parse_pipe,
     parse_token,
-    peek,
     wait_job,
 )
 from tinybird.feedback_manager import FeedbackManager
@@ -753,84 +747,6 @@ async def dependencies(
         raise CLIException(FeedbackManager.error_partial_replace_cant_be_executed(datasource=datasource))
 
 
-@cli.command()
-@click.argument("filenames", type=click.Path(exists=True), nargs=-1, required=True)
-@click.option(
-    "--line-length",
-    is_flag=False,
-    default=100,
-    help="A number indicating the maximum characters per line in the node SQL, lines will be splitted based on the SQL syntax and the number of characters passed as a parameter",
-)
-@click.option("--dry-run", is_flag=True, default=False, help="Don't ask to override the local file")
-@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation to overwrite the local file")
-@click.option(
-    "--diff",
-    is_flag=True,
-    default=False,
-    help="Formats local file, prints the diff and exits 1 if different, 0 if equal",
-)
-@click.pass_context
-@coro
-async def fmt(
-    ctx: Context, filenames: List[str], line_length: int, dry_run: bool, yes: bool, diff: bool
-) -> Optional[str]:
-    """
-    Formats a .datasource, .pipe or .incl file
-
-    This command removes comments starting with # from the file, use DESCRIPTION instead.
-
-    The format command tries to parse the datafile so syntax errors might rise.
-
-    .incl files must contain a NODE definition
-    """
-
-    result = ""
-    failed = []
-    for filename in filenames:
-        if not diff:
-            click.echo(filename)
-        extensions = Path(filename).suffixes
-        if is_file_a_datasource(filename):
-            result = await format_datasource(filename, skip_eval=True)
-        elif (".pipe" in extensions) or (".incl" in extensions):
-            result = await format_pipe(filename, line_length, skip_eval=True)
-        else:
-            click.echo("Unsupported file type. Supported files types are: .pipe, .incl and .datasource")
-            return None
-
-        if diff:
-            result = result.rstrip("\n")
-            lines_fmt = [f"{line}\n" for line in result.split("\n")]
-            with open(filename, "r") as file:
-                lines_file = file.readlines()
-            diff_result = difflib.unified_diff(
-                lines_file, lines_fmt, fromfile=f"{Path(filename).name} local", tofile="fmt datafile"
-            )
-            diff_result = color_diff(diff_result)
-            not_empty, diff_lines = peek(diff_result)
-            if not_empty:
-                sys.stdout.writelines(diff_lines)
-                failed.append(filename)
-                click.echo("")
-        else:
-            click.echo(result)
-            if dry_run:
-                return None
-
-            if yes or click.confirm(FeedbackManager.prompt_override_local_file(name=filename)):
-                with open(f"{filename}", "w") as file:
-                    file.write(result)
-
-                click.echo(FeedbackManager.success_generated_local_file(file=filename))
-
-    if len(failed):
-        click.echo(FeedbackManager.error_failed_to_format_files(number=len(failed)))
-        for f in failed:
-            click.echo(f"tb fmt {f} --yes")
-        sys.exit(1)
-    return result
-
-
 @cli.command(
     name="diff",
     short_help="Diffs local datafiles to the corresponding remote files in the workspace. For the case of .datasource files it just diffs VERSION and SCHEMA, since ENGINE, KAFKA or other metadata is considered immutable.",

{tinybird-cli-5.7.0 → tinybird-cli-5.8.0}/tinybird/tb_cli_modules/datasource.py

@@ -805,3 +805,65 @@ async def datasource_copy_from_main(
     if wait:
         base_msg = "Copy from Main Workspace" if sql_from_main else f"Copy from {sql}"
         await wait_job(client, job_id, job_url, f"{base_msg} to {datasource_name}")
+
+
+@datasource.group(name="scheduling")
+@click.pass_context
+def datasource_scheduling(ctx: Context) -> None:
+    """Data Source scheduling commands."""
+
+
+@datasource_scheduling.command(name="state")
+@click.argument("datasource_name")
+@click.pass_context
+@coro
+async def datasource_scheduling_state(ctx: Context, datasource_name: str) -> None:
+    """Get the scheduling state of a Data Source."""
+    client: TinyB = ctx.obj["client"]
+    try:
+        state = await client.datasource_scheduling_state(datasource_name)
+        click.echo(FeedbackManager.info_datasource_scheduling_state(datasource=datasource_name, state=state))
+    except Exception as e:
+        raise CLIDatasourceException(
+            FeedbackManager.error_datasource_scheduling_state(datasource=datasource_name, error=e)
+        )
+
+
+@datasource_scheduling.command(name="pause")
+@click.argument("datasource_name")
+@click.pass_context
+@coro
+async def datasource_scheduling_pause(ctx: Context, datasource_name: str) -> None:
+    """Pause the scheduling of a Data Source."""
+
+    click.echo(FeedbackManager.info_datasource_scheduling_pause())
+    client: TinyB = ctx.ensure_object(dict)["client"]
+
+    try:
+        await client.datasource_scheduling_pause(datasource_name)
+        click.echo(FeedbackManager.success_datasource_scheduling_paused(datasource=datasource_name))
+
+    except Exception as e:
+        raise CLIDatasourceException(
+            FeedbackManager.error_pausing_datasource_scheduling(datasource=datasource_name, error=e)
+        )
+
+
+@datasource_scheduling.command(name="resume")
+@click.argument("datasource_name")
+@click.pass_context
+@coro
+async def datasource_scheduling_resume(ctx: Context, datasource_name: str) -> None:
+    """Resume the scheduling of a Data Source."""
+
+    click.echo(FeedbackManager.info_datasource_scheduling_resume())
+    client: TinyB = ctx.ensure_object(dict)["client"]
+
+    try:
+        await client.datasource_scheduling_resume(datasource_name)
+        click.echo(FeedbackManager.success_datasource_scheduling_resumed(datasource=datasource_name))
+
+    except Exception as e:
+        raise CLIDatasourceException(
+            FeedbackManager.error_resuming_datasource_scheduling(datasource=datasource_name, error=e)
+        )

tinybird-cli-5.8.0/tinybird/tb_cli_modules/fmt.py

@@ -0,0 +1,90 @@
+import difflib
+import sys
+from pathlib import Path
+from typing import List, Optional
+
+import click
+from click import Context
+
+from tinybird.datafile import color_diff, format_datasource, format_pipe, is_file_a_datasource, peek
+from tinybird.feedback_manager import FeedbackManager
+from tinybird.tb_cli_modules.cli import cli
+from tinybird.tb_cli_modules.common import coro
+
+
+@cli.command()
+@click.argument("filenames", type=click.Path(exists=True), nargs=-1, required=True)
+@click.option(
+    "--line-length",
+    is_flag=False,
+    default=100,
+    help="A number indicating the maximum characters per line in the node SQL, lines will be splitted based on the SQL syntax and the number of characters passed as a parameter",
+)
+@click.option("--dry-run", is_flag=True, default=False, help="Don't ask to override the local file")
+@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation to overwrite the local file")
+@click.option(
+    "--diff",
+    is_flag=True,
+    default=False,
+    help="Formats local file, prints the diff and exits 1 if different, 0 if equal",
+)
+@click.pass_context
+@coro
+async def fmt(
+    ctx: Context, filenames: List[str], line_length: int, dry_run: bool, yes: bool, diff: bool
+) -> Optional[str]:
+    """
+    Formats a .datasource, .pipe or .incl file
+
+    This command removes comments starting with # from the file, use DESCRIPTION instead.
+
+    The format command tries to parse the datafile so syntax errors might rise.
+
+    .incl files must contain a NODE definition
+    """
+
+    result = ""
+    failed = []
+    for filename in filenames:
+        if not diff:
+            click.echo(filename)
+        extensions = Path(filename).suffixes
+        if is_file_a_datasource(filename):
+            result = await format_datasource(filename, skip_eval=True)
+        elif (".pipe" in extensions) or (".incl" in extensions):
+            result = await format_pipe(filename, line_length, skip_eval=True)
+        else:
+            click.echo("Unsupported file type. Supported files types are: .pipe, .incl and .datasource")
+            return None
+
+        if diff:
+            result = result.rstrip("\n")
+            lines_fmt = [f"{line}\n" for line in result.split("\n")]
+            with open(filename, "r") as file:
+                lines_file = file.readlines()
+            diff_result = difflib.unified_diff(
+                lines_file, lines_fmt, fromfile=f"{Path(filename).name} local", tofile="fmt datafile"
+            )
+            diff_result = color_diff(diff_result)
+            not_empty, diff_lines = peek(diff_result)
+            if not_empty:
+                sys.stdout.writelines(diff_lines)
+                failed.append(filename)
+                click.echo("")
+        else:
+            click.echo(result)
+            if dry_run:
+                return None
+
+            if yes or click.confirm(FeedbackManager.prompt_override_local_file(name=filename)):
+                with open(f"{filename}", "w") as file:
+                    file.write(result)
+
+                click.echo(FeedbackManager.success_generated_local_file(file=filename))
+
+    if len(failed):
+        click.echo(FeedbackManager.error_failed_to_format_files(number=len(failed)))
+        for f in failed:
+            click.echo(f"tb fmt {f} --yes")
+        sys.exit(1)
+    return result

tinybird-cli-5.8.0/tinybird/tb_cli_modules/tag.py

@@ -0,0 +1,116 @@
+from typing import Optional
+
+import click
+from click import Context
+
+from tinybird.feedback_manager import FeedbackManager
+from tinybird.tb_cli_modules.cli import cli
+from tinybird.tb_cli_modules.common import (
+    coro,
+    echo_safe_humanfriendly_tables_format_smart_table,
+    get_current_main_workspace,
+)
+
+
+@cli.group()
+@click.pass_context
+def tag(ctx: Context) -> None:
+    """Tag commands"""
+
+
+@tag.command(name="ls")
+@click.argument("tag_name", required=False)
+@click.pass_context
+@coro
+async def tag_ls(ctx: Context, tag_name: Optional[str]) -> None:
+    """List all the tags of the current Workspace or the resources associated to a specific tag."""
+
+    client = ctx.ensure_object(dict)["client"]
+    config = ctx.ensure_object(dict)["config"]
+    main_workspace = await get_current_main_workspace(client, config)
+    token = main_workspace.get("token") if main_workspace else None
+
+    response = await client.get_all_tags(token=token)
+
+    if tag_name:
+        the_tag = [tag for tag in response["tags"] if tag["name"] == tag_name]
+
+        columns = ["name", "id", "type"]
+        table = []
+
+        if len(the_tag) > 0:
+            for resource in the_tag[0]["resources"]:
+                table.append([resource["name"], resource["id"], resource["type"]])
+
+        click.echo(FeedbackManager.info_tag_resources(tag_name=tag_name))
+        echo_safe_humanfriendly_tables_format_smart_table(table, column_names=columns)
+        return
+
+    columns = ["tag", "resources"]
+    table = []
+
+    for tag in response["tags"]:
+        unique_resources = []
+        for resource in tag["resources"]:
+            if resource.get("name", "") not in unique_resources:
+                unique_resources.append(resource)  # Reducing by name in case there are duplicates.
+        table.append([tag["name"], len(unique_resources)])
+
+    click.echo(FeedbackManager.info_tag_list())
+    echo_safe_humanfriendly_tables_format_smart_table(table, column_names=columns)
+
+
+@tag.command(name="create")
+@click.argument("tag_name")
+@click.pass_context
+@coro
+async def tag_create(ctx: Context, tag_name: str) -> None:
+    """Create a tag in the current Workspace."""
+
+    client = ctx.ensure_object(dict)["client"]
+    config = ctx.ensure_object(dict)["config"]
+    main_workspace = await get_current_main_workspace(client, config)
+    token = main_workspace.get("token") if main_workspace else None
+
+    await client.create_tag(name=tag_name, token=token)
+
+    click.echo(FeedbackManager.success_tag_created(tag_name=tag_name))
+
+
+@tag.command(name="rm")
+@click.argument("tag_name")
+@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation to delete the tag.")
+@click.pass_context
+@coro
+async def tag_rm(ctx: Context, tag_name: str, yes: bool) -> None:
+    """Remove a tag from the current Workspace."""
+
+    client = ctx.ensure_object(dict)["client"]
+    config = ctx.ensure_object(dict)["config"]
+    main_workspace = await get_current_main_workspace(client, config)
+    token = main_workspace.get("token") if main_workspace else None
+
+    remove_tag = True
+
+    if not yes:
+        all_tags = await client.get_all_tags(token=token)
+        the_tag = [tag for tag in all_tags["tags"] if tag["name"] == tag_name]
+        if len(the_tag) > 0:
+            unique_resources = []
+            for resource in the_tag[0]["resources"]:
+                if resource.get("name", "") not in unique_resources:
+                    unique_resources.append(resource)  # Reducing by name in case there are duplicates.
+
+            if len(unique_resources) > 0:
+                remove_tag = click.confirm(
+                    FeedbackManager.warning_tag_remove(tag_name=tag_name, resources_len=len(unique_resources))
+                )
+            else:
+                remove_tag = click.confirm(FeedbackManager.warning_tag_remove_no_resources(tag_name=tag_name))
+        else:
+            remove_tag = False
+            click.echo(FeedbackManager.error_tag_not_found(tag_name=tag_name))
+
+    if remove_tag:
+        await client.delete_tag(tag_name, token=token)
+        click.echo(FeedbackManager.success_tag_removed(tag_name=tag_name))

{tinybird-cli-5.7.0 → tinybird-cli-5.8.0}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.7.0
+Version: 5.8.0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,6 +18,14 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.8.0
+**********
+
+- `Added` `tb datasource scheduling` commands to manage the scheduling of a Data Source
+- `Added` `tb tag` commands to manage tags from the CLI.
+- `Added` support to `TAGS` in `tb fmt`.
+- `Added` support to `TAGS` in `tb pull` and `tb push`. Allows tagging resources for filtering in the UI.
+
 5.7.0
 **********
 

{tinybird-cli-5.7.0 → tinybird-cli-5.8.0}/tinybird_cli.egg-info/SOURCES.txt

@@ -26,9 +26,11 @@ tinybird/tb_cli_modules/config.py
 tinybird/tb_cli_modules/connection.py
 tinybird/tb_cli_modules/datasource.py
 tinybird/tb_cli_modules/exceptions.py
+tinybird/tb_cli_modules/fmt.py
 tinybird/tb_cli_modules/job.py
 tinybird/tb_cli_modules/pipe.py
 tinybird/tb_cli_modules/regions.py
+tinybird/tb_cli_modules/tag.py
 tinybird/tb_cli_modules/telemetry.py
 tinybird/tb_cli_modules/test.py
 tinybird/tb_cli_modules/token.py