tinybird-cli 2.0.1.dev0__tar.gz → 2.1.0.dev2__tar.gz

{tinybird-cli-2.0.1.dev0 → tinybird-cli-2.1.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 2.0.1.dev0
+Version: 2.1.0.dev2
 Summary: Tinybird Command Line Tool
 Home-page: https://docs.tinybird.co/cli.html
 Author: Tinybird
@@ -19,6 +19,16 @@ Changelog
 
 ---------
 
+2.1.0.dev2
+************
+
+- `Changed` Internal changes in `tb deploy`
+
+2.1.0.dev1
+************
+
+- `Added` `tb deploy` removes resources from datafiles deleted
+
 2.0.0
 ************
 

{tinybird-cli-2.0.1.dev0 → tinybird-cli-2.1.0.dev2}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://docs.tinybird.co/cli.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '2.0.1.dev0'
-__revision__ = '40c7bee'
+__version__ = '2.1.0.dev2'
+__revision__ = '8400007'

{tinybird-cli-2.0.1.dev0 → tinybird-cli-2.1.0.dev2}/tinybird/client.py

@@ -1084,6 +1084,9 @@ class TinyB(object):
         }
         return await self._req(f"/v0/workspaces/{workspace_id}/releases/?{urlencode(params)}", method="POST", data="")
 
+    async def release_failed(self, workspace_id: str, semver: str):
+        return await self._req(f"/v0/workspaces/{workspace_id}/releases/{semver}?status=failed", method="PUT")
+
     async def release_preview(self, workspace_id: str, semver: str):
         return await self._req(f"/v0/workspaces/{workspace_id}/releases/{semver}?status=preview", method="PUT")
 

{tinybird-cli-2.0.1.dev0 → tinybird-cli-2.1.0.dev2}/tinybird/datafile.py

@@ -52,7 +52,7 @@ from tinybird.tb_cli_modules.common import _get_tb_client, get_current_main_work
 
 from .tornado_template import UnClosedIfError
 from .sql import parse_table_structure, schema_to_sql_columns
-from .client import TinyB, DoesNotExistException
+from .client import TinyB, DoesNotExistException, AuthException, CanNotBeDeletedException
 from .sql_template import render_sql_template, get_used_tables_in_template
 from tinybird.sql_template_fmt import format_sql_template
 from .feedback_manager import FeedbackManager
@@ -410,6 +410,9 @@ class CLIGitRelease:
             for filename in filenames
         }
 
+    def get_deleted_from_diffs(self, diffs: List[Diff]) -> List[str]:
+        return [diff.a_path for diff in diffs if diff.change_type == self.ChangeType.DELETED.value and diff.a_path]
+
     async def update_release(
         self, tb_client: TinyB, current_ws: Dict[str, Any], commit: Optional[str] = None
     ) -> Dict[str, Any]:
@@ -454,6 +457,7 @@ class Deployment:
             if not diffs:
                 click.echo(FeedbackManager.info_git_release_no_diffs())
             changed = self.cli_git_release.get_changes_from_diffs(diffs, filenames)
+            deleted = self.cli_git_release.get_deleted_from_diffs(diffs)
 
         else:
             # only changes flow
@@ -462,7 +466,8 @@ class Deployment:
                 if config
                 else None
             )
-        return changed
+            deleted = []
+        return changed, deleted
 
     def preparing_release(self):
         if not self.is_git_release:
@@ -499,11 +504,70 @@ class Deployment:
         if not self.is_git_release or self.dry_run or has_semver:
             return
         self.cli_git_release = self.cli_git_release or CLIGitRelease()
-        release = await self.tb_client.workspace_commit_update(
-            self.current_ws["id"], self.cli_git_release.head().commit.hexsha if not commit else commit
-        )
+        release = await self.cli_git_release.update_release(self.tb_client, self.current_ws, commit)
         click.echo(FeedbackManager.success_git_release(release_commit=release["commit"]))
 
+    async def delete_resources(self, deleted: List[str], remote_pipes: List[Dict[str, Any]], dry_run: bool = False):
+        async def delete_pipe(resource_name_or_id: str, dry_run: bool = False):
+            if dry_run:
+                click.echo(
+                    FeedbackManager.info_deleting_resource(dry_run="[DRY RUN] ", resource_name=resource_name_or_id)
+                )
+            else:
+                click.echo(FeedbackManager.info_deleting_resource(dry_run="", resource_name=resource_name_or_id))
+                try:
+                    await self.tb_client.pipe_delete(resource_name_or_id)
+                except AuthException as e:
+                    raise CLIGitReleaseException(str(e))
+                else:
+                    click.echo(FeedbackManager.success_delete(name=resource_name_or_id))
+
+        async def delete_datasource(resource_name_or_id: str, dry_run: bool = False):
+            if dry_run:
+                click.echo(
+                    FeedbackManager.info_deleting_resource(dry_run="[DRY RUN] ", resource_name=resource_name_or_id)
+                )
+            else:
+                click.echo(FeedbackManager.info_deleting_resource(dry_run="", resource_name=resource_name_or_id))
+                try:
+                    await self.tb_client.datasource_delete(resource_name_or_id)
+                except CanNotBeDeletedException as e:
+                    raise CLIGitReleaseException(str(e))
+                else:
+                    click.echo(FeedbackManager.success_delete(name=resource_name_or_id))
+
+        resource_extension_processor = {".datasource": delete_datasource, ".pipe": delete_pipe}
+
+        # builds pipes deps the other way around from API to use with toposort
+        # i.e {'analytics_hits': ['analytics_pages', 'trend', 'analytics_sources', 'analytics_sessions']}
+        pipes_deps: Dict[str, List[str]] = {}
+        remote_pipes_type: Dict[str, str] = {}
+        for pipe in remote_pipes:
+            pipes_deps[pipe["name"]] = []
+            remote_pipes_type[pipe["name"]] = pipe["type"]
+            for node in pipe["nodes"]:
+                for dep in node["dependencies"]:
+                    if dep in [pipe["name"] for pipe in remote_pipes]:
+                        pipes_deps.setdefault(dep, []).append(pipe["name"])
+        deleted_resources: Dict[str, str] = {Path(resource).resolve().stem: resource for resource in deleted}
+
+        for group in toposort(pipes_deps):
+            # for each level keep materialized for the end
+            for pipe_name in sorted(group, key=lambda x: remote_pipes_type[x] == "materialized"):
+                if pipe_name in deleted_resources:
+                    resource_path = deleted_resources.pop(pipe_name)
+                    try:
+                        await resource_extension_processor[Path(resource_path).suffix](pipe_name, dry_run)
+                    except KeyError:
+                        raise CLIGitReleaseException(FeedbackManager.error_file_extension(resource_path))
+
+        # Delete pending resources (datasources)
+        for name, path in deleted_resources.items():
+            try:
+                await resource_extension_processor[Path(path).suffix](name, dry_run)
+            except KeyError:
+                raise CLIGitReleaseException(FeedbackManager.error_file_extension(path))
+
 
 class Datafile:
     def __init__(self) -> None:
@@ -3407,7 +3471,7 @@ async def folder_push(
                 workspace_lib_paths.append((x.name, x))
 
     datasources: List[Dict[str, Any]] = await tb_client.datasources()
-    pipes: List[Dict[str, Any]] = await tb_client.pipes()
+    pipes: List[Dict[str, Any]] = await tb_client.pipes(dependencies=True)
 
     existing_resources: List[str] = [x["name"] for x in datasources] + [x["name"] for x in pipes]
     # replace workspace mapping names
@@ -3429,7 +3493,7 @@ async def folder_push(
         filenames = get_project_filenames(folder)
 
     # get the list of changes
-    changed = await deployment.detect_changes(filenames, only_changes, config)
+    changed, deleted = await deployment.detect_changes(filenames, only_changes, config)
 
     deployment.preparing_release()
 
@@ -3668,9 +3732,12 @@ async def folder_push(
     if deployment.is_git_release:
         deployment.deploying_dry_run()
         await push_files(dry_run=True)
+
+        await deployment.delete_resources(deleted, pipes, dry_run=True)
         if not deployment.dry_run:
             deployment.deploying()
             await push_files(dry_run)
+            await deployment.delete_resources(deleted, pipes)
     else:
         await push_files(dry_run)
 
@@ -4427,3 +4494,23 @@ def is_datasource(resource: Dict[str, Any]) -> bool:
     if resource and resource.get("resource") == "datasources":
         return True
     return False
+
+
+async def create_release(client: TinyB, config: Dict[str, Any], semver: str, folder: Optional[str] = None) -> None:
+    if not folder:
+        folder = getcwd()
+    cli_git_release = None
+    try:
+        cli_git_release = CLIGitRelease(path=folder)
+        commit = cli_git_release.head_commit()
+    except CLIGitReleaseException:
+        raise CLIGitReleaseException(FeedbackManager.error_no_git_repo_for_init(repo_path=folder))
+
+    await client.release_new(config["id"], semver, commit)
+    click.echo(FeedbackManager.success_deployment_release(semver=semver))
+
+
+def has_internal_datafiles(folder: str) -> bool:
+    folder = folder or "."
+    filenames = get_project_filenames(folder)
+    return any([f for f in filenames if "spans" in str(f) and "vendor" not in str(f)])

{tinybird-cli-2.0.1.dev0 → tinybird-cli-2.1.0.dev2}/tinybird/feedback_manager.py

@@ -313,6 +313,15 @@ class FeedbackManager:
         "There are resources renamed. `tb deploy` can't deploy renamed resources, create new resources instead."
     )
 
+    info_create_release = info_message(
+        "** Release semver is bumped. Will use --fork-downstream to deploy changes. 💡 Hint: --fork-downstream creates new versions of the dependent downstream resources of the changed resources."
+    )
+    info_alter_release = info_message(
+        "** Release semver is not bumped. Changes are deployed to the current live Release. 💡 Hint: To deploy to a new Release edit the VERSION env variable in the .tinyenv file of the Data Project."
+    )
+    info_releases_detected = info_message(
+        "** Live Release: {current_semver} => New Release: {semver} 💡 Hint: To deploy to a new Release edit the VERSION env variable in the .tinyenv file of the Data Project."
+    )
     info_pre_prompt_auth_login_user_token = info_message(
         "ℹ️ In order to log into Tinybird, you need your user token. Please, go to {host}/tokens/ and paste your User Token here."
     )
@@ -436,7 +445,9 @@ Ready? """
     )
     warning_for_cicd_file = warning_message("** 🚨 Warning 🚨: {warning_message}")
     warning_unknown_response = warning_message("** Warning. Unknown response from server: {response}")
+    warning_resource_not_in_workspace = warning_message("** Warning: '{resource_name}' not found in workspace")
 
+    info_fixtures_branch = info_message("** Data Fixtures are only pushed to Branches")
     info_materialize_push_datasource_exists = warning_message("** Data Source {name} already exists")
     info_materialize_push_datasource_override = prompt_message(
         "Delete the Data Source from the workspace and push {name} again?"
@@ -589,6 +600,7 @@ Ready? """
     info_detected_changes_from_includes = info_message("** Changes from includes:")
     info_processing_from_include = info_message("\t{include_filename} => {filename}")
     info_deps_for_resource = info_message("\t{resource} => '{dep}'")
+    info_deleting_resource = info_message("** {dry_run}Deleting '{resource_name}'")
 
     info_cicd_generation_cancelled_by_user = info_message("** CI/CD files generation cancelled by user.")
     info_skipping_sharing_datasources_environment = info_message(
@@ -697,6 +709,7 @@ Ready? """
     success_sink_job_finished = success_message("""** Data sinked to '{bucket_path}'""")
     success_print_pipe = success_message("** Pipe: {pipe}")
     success_create = success_message("** '{name}' created")
+    success_delete = success_message("** '{name}' deleted")
     success_progress_blocks = success_message("** \N{front-facing baby chick} done")
     success_now_using_config = success_message("** Now using {name} ({id})")
     success_connector_config = success_message(

{tinybird-cli-2.0.1.dev0 → tinybird-cli-2.1.0.dev2}/tinybird/tb_cli_modules/branch.py

@@ -34,7 +34,7 @@ from tinybird.tb_cli_modules.common import (
     get_current_workspace,
 )
 from tinybird.feedback_manager import FeedbackManager
-from tinybird.datafile import wait_job, CLIGitRelease, CLIGitReleaseException
+from tinybird.datafile import wait_job, CLIGitRelease, CLIGitReleaseException, create_release
 from tinybird.tb_cli_modules.exceptions import CLIBranchException, CLIException, CLIReleaseException
 
 
@@ -145,15 +145,7 @@ async def release_create(ctx: Context, semver: str) -> None:
     client: TinyB = ctx.ensure_object(dict)["client"]
     config = ctx.ensure_object(dict)["config"]
     folder = getcwd()
-    cli_git_release = None
-    try:
-        cli_git_release = CLIGitRelease(path=folder)
-        commit = cli_git_release.head_commit()
-    except CLIGitReleaseException:
-        raise CLIGitReleaseException(FeedbackManager.error_no_git_repo_for_init(repo_path=folder))
-
-    await client.release_new(config["id"], semver, commit)
-    click.echo(FeedbackManager.success_deployment_release(semver=semver))
+    await create_release(client, config, semver, folder)
 
 
 @release.command(name="promote", short_help="Promotes to live status a preview Release")
@@ -177,8 +169,9 @@ async def release_promote(ctx: Context, semver: str) -> None:
     client = _get_tb_client(config["token"], config["host"])
 
     try:
-        await client.release_promote(config["id"], semver)
+        release = await client.release_promote(config["id"], semver)
         click.echo(FeedbackManager.success_release_promote(semver=semver))
+        click.echo(FeedbackManager.success_git_release(release_commit=release["commit"]))
     except Exception as e:
         raise CLIReleaseException(FeedbackManager.error_exception(error=str(e)))
 

{tinybird-cli-2.0.1.dev0 → tinybird-cli-2.1.0.dev2}/tinybird/tb_cli_modules/cli.py

@@ -14,6 +14,7 @@ import re
 import shutil
 import sys
 import click
+from packaging import version
 from click import Context
 
 from pathlib import Path
@@ -29,6 +30,7 @@ from tinybird.datafile import (
     AlreadyExistsException,
     Datafile,
     ParseException,
+    create_release,
     build_graph,
     folder_push,
     folder_pull,
@@ -45,6 +47,7 @@ from tinybird.datafile import (
     peek,
     color_diff,
     parse_token,
+    has_internal_datafiles,
 )
 import tinybird.context as context
 from tinybird.config import FeatureFlags, get_config, VERSION, CURRENT_VERSION, SUPPORTED_CONNECTORS
@@ -114,12 +117,6 @@ DEFAULT_PATTERNS: List[Tuple[str, Union[str, Callable[[str], str]]]] = [
     default=True,
     help="Don't print version warning message if there's a new available version. You can use TB_VERSION_WARNING envar",
 )
-@click.option(
-    "--hide-tokens",
-    is_flag=True,
-    default=False,
-    help="Disable the output of tokens [DEPRECATED: tokens are hidden by default]",
-)
 @click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens")
 @click.version_option(version=VERSION)
 @click.pass_context
@@ -144,18 +141,11 @@ async def cli(
     sf_stage,
     with_headers: bool,
     version_warning: bool,
-    hide_tokens: bool,
     show_tokens: bool,
 ) -> None:
     """
     Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
     """
-    if hide_tokens:
-        click.echo(
-            FeedbackManager.warning_deprecated(
-                warning="The `--hide-tokens` flag is deprecated. Tokens are hidden by default, see `tb --help` for more info."
-            )
-        )
 
     # We need to unpatch for our tests not to break
     if show_tokens:
@@ -586,13 +576,6 @@ def check(filenames: List[str], debug: bool) -> None:
     help="The user token is required for sharing a datasource that contains the SHARED_WITH entry.",
     type=click.types.STRING,
 )
-@click.option(
-    "--is-internal",
-    is_flag=True,
-    default=False,
-    help="Use when working with the Internal workspace as by default users can not create MV from internal datasources",
-    hidden=True,
-)
 @click.pass_context
 @coro
 async def push(
@@ -625,12 +608,14 @@ async def push(
     check_requests_from_main: bool,
     folder: str,
     user_token: Optional[str],
-    is_internal: bool,
 ) -> None:
     """Push files to Tinybird."""
 
     ignore_sql_errors = FeatureFlags.ignore_sql_errors()
     context.disable_template_security_validation.set(True)
+
+    is_internal = has_internal_datafiles(folder)
+
     await folder_push(
         create_tb_client(ctx),
         filenames,
@@ -833,25 +818,14 @@ async def fmt(
     help="Diffs local datafiles to the corresponding remote files in the main workspace. Only works when authenticated on an Environment.",
     hidden=True,
 )
-@click.option(
-    "--production",
-    is_flag=True,
-    default=False,
-    help="Diffs local datafiles to the corresponding remote files in the main workspace. Only works when authenticated on an Environment. (deprecated)",
-    hidden=True,
-)
 @click.pass_context
 @coro
 async def diff(
-    ctx: Context, filename: Optional[Tuple], fmt: bool, no_color: bool, no_verbose: bool, main: bool, production: bool
+    ctx: Context, filename: Optional[Tuple], fmt: bool, no_color: bool, no_verbose: bool, main: bool
 ) -> None:
     only_resources_changed = no_verbose
     client: TinyB = ctx.ensure_object(dict)["client"]
 
-    if production:
-        click.echo(FeedbackManager.warning_deprecated(warning="--production flag is deprecated, use --main instead"))
-
-    main = main or production
     if not main:
         changed = await diff_command(
             list(filename) if filename else None, fmt, client, no_color, with_print=not only_resources_changed
@@ -1420,17 +1394,17 @@ async def prompt(_ctx: Context) -> None:
     hidden=True,
 )
 @click.option(
-    "--is-internal",
+    "--only-changes",
     is_flag=True,
     default=False,
-    help="Use when working with the Internal workspace as by default users can not create MV from internal datasources",
+    help="Deprecated, remove when we use `tb deploy` to deploy to main. See => https://gitlab.com/tinybird/analytics/-/issues/9669",
     hidden=True,
 )
 @click.option(
-    "--only-changes",
+    "--v3",
     is_flag=True,
     default=False,
-    help="Deprecated, remove when we use `tb deploy` to deploy to main. See => https://gitlab.com/tinybird/analytics/-/issues/9669",
+    help="Some specific logic to deploy using Releases. Will remove once we go to GA",
     hidden=True,
 )
 @click.pass_context
@@ -1455,13 +1429,80 @@ async def deploy(
     user_token: Optional[str],
     fork_downstream: bool,
     fork: bool,
-    is_internal: bool,
     only_changes: bool,
+    v3: bool,
 ) -> None:
     """Deploy in tinybird pushing resources changed from previous release using git"""
 
     ignore_sql_errors = FeatureFlags.ignore_sql_errors()
     context.disable_template_security_validation.set(True)
+
+    is_internal = has_internal_datafiles(folder)
+
+    if not v3:
+        await folder_push(
+            tb_client=create_tb_client(ctx),
+            dry_run=dry_run,
+            check=False,
+            push_deps=True,
+            debug=debug,
+            force=True,
+            git_release=True,
+            only_changes=only_changes,
+            override_datasource=override_datasource,
+            populate=populate,
+            populate_subset=subset,
+            populate_condition=sql_condition,
+            unlink_on_populate_error=unlink_on_populate_error,
+            upload_fixtures=fixtures,
+            wait=wait,
+            ignore_sql_errors=ignore_sql_errors,
+            skip_confirmation=yes,
+            workspace_map=dict(workspace_map),
+            workspace_lib_paths=workspace,
+            timeout=timeout,
+            run_tests=False,
+            folder=folder,
+            config=ctx.ensure_object(dict)["config"],
+            user_token=user_token,
+            fork_downstream=fork_downstream,
+            fork=fork,
+            is_internal=is_internal,
+        )
+        return
+
+    client: TinyB = ctx.ensure_object(dict)["client"]
+    config = ctx.ensure_object(dict)["config"]
+    workspaces: List[Dict[str, Any]] = (await client.user_workspaces_and_branches()).get("workspaces", [])
+    current_ws: Dict[str, Any] = next(
+        (workspace for workspace in workspaces if config and workspace.get("id", ".") == config.get("id", "..")), {}
+    )
+
+    semver = config.get("semver")
+    is_environment = current_ws.get("is_branch")
+    # FIXME decide when to auto-promote
+    auto_promote = False
+    release = current_ws.get("release", {})
+    current_semver: Optional[str] = "no release in remote Workspace"
+    if release and isinstance(release, dict):
+        current_semver = release.get("semver")
+
+    if not is_environment:
+        # TODO upload fixtures on tb test
+        fixtures = False
+        click.echo(FeedbackManager.info_fixtures_branch())
+
+    release_created = False
+    if semver and current_semver:
+        click.echo(FeedbackManager.info_releases_detected(current_semver=current_semver, semver=semver))
+        if version.parse(semver.split("-snapshot")[0]) > version.parse(current_semver.split("-snapshot")[0]):
+            click.echo(FeedbackManager.info_create_release())
+            await create_release(client, config, semver)
+            release_created = True
+            fork_downstream = True
+        else:
+            click.echo(FeedbackManager.info_alter_release())
+            yes = True
     try:
         await folder_push(
             tb_client=create_tb_client(ctx),
@@ -1492,9 +1533,9 @@ async def deploy(
             fork=fork,
             is_internal=is_internal,
         )
-    except click.ClickException as e:
-        # custom exceptions coming from the API for the git workflow
-        # FIXME: change this in the API
+    except Exception as e:
+        if release_created:
+            await client.release_failed(config["id"], semver)
         if "--override-datasource" in str(e):
             new_exception = type(e)(
                 f"{str(e).replace('. If you want to try to force override the Materialized View, please use the `--override-datasource` flag', '')} If you are creating a new Release update the related .datasource files and use --fork-downstream"
@@ -1502,3 +1543,18 @@ async def deploy(
             raise new_exception
         else:
             raise e
+
+    if release_created:
+        try:
+            await client.release_preview(config["id"], semver)
+            click.echo(FeedbackManager.success_release_preview(semver=semver))
+            if auto_promote:
+                # TODO: define when auto-promote in the main Environment
+                try:
+                    release = await client.release_promote(config["id"], semver)
+                    click.echo(FeedbackManager.success_release_promote(semver=semver))
+                    click.echo(FeedbackManager.success_git_release(release_commit=release["commit"]))
+                except Exception as e:
+                    raise CLIException(FeedbackManager.error_exception(error=str(e)))
+        except Exception as e:
+            raise CLIException(FeedbackManager.error_exception(error=str(e)))

{tinybird-cli-2.0.1.dev0 → tinybird-cli-2.1.0.dev2}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 2.0.1.dev0
+Version: 2.1.0.dev2
 Summary: Tinybird Command Line Tool
 Home-page: https://docs.tinybird.co/cli.html
 Author: Tinybird
@@ -19,6 +19,16 @@ Changelog
 
 ---------
 
+2.1.0.dev2
+************
+
+- `Changed` Internal changes in `tb deploy`
+
+2.1.0.dev1
+************
+
+- `Added` `tb deploy` removes resources from datafiles deleted
+
 2.0.0
 ************