tinybird-cli 2.0.1.dev0__tar.gz → 2.1.0.dev1__tar.gz

{tinybird-cli-2.0.1.dev0 → tinybird-cli-2.1.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 2.0.1.dev0
+Version: 2.1.0.dev1
 Summary: Tinybird Command Line Tool
 Home-page: https://docs.tinybird.co/cli.html
 Author: Tinybird
@@ -19,6 +19,12 @@ Changelog
 
 ---------
 
+2.1.0.dev1
+************
+
+- `Added` `tb deploy` removes resources from datafiles deleted
+
+
 2.0.0
 ************
 

{tinybird-cli-2.0.1.dev0 → tinybird-cli-2.1.0.dev1}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://docs.tinybird.co/cli.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '2.0.1.dev0'
-__revision__ = '40c7bee'
+__version__ = '2.1.0.dev1'
+__revision__ = '877be29'

{tinybird-cli-2.0.1.dev0 → tinybird-cli-2.1.0.dev1}/tinybird/datafile.py

@@ -52,7 +52,7 @@ from tinybird.tb_cli_modules.common import _get_tb_client, get_current_main_work
 
 from .tornado_template import UnClosedIfError
 from .sql import parse_table_structure, schema_to_sql_columns
-from .client import TinyB, DoesNotExistException
+from .client import TinyB, DoesNotExistException, AuthException, CanNotBeDeletedException
 from .sql_template import render_sql_template, get_used_tables_in_template
 from tinybird.sql_template_fmt import format_sql_template
 from .feedback_manager import FeedbackManager
@@ -410,6 +410,9 @@ class CLIGitRelease:
             for filename in filenames
         }
 
+    def get_deleted_from_diffs(self, diffs: List[Diff]) -> List[str]:
+        return [diff.a_path for diff in diffs if diff.change_type == self.ChangeType.DELETED.value and diff.a_path]
+
     async def update_release(
         self, tb_client: TinyB, current_ws: Dict[str, Any], commit: Optional[str] = None
     ) -> Dict[str, Any]:
@@ -454,6 +457,7 @@ class Deployment:
             if not diffs:
                 click.echo(FeedbackManager.info_git_release_no_diffs())
             changed = self.cli_git_release.get_changes_from_diffs(diffs, filenames)
+            deleted = self.cli_git_release.get_deleted_from_diffs(diffs)
 
         else:
             # only changes flow
@@ -462,7 +466,8 @@ class Deployment:
                 if config
                 else None
             )
-        return changed
+            deleted = []
+        return changed, deleted
 
     def preparing_release(self):
         if not self.is_git_release:
@@ -504,6 +509,67 @@ class Deployment:
         )
         click.echo(FeedbackManager.success_git_release(release_commit=release["commit"]))
 
+    async def delete_resources(self, deleted: List[str], remote_pipes: List[Dict[str, Any]], dry_run: bool = False):
+        async def delete_pipe(resource_name_or_id: str, dry_run: bool = False):
+            if dry_run:
+                click.echo(
+                    FeedbackManager.info_deleting_resource(dry_run="[DRY RUN] ", resource_name=resource_name_or_id)
+                )
+            else:
+                click.echo(FeedbackManager.info_deleting_resource(dry_run="", resource_name=resource_name_or_id))
+                try:
+                    await self.tb_client.pipe_delete(resource_name_or_id)
+                except AuthException as e:
+                    raise CLIGitReleaseException(str(e))
+                else:
+                    click.echo(FeedbackManager.success_delete(name=resource_name_or_id))
+
+        async def delete_datasource(resource_name_or_id: str, dry_run: bool = False):
+            if dry_run:
+                click.echo(
+                    FeedbackManager.info_deleting_resource(dry_run="[DRY RUN] ", resource_name=resource_name_or_id)
+                )
+            else:
+                click.echo(FeedbackManager.info_deleting_resource(dry_run="", resource_name=resource_name_or_id))
+                try:
+                    await self.tb_client.datasource_delete(resource_name_or_id)
+                except CanNotBeDeletedException as e:
+                    raise CLIGitReleaseException(str(e))
+                else:
+                    click.echo(FeedbackManager.success_delete(name=resource_name_or_id))
+
+        resource_extension_processor = {".datasource": delete_datasource, ".pipe": delete_pipe}
+
+        # builds pipes deps the other way around from API to use with toposort
+        # i.e {'analytics_hits': ['analytics_pages', 'trend', 'analytics_sources', 'analytics_sessions']}
+        pipes_deps: Dict[str, List[str]] = {}
+        remote_pipes_type: Dict[str, str] = {}
+        for pipe in remote_pipes:
+            pipes_deps[pipe["name"]] = []
+            remote_pipes_type[pipe["name"]] = pipe["type"]
+            for node in pipe["nodes"]:
+                for dep in node["dependencies"]:
+                    if dep in [pipe["name"] for pipe in remote_pipes]:
+                        pipes_deps.setdefault(dep, []).append(pipe["name"])
+        deleted_resources: Dict[str, str] = {Path(resource).resolve().stem: resource for resource in deleted}
+
+        for group in toposort(pipes_deps):
+            # for each level keep materialized for the end
+            for pipe_name in sorted(group, key=lambda x: remote_pipes_type[x] == "materialized"):
+                if pipe_name in deleted_resources:
+                    resource_path = deleted_resources.pop(pipe_name)
+                    try:
+                        await resource_extension_processor[Path(resource_path).suffix](pipe_name, dry_run)
+                    except KeyError:
+                        raise CLIGitReleaseException(FeedbackManager.error_file_extension(resource_path))
+
+        # Delete pending resources (datasources)
+        for name, path in deleted_resources.items():
+            try:
+                await resource_extension_processor[Path(path).suffix](name, dry_run)
+            except KeyError:
+                raise CLIGitReleaseException(FeedbackManager.error_file_extension(path))
+
 
 class Datafile:
     def __init__(self) -> None:
@@ -3407,7 +3473,7 @@ async def folder_push(
                 workspace_lib_paths.append((x.name, x))
 
     datasources: List[Dict[str, Any]] = await tb_client.datasources()
-    pipes: List[Dict[str, Any]] = await tb_client.pipes()
+    pipes: List[Dict[str, Any]] = await tb_client.pipes(dependencies=True)
 
     existing_resources: List[str] = [x["name"] for x in datasources] + [x["name"] for x in pipes]
     # replace workspace mapping names
@@ -3429,7 +3495,7 @@ async def folder_push(
         filenames = get_project_filenames(folder)
 
     # get the list of changes
-    changed = await deployment.detect_changes(filenames, only_changes, config)
+    changed, deleted = await deployment.detect_changes(filenames, only_changes, config)
 
     deployment.preparing_release()
 
@@ -3668,9 +3734,12 @@ async def folder_push(
     if deployment.is_git_release:
         deployment.deploying_dry_run()
         await push_files(dry_run=True)
+
+        await deployment.delete_resources(deleted, pipes, dry_run=True)
         if not deployment.dry_run:
             deployment.deploying()
             await push_files(dry_run)
+            await deployment.delete_resources(deleted, pipes)
     else:
         await push_files(dry_run)
 

{tinybird-cli-2.0.1.dev0 → tinybird-cli-2.1.0.dev1}/tinybird/feedback_manager.py

@@ -436,6 +436,7 @@ Ready? """
     )
     warning_for_cicd_file = warning_message("** 🚨 Warning 🚨: {warning_message}")
     warning_unknown_response = warning_message("** Warning. Unknown response from server: {response}")
+    warning_resource_not_in_workspace = warning_message("** Warning: '{resource_name}' not found in workspace")
 
     info_materialize_push_datasource_exists = warning_message("** Data Source {name} already exists")
     info_materialize_push_datasource_override = prompt_message(
@@ -589,6 +590,7 @@ Ready? """
     info_detected_changes_from_includes = info_message("** Changes from includes:")
     info_processing_from_include = info_message("\t{include_filename} => {filename}")
     info_deps_for_resource = info_message("\t{resource} => '{dep}'")
+    info_deleting_resource = info_message("** {dry_run}Deleting '{resource_name}'")
 
     info_cicd_generation_cancelled_by_user = info_message("** CI/CD files generation cancelled by user.")
     info_skipping_sharing_datasources_environment = info_message(
@@ -697,6 +699,7 @@ Ready? """
     success_sink_job_finished = success_message("""** Data sinked to '{bucket_path}'""")
     success_print_pipe = success_message("** Pipe: {pipe}")
     success_create = success_message("** '{name}' created")
+    success_delete = success_message("** '{name}' deleted")
     success_progress_blocks = success_message("** \N{front-facing baby chick} done")
     success_now_using_config = success_message("** Now using {name} ({id})")
     success_connector_config = success_message(

{tinybird-cli-2.0.1.dev0 → tinybird-cli-2.1.0.dev1}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 2.0.1.dev0
+Version: 2.1.0.dev1
 Summary: Tinybird Command Line Tool
 Home-page: https://docs.tinybird.co/cli.html
 Author: Tinybird
@@ -19,6 +19,12 @@ Changelog
 
 ---------
 
+2.1.0.dev1
+************
+
+- `Added` `tb deploy` removes resources from datafiles deleted
+
+
 2.0.0
 ************