tinybird-cli 3.13.1.dev1__tar.gz → 4.0.1.dev0__tar.gz

{tinybird-cli-3.13.1.dev1 → tinybird-cli-4.0.1.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 3.13.1.dev1
+Version: 4.0.1.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,6 +18,19 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+4.0.0
+************
+
+This is a major release, please read the commands affected below and consider updating your scripts and workflow before upgrading to these version.
+
+- `Deprecated` `--semver` flag and `tb release` commands are now deprecated. You can keep using `tb deploy` to integrate and deploy from git. Changes are deployed to the main Workspace instead of to a Release.
+- `Removed` `--cicd` flag and CI/CD templates generation from `tb init`. You can still use the git integration, just create your own pipelines. You can use the ones in this repo as an exmple https://github.com/tinybirdco/ci
+- `Removed` `tb env` command is removed, use `tb branch` instead.
+- `Deprecated` .datasource files with `ENGINE "Join"` are deprecated, use `Engine "MergeeTree"` instead.
+- `Deprecated` `tb materialize`
+- `Removed` Drop the `--timeout` flag from `tb push` which made the populate job to timeout. You can use now `--wait` to wait for the job to finish or nothing to just create the job and return.
+- `Removed` Support for `KEY` directive is removed. The `KEY` was used to create a Data Source with Join engine by the given `KEY` column name. Join engines are also deprecated, you can use a regular `MergeTree` Data Source instead and adapt the pipes SQL accordingly.
+
 3.12.0
 ************
 

{tinybird-cli-3.13.1.dev1 → tinybird-cli-4.0.1.dev0}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '3.13.1.dev1'
-__revision__ = 'bfd6dd2'
+__version__ = '4.0.1.dev0'
+__revision__ = '2edda96'

{tinybird-cli-3.13.1.dev1 → tinybird-cli-4.0.1.dev0}/tinybird/datafile.py

@@ -706,7 +706,6 @@ class Datafile:
         self.sources: List[str] = []
         self.nodes: List[Dict[str, Any]] = []
         self.tokens: List[Dict[str, Any]] = []
-        self.keys: List[Dict[str, Any]] = []
         self.version: Optional[int] = None
         self.description: Optional[str] = None
         self.raw: Optional[List[str]] = None
@@ -967,11 +966,6 @@ def parse(
             raise ParseException('TOKEN gets two params, token name and permissions e.g TOKEN "read api token" READ')
         doc.tokens.append({"token_name": _unquote(args[0]), "permissions": args[1]})
 
-    def add_key(*args: str, **kwargs: Any) -> None:  # token_name, permissions):
-        if len(args) < 1:
-            raise ParseException("KEY gets one params")
-        doc.keys.append({"column": _unquote(args[0])})
-
     def test(*args: str, **kwargs: Any) -> None:
         print("test", args, kwargs)
 
@@ -1083,7 +1077,6 @@ def parse(
         "resource": assign_node_var("resource"),
         "filter": assign_node_var("filter"),
         "token": add_token,
-        "key": add_key,
         "test": test,
         "include": include,
         "sql": sql("sql"),
@@ -1194,46 +1187,6 @@ def parse(
     return doc
 
 
-def generate_resource_for_key(key: str, name: str, schema: str, version: str, res_name: str) -> List[Dict[str, Any]]:
-    resources: List[Dict[str, Any]] = []
-    # datasource
-    ds_name = f"{name}_join_by_{key}{version}"
-    params = {
-        "name": ds_name,
-        "schema": schema,
-        "engine": "Join",
-        "engine_join_strictness": "ANY",
-        "engine_join_type": "LEFT",
-        "engine_key_columns": key,
-    }
-    resources.append(
-        {
-            "resource_name": f"{name}_join_by_{key}",
-            "resource": "datasources",
-            "params": params,
-            "filename": f"{params['name']}.datasource",
-        }
-    )
-    resources.append(
-        {
-            "resource_name": f"{name}_join_by_{key}_pipe",
-            "resource": "pipes",
-            "filename": f"{params['name']}.pipe",
-            "name": f"{name}_join_by_{key}_pipe{version}",
-            "schema": schema,
-            "nodes": [
-                {
-                    "sql": f"select * from {name}{version}",
-                    "params": {"name": f"{name}_join_by_{key}_view", "type": "materialized", "datasource": ds_name},
-                }
-            ],
-            "deps": [res_name],  # [ds_name],
-            "tokens": [],
-        }
-    )
-    return resources
-
-
 async def process_file(
     filename: str,
     tb_client: TinyB,
@@ -1394,7 +1347,6 @@ async def process_file(
                     # the path was not relative, not inside workspace
                     pass
 
-        res_name = name
         version = f"__v{doc.version}" if doc.version is not None else ""
 
         def append_version_to_name(name: str, version: str) -> str:
@@ -1505,21 +1457,12 @@ async def process_file(
                 "version": doc.version,
                 "params": params,
                 "filename": filename,
-                "keys": doc.keys,
                 "deps": deps,
                 "tokens": doc.tokens,
                 "shared_with": doc.shared_with,
             }
         )
 
-        # generate extra resources in case of the key
-        if doc.keys:
-            for k in doc.keys:
-                resources += generate_resource_for_key(k["column"], name, params["schema"], version, res_name)
-                # set derived resources version the same the parent
-                for x in resources:
-                    x["version"] = doc.version
-
         return resources
 
     elif DataFileExtensions.PIPE in filename:
@@ -1722,7 +1665,7 @@ def find_file_by_name(
     if not is_raw:
         f, raw = find_file_by_name(
             folder,
-            get_dep_from_raw_tables(name),
+            name,
             verbose=verbose,
             is_raw=True,
             workspace_lib_paths=workspace_lib_paths,
@@ -2610,7 +2553,6 @@ async def new_pipe(
     skip_tokens: bool = False,
     ignore_sql_errors: bool = False,
     only_response_times: bool = False,
-    timeout=None,
     run_tests: bool = False,
     as_standard: bool = False,
     tests_to_run: int = 0,
@@ -2771,7 +2713,7 @@ async def new_pipe(
             click.echo(FeedbackManager.info_populate_job_url(url=job_url))
 
         if wait_populate:
-            result = await wait_job(tb_client, job_id, job_url, "Populating", timeout)
+            result = await wait_job(tb_client, job_id, job_url, "Populating")
             click.echo(FeedbackManager.info_populate_job_result(result=result))
     else:
         if data.get("type") == "default" and not skip_tokens and not as_standard and not copy_node and not sink_node:
@@ -2969,6 +2911,12 @@ async def new_ds(
     except DoesNotExistException:
         datasource_exists = False
 
+    if ds["params"].get("engine", "").lower() == "join":
+        deprecation_notice = FeedbackManager.warning_deprecated(
+            warning="Data Sources with Join engine are deprecated and will be removed in the next major release of tinybird-cli. Use MergeTree instead."
+        )
+        click.echo(deprecation_notice)
+
     if not datasource_exists or fork_downstream or fork:
         params = ds["params"]
         params["branch_mode"] = "fork" if fork_downstream or fork else "None"
@@ -3255,7 +3203,6 @@ async def exec_file(
     ignore_sql_errors: bool = False,
     skip_confirmation: bool = False,
     only_response_times: bool = False,
-    timeout=None,
     run_tests=False,
     as_standard=False,
     tests_to_run: int = 0,
@@ -3285,7 +3232,6 @@ async def exec_file(
             wait_populate,
             ignore_sql_errors=ignore_sql_errors,
             only_response_times=only_response_times,
-            timeout=timeout,
             run_tests=run_tests,
             as_standard=as_standard,
             tests_to_run=tests_to_run,
@@ -3417,21 +3363,6 @@ def get_remote_resource_name_without_version(remote_resource_name: str) -> str:
     return parts["name"]
 
 
-def get_dep_from_raw_tables(x: str) -> str:
-    """
-    datasources KEY command generates tables, this transform the used table with the source file
-    >>> get_dep_from_raw_tables('test')
-    'test'
-    >>> get_dep_from_raw_tables('test_join_by_column')
-    'test'
-    """
-
-    try:
-        return x[: x.index("_join_by_")]
-    except ValueError:
-        return x
-
-
 def create_downstream_dependency_graph(dependency_graph: Dict[str, Set[str]], all_resources: Dict[str, Dict[str, Any]]):
     """
     This function reverses the dependency graph obtained from build_graph so you have downstream dependencies for each node in the graph.
@@ -3869,7 +3800,6 @@ async def folder_push(
     workspace_map=None,
     workspace_lib_paths=None,
     no_versions: bool = False,
-    timeout=None,
     run_tests: bool = False,
     as_standard: bool = False,
     raise_on_exists: bool = False,
@@ -4058,7 +3988,6 @@ async def folder_push(
                             ignore_sql_errors,
                             skip_confirmation,
                             only_response_times,
-                            timeout,
                             run_tests,
                             as_standard,
                             tests_to_run,

{tinybird-cli-3.13.1.dev1 → tinybird-cli-4.0.1.dev0}/tinybird/feedback_manager.py

@@ -293,15 +293,15 @@ class FeedbackManager:
     error_head_outdated = error_message(
         "Current HEAD commit '{commit}' is outdated.\n\n    💡Hint: consider to rebase your Git branch with base branch."
     )
-    error_head_already_released = error_message("Current HEAD commit '{commit}' is already released")
+    error_head_already_released = error_message("Current HEAD commit '{commit}' is already deployed")
     error_in_git_ancestor_commits = error_message(
         "Error checking relationship between HEAD '{head_commit}' and Workspace commit '{workspace_commit}'.\n\n    Current Workspace commit needs to be an ancestor of the commit being deployed.\n\n    💡 Hint: consider to rebase your Git branch. If the problem persists, double check the Workspace commit exists in the git log, if it does not exist you can override the Workspace commit to an ancestor of the commit being deployed using `tb init --override-commit <commit>`"
     )
     error_init_release = error_message(
-        "No release on Workspace '{workspace}'. Hint: use 'tb init --git' to start working with git releases"
+        "No release on Workspace '{workspace}'. Hint: use 'tb init --git' to start working with git"
     )
     error_branch_init_release = error_message(
-        "Branch '{workspace}' not ready for deploy. Hint: use 'tb init --git' to start working with git releases and assure Branch is completely created using '--wait' on 'tb branch create'"
+        "Branch '{workspace}' not ready for deploy. Hint: use 'tb init --git' to start working with git and assure Branch is completely created using '--wait' on 'tb branch create'"
     )
     error_commit_changes_to_init_release = error_message(
         "Data project has changes: 'git status -s {path}'\n\n{git_output} \n\n You need to commit your changes in the data project, then re-run 'tb init --git' to finish git initialization."
@@ -318,7 +318,7 @@ class FeedbackManager:
     )
     error_token_does_not_exist = error_message("Token '{token_id}' does not exist")
     error_no_correct_token_for_init = error_message(
-        "No access to git releases. Use a Workspace admin token. See https://www.tinybird.co/docs/version-control/working-with-version-control.html"
+        "No access to git. Use a Workspace admin token. See https://www.tinybird.co/docs/production/working-with-version-control.html"
     )
     error_diff_resources_for_git_init = error_message(
         "Error initializing Git, there are diffs between local Data Project and Workspace '{workspace}'.\nHint: Execute '{pull_command}' to download  Datafiles"
@@ -326,7 +326,7 @@ class FeedbackManager:
     error_remove_oldest_rollback = error_message(
         "Removal of oldest rollback Release failed with error: {error}. Release {semver} is in preview status and has not been promoted, remove the rollback Release and promote to finish the deployment."
     )
-    error_release_already_set = error_message("Can't init '{workspace}', it has already a release '{commit}'")
+    error_release_already_set = error_message("Can't init '{workspace}', it has already a commit '{commit}'")
     error_release_rm_param = error_message("One of --semver <semver> or --oldest-rollback are required.")
     error_release_rollback_live_not_found = error_message("Live Release not found. Rollback can't be applied")
     error_git_provider_index = error_message(
@@ -341,11 +341,11 @@ class FeedbackManager:
         "Materialized view {pipe} has the datasource definition in the same file as the SQL transformation. This is no longer supported while using Versions. Please, split the datasource definition into a separate file."
     )
     error_check_backfill_required = error_message(
-        "Not safe to deploy to live resource '{resource_name}', backfill might be required. Consider deploying to preview bumping SemVer to major or disabling 'TB_AUTO_PROMOTE'. In case you want to disable this check use 'TB_CHECK_BACKFILL_REQUIRED=0'. For more information about the deployment process, please visit https://www.tinybird.co/docs/version-control/deployment-strategies"
+        "Not safe to deploy to live resource '{resource_name}', backfill might be required. Consider deploying to preview bumping SemVer to major or disabling 'TB_AUTO_PROMOTE'. In case you want to disable this check use 'TB_CHECK_BACKFILL_REQUIRED=0'. For more information about the deployment process, please visit https://www.tinybird.co/docs/production/deployment-strategies"
     )
 
     error_connector_require_post_release = error_message(
-        "{connector} Data sources require a post-release deployment. Increment the post-release number of the semver (for example: 0.0.1 -> 0.0.1-1) to do so. You can read more about post-releases at https://www.tinybird.co/docs/version-control/deployment-strategies"
+        "{connector} Data sources require a post-release deployment. Increment the post-release number of the semver (for example: 0.0.1 -> 0.0.1-1) to do so. You can read more about post-releases at https://www.tinybird.co/docs/production/deployment-strategies"
     )
 
     info_incl_relative_path = info_message("** Relative path {path} does not exist, skipping.")
@@ -419,7 +419,7 @@ Ready? """
     )
     prompt_init_cicd = prompt_message("\n Do you want to generate CI/CD config files?")
     prompt_init_git_release_force = prompt_message(
-        "You are going to manually update workspace release commit reference manually, this is just for special ocasions. Do you want to update current commit release reference '{current_commit}' to '{new_commit}'?"
+        "You are going to manually update workspace commit reference manually, this is just for special occasions. Do you want to update current commit reference '{current_commit}' to '{new_commit}'?"
     )
 
     warning_no_test_results = warning_message("Warning: No test results to show")
@@ -462,6 +462,9 @@ Ready? """
         "Auth found for {connector} connector but it's not properly installed. Please run `pip install tinybird-cli[{connector}]` to use it"
     )
     warning_deprecated = warning_message("** 🚨🚨🚨 [DEPRECATED]: {warning}")
+    warning_deprecated_releases = warning_message(
+        "** 🚨🚨🚨 [DEPRECATED]: --semver and Releases are deprecated. You can keep using `tb deploy` to deploy the changed resources from a git commit to the main Workspace and any other command without the --semver flag."
+    )
     warning_token_pipe = warning_message("** There's no read token for pipe {pipe}")
     warning_file_not_found_inside = warning_message(
         "** Warning: {name} not found inside: \n   - {folder}\n   - {folder}/datasources\n   - {folder}/endpoints"
@@ -657,7 +660,7 @@ Ready? """
     info_ask_for_datasource_confirmation = info_message("** Please type the Data Source name to be replaced")
     info_datasource_doesnt_match = info_message("** The description or schema of '{datasource}' has changed.")
     info_custom_deployment = info_message(
-        "** 🚨 Warning 🚨: The changes in the branch require the `--yes` flag to the `tb deploy` command to overwrite resources. Run `tb release generate --semver <semver> to generate a custom deployment and include the `--yes` flag. Read more about custom deployments => https://www.tinybird.co/docs/version-control/deployment-strategies.html#customizing-deployments"
+        "** 🚨 Warning 🚨: The changes in the branch require the `--yes` flag to the `tb deploy` command to overwrite resources. Run `tb release generate --semver <semver> to generate a custom deployment and include the `--yes` flag. Read more about custom deployments => https://www.tinybird.co/docs/production/deployment-strategies.html#customizing-deployments"
     )
     info_ask_for_alter_confirmation = info_message("** Please confirm you want to apply the changes above y/N")
     info_available_regions = info_message("** List of available regions:")
@@ -681,20 +684,20 @@ Ready? """
     info_workspace_branch_create_greeting = info_message(
         "Please enter the name for your new Branch. Remember the name you choose must be unique, you can add a suffix in case of collision.\nYou can bypass this step by supplying a value for the --name option."
     )
-    info_no_git_release_yet = info_message("\n** Initializing releases based on git for Workspace '{workspace}'")
+    info_no_git_release_yet = info_message("\n** Initializing based on git for Workspace '{workspace}'")
     info_diff_resources_for_git_init = info_message(
         "** Checking diffs between remote Workspace and local. Hint: use 'tb diff' to check if your Data Project and Workspace synced"
     )
     info_cicd_file_generated = info_message("** File {file_path} generated for CI/CD")
     info_available_git_providers = info_message("** List of available providers:")
     info_git_release_init_without_diffs = info_message("** No diffs detected for '{workspace}'")
-    info_deployment_detecting_changes_header = info_message("\n** Detecting changes from last release ...")
-    info_deployment_preparing_release_header = info_message("\n** Preparing release ...")
-    info_deployment_deploying_release_dry_run_header = info_message("\n** [DRY RUN] Deploying release ...")
-    info_deployment_deploying_release_header = info_message("\n** Deploying release ...")
+    info_deployment_detecting_changes_header = info_message("\n** Detecting changes from last commit ...")
+    info_deployment_preparing_release_header = info_message("\n** Preparing commit ...")
+    info_deployment_deploying_release_dry_run_header = info_message("\n** [DRY RUN] Deploying commit ...")
+    info_deployment_deploying_release_header = info_message("\n** Deploying commit ...")
     info_deployment_deploy_deps = info_message("** Deploying downstream dependent pipes ...")
     info_git_release_diffs = info_message(
-        "** Diffs from current release '{current_release_commit}' and new '{new_release_commit}':"
+        "** Diffs from current commit '{current_release_commit}' and new '{new_release_commit}':"
     )
     info_git_release_diff = info_message("\t{change_type}:\t{datafile_changed}")
     info_git_release_no_diffs = info_message("\tno diffs detected")
@@ -713,7 +716,7 @@ Ready? """
         "** Skipping `SHARED_WITH` entry as the flag --user_token was not used"
     )
     info_generate_cicd_config = info_message(
-        "** {provider} CI/CD config files generated. Read this guide to learn how to run CI/CD pipelines: https://www.tinybird.co/docs/version-control/continuous-integration.html"
+        "** {provider} CI/CD config files generated. Read this guide to learn how to run CI/CD pipelines: https://www.tinybird.co/docs/production/continuous-integration.html"
     )
     info_release_no_rollback = info_message("** No action")
     info_no_release_deleted = info_message(" ** No Release deleted")
@@ -886,10 +889,10 @@ Ready? """
 
     success_test_added = success_message("** Test added successfully")
     success_remember_api_host = success_message("** Remember to use {api_host} in all your API calls.")
-    success_git_release = success_message("** New release deployed: '{release_commit}'")
-    success_dry_git_release = success_message("** [DRY RUN] New release deployed: '{release_commit}'")
+    success_git_release = success_message("** New commit deployed: '{release_commit}'")
+    success_dry_git_release = success_message("** [DRY RUN] New commit deployed: '{release_commit}'")
     success_init_git_release = success_message(
-        "** Workspace '{workspace_name}' release initialized to commit '{release_commit}'.\n Now start working with git, pushing changes to pull requests and let the CI/CD work for you. More details in this guide: https://www.tinybird.co/docs/version-control/working-with-version-control.html."
+        "** Workspace '{workspace_name}' initialized to commit '{release_commit}'.\n Now start working with git, pushing changes to pull requests and let the CI/CD work for you. More details in this guide: https://www.tinybird.co/docs/production/working-with-version-control.html."
     )
     success_release_update = success_message("** Live Release {semver} updated to {new_semver}")
     success_dry_release_update = success_message("** [DRY RUN] Live Release {semver} updated to {new_semver}")

{tinybird-cli-3.13.1.dev1 → tinybird-cli-4.0.1.dev0}/tinybird/tb_cli_modules/branch.py

@@ -47,7 +47,7 @@ def release() -> None:
 @coro
 async def release_ls() -> None:
     """List current available Releases in the Workspace"""
-
+    click.echo(FeedbackManager.warning_deprecated_releases())
     config = CLIConfig.get_project_config()
     _ = await try_update_config_with_remote(config, only_if_needed=True)
 
@@ -78,6 +78,7 @@ async def print_releases(config: CLIConfig):
 )
 @coro
 async def release_generate(semver: str) -> None:
+    click.echo(FeedbackManager.warning_deprecated_releases())
     if os.path.exists(".tinyenv"):
         with open(".tinyenv", "r") as env_file:
             lines = env_file.readlines()
@@ -137,6 +138,7 @@ set -euxo pipefail
 )
 @coro
 async def release_create(semver: str) -> None:
+    click.echo(FeedbackManager.warning_deprecated_releases())
     config = CLIConfig.get_project_config()
     _ = await try_update_config_with_remote(config, only_if_needed=True)
 
@@ -154,6 +156,7 @@ async def release_promote(semver: str) -> None:
     """
     The oldest rollback Release will be automatically removed if no usage, otherwise export TB_FORCE_REMOVE_OLDEST_ROLLBACK="1" to force deletion
     """
+    click.echo(FeedbackManager.warning_deprecated_releases())
     config = CLIConfig.get_project_config()
     _ = await try_update_config_with_remote(config, only_if_needed=True)
 
@@ -179,6 +182,7 @@ async def release_promote(semver: str) -> None:
 )
 @coro
 async def release_preview(semver: str) -> None:
+    click.echo(FeedbackManager.warning_deprecated_releases())
     config = CLIConfig.get_project_config()
     _ = await try_update_config_with_remote(config, only_if_needed=True)
 
@@ -195,6 +199,7 @@ async def release_preview(semver: str) -> None:
 @click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
 @coro
 async def release_rollback(yes: bool) -> None:
+    click.echo(FeedbackManager.warning_deprecated_releases())
     config = CLIConfig.get_project_config()
     _ = await try_update_config_with_remote(config, only_if_needed=False)
 
@@ -238,6 +243,7 @@ async def release_rollback(yes: bool) -> None:
 )
 @coro
 async def release_rm(semver: str, oldest_rollback: bool, force: bool, yes: bool, dry_run: bool) -> None:
+    click.echo(FeedbackManager.warning_deprecated_releases())
     if (not semver and not oldest_rollback) or (semver and oldest_rollback):
         raise CLIException(FeedbackManager.error_release_rm_param())
 

{tinybird-cli-3.13.1.dev1 → tinybird-cli-4.0.1.dev0}/tinybird/tb_cli_modules/cicd.py

@@ -209,15 +209,10 @@ EXEC_TEST_SH = """
 set -euxo pipefail
 
 export TB_VERSION_WARNING=0
-export VERSION=$1
 
 run_test() {
     t=$1
     echo "** Running $t **"
-    # Check if VERSION is provided
-    if [[ -n $VERSION ]]; then
-        sed -i "s/tb/tb --semver $VERSION/" $t
-    fi
     echo "** $(cat $t)"
     tmpfile=$(mktemp)
     retries=0
@@ -262,7 +257,7 @@ run_test() {
 export -f run_test
 
 fail=0
-find ./tests -name "*.test" -print0 | xargs -0 -I {} -P 4 bash -c 'run_test "$@"' _ {} $VERSION || fail=1
+find ./tests -name "*.test" -print0 | xargs -0 -I {} -P 4 bash -c 'run_test "$@"' _ {} || fail=1
 
 if [ $fail == 1 ]; then
   exit -1;
@@ -273,8 +268,6 @@ APPEND_FIXTURES_SH = """
 #!/usr/bin/env bash
 set -euxo pipefail
 
-VERSION=$1
-
 directory="datasources/fixtures"
 extensions=("csv" "ndjson")
 
@@ -287,7 +280,7 @@ for extension in "${extensions[@]}"; do
     file_name=$(basename "$file_path")
     file_name_without_extension="${file_name%.*}"
 
-    command="tb --semver $VERSION datasource append $file_name_without_extension datasources/fixtures/$file_name"
+    command="tb datasource append $file_name_without_extension datasources/fixtures/$file_name"
     echo $command
     $command
   done

{tinybird-cli-3.13.1.dev1 → tinybird-cli-4.0.1.dev0}/tinybird/tb_cli_modules/cli.py

@@ -90,7 +90,7 @@ DEFAULT_PATTERNS: List[Tuple[str, Union[str, Callable[[str], str]]]] = [
 )
 @click.option("--token", help="Use auth token, defaults to TB_TOKEN envvar, then to the .tinyb file")
 @click.option("--host", help="Use custom host, defaults to TB_HOST envvar, then to https://api.tinybird.co")
-@click.option("--semver", help="Semver of a Release to run the command. Example: 1.0.0")
+@click.option("--semver", help="Semver of a Release to run the command. Example: 1.0.0", hidden=True)
 @click.option("--gcp-project-id", help="The Google Cloud project ID", hidden=True)
 @click.option(
     "--gcs-bucket", help="The Google Cloud Storage bucket to write temp files when using the connectors", hidden=True
@@ -296,10 +296,7 @@ async def cli(
     help="Use this option to manually override the reference commit of your workspace. This is useful if a commit is not recognized in your git log, such as after a force push (git push -f).",
 )
 @click.option(
-    "--cicd",
-    is_flag=True,
-    default=False,
-    help="Generates only CI/CD files for your git provider",
+    "--cicd", is_flag=True, default=False, help="Generates only CI/CD files for your git provider", hidden=True
 )
 @click.pass_context
 @coro
@@ -437,7 +434,7 @@ async def init(
             else:
                 click.echo(FeedbackManager.info_cicd_already_exists(provider=cicd_provider.name))
         else:
-            if cicd or click.confirm(FeedbackManager.prompt_init_cicd(), default=True):
+            if cicd:
                 data_project_dir = os.path.relpath(folder, cli_git_release.working_dir())
                 await init_cicd(
                     client, current_ws, config, path=cli_git_release.working_dir(), data_project_dir=data_project_dir
@@ -567,7 +564,6 @@ def check(filenames: List[str], debug: bool) -> None:
     default=False,
     help="when set, resource dependency versions are not used, it pushes the dependencies as-is",
 )
-@click.option("--timeout", type=float, default=None, help="timeout you want to use for the job populate")
 @click.option(
     "-l", "--limit", type=click.IntRange(0, 100), default=0, required=False, help="Number of requests to validate"
 )
@@ -634,7 +630,6 @@ async def push(
     workspace_map,
     workspace,
     no_versions: bool,
-    timeout: Optional[float],
     limit: int,
     sample_by_params: int,
     failfast: bool,
@@ -673,7 +668,6 @@ async def push(
         workspace_map=dict(workspace_map),
         workspace_lib_paths=workspace,
         no_versions=no_versions,
-        timeout=timeout,
         run_tests=False,
         tests_to_run=limit,
         tests_sample_by_params=sample_by_params,
@@ -1068,13 +1062,10 @@ async def materialize(
     override_datasource: bool,
     target_datasource: Optional[str] = None,
 ) -> None:
-    """[BETA] Given a local Pipe datafile path (FILENAME) and optionally a Materialized View name (TARGET_DATASOURCE), choose one of its nodes to materialize.
-
-    This command guides you to generate the Materialized View with name TARGET_DATASOURCE, the only requirement is having a valid Pipe datafile locally. Use `tb pull` to download resources from your workspace when needed.
-
-    Syntax: tb materialize path/to/pipe.pipe
-    """
-
+    deprecation_notice = FeedbackManager.warning_deprecated(
+        warning="'tb materialize' is deprecated. To create a Materialized View in a guided way you can use the UI: `Create Pipe` and use the `Create Materialized View` wizard. Finally download the resulting `.pipe` and `.datasource` files."
+    )
+    click.echo(deprecation_notice)
     cl = create_tb_client(ctx)
 
     async def _try_push_pipe_to_analyze(pipe_name):
@@ -1402,7 +1393,6 @@ async def prompt(_ctx: Context) -> None:
     multiple=True,
     help="add a workspace path to the list of external workspaces, usage: --workspace name path/to/folder",
 )
-@click.option("--timeout", type=float, default=None, help="timeout you want to use for the job populate")
 @click.option(
     "--folder",
     default=".",
@@ -1454,28 +1444,15 @@ async def deploy(
     yes: bool,
     workspace_map,
     workspace,
-    timeout: Optional[float],
     folder: str,
     user_token: Optional[str],
     fork_downstream: bool,
     fork: bool,
     use_main: bool,
 ) -> None:
-    """Deploy in Tinybird pushing resources changed from previous Release using git.
-
-    Usage: tb --semver <semver> deploy
-
-    Semantic versioning logic is as follows:
-
-    --semver <semver> format is major.minor.patch-post where major, minor, patch and post are integer numbers
+    """Deploy in Tinybird pushing resources changed from last git commit deployed.
 
-      - bump post to deploy to the current live Release, rollback to previous post version is not available
-
-      - bump patch or minor to deploy a new Release and auto-promote it to live. Add TB_AUTO_PROMOTE=0 to create the Release in preview status
-
-      - bump major to deploy a new Release in preview status
-
-    When auto-promote the oldest rollback Release will be automatically removed if no usage, otherwise export TB_FORCE_REMOVE_OLDEST_ROLLBACK="1" to force deletion
+    Usage: tb deploy
     """
     try:
         ignore_sql_errors = FeatureFlags.ignore_sql_errors()
@@ -1504,51 +1481,70 @@ async def deploy(
         release_created = False
         new_release = False
         check_backfill_required = False
+
+        # semver is now optional, if not sent force the bump
+        if not semver:
+            semver = current_semver
+        else:
+            click.echo(FeedbackManager.warning_deprecated_releases())
+
         if semver and current_semver:
             new_version = version.parse(semver.split("-snapshot")[0])
             current_version = version.parse(current_semver.split("-snapshot")[0])
-            if dry_run:
-                click.echo(
-                    FeedbackManager.info_dry_releases_detected(current_semver=current_version, semver=new_version)
-                )
-            else:
-                click.echo(FeedbackManager.info_releases_detected(current_semver=current_version, semver=new_version))
-            if new_version <= current_version:
-                click.echo(FeedbackManager.error_bump_release(command="deploy [OPTIONS]"))
-                sys.exit(1)
-
-            if is_post_semver(new_version, current_version):
+            show_feedback = new_version != current_version
+            if show_feedback:
                 if dry_run:
-                    click.echo(FeedbackManager.info_dry_local_release(version=new_version))
+                    click.echo(
+                        FeedbackManager.info_dry_releases_detected(current_semver=current_version, semver=new_version)
+                    )
+                else:
+                    click.echo(
+                        FeedbackManager.info_releases_detected(current_semver=current_version, semver=new_version)
+                    )
+
+            if new_version == current_version:
+                if current_version.post is None:
+                    semver = str(current_version) + "-1"
+                    new_version = version.Version(semver)
                 else:
-                    click.echo(FeedbackManager.info_local_release(version=new_version))
+                    semver = f"{current_version.major}.{current_version.minor}.{current_version.micro}-{current_version.post + 1}"
+                    new_version = version.Version(semver)
+
+            if is_post_semver(new_version, current_version):
+                if show_feedback:
+                    if dry_run:
+                        click.echo(FeedbackManager.info_dry_local_release(version=new_version))
+                    else:
+                        click.echo(FeedbackManager.info_local_release(version=new_version))
                 yes = True
                 auto_promote = False
                 new_release = False
             elif is_major_semver(new_version, current_version):
-                if dry_run:
-                    click.echo(FeedbackManager.info_dry_major_release(version=new_version))
-                else:
-                    click.echo(FeedbackManager.info_major_release(version=new_version))
+                if show_feedback:
+                    if dry_run:
+                        click.echo(FeedbackManager.info_dry_major_release(version=new_version))
+                    else:
+                        click.echo(FeedbackManager.info_major_release(version=new_version))
                 auto_promote = False
                 new_release = True
             else:
                 # allows TB_AUTO_PROMOTE=0 so release is left in preview
                 auto_promote = getenv_bool("TB_AUTO_PROMOTE", True)
                 if auto_promote:
-                    if dry_run:
-                        click.echo(FeedbackManager.info_dry_minor_patch_release_with_autopromote(version=new_version))
-                    else:
-                        click.echo(FeedbackManager.info_minor_patch_release_with_autopromote(version=new_version))
+                    if show_feedback:
+                        if dry_run:
+                            click.echo(
+                                FeedbackManager.info_dry_minor_patch_release_with_autopromote(version=new_version)
+                            )
+                        else:
+                            click.echo(FeedbackManager.info_minor_patch_release_with_autopromote(version=new_version))
                 else:
-                    if dry_run:
-                        click.echo(FeedbackManager.info_dry_minor_patch_release_no_autopromote(version=new_version))
-                    else:
-                        click.echo(FeedbackManager.info_minor_patch_release_no_autopromote(version=new_version))
+                    if show_feedback:
+                        if dry_run:
+                            click.echo(FeedbackManager.info_dry_minor_patch_release_no_autopromote(version=new_version))
+                        else:
+                            click.echo(FeedbackManager.info_minor_patch_release_no_autopromote(version=new_version))
                 new_release = True
-        else:
-            click.echo(FeedbackManager.error_bump_release(command="deploy [OPTIONS]"))
-            sys.exit(1)
 
         if new_release:
             if not dry_run:
@@ -1584,7 +1580,6 @@ async def deploy(
                 skip_confirmation=yes,
                 workspace_map=dict(workspace_map),
                 workspace_lib_paths=workspace,
-                timeout=timeout,
                 run_tests=False,
                 folder=folder,
                 config=ctx.ensure_object(dict)["config"],
@@ -1636,10 +1631,12 @@ async def deploy(
 
         if not new_release:
             if dry_run:
-                click.echo(FeedbackManager.success_dry_release_update(semver=current_semver, new_semver=semver))
+                if show_feedback:
+                    click.echo(FeedbackManager.success_dry_release_update(semver=current_semver, new_semver=semver))
             else:
                 client.semver = None
                 await client.update_release_semver(config["id"], current_semver, semver)
-                click.echo(FeedbackManager.success_release_update(semver=current_semver, new_semver=semver))
+                if show_feedback:
+                    click.echo(FeedbackManager.success_release_update(semver=current_semver, new_semver=semver))
     except Exception as e:
         raise CLIException(str(e))

{tinybird-cli-3.13.1.dev1 → tinybird-cli-4.0.1.dev0}/tinybird/tb_cli_modules/common.py

@@ -233,14 +233,6 @@ class AliasedGroup(click.Group):
         if cm is not None:
             return cm
 
-        if cmd_name == "env":
-            deprecation_notice = FeedbackManager.warning_deprecated(
-                warning="'tb env' is deprecated, use 'tb branch' instead. 'tb env' will be removed in the next 3.0.0 release, please update your scripts."
-            )
-            click.echo(deprecation_notice)
-
-            return click.Group.get_command(self, ctx, "branch")
-
     def resolve_command(self, ctx, args):
         # always return the command's name, not the alias
         _, cmd, args = super().resolve_command(ctx, args)
@@ -431,7 +423,7 @@ async def folder_init(
                 t_file.write(
                     """
 ##############################################################################################################################
-###   Visit https://www.tinybird.co/docs/version-control/implementing-test-strategies.html#data-quality-tests              ###
+###   Visit https://www.tinybird.co/docs/production/implementing-test-strategies.html#data-quality-tests              ###
 ###   for more details on Data Quality tests                                                                               ###
 ##############################################################################################################################
 
@@ -467,7 +459,7 @@ async def folder_init(
                 t_file.write(
                     """
 ############################################################################################################################
-###   Visit https://www.tinybird.co/docs/version-control/implementing-test-strategies.html#regression-tests              ###
+###   Visit https://www.tinybird.co/docs/production/implementing-test-strategies.html#regression-tests              ###
 ###   for more details on Regression tests                                                                               ###
 ############################################################################################################################
 
@@ -1171,7 +1163,7 @@ async def print_branch_regression_tests_summary(client, job_id, host, response=N
                     click.echo(f"❌ FAILED {failure['name']}\n")
     if failed:
         raise CLIException(
-            "Check Failures Detail above for more information. If the results are expected, skip asserts or increase thresholds, see 💡 Hints above (note skip asserts flags are applied to all regression tests, so use them when it makes sense).\n\nIf you are using the CI template for GitHub or GitLab you can add skip asserts flags as labels to the MR and they are automatically applied. Find available flags to skip asserts and thresholds here => https://www.tinybird.co/docs/version-control/implementing-test-strategies.html#fixture-tests"
+            "Check Failures Detail above for more information. If the results are expected, skip asserts or increase thresholds, see 💡 Hints above (note skip asserts flags are applied to all regression tests, so use them when it makes sense).\n\nIf you are using the CI template for GitHub or GitLab you can add skip asserts flags as labels to the MR and they are automatically applied. Find available flags to skip asserts and thresholds here => https://www.tinybird.co/docs/production/implementing-test-strategies.html#fixture-tests"
         )
 
 
@@ -1902,7 +1894,6 @@ async def wait_job(
     job_id: str,
     job_url: str,
     label: str,
-    timeout: Optional[int] = None,
     wait_observer: Optional[Callable[[Dict[str, Any], ProgressBar], None]] = None,
 ) -> Dict[str, Any]:
     progress_bar: ProgressBar
@@ -1926,7 +1917,7 @@ async def wait_job(
 
         try:
             # TODO: Simplify this as it's not needed to use two functions for
-            result = await wait_job_no_ui(tb_client, job_id, timeout, progressbar_cb)
+            result = await wait_job_no_ui(tb_client, job_id, progressbar_cb)
             if result["status"] != "done":
                 raise CLIException(FeedbackManager.error_while_running_job(error=result["error"]))
             return result
@@ -1941,11 +1932,10 @@ async def wait_job(
 async def wait_job_no_ui(
     tb_client: TinyB,
     job_id: str,
-    timeout: Optional[int] = None,
     status_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
 ) -> Dict[str, Any]:
     try:
-        result = await asyncio.wait_for(tb_client.wait_for_job(job_id, status_callback=status_callback), timeout)
+        result = await asyncio.wait_for(tb_client.wait_for_job(job_id, status_callback=status_callback), None)
         if result["status"] != "done":
             raise JobException(result.get("error"))
         return result

{tinybird-cli-3.13.1.dev1 → tinybird-cli-4.0.1.dev0}/tinybird/tb_cli_modules/pipe.py

@@ -581,7 +581,6 @@ async def regression_test(
         workspace_map=dict(workspace_map),
         workspace_lib_paths=workspace,
         no_versions=no_versions,
-        timeout=False,
         run_tests=True,
         tests_to_run=limit,
         tests_sample_by_params=sample_by_params,

{tinybird-cli-3.13.1.dev1 → tinybird-cli-4.0.1.dev0}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 3.13.1.dev1
+Version: 4.0.1.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,6 +18,19 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+4.0.0
+************
+
+This is a major release, please read the commands affected below and consider updating your scripts and workflow before upgrading to these version.
+
+- `Deprecated` `--semver` flag and `tb release` commands are now deprecated. You can keep using `tb deploy` to integrate and deploy from git. Changes are deployed to the main Workspace instead of to a Release.
+- `Removed` `--cicd` flag and CI/CD templates generation from `tb init`. You can still use the git integration, just create your own pipelines. You can use the ones in this repo as an exmple https://github.com/tinybirdco/ci
+- `Removed` `tb env` command is removed, use `tb branch` instead.
+- `Deprecated` .datasource files with `ENGINE "Join"` are deprecated, use `Engine "MergeeTree"` instead.
+- `Deprecated` `tb materialize`
+- `Removed` Drop the `--timeout` flag from `tb push` which made the populate job to timeout. You can use now `--wait` to wait for the job to finish or nothing to just create the job and return.
+- `Removed` Support for `KEY` directive is removed. The `KEY` was used to create a Data Source with Join engine by the given `KEY` column name. Join engines are also deprecated, you can use a regular `MergeTree` Data Source instead and adapt the pipes SQL accordingly.
+
 3.12.0
 ************