tinybird-cli 2.1.1.dev6__tar.gz → 2.1.1.dev8__tar.gz

{tinybird-cli-2.1.1.dev6 → tinybird-cli-2.1.1.dev8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 2.1.1.dev6
+Version: 2.1.1.dev8
 Summary: Tinybird Command Line Tool
 Home-page: https://docs.tinybird.co/cli.html
 Author: Tinybird
@@ -19,7 +19,17 @@ Changelog
 
 ---------
 
-2.1.1.dev5
+2.1.1.dev8
+************
+
+- `Changed` Use `tb branch` instead of `tb env`. `tb env` is still usable but deprecated and will be removed in the next version.
+
+2.1.1.dev7
+************
+
+- `Fixed` Be smarter in `tb fmt` when detecting if `.incl` file contains Data Source or Pipe definitions.
+
+2.1.1.dev6
 ************
 
 - `Fixed` `tb fmt` adds empty quoted strings in .datasource connection section.

{tinybird-cli-2.1.1.dev6 → tinybird-cli-2.1.1.dev8}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://docs.tinybird.co/cli.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '2.1.1.dev6'
-__revision__ = '209e97a'
+__version__ = '2.1.1.dev8'
+__revision__ = '96a93ca'

tinybird-cli-2.1.1.dev8/tinybird/connector_settings.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+from enum import Enum
+from typing import List, Optional
+
+from pydantic import BaseModel
+
+
+class DataConnectorType(str, Enum):
+    KAFKA = "kafka"
+    GCLOUD_SCHEDULER = "gcscheduler"
+    SNOWFLAKE = "snowflake"
+    BIGQUERY = "bigquery"
+    GCLOUD_STORAGE = "gcs"
+    GCLOUD_STORAGE_HMAC = "gcs_hmac"
+    AMAZON_S3 = "s3"
+    AMAZON_S3_IAMROLE = "s3_iamrole"
+
+    def __str__(self) -> str:
+        return self.value
+
+
+DataConnectors = DataConnectorType  # this is just to make picking happy
+
+
+class DataConnectorSetting(BaseModel):
+    ...
+
+
+class KafkaConnectorSetting(DataConnectorSetting):
+    kafka_bootstrap_servers: str
+    kafka_sasl_plain_username: str
+    kafka_sasl_plain_password: str
+    cli_version: Optional[str] = None
+    tb_endpoint: Optional[str] = None
+    kafka_security_protocol: Optional[str] = None
+    kafka_sasl_mechanism: Optional[str] = None
+    kafka_schema_registry_url: Optional[str] = None
+
+
+class S3ConnectorSetting(DataConnectorSetting):
+    s3_access_key_id: str
+    s3_secret_access_key: str
+    s3_region: str
+
+
+class S3IAMConnectorSetting(DataConnectorSetting):
+    s3_iamrole_arn: str
+    s3_iamrole_region: str
+    s3_iamrole_external_id: Optional[str] = None
+
+
+class SnowflakeConnectorSetting(DataConnectorSetting):
+    account: str
+    username: str
+    password: str
+    role: str
+    warehouse: str
+    warehouse_size: Optional[str] = None
+    stage: Optional[str] = None
+    integration: Optional[str] = None
+
+
+class GCSchedulerConnectorSetting(DataConnectorSetting):
+    gcscheduler_region: Optional[str] = None
+
+
+class BigQueryConnectorSetting(DataConnectorSetting):
+    account: Optional[str] = None
+
+
+class GCSHmacConnectorSetting(DataConnectorSetting):
+    gcs_hmac_access_id: str
+    gcs_hmac_secret: str
+
+
+class GCSConnectorSetting(DataConnectorSetting):
+    gcs_private_key_id: str
+    gcs_client_x509_cert_url: str
+    gcs_project_id: str
+    gcs_client_id: str
+    gcs_client_email: str
+    gcs_private_key: str
+
+
+DATA_CONNECTOR_SETTINGS: dict[DataConnectors, type[DataConnectorSetting]] = {
+    DataConnectors.KAFKA: KafkaConnectorSetting,
+    DataConnectors.GCLOUD_SCHEDULER: GCSchedulerConnectorSetting,
+    DataConnectors.SNOWFLAKE: SnowflakeConnectorSetting,
+    DataConnectors.BIGQUERY: BigQueryConnectorSetting,
+    DataConnectors.GCLOUD_STORAGE: GCSConnectorSetting,
+    DataConnectors.GCLOUD_STORAGE_HMAC: GCSHmacConnectorSetting,
+    DataConnectors.AMAZON_S3: S3ConnectorSetting,
+    DataConnectors.AMAZON_S3_IAMROLE: S3IAMConnectorSetting,
+}
+
+
+class DataLinkerSettings:
+    kafka = [
+        "tb_datasource",
+        "tb_token",
+        "kafka_topic",
+        "kafka_group_id",
+        "kafka_auto_offset_reset",
+        "kafka_store_raw_value",
+        "kafka_store_headers",
+    ]
+
+
+class DataSinkSettings:
+    gcscheduler = [
+        "cron",
+        "timezone",
+        "status",
+        "gcscheduler_target_url",
+        "gcscheduler_job_name",
+        "gcscheduler_region",
+    ]
+    gcs_hmac = [
+        "bucket_path",
+        "file_template",
+        "partition_node",
+        "format",
+        "compression",
+    ]
+    s3 = [
+        "bucket_path",
+        "file_template",
+        "partition_node",
+        "format",
+        "compression",
+    ]
+    s3_iamrole = [
+        "bucket_path",
+        "file_template",
+        "partition_node",
+        "format",
+        "compression",
+    ]
+
+
+class DataSensitiveSettings:
+    kafka = ["kafka_sasl_plain_password"]
+    gcscheduler = [
+        "gcscheduler_target_url",
+        "gcscheduler_job_name",
+        "gcscheduler_region",
+    ]
+    bigquery: List[str] = []
+    snowflake: List[str] = []
+    gcs_hmac: List[str] = ["gcs_hmac_secret"]
+    s3: List[str] = ["s3_secret_access_key"]
+    s3_iamrole: List[str] = ["s3_iamrole_arn"]

{tinybird-cli-2.1.1.dev6 → tinybird-cli-2.1.1.dev8}/tinybird/datafile.py

@@ -2682,7 +2682,7 @@ async def share_and_unshare_datasource(
     datasource_id = datasource.get("id", "")
     workspaces: List[Dict[str, Any]]
 
-    # In case we are pushing to an enviroment, we don't share the datasource
+    # In case we are pushing to a branch, we don't share the datasource
     # FIXME: Have only once way to get the current workspace
     if current_ws:
         # Force to get all the workspaces the user can access
@@ -2693,7 +2693,7 @@ async def share_and_unshare_datasource(
         workspaces = workspace.get("workspaces", [])
 
     if workspace.get("is_branch", False):
-        click.echo(FeedbackManager.info_skipping_sharing_datasources_environment(datasource=datasource["name"]))
+        click.echo(FeedbackManager.info_skipping_sharing_datasources_branch(datasource=datasource["name"]))
         return
 
     # We duplicate the client to use the user_token
@@ -3550,7 +3550,7 @@ async def folder_push(
     current_ws: Dict[str, Any] = next(
         (workspace for workspace in workspaces if config and workspace.get("id", ".") == config.get("id", "..")), {}
     )
-    is_environment = current_ws.get("is_branch", False)
+    is_branch = current_ws.get("is_branch", False)
     has_semver = False
     if config and config.get("semver"):
         has_semver = True
@@ -3639,7 +3639,7 @@ async def folder_push(
         current_ws=current_ws,
         changed=changed,
         only_changes=only_changes or (deployment.is_git_release and has_semver),
-        skip_connectors=is_environment,
+        skip_connectors=is_branch,
         fork_downstream=fork_downstream,
         is_internal=is_internal,
     )
@@ -3864,7 +3864,7 @@ async def folder_push(
             click.echo(FeedbackManager.info_pushing_fixtures())
 
             # We need to upload the fixtures even if there is no change
-            if is_environment:
+            if is_branch:
                 filenames = get_project_filenames(folder, with_vendor=True)
                 to_run, dep_map, back_to_run, back_dep_map = await build_graph(
                     filenames,
@@ -3876,7 +3876,7 @@ async def folder_push(
                     verbose=verbose,
                     workspace_lib_paths=workspace_lib_paths,
                     current_ws=current_ws,
-                    skip_connectors=is_environment,
+                    skip_connectors=is_branch,
                 )
 
             processed = set()
@@ -3891,13 +3891,13 @@ async def folder_push(
                                 debug,
                                 folder,
                                 force,
-                                mode="append" if is_environment else "replace",
+                                mode="append" if is_branch else "replace",
                             )
                             processed.add(name)
             for f in to_run:
                 if f not in processed:
                     await check_fixtures_data(
-                        tb_client, to_run[f], debug, folder, force, mode="append" if is_environment else "replace"
+                        tb_client, to_run[f], debug, folder, force, mode="append" if is_branch else "replace"
                     )
         else:
             if verbose:
@@ -4534,7 +4534,7 @@ async def diff_files(
     async def parse(filename, with_format=True, unroll_includes=False):
         extensions = Path(filename).suffixes
         lines = None
-        if ".datasource" in extensions:  # Accepts '.datasource' and '.datasource.incl'
+        if is_file_a_datasource(filename):
             lines = (
                 await format_datasource(
                     filename, unroll_includes=unroll_includes, for_diff=True, client=client, replace_includes=True
@@ -4649,3 +4649,21 @@ def has_internal_datafiles(folder: str) -> bool:
     folder = folder or "."
     filenames = get_project_filenames(folder)
     return any([f for f in filenames if "spans" in str(f) and "vendor" not in str(f)])
+
+
+def is_file_a_datasource(filename: str) -> bool:
+    extensions = Path(filename).suffixes
+    if ".datasource" in extensions:  # Accepts '.datasource' and '.datasource.incl'
+        return True
+
+    if ".incl" in extensions:
+        lines = []
+        with open(filename) as file:
+            lines = file.readlines()
+
+        for line in lines:
+            trimmed_line = line.strip().lower()
+            if trimmed_line.startswith("schema") or trimmed_line.startswith("engine"):
+                return True
+
+    return False

{tinybird-cli-2.1.1.dev6 → tinybird-cli-2.1.1.dev8}/tinybird/feedback_manager.py

@@ -194,15 +194,15 @@ class FeedbackManager:
     error_connection_create = error_message("Connection {connection_name} could not be created: {error}")
     error_workspace = error_message("Workspace {workspace} not found. use 'tb workspace ls' to list your workspaces")
     error_branch = error_message(
-        "Environment {branch} not found. use 'tb env ls' to list your Environments, make sure you are authenticated using the right workspace token"
+        "Branch {branch} not found. use 'tb branch ls' to list your Branches, make sure you are authenticated using the right workspace token"
     )
     error_not_a_branch = error_message(
-        "To use this command you need to be authenticated on an Environment. Use 'tb env ls' and 'tb env use' and retry the command."
+        "To use this command you need to be authenticated on a Branch. Use 'tb branch ls' and 'tb branch use' and retry the command."
     )
     error_not_allowed_in_branch = error_message(
-        "You need to be in Main to run this command. Hint: run tb env use main to switch to Main"
+        "You need to be in Main to run this command. Hint: run `tb branch use main` to switch to Main"
     )
-    error_not_allowed_in_main_branch = error_message("Command disabled for 'main' Environment")
+    error_not_allowed_in_main_branch = error_message("Command disabled for 'main' Branch")
     error_getting_region_by_index = error_message(
         "Unable to get region by index, list available regions using 'tb auth ls'"
     )
@@ -281,8 +281,8 @@ class FeedbackManager:
     error_init_release = error_message(
         "No release on workspace '{workspace}'. Hint: use 'tb init --git' to start working with git releases"
     )
-    error_env_init_release = error_message(
-        "Environment '{workspace}' not ready for deploy. Hint: use 'tb init --git' to start working with git releases and assure Environment is completely created using '--wait' on 'tb env create'"
+    error_branch_init_release = error_message(
+        "Branch '{workspace}' not ready for deploy. Hint: use 'tb init --git' to start working with git releases and assure Branch is completely created using '--wait' on 'tb branch create'"
     )
     error_commit_changes_to_init_release = error_message(
         "Data project has changes: 'git status -s {path}'\n\n{git_output} \n\n You need to commit your changes in the data project, then re-run 'tb init --git' to finish git initialization."
@@ -357,8 +357,8 @@ Ready? """
     )
 
     warning_no_test_results = warning_message("Warning: No test results to show")
-    warning_using_environment_token = warning_message("** You're using the token defined in $TB_TOKEN.")
-    warning_using_environment_host = warning_message("** You're using the token defined in $TB_HOST.")
+    warning_using_branch_token = warning_message("** You're using the token defined in $TB_TOKEN.")
+    warning_using_branch_host = warning_message("** You're using the token defined in $TB_HOST.")
 
     error_bigquery_improper_permissions = info_message(
         "** Error: no access detected. It might take a minute to detect the new permissions.\n"
@@ -420,7 +420,7 @@ Ready? """
         "Do you want to remove all pipes and Data Sources from this workspace?"
     )
     warning_confirm_delete_workspace = prompt_message("Do you want to remove {workspace_name} workspace?")
-    warning_confirm_delete_branch = prompt_message("Do you want to remove '{branch}' Environment?")
+    warning_confirm_delete_branch = prompt_message("Do you want to remove '{branch}' Branch?")
     warning_confirm_delete_release = prompt_message("Do you want to remove Release {semver}?")
     warning_confirm_rollback_release = prompt_message("Do you want to rollback current Release {semver} to {rollback}?")
 
@@ -476,9 +476,9 @@ Ready? """
     )
     info_populate_job_result = info_message("** Populating job result\n {result}")
     info_populate_job_url = info_message("** Populating job url {url}")
-    info_data_branch_job_url = info_message("** Environment job url {url}")
-    info_regression_tests_branch_job_url = info_message("** Environment regression tests job url {url}")
-    info_merge_branch_job_url = info_message("** Merge Environment deployment job url {url}")
+    info_data_branch_job_url = info_message("** Branch job url {url}")
+    info_regression_tests_branch_job_url = info_message("** Branch regression tests job url {url}")
+    info_merge_branch_job_url = info_message("** Merge Branch deployment job url {url}")
     info_copy_from_main_job_url = info_message("** Copy from 'main' Workspace to '{datasource_name}' job url {url}")
     info_copy_with_sql_job_url = info_message("** Copy with --sql `{sql}` to '{datasource_name}' job url {url}")
     info_populate_subset_job_url = info_message("** Populating (subset {subset}) job url {url}")
@@ -542,14 +542,14 @@ Ready? """
     info_progress_blocks = info_message("\N{egg} blocks")
     info_progress_current_blocks = info_message("\N{hatching chick} blocks")
     info_release_generated = info_message(
-        "** Custom deployment files for Release {semver} generated. Edit the `ci-deploy.sh` file with the instructions to deploy the branch to a CI Environment and the `cd-deploy.sh` file with the instructions to deploy to the Main Environment after merge."
+        "** Custom deployment files for Release {semver} generated. Edit the `ci-deploy.sh` file with the instructions to deploy the branch to a CI Branch and the `cd-deploy.sh` file with the instructions to deploy to the Main Branch after merge."
     )
     info_jobs = info_message("** Jobs:")
     info_workspaces = info_message("** Workspaces:")
-    info_branches = info_message("** Environments:")
+    info_branches = info_message("** Branches:")
     info_releases = info_message("** Releases:")
     info_current_workspace = info_message("** Current workspace:")
-    info_current_branch = info_message("** Current Environment:")
+    info_current_branch = info_message("** Current Branch:")
     info_job = info_message("  ** Job: {job}")
     info_data_pushed = info_message("** Data pushed to {datasource}")
     info_materialized_datasource_created = info_message(
@@ -587,7 +587,7 @@ Ready? """
         "Now let's pick a starter template! 🐣\nStarter template are pre-built data projects for different use cases, that you can use as a starting point and then build on top of that.\nYou can bypass this step by supplying a value for the --starter-kit option."
     )
     info_workspace_branch_create_greeting = info_message(
-        "Please enter the name for your new Environment. Remember the name you choose must be unique, you can add a suffix in case of collision.\nYou can bypass this step by supplying a value for the --name option."
+        "Please enter the name for your new Branch. Remember the name you choose must be unique, you can add a suffix in case of collision.\nYou can bypass this step by supplying a value for the --name option."
     )
     info_no_git_release_yet = info_message("\n** Initializing releases based on git for Workspace '{workspace}'")
     info_diff_resources_for_git_init = info_message(
@@ -612,8 +612,8 @@ Ready? """
     info_deleting_resource = info_message("** {dry_run}Deleting '{resource_name}'")
 
     info_cicd_generation_cancelled_by_user = info_message("** CI/CD files generation cancelled by user.")
-    info_skipping_sharing_datasources_environment = info_message(
-        "** Skipping sharing the datasoure {datasource} in an environment"
+    info_skipping_sharing_datasources_branch = info_message(
+        "** Skipping sharing the datasoure {datasource} in a Branch"
     )
     info_skipping_shared_with_entry = info_message(
         "** Skipping `SHARED_WITH` entry as the flag --user-token was not used"
@@ -755,20 +755,20 @@ Ready? """
     success_using_host = success_message("** Using host: {host} ({name})")
     success_workspace_created = success_message("** Workspace '{workspace_name}' has been created")
     success_workspace_branch_created = success_message(
-        "** Environment '{branch_name}' from '{workspace_name}' has been created"
+        "** Branch '{branch_name}' from '{workspace_name}' has been created"
     )
     success_workspace_data_branch = success_message(
-        "** Partitions from 'main' Workspace have been attached to the Environment"
+        "** Partitions from 'main' Workspace have been attached to the Branch"
     )
     success_workspace_data_branch_in_progress = success_message(
-        "** Partitions from 'main' Workspace are being attached to the Environment in job {job_url}"
+        "** Partitions from 'main' Workspace are being attached to the Branch in job {job_url}"
     )
 
     success_workspace_deploying_template = success_message(
         "Deploying your new '{workspace_name}' workspace, using the '{template}' template:"
     )
     success_workspace_deleted = success_message("** Workspace '{workspace_name}' deleted")
-    success_branch_deleted = success_message("** Environment '{branch_name}' deleted")
+    success_branch_deleted = success_message("** Branch '{branch_name}' deleted")
     success_workspace_user_added = success_message("** User {user} added to workspace '{workspace_name}'")
     success_workspace_users_added = success_message("** Users added to workspace '{workspace_name}'")
     success_workspace_user_removed = success_message("** User {user} removed from workspace '{workspace_name}'")

{tinybird-cli-2.1.1.dev6 → tinybird-cli-2.1.1.dev8}/tinybird/tb_cli_modules/branch.py

@@ -266,15 +266,15 @@ async def release_rm(semver: str, force: bool, yes: bool, dry_run: bool) -> None
 
 
 @cli.group()
-def env() -> None:
-    """Environment commands. Environments is an experimental feature only available in beta. Running env commands without activation will return an error"""
+def branch() -> None:
+    """Branch commands. Branches are an experimental feature only available in beta. Running branch commands without activation will return an error"""
     pass
 
 
-@env.command(name="ls")
+@branch.command(name="ls")
 @coro
 async def branch_ls() -> None:
-    """List all the environments available using the workspace token"""
+    """List all the branches available using the workspace token"""
 
     config = CLIConfig.get_project_config()
     _ = await try_update_config_with_remote(config, only_if_needed=True)
@@ -303,43 +303,43 @@ async def branch_ls() -> None:
     echo_safe_humanfriendly_tables_format_smart_table(table, column_names=columns)
 
 
-@env.command(name="use")
-@click.argument("env_name_or_id")
+@branch.command(name="use")
+@click.argument("branch_name_or_id")
 @coro
-async def branch_use(env_name_or_id: str) -> None:
-    """Switch to another Environment (requires an admin token associated with a user). Use 'tb env ls' to list the Environments you can access"""
+async def branch_use(branch_name_or_id: str) -> None:
+    """Switch to another Branch (requires an admin token associated with a user). Use 'tb branch ls' to list the Branches you can access"""
 
     config = CLIConfig.get_project_config()
 
-    if env_name_or_id == MAIN_BRANCH:
+    if branch_name_or_id == MAIN_BRANCH:
         current_main_workspace = await get_current_main_workspace_temp(config)
         assert isinstance(current_main_workspace, dict)
         await switch_to_workspace_by_user_workspace_data(config, current_main_workspace)
     else:
-        await switch_workspace(config, env_name_or_id, only_environments=True)
+        await switch_workspace(config, branch_name_or_id, only_environments=True)
 
 
-@env.command(name="current")
+@branch.command(name="current")
 @coro
 async def branch_current() -> None:
-    """Show the Environment you're currently authenticated to"""
+    """Show the Branch you're currently authenticated to"""
     config = CLIConfig.get_project_config()
     await print_current_branch(config)
 
 
-@env.command(name="create", short_help="Create a new Environment in the current 'main' Workspace")
-@click.argument("env_name", required=False)
+@branch.command(name="create", short_help="Create a new Branch in the current 'main' Workspace")
+@click.argument("branch_name", required=False)
 @click.option(
     "--last-partition",
     is_flag=True,
     default=False,
-    help="Attach the last modified partition from 'main' to the new Environment",
+    help="Attach the last modified partition from 'main' to the new Branch",
 )
 @click.option(
     "--all",
     is_flag=True,
     default=False,
-    help="Attach all data from 'main' to the new Environment. Use only if you actually need all the data in the Branch",
+    help="Attach all data from 'main' to the new Branch. Use only if you actually need all the data in the Branch",
 )
 @click.option(
     "-i",
@@ -357,26 +357,26 @@ async def branch_current() -> None:
 )
 @coro
 async def create_branch(
-    env_name: Optional[str], last_partition: bool, all: bool, ignore_datasources: List[str], wait: bool
+    branch_name: Optional[str], last_partition: bool, all: bool, ignore_datasources: List[str], wait: bool
 ) -> None:
     if last_partition and all:
         raise CLIException(FeedbackManager.error_exception(error="Use --last-partition or --all but not both"))
-    await create_workspace_branch(env_name, last_partition, all, list(ignore_datasources), wait)
+    await create_workspace_branch(branch_name, last_partition, all, list(ignore_datasources), wait)
 
 
-@env.command(name="rm", short_help="Removes an Environment from the Workspace. It can't be recovered.")
-@click.argument("env_name_or_id")
+@branch.command(name="rm", short_help="Removes a Branch from the Workspace. It can't be recovered.")
+@click.argument("branch_name_or_id")
 @click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
 @coro
-async def delete_branch(env_name_or_id: str, yes: bool) -> None:
-    """Remove an Environment (not Main)"""
+async def delete_branch(branch_name_or_id: str, yes: bool) -> None:
+    """Remove an Branch (not Main)"""
 
     config = CLIConfig.get_project_config()
     _ = await try_update_config_with_remote(config)
 
     client = config.get_client()
 
-    if env_name_or_id == MAIN_BRANCH:
+    if branch_name_or_id == MAIN_BRANCH:
         raise CLIException(FeedbackManager.error_not_allowed_in_main_branch())
 
     try:
@@ -385,7 +385,7 @@ async def delete_branch(env_name_or_id: str, yes: bool) -> None:
             (
                 workspace
                 for workspace in workspace_branches
-                if workspace["name"] == env_name_or_id or workspace["id"] == env_name_or_id
+                if workspace["name"] == branch_name_or_id or workspace["id"] == branch_name_or_id
             ),
             None,
         )
@@ -393,7 +393,7 @@ async def delete_branch(env_name_or_id: str, yes: bool) -> None:
         raise CLIBranchException(FeedbackManager.error_exception(error=str(e)))
 
     if not workspace_to_delete:
-        raise CLIBranchException(FeedbackManager.error_branch(branch=env_name_or_id))
+        raise CLIBranchException(FeedbackManager.error_branch(branch=branch_name_or_id))
 
     if yes or click.confirm(FeedbackManager.warning_confirm_delete_branch(branch=workspace_to_delete["name"])):
         need_to_switch_to_main = workspace_to_delete.get("main") and config["id"] == workspace_to_delete["id"]
@@ -419,21 +419,21 @@ async def delete_branch(env_name_or_id: str, yes: bool) -> None:
                     raise CLIException(FeedbackManager.error_switching_to_main())
 
 
-@env.command(
+@branch.command(
     name="data",
-    short_help="Perform a data branch operation to bring data into the current Environment. Check flags for details",
+    short_help="Perform a data branch operation to bring data into the current Branch. Check flags for details",
 )
 @click.option(
     "--last-partition",
     is_flag=True,
     default=False,
-    help="Attach the last modified partition from 'main' to the new Environment",
+    help="Attach the last modified partition from 'main' to the new Branch",
 )
 @click.option(
     "--all",
     is_flag=True,
     default=False,
-    help="Attach all data from 'main' to the new Environment. Use only if you actually need all the data in the Environment",
+    help="Attach all data from 'main' to the new Branch. Use only if you actually need all the data in the Branch",
 )
 @click.option(
     "-i",
@@ -482,7 +482,7 @@ async def data_branch(last_partition: bool, all: bool, ignore_datasources: List[
             job_id = response["job"]["job_id"]
             job_url = response["job"]["job_url"]
             click.echo(FeedbackManager.info_data_branch_job_url(url=job_url))
-            job_response = await wait_job(client, job_id, job_url, "Environment creation")
+            job_response = await wait_job(client, job_id, job_url, "Branch creation")
             response = job_response["result"]
             is_job = False
             is_summary = "partitions" in response
@@ -500,7 +500,7 @@ async def data_branch(last_partition: bool, all: bool, ignore_datasources: List[
         raise CLIBranchException(FeedbackManager.error_exception(error=str(e)))
 
 
-@env.group("regression-tests", invoke_without_command=True)
+@branch.group("regression-tests", invoke_without_command=True)
 @click.option(
     "-f",
     "--filename",
@@ -518,20 +518,20 @@ async def data_branch(last_partition: bool, all: bool, ignore_datasources: List[
     "--skip-regression-tests/--no-skip-regression-tests",
     envvar="TB_SKIP_REGRESSION",
     default=False,
-    help="Flag to skip execution of regression tests. This is handy for CI environments where regression might be flaky",
+    help="Flag to skip execution of regression tests. This is handy for CI branches where regression might be flaky",
 )
 @click.option(
     "--main",
     is_flag=True,
     default=False,
-    help="Run regression tests in the main Environment. For this flag to work all the resources in the Environment pipe endpoints need to exist in the main Environment.",
+    help="Run regression tests in the main Branch. For this flag to work all the resources in the Branch pipe endpoints need to exist in the main Branch.",
 )
 @click.pass_context
 @coro
 async def regression_tests(
     ctx, filename: str, wait: bool, skip_regression_tests: Optional[bool] = False, main: Optional[bool] = False
 ):
-    """Regression test commands for Environments"""
+    """Regression test commands for Branches"""
     if skip_regression_tests:
         click.echo(FeedbackManager.warning_regression_skipped())
         return
@@ -633,7 +633,7 @@ async def _run_regression(
 
 @regression_tests.command(
     name="coverage",
-    short_help="Run regression tests using coverage requests for Environment vs Main Workspace. It creates a regression-tests job. The argument pipe_name supports regular expressions. Using '.*' if no pipe_name is provided",
+    short_help="Run regression tests using coverage requests for Branch vs Main Workspace. It creates a regression-tests job. The argument pipe_name supports regular expressions. Using '.*' if no pipe_name is provided",
 )
 @click.argument("pipe_name", required=False)
 @click.option(
@@ -694,13 +694,13 @@ async def _run_regression(
     "--skip-regression-tests/--no-skip-regression-tests",
     envvar="TB_SKIP_REGRESSION",
     default=False,
-    help="Flag to skip execution of regression tests. This is handy for CI environments where regression might be flaky",
+    help="Flag to skip execution of regression tests. This is handy for CI branches where regression might be flaky",
 )
 @click.option(
     "--main",
     is_flag=True,
     default=False,
-    help="Run regression tests in the main Environment. For this flag to work all the resources in the Environment pipe endpoints need to exist in the main Environment.",
+    help="Run regression tests in the main Branch. For this flag to work all the resources in the Branch pipe endpoints need to exist in the main Branch.",
 )
 @coro
 async def coverage(
@@ -736,7 +736,7 @@ async def coverage(
 
 @regression_tests.command(
     name="last",
-    short_help="Run regression tests using last requests for Environment vs Main Workspace. It creates a regression-tests job. The argument pipe_name supports regular expressions. Using '.*' if no pipe_name is provided",
+    short_help="Run regression tests using last requests for Branch vs Main Workspace. It creates a regression-tests job. The argument pipe_name supports regular expressions. Using '.*' if no pipe_name is provided",
 )
 @click.argument("pipe_name", required=False)
 @click.option(
@@ -805,7 +805,7 @@ async def coverage(
     "--skip-regression-tests/--no-skip-regression-tests",
     envvar="TB_SKIP_REGRESSION",
     default=False,
-    help="Flag to skip execution of regression tests. This is handy for CI environments where regression might be flaky",
+    help="Flag to skip execution of regression tests. This is handy for CI branches where regression might be flaky",
 )
 @coro
 async def last(
@@ -841,7 +841,7 @@ async def last(
 
 @regression_tests.command(
     name="manual",
-    short_help="Run regression tests using manual requests for Environment vs Main Workspace. It creates a regression-tests job. The argument pipe_name supports regular expressions. Using '.*' if no pipe_name is provided",
+    short_help="Run regression tests using manual requests for Branch vs Main Workspace. It creates a regression-tests job. The argument pipe_name supports regular expressions. Using '.*' if no pipe_name is provided",
     context_settings=dict(allow_extra_args=True, ignore_unknown_options=True),
 )
 @click.argument("pipe_name", required=False)
@@ -903,7 +903,7 @@ async def last(
     "--skip-regression-tests/--no-skip-regression-tests",
     envvar="TB_SKIP_REGRESSION",
     default=False,
-    help="Flag to skip execution of regression tests. This is handy for CI environments where regression might be flaky",
+    help="Flag to skip execution of regression tests. This is handy for CI branches where regression might be flaky",
 )
 @click.pass_context
 @coro
@@ -939,9 +939,9 @@ async def manual(
     )
 
 
-@env.group()
+@branch.group()
 def datasource() -> None:
-    """Environment data source commands."""
+    """Branch data source commands."""
 
 
 @datasource.command(name="copy")
@@ -949,7 +949,7 @@ def datasource() -> None:
 @click.option(
     "--sql",
     default=None,
-    help="Freeform SQL query to select what is copied from Main into the Environment Data Source",
+    help="Freeform SQL query to select what is copied from Main into the Branch Data Source",
     required=False,
 )
 @click.option(

{tinybird-cli-2.1.1.dev6 → tinybird-cli-2.1.1.dev8}/tinybird/tb_cli_modules/cicd.py

@@ -16,6 +16,10 @@ class Provider(Enum):
     GitLab = 1
 
 
+WORKFLOW_VERSION = "v2.5.1"
+
+TB_DEPLOY = "false"
+
 GITHUB_CI_YML = """
     ##################################################
     ###   Visit https://github.com/tinybirdco/ci   ###
@@ -37,10 +41,10 @@ GITHUB_CI_YML = """
     concurrency: ${{! github.workflow }}-${{! github.event.pull_request.number }}
 
     jobs:
-        ci: # ci using environments from workspace '{{ workspace_name }}'
-          uses: tinybirdco/ci/.github/workflows/ci.yml@v2.5.1
+        ci: # ci using branches from workspace '{{ workspace_name }}'
+          uses: tinybirdco/ci/.github/workflows/ci.yml@{{ workflow_version }}
           with:
-            tb_deploy: false
+            tb_deploy: {{ tb_deploy }}
             data_project_dir: {{ data_project_dir }}
           secrets:
             tb_admin_token: ${{! secrets.TB_ADMIN_TOKEN }}  # set the Workspace admin token in GitHub secrets
@@ -65,9 +69,9 @@ GITHUB_CD_YML = """
           - '{{ data_project_dir }}/**'{% end %}
     jobs:
       cd:  # deploy changes to workspace '{{ workspace_name }}'
-        uses: tinybirdco/ci/.github/workflows/cd.yml@v2.5.1
+        uses: tinybirdco/ci/.github/workflows/cd.yml@{{ workflow_version }}
         with:
-          tb_deploy: false
+          tb_deploy: {{ tb_deploy }}
           data_project_dir: {{ data_project_dir }}
         secrets:
           tb_admin_token: ${{! secrets.TB_ADMIN_TOKEN }}  # set the Workspace admin token in GitHub secrets
@@ -92,11 +96,11 @@ GITHUB_RELEASES_YML = """
 
     jobs:
       cd:  # manage releases for workspace '{{ workspace_name }}'
-        uses: tinybirdco/ci/.github/workflows/release.yml@v2.5.0
+        uses: tinybirdco/ci/.github/workflows/release.yml@{{ workflow_version }}
         with:
           job_to_run: ${{! inputs.job_to_run }}
           data_project_dir: {{ data_project_dir }}
-          tb_deploy: false
+          tb_deploy: {{ tb_deploy }}
         secrets:
           tb_admin_token: ${{! secrets.TB_ADMIN_TOKEN }}  # set the Workspace admin token in GitHub secrets
           tb_host: {{ tb_host }}
@@ -109,7 +113,7 @@ GITLAB_YML = """
     ###   for more details or custom CI/CD         ###
     ##################################################
 
-    include: "https://raw.githubusercontent.com/tinybirdco/ci/v2.5.1/.gitlab/ci_cd.yaml"
+    include: "https://raw.githubusercontent.com/tinybirdco/ci/{{ workflow_version }}/.gitlab/ci_cd.yaml"
 
     .ci_config_rules:
       - &ci_config_rule
@@ -136,9 +140,9 @@ GITLAB_YML = """
         TB_HOST: "{{ tb_host }}"
         TB_ADMIN_TOKEN: $TB_ADMIN_TOKEN  # set the Workspace admin token in GitLab CI/CD Variables
         DATA_PROJECT_DIR: "{{ data_project_dir }}"
-        TB_DEPLOY: "false"
+        TB_DEPLOY: "{{ tb_deploy }}"
 
-    run_ci:  # ci using environments from workspace '{{ workspace_name }}'
+    run_ci:  # ci using branches from workspace '{{ workspace_name }}'
       extends: .run_ci
       rules:
         - *ci_config_rule
@@ -305,6 +309,8 @@ async def init_cicd(
             "token_name": token["name"],
             "token_id": token["id"],
             "data_project_dir": data_project_dir,
+            "workflow_version": WORKFLOW_VERSION,
+            "tb_deploy": TB_DEPLOY,
         }
         warning_message = generator(path, params)
         if warning_message:

{tinybird-cli-2.1.1.dev6 → tinybird-cli-2.1.1.dev8}/tinybird/tb_cli_modules/cli.py

@@ -40,6 +40,7 @@ from tinybird.datafile import (
     get_project_filenames,
     get_resource_versions,
     has_internal_datafiles,
+    is_file_a_datasource,
     parse_datasource,
     parse_pipe,
     parse_token,
@@ -764,7 +765,7 @@ async def fmt(
         if not diff:
             click.echo(filename)
         extensions = Path(filename).suffixes
-        if ".datasource" in extensions:  # Accepts '.datasource' and '.datasource.incl'
+        if is_file_a_datasource(filename):
             result = await format_datasource(filename)
         elif (".pipe" in extensions) or (".incl" in extensions):
             result = await format_pipe(filename, line_length)
@@ -819,7 +820,7 @@ async def fmt(
     "--main",
     is_flag=True,
     default=False,
-    help="Diffs local datafiles to the corresponding remote files in the main workspace. Only works when authenticated on an Environment.",
+    help="Diffs local datafiles to the corresponding remote files in the main workspace. Only works when authenticated on a Branch.",
     hidden=True,
 )
 @click.pass_context
@@ -1482,15 +1483,15 @@ async def deploy(
     )
 
     semver = config.get("semver")
-    is_environment = current_ws.get("is_branch")
+    is_branch = current_ws.get("is_branch")
     # FIXME: we need a better way
-    auto_promote = (is_environment and current_ws.get("name", "").startswith("tmp_ci")) or auto_promote
+    auto_promote = (is_branch and current_ws.get("name", "").startswith("tmp_ci")) or auto_promote
     release = current_ws.get("release", {})
     current_semver: Optional[str] = "no release in remote Workspace"
     if release and isinstance(release, dict):
         current_semver = release.get("semver")
 
-    if not is_environment:
+    if not is_branch:
         # TODO upload fixtures on tb test
         fixtures = False
         click.echo(FeedbackManager.info_fixtures_branch())

{tinybird-cli-2.1.1.dev6 → tinybird-cli-2.1.1.dev8}/tinybird/tb_cli_modules/common.py

@@ -226,7 +226,28 @@ async def get_current_workspace_branches(config: CLIConfig) -> List[Dict[str, An
     return [branch for branch in branches if branch.get("main") == current_main_workspace["id"]]
 
 
-class CatchAuthExceptions(click.Group):
+class AliasedGroup(click.Group):
+    def get_command(self, ctx, cmd_name):
+        # Step one: built-in commands as normal
+        cm = click.Group.get_command(self, ctx, cmd_name)
+        if cm is not None:
+            return cm
+
+        if cmd_name == "env":
+            deprecation_notice = FeedbackManager.warning_deprecated(
+                warning="'tb env' is deprecated, use 'tb branch' instead. 'tb env' will be removed in the next 3.0.0 release, please update your scripts."
+            )
+            click.echo(deprecation_notice)
+
+            return click.Group.get_command(self, ctx, "branch")
+
+    def resolve_command(self, ctx, args):
+        # always return the command's name, not the alias
+        _, cmd, args = super().resolve_command(ctx, args)
+        return cmd.name, cmd, args
+
+
+class CatchAuthExceptions(AliasedGroup):
     """utility class to get all the auth exceptions"""
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
@@ -953,7 +974,7 @@ async def create_workspace_branch(
         if not branch_name:
             click.echo(FeedbackManager.info_workspace_branch_create_greeting())
             default_name = f"{workspace['name']}_{uuid.uuid4().hex[0:4]}"
-            branch_name = click.prompt("\Environment name", default=default_name, err=True, type=str)
+            branch_name = click.prompt("\Branch name", default=default_name, err=True, type=str)
         assert isinstance(branch_name, str)
 
         response = await config.get_client().create_workspace_branch(
@@ -988,7 +1009,7 @@ async def create_workspace_branch(
             assert isinstance(job_id, str)
 
             # Await the job to finish and get the result dict
-            job_response = await wait_job(config.get_client(), job_id, job_url, "Environment creation")
+            job_response = await wait_job(config.get_client(), job_id, job_url, "Branch creation")
             if job_response is None:
                 raise CLIException(f"Empty job API response (job_id: {job_id}, job_url: {job_url})")
             else:
@@ -1056,7 +1077,7 @@ async def print_branch_regression_tests_summary(client, job_id, host, response=N
     for step in response["progress"]:
         run = step["run"]
         if run.get("metrics_summary") and run.get("metrics_timing"):
-            column_names = [f"{run['pipe_name']}({run['test_type']})", "Origin", "Environment", "Delta"]
+            column_names = [f"{run['pipe_name']}({run['test_type']})", "Origin", "Branch", "Delta"]
 
             click.echo(
                 format_pretty_table(

{tinybird-cli-2.1.1.dev6 → tinybird-cli-2.1.1.dev8}/tinybird/tb_cli_modules/connection.py

@@ -12,6 +12,7 @@ import click
 from click import Context
 
 from tinybird.client import DoesNotExistException, TinyB
+from tinybird.connector_settings import DATA_CONNECTOR_SETTINGS, DataConnectors, DataSensitiveSettings
 from tinybird.feedback_manager import FeedbackManager
 from tinybird.tb_cli_modules.cli import cli
 from tinybird.tb_cli_modules.common import (
@@ -339,11 +340,8 @@ async def connection_rm(ctx: Context, connection_id: str, force: bool) -> None:
 @click.option("--connector", help="Filter by connector")
 @click.pass_context
 @coro
-async def connection_ls(ctx: Context, connector: str) -> None:
+async def connection_ls(ctx: Context, connector: Optional[DataConnectors] = None) -> None:
     """List connections."""
-
-    from tinybird.connector_settings import DataConnectorSettings, DataSensitiveSettings
-
     obj: Dict[str, Any] = ctx.ensure_object(dict)
     client: TinyB = obj["client"]
 
@@ -358,9 +356,9 @@ async def connection_ls(ctx: Context, connector: str) -> None:
         columns = ["service", "name", "id", "connected_datasources"]
     else:
         sensitive_settings = getattr(DataSensitiveSettings, connector)
-        columns = ["service", "name", "id", "connected_datasources"] + [
-            setting.replace("tb_", "") for setting in getattr(DataConnectorSettings, connector)
-        ]
+        columns = ["service", "name", "id", "connected_datasources"]
+        if connector_settings := DATA_CONNECTOR_SETTINGS.get(connector):
+            columns += [setting.replace("tb_", "") for setting in connector_settings.model_fields.keys()]
 
     for connection in connections:
         row = [_get_setting_value(connection, setting, sensitive_settings) for setting in columns]

{tinybird-cli-2.1.1.dev6 → tinybird-cli-2.1.1.dev8}/tinybird/tb_cli_modules/datasource.py

@@ -741,7 +741,7 @@ async def datasource_exchange(ctx, datasource_a, datasource_b):
 @click.option(
     "--sql",
     default=None,
-    help="Freeform SQL query to select what is copied from Main into the Environment Data Source",
+    help="Freeform SQL query to select what is copied from Main into the Branch Data Source",
     required=False,
 )
 @click.option(

{tinybird-cli-2.1.1.dev6 → tinybird-cli-2.1.1.dev8}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 2.1.1.dev6
+Version: 2.1.1.dev8
 Summary: Tinybird Command Line Tool
 Home-page: https://docs.tinybird.co/cli.html
 Author: Tinybird
@@ -19,7 +19,17 @@ Changelog
 
 ---------
 
-2.1.1.dev5
+2.1.1.dev8
+************
+
+- `Changed` Use `tb branch` instead of `tb env`. `tb env` is still usable but deprecated and will be removed in the next version.
+
+2.1.1.dev7
+************
+
+- `Fixed` Be smarter in `tb fmt` when detecting if `.incl` file contains Data Source or Pipe definitions.
+
+2.1.1.dev6
 ************
 
 - `Fixed` `tb fmt` adds empty quoted strings in .datasource connection section.

{tinybird-cli-2.1.1.dev6 → tinybird-cli-2.1.1.dev8}/tinybird_cli.egg-info/requires.txt

@@ -5,6 +5,7 @@ cryptography>=3.4.8
 croniter==1.3.8
 GitPython==3.1.29
 humanfriendly==8.2
+pydantic==2.5.2
 pyperclip==1.8.2
 pyyaml<6.1,>=6.0
 requests<3,>=2.28.1

tinybird-cli-2.1.1.dev6/tinybird/connector_settings.py

@@ -1,102 +0,0 @@
-from enum import Enum
-from typing import List
-
-
-class DataConnectorType(str, Enum):
-    KAFKA = "kafka"
-    GCLOUD_SCHEDULER = "gcscheduler"
-    SNOWFLAKE = "snowflake"
-    BIGQUERY = "bigquery"
-    GCLOUD_STORAGE = "gcs"
-    GCLOUD_STORAGE_HMAC = "gcs_hmac"
-    AMAZON_S3 = "s3"
-    AMAZON_S3_IAMROLE = "s3_iamrole"
-
-    def __str__(self) -> str:
-        return self.value
-
-
-class DataConnectorSettings:
-    kafka = [
-        "cli_version",
-        "tb_endpoint",
-        "kafka_bootstrap_servers",
-        "kafka_sasl_plain_username",
-        "kafka_sasl_plain_password",
-        "kafka_security_protocol",
-        "kafka_sasl_mechanism",
-        "kafka_schema_registry_url",
-    ]
-    gcscheduler = ["gcscheduler_region"]
-    bigquery = ["account"]
-    snowflake = ["account", "username", "password", "warehouse", "warehouse_size", "role", "stage", "integration"]
-    gcs_hmac = ["gcs_hmac_access_id", "gcs_hmac_secret"]
-    s3 = [
-        "s3_access_key_id",
-        "s3_secret_access_key",
-        "s3_region",
-    ]
-    s3_iamrole = [
-        "s3_iamrole_arn",
-        "s3_iamrole_external_id",
-        "s3_iamrole_region",
-    ]
-    gcs = [
-        "gcs_private_key_id",
-        "gcs_client_x509_cert_url",
-        "gcs_project_id",
-        "gcs_client_id",
-        "gcs_client_email",
-        "gcs_private_key",
-    ]
-
-
-class DataLinkerSettings:
-    kafka = [
-        "tb_datasource",
-        "tb_token",
-        "kafka_topic",
-        "kafka_group_id",
-        "kafka_auto_offset_reset",
-        "kafka_store_raw_value",
-        "kafka_store_headers",
-    ]
-
-
-class DataSinkSettings:
-    gcscheduler = ["cron", "timezone", "status", "gcscheduler_target_url", "gcscheduler_job_name", "gcscheduler_region"]
-    gcs_hmac = [
-        "bucket_path",
-        "file_template",
-        "partition_node",
-        "format",
-        "compression",
-    ]
-    s3 = [
-        "bucket_path",
-        "file_template",
-        "partition_node",
-        "format",
-        "compression",
-    ]
-    s3_iamrole = [
-        "bucket_path",
-        "file_template",
-        "partition_node",
-        "format",
-        "compression",
-    ]
-
-
-class DataSensitiveSettings:
-    kafka = ["kafka_sasl_plain_password"]
-    gcscheduler = [
-        "gcscheduler_target_url",
-        "gcscheduler_job_name",
-        "gcscheduler_region",
-    ]
-    bigquery: List[str] = []
-    snowflake: List[str] = []
-    gcs_hmac: List[str] = ["gcs_hmac_secret"]
-    s3: List[str] = ["s3_secret_access_key"]
-    s3_iamrole: List[str] = ["s3_iamrole_arn"]