tinybird 4.5.4.dev0__tar.gz → 4.5.5.dev0__tar.gz

{tinybird-4.5.4.dev0 → tinybird-4.5.5.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 4.5.4.dev0
+Version: 4.5.5.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -52,6 +52,11 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+4.5.4
+*******
+
+- `Added` `tb migrate-to-forward` to help users migrate Classic workspaces to Forward
+
 4.5.3
 *******
 

{tinybird-4.5.4.dev0 → tinybird-4.5.5.dev0}/tinybird/service_datasources.py

@@ -507,6 +507,8 @@ def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "ch_written_bytes", "type": "UInt64"},
                 {"name": "ch_cpu_time", "type": "Float32"},
                 {"name": "rate_limited", "type": "UInt8"},
+                {"name": "ips", "type": "SimpleAggregateFunction(groupUniqArrayArray, Array(String))"},
+                {"name": "tokens", "type": "SimpleAggregateFunction(groupUniqArrayArray, Array(String))"},
             ],
         },
         {

{tinybird-4.5.4.dev0 → tinybird-4.5.5.dev0}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '4.5.4.dev0'
-__revision__ = '23af8bd'
+__version__ = '4.5.5.dev0'
+__revision__ = 'eebce0e'

{tinybird-4.5.4.dev0 → tinybird-4.5.5.dev0}/tinybird/tb/modules/cli.py

@@ -63,7 +63,7 @@ DEV_MODE_VALUES = {DEV_MODE_MANUAL, DEV_MODE_LOCAL, DEV_MODE_BRANCH}
 DEV_MODE_ROUTED_COMMANDS = {"build", "deploy"}
 SDK_PROJECT_ROUTED_COMMANDS = {"build", "deploy", "preview"}
 TS_PROJECT_ROUTED_COMMANDS = SDK_PROJECT_ROUTED_COMMANDS
-COMMANDS_ALWAYS_CLOUD = {"infra", "branch", "environment", "workspace", "preview"}
+COMMANDS_ALWAYS_CLOUD = {"infra", "branch", "environment", "workspace", "preview", "migrate-to-forward"}
 PROJECT_TYPE_TYPESCRIPT = "ts-sdk"
 PROJECT_TYPE_PYTHON = "python-sdk"
 PROJECT_TYPE_CLI = "cli"

{tinybird-4.5.4.dev0 → tinybird-4.5.5.dev0}/tinybird/tb/modules/deployment.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import os
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Any, Dict, Optional
@@ -7,14 +8,17 @@ from typing import Any, Dict, Optional
 import click
 import requests
 
+from tinybird.tb.client import TinyB
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import (
     echo_safe_humanfriendly_tables_format_smart_table,
     sys_exit,
 )
+from tinybird.tb.modules.create import persist_tinybird_config
 from tinybird.tb.modules.deployment_common import (
     create_deployment,
     discard_deployment,
+    migrate_to_forward_workspace,
     promote_deployment,
 )
 from tinybird.tb.modules.feedback_manager import FeedbackManager
@@ -151,6 +155,63 @@ def api_fetch(url: str, headers: dict, request_from: Optional[str] = None) -> di
     return {}
 
 
+def _get_classic_workspace_branches(client: TinyB, workspace_id: str) -> list[dict[str, Any]]:
+    branches: list[dict[str, Any]] = client.user_workspace_branches(version="v0").get("workspaces", [])
+    return [branch for branch in branches if str(branch.get("main")) == workspace_id]
+
+
+def _get_non_live_classic_releases(client: TinyB, workspace_id: str) -> list[dict[str, Any]]:
+    releases: list[dict[str, Any]] = client.releases(workspace_id=workspace_id).get("releases", [])
+    return [release for release in releases if release.get("status") != "live"]
+
+
+def _cleanup_classic_migration_blockers(client: TinyB, config: Dict[str, Any]) -> None:
+    workspace_id = str(config["id"])
+    workspace_name = str(config["name"])
+    branches = _get_classic_workspace_branches(client, workspace_id)
+    releases = _get_non_live_classic_releases(client, workspace_id)
+
+    if not branches and not releases:
+        return
+
+    try:
+        for branch in branches:
+            client.delete_branch(id=str(branch["id"]))
+
+        for release in releases:
+            client.release_rm(
+                workspace_id=workspace_id,
+                semver=str(release["semver"]),
+                confirmation=workspace_name,
+                dry_run=False,
+                force=False,
+            )
+    except Exception as e:
+        message = f"Error cleaning up Classic branches or releases before migration: {str(e)}"
+        click.echo(FeedbackManager.error(message=message))
+        sys_exit("migration_error", message)
+
+
+def _persist_migrate_to_forward_config(project: Project) -> None:
+    root_folder = os.getcwd()
+    project_folder = os.path.relpath(project.path.resolve(), root_folder)
+
+    config_changed, config_created = persist_tinybird_config(
+        root_folder=root_folder,
+        project_type="cli",
+        dev_mode="manual",
+        folder=project_folder,
+    )
+
+    if not config_changed:
+        return
+
+    message = "Created tinybird.config.json for the Forward CLI"
+    if not config_created:
+        message = "Updated tinybird.config.json for the Forward CLI"
+    click.echo(FeedbackManager.info(message=message))
+
+
 @cli.group(name="deployment")
 def deployment_group() -> None:
     """
@@ -371,6 +432,104 @@ def deploy(
     create_deployment_cmd(ctx, wait, auto, check, allow_destructive_operations, template, verbose)
 
 
+@cli.command(name="migrate-to-forward")
+@click.option(
+    "--allow-destructive-operations/--no-allow-destructive-operations",
+    is_flag=True,
+    default=False,
+    help="Allow destructive operations in deployments (for example replacing a Pipe with a Data Source).",
+)
+@click.pass_context
+def migrate_to_forward(ctx: click.Context, allow_destructive_operations: bool) -> None:
+    """Migrate a Tinybird Classic cloud workspace to Tinybird Forward."""
+    client = ctx.ensure_object(dict)["client"]
+    project: Project = ctx.ensure_object(dict)["project"]
+    config: Dict[str, Any] = ctx.ensure_object(dict)["config"]
+    env = ctx.ensure_object(dict)["env"]
+    output = ctx.ensure_object(dict)["output"]
+
+    try:
+        workspace_info = client.workspace_info(version="v0")
+    except Exception as e:
+        message = f"Error checking workspace status: {str(e)}"
+        click.echo(FeedbackManager.error(message=message))
+        sys_exit("migration_error", message)
+
+    if workspace_info.get("is_forward", False):
+        message = "This command is unavailable for Tinybird Forward workspaces."
+        click.echo(FeedbackManager.error(message=message))
+        sys_exit("migration_error", message)
+
+    click.echo(
+        FeedbackManager.warning(
+            message=(
+                "This operation is irreversible: once your workspace is migrated to Tinybird Forward, "
+                "you cannot switch it back to Tinybird Classic. It will also run your first Forward deployment."
+            )
+        )
+    )
+
+    if not click.confirm("Do you want to proceed and run the deployment check now?", default=False):
+        click.echo(FeedbackManager.info(message="Migration cancelled."))
+        return
+
+    check_result = create_deployment(
+        project,
+        client,
+        config,
+        wait=False,
+        auto=False,
+        verbose=False,
+        check=True,
+        allow_destructive_operations=allow_destructive_operations,
+        output=output,
+        env=env,
+        show_migrate_to_forward_hint=False,
+        return_check_result=True,
+        skip_forward_workspace_validation=True,
+    )
+    if not check_result:
+        message = "Deployment check did not complete. Migration cancelled."
+        click.echo(FeedbackManager.error(message=message))
+        sys_exit("migration_error", message)
+
+    if check_result and check_result.get("status") == "no_changes":
+        click.echo(
+            FeedbackManager.warning(
+                message=(
+                    "No deployment changes were detected. Add this dummy pipe to your workspace and run "
+                    "`tb migrate-to-forward` again:"
+                )
+            )
+        )
+        click.echo("NODE n\nSQL >\n    select 'Forward'")
+        return
+
+    if not click.confirm(
+        "Do you want to continue with the migration? This will also delete your branches, releases and switch your workspace from Classic to Forward.",
+        default=False,
+    ):
+        click.echo(FeedbackManager.info(message="Migration cancelled."))
+        return
+
+    _persist_migrate_to_forward_config(project)
+    _cleanup_classic_migration_blockers(client, config)
+    migrate_to_forward_workspace(client=client, output=output, dry_run=False)
+    create_deployment(
+        project,
+        client,
+        config,
+        wait=True,
+        auto=True,
+        verbose=False,
+        check=False,
+        allow_destructive_operations=allow_destructive_operations,
+        output=output,
+        env=env,
+        skip_forward_workspace_validation=True,
+    )
+
+
 def create_deployment_cmd(
     ctx: click.Context,
     wait: bool,
@@ -381,6 +540,7 @@ def create_deployment_cmd(
     verbose: bool = False,
 ) -> None:
     output = ctx.ensure_object(dict)["output"]
+    env = ctx.ensure_object(dict)["env"]
     project: Project = ctx.ensure_object(dict)["project"]
     if template:
         if project.get_project_files():
@@ -416,6 +576,7 @@ def create_deployment_cmd(
         allow_destructive_operations,
         ingest_hint=not is_web_analytics_starter_kit,
         output=output,
+        env=env,
     )
     show_web_analytics_starter_kit_hints(client, is_web_analytics_starter_kit)
 

{tinybird-4.5.4.dev0 → tinybird-4.5.5.dev0}/tinybird/tb/modules/deployment_common.py

@@ -99,6 +99,22 @@ def api_post(
     return {}
 
 
+def _get_migrate_to_forward_error_message(result: dict[str, Any]) -> str:
+    error = result.get("error")
+    if error:
+        return str(error)
+
+    deployment = result.get("deployment") or {}
+    deployment_errors = deployment.get("errors") or []
+    error_messages = [
+        str(item.get("error")) for item in deployment_errors if isinstance(item, dict) and item.get("error")
+    ]
+    if error_messages:
+        return "; ".join(error_messages)
+
+    return "Migration to Tinybird Forward failed"
+
+
 def _is_first_deployment_with_seed_live(host: Optional[str], headers: dict) -> bool:
     """Best-effort check for first real deployment when seed deployment (id=0) is still live."""
     try:
@@ -134,8 +150,43 @@ def _get_deployment_job(client: TinyB, deployment_id: Optional[Union[str, int]])
         return None
 
 
+def migrate_to_forward_workspace(client: TinyB, output: str = "human", dry_run: bool = False) -> None:
+    headers = {"Authorization": f"Bearer {client.token}"}
+    params = {"dry_run": dry_run}
+    result = api_post(f"{client.host}/v1/migrate-to-forward", headers=headers, params=params)
+
+    if result.get("result") != "success":
+        error_message = _get_migrate_to_forward_error_message(result)
+        if output == "json":
+            echo_json(result)
+        else:
+            click.echo(FeedbackManager.error(message=error_message))
+        sys_exit("deployment_error", error_message)
+
+    if dry_run:
+        return
+
+    if output == "json":
+        echo_json(result)
+        return
+
+    click.echo(FeedbackManager.success(message="✓ Workspace migrated to Tinybird Forward"))
+
+
+def _should_show_migrate_to_forward_hint(client: TinyB, env: Optional[str]) -> bool:
+    if env != "cloud":
+        return False
+
+    try:
+        workspace_info = client.workspace_info(version="v0")
+    except Exception:
+        logging.exception("Error reading workspace info while deciding migrate-to-forward hint")
+        return False
+
+    return not workspace_info.get("is_forward", False) and not workspace_info.get("is_branch", False)
+
+
 # TODO(eclbg): This logic should be in the server, and there should be a dedicated endpoint for promoting a deployment
-# potato
 def promote_deployment(
     host: Optional[str],
     headers: dict,
@@ -285,9 +336,12 @@ def create_deployment(
     check: Optional[bool] = None,
     allow_destructive_operations: Optional[bool] = None,
     ingest_hint: Optional[bool] = True,
+    show_migrate_to_forward_hint: bool = True,
     output: Optional[str] = "human",
     env: Optional[str] = "cloud",
-) -> None:
+    return_check_result: bool = False,
+    skip_forward_workspace_validation: bool = False,
+) -> Optional[Dict[str, Any]]:
     # TODO: This code is duplicated in build_server.py
     # Should be refactored to be shared
     MULTIPART_BOUNDARY_DATA_PROJECT = "data_project://"
@@ -342,6 +396,8 @@ def create_deployment(
             params["auto_promote"] = "true"
         if allow_destructive_operations:
             params["allow_destructive_operations"] = "true"
+        if skip_forward_workspace_validation:
+            params["skip_forward_workspace_validation"] = "true"
 
         deployment_request_sent = True
         result = api_post(
@@ -388,10 +444,27 @@ def create_deployment(
 
         status = result.get("result")
         if check:
-            if status == "success":
-                click.echo(FeedbackManager.success(message="\n✓ Deployment is valid"))
-                sys.exit(0)
-            elif status == "no_changes":
+            if status in {"success", "no_changes"}:
+                if status == "success":
+                    click.echo(FeedbackManager.success(message="\n✓ Deployment is valid"))
+
+                if (
+                    output == "human"
+                    and show_migrate_to_forward_hint
+                    and _should_show_migrate_to_forward_hint(client, env)
+                ):
+                    click.echo(
+                        FeedbackManager.info(
+                            message=(
+                                "You can now migrate this Tinybird Classic workspace to Forward "
+                                "with `tb migrate-to-forward`."
+                            )
+                        )
+                    )
+
+                if return_check_result:
+                    return {"status": status, "deployment": deployment}
+
                 sys.exit(0)
 
             click.echo(FeedbackManager.error(message="\n✗ Deployment is not valid"))
@@ -439,7 +512,10 @@ def create_deployment(
     except Exception as e:
         click.echo(FeedbackManager.error_exception(error=e))
 
-        if not deployment and not check:
+        if check:
+            sys_exit("deployment_error", "Deployment check failed")
+
+        if not deployment:
             sys_exit("deployment_error", "Deployment failed")
     except KeyboardInterrupt:
         if deployment_request_sent and not check:
@@ -468,7 +544,7 @@ def create_deployment(
             if not deployment:
                 click.echo(FeedbackManager.error(message="Error parsing deployment from response"))
                 sys_exit("deployment_error", "Error parsing deployment from response")
-                return
+                return None
 
             status = deployment.get("status")
             errors = deployment.get("errors")
@@ -563,6 +639,8 @@ def create_deployment(
     if output == "json" and deployment:
         echo_json(deployment, 8)
 
+    return None
+
 
 def _build_data_movement_message(kind: str, source_mv_name: Optional[str]) -> str:
     if kind == "backfill_with_mv_queries":

{tinybird-4.5.4.dev0 → tinybird-4.5.5.dev0}/tinybird/tb_cli_modules/branch.py

@@ -58,11 +58,10 @@ async def release_ls() -> None:
 async def print_releases(config: CLIConfig):
     response = await config.get_client().releases(config["id"])
 
-    table: List[Tuple[str, str, str, str, str]] = []
-    for release in response["releases"]:
-        table.append(
-            (release["created_at"], release["semver"], release["status"], release["commit"], release["rollback"])
-        )
+    table: List[Tuple[str, str, str, str, str]] = [
+        (release["created_at"], release["semver"], release["status"], release["commit"], release["rollback"])
+        for release in response["releases"]
+    ]
 
     columns = ["created_at", "semver", "status", "commit", "rollback release"]
     click.echo(FeedbackManager.info_releases())

{tinybird-4.5.4.dev0 → tinybird-4.5.5.dev0}/tinybird/tb_cli_modules/cli.py

@@ -867,9 +867,7 @@ async def sql(
         if format_ == "json":
             click.echo(json.dumps(res, indent=8))
         else:
-            dd = []
-            for d in res["data"]:
-                dd.append(d.values())
+            dd = [d.values() for d in res["data"]]
             echo_safe_humanfriendly_tables_format_smart_table(dd, column_names=res["data"][0].keys())
     else:
         click.echo(FeedbackManager.info_no_rows())

{tinybird-4.5.4.dev0 → tinybird-4.5.5.dev0}/tinybird/tb_cli_modules/common.py

@@ -747,10 +747,12 @@ async def create_workspace_branch(
 async def print_data_branch_summary(client, job_id, response=None):
     response = await client.job(job_id) if job_id else response or {"partitions": []}
     columns = ["Data Source", "Partition", "Status", "Error"]
-    table = []
+    table: list[list] = []
     for partition in response["partitions"]:
-        for p in partition["partitions"]:
-            table.append([partition["datasource"]["name"], p["partition"], p["status"], p.get("error", "")])
+        table.extend(
+            [partition["datasource"]["name"], p["partition"], p["status"], p.get("error", "")]
+            for p in partition["partitions"]
+        )
     echo_safe_humanfriendly_tables_format_smart_table(table, column_names=columns)
 
 

{tinybird-4.5.4.dev0 → tinybird-4.5.5.dev0}/tinybird/tb_cli_modules/job.py

@@ -36,9 +36,7 @@ async def jobs_ls(ctx: Context, status: str) -> None:
     jobs = await client.jobs(status=status)
     columns = ["id", "kind", "status", "created at", "updated at", "job url"]
     click.echo(FeedbackManager.info_jobs())
-    table = []
-    for j in jobs:
-        table.append([j[c.replace(" ", "_")] for c in columns])
+    table = [[j[c.replace(" ", "_")] for c in columns] for j in jobs]
     echo_safe_humanfriendly_tables_format_smart_table(table, column_names=columns)
     click.echo("\n")
 

{tinybird-4.5.4.dev0 → tinybird-4.5.5.dev0}/tinybird/tb_cli_modules/tag.py

@@ -28,11 +28,10 @@ async def tag_ls(ctx: Context, tag_name: Optional[str]) -> None:
         the_tag = [tag for tag in response["tags"] if tag["name"] == tag_name]
 
         columns = ["name", "id", "type"]
-        table = []
+        table: list[list] = []
 
         if len(the_tag) > 0:
-            for resource in the_tag[0]["resources"]:
-                table.append([resource["name"], resource["id"], resource["type"]])
+            table.extend([resource["name"], resource["id"], resource["type"]] for resource in the_tag[0]["resources"])
 
         click.echo(FeedbackManager.info_tag_resources(tag_name=tag_name))
         echo_safe_humanfriendly_tables_format_smart_table(table, column_names=columns)

{tinybird-4.5.4.dev0 → tinybird-4.5.5.dev0}/tinybird/tb_cli_modules/workspace.py

@@ -55,19 +55,18 @@ async def workspace_ls(ctx: Context) -> None:
         raise CLIWorkspaceException(FeedbackManager.error_unable_to_identify_main_workspace())
 
     columns = ["name", "id", "role", "plan", "current"]
-    table = []
     click.echo(FeedbackManager.info_workspaces())
 
-    for workspace in response["workspaces"]:
-        table.append(
-            [
-                workspace["name"],
-                workspace["id"],
-                workspace["role"],
-                _get_workspace_plan_name(workspace["plan"]),
-                current_main_workspace["id"] == workspace["id"],
-            ]
-        )
+    table = [
+        [
+            workspace["name"],
+            workspace["id"],
+            workspace["role"],
+            _get_workspace_plan_name(workspace["plan"]),
+            current_main_workspace["id"] == workspace["id"],
+        ]
+        for workspace in response["workspaces"]
+    ]
 
     echo_safe_humanfriendly_tables_format_smart_table(table, column_names=columns)
 

{tinybird-4.5.4.dev0 → tinybird-4.5.5.dev0}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 4.5.4.dev0
+Version: 4.5.5.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -52,6 +52,11 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+4.5.4
+*******
+
+- `Added` `tb migrate-to-forward` to help users migrate Classic workspaces to Forward
+
 4.5.3
 *******