tinybird 4.6.16.dev0__tar.gz → 4.6.18.dev0__tar.gz

{tinybird-4.6.16.dev0 → tinybird-4.6.18.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 4.6.16.dev0
+Version: 4.6.18.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -51,11 +51,27 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 
 Changelog
 ----------
+
+4.6.16
+********
+
+- `Fixed` ``tb deployment promote --wait`` and ``tb deployment discard --wait`` no longer hang forever when the server reports the old/discarded deployment removal as ``failed``; the CLI now detects the failure, prints the deployment errors, and exits non-zero.
+
+4.6.15
+********
+
+- `Fixed` ``tb deploy`` waits for the old deployment to delete to avoid clashing with a new deployment submission.
+
 4.6.14
 ********
 
 - `Changed` ``tb datasource append`` and ``tb datasource replace`` submit local CSV, NDJSON, and Parquet files through the v1 import endpoint by default. ``tb datasource append --events`` also submits its request body through v1. These asynchronous imports print the queued job ID; pass ``--wait`` to wait for completion. Remote URL imports continue to use their existing route.
 
+4.6.14
+********
+
+- `Fixed` `tb migrate-to-forward` correctly finds and offers to delete all branches, not just those created by the user running the migration.
+
 4.6.13
 ********
 

{tinybird-4.6.16.dev0 → tinybird-4.6.18.dev0}/tinybird/ch_utils/engine.py

@@ -476,6 +476,9 @@ def engine_options(columns, options: List[EngineOption], args: Dict):
         if option_value:
             if o.sql.lower() == "settings":
                 engine_settings = f"{o.sql} {option_value}"
+            # Conditional TTLs fail if we wrap the value in parentheses
+            elif o.sql.lower() == "ttl":
+                options_values.append(f"{o.sql} {option_value}")
             else:
                 options_values.append(f"{o.sql} ({option_value})")
 
@@ -536,7 +539,9 @@ def engine_full_from_dict(
     >>> engine_full_from_dict('MergeTree', {'sorting_key': 'local_date, cod_store'}, schema=schema)
     'MergeTree() ORDER BY (local_date, cod_store)'
     >>> engine_full_from_dict('MergeTree', {'partition_key': 'toDate(timestamp)', 'sorting_key': 'local_date, cod_store', 'settings': 'index_granularity = 32, index_granularity_bytes = 2048', 'ttl': 'toDate(local_date) + INTERVAL 1 DAY'}, schema=schema)
-    'MergeTree() PARTITION BY (toDate(timestamp)) ORDER BY (local_date, cod_store) TTL (toDate(local_date) + INTERVAL 1 DAY) SETTINGS index_granularity = 32, index_granularity_bytes = 2048'
+    'MergeTree() PARTITION BY (toDate(timestamp)) ORDER BY (local_date, cod_store) TTL toDate(local_date) + INTERVAL 1 DAY SETTINGS index_granularity = 32, index_granularity_bytes = 2048'
+    >>> engine_full_from_dict('MergeTree', {'sorting_key': 'timestamp', 'ttl': 'timestamp + INTERVAL 1 DAY DELETE WHERE is_deleted = 1'}, schema=schema)
+    'MergeTree() ORDER BY (timestamp) TTL timestamp + INTERVAL 1 DAY DELETE WHERE is_deleted = 1'
 
     >>> schema = ''
     >>> engine_full_from_dict('CollapsingMergeTree', {'sign': 'sign_column'}, schema=schema)

{tinybird-4.6.16.dev0 → tinybird-4.6.18.dev0}/tinybird/config.py

@@ -49,11 +49,7 @@ LEGACY_HOSTS = {
     "https://ui.europe-west2.gcp.tinybird.co": "https://cloud.tinybird.co/gcp/europe-west2",
     "https://ui.ap-east.aws.tinybird.co": "https://app.tinybird.co/aws/ap-east-1",
     "https://ui.ap-east-1.aws.tinybird.co": "https://app.tinybird.co/aws/ap-east-1",
-    "https://ui.split.tinybird.co": "https://app.tinybird.co/aws/split-us-east",
-    "https://ui.split.us-west-2.aws.tinybird.co": "https://app.tinybird.co/aws/split-us-west-2",
     "https://ui.northamerica-northeast2.gcp.tinybird.co": "https://app.tinybird.co/gcp/northamerica-northeast2",
-    "https://api.split.tinybird.co": "https://app.tinybird.co/aws/split-us-east",
-    "https://api.split.us-west-2.aws.tinybird.co": "https://app.tinybird.co/aws/split-us-west-2",
     "https://ui.wadus1.gcp.tinybird.co": "https://app.wadus.tinybird.co/gcp/wadus1",
     "https://ui.wadus2.gcp.tinybird.co": "https://app.wadus.tinybird.co/gcp/wadus2",
     "https://ui.wadus3.gcp.tinybird.co": "https://app.wadus.tinybird.co/gcp/wadus3",

{tinybird-4.6.16.dev0 → tinybird-4.6.18.dev0}/tinybird/datafile/common.py

@@ -278,6 +278,12 @@ REQUIRED_DYNAMODB_PARAMS = {
 }
 DYNAMODB_PARAMS = REQUIRED_DYNAMODB_PARAMS
 
+REQUIRED_POSTGRES_PARAMS = {
+    "pg_connection_name",
+    "pg_table",
+}
+POSTGRES_PARAMS = REQUIRED_POSTGRES_PARAMS
+
 
 def extract_column_names_from_sorting_key_part(part: str) -> List[str]:
     """
@@ -602,6 +608,13 @@ class Datafile:
                     raise DatafileValidationError(
                         f"Invalid import schedule '{node['import_schedule']}'. Only {sorted(VALID_BLOB_STORAGE_CRON_VALUES)} values are allowed"
                     )
+            # Validate Postgres params
+            if any(param in node for param in POSTGRES_PARAMS) and (
+                missing := [param for param in REQUIRED_POSTGRES_PARAMS if param not in node]
+            ):
+                raise DatafileValidationError(
+                    f"Some Postgres params have been provided, but the following required ones are missing: {missing}"
+                )
 
         else:
             # We cannot validate a datafile whose kind is unknown
@@ -2068,6 +2081,8 @@ def parse(
             "import_query": assign_var("import_query"),  # Deprecated, BQ and SFK
             "import_table_arn": assign_var("import_table_arn"),  # Only for DynamoDB
             "import_export_bucket": assign_var("import_export_bucket"),  # For DynamoDB
+            "pg_connection_name": assign_var("pg_connection_name"),
+            "pg_table": assign_var("pg_table"),
             "shared_with": shared_with,
             "export_service": export_service,  # Deprecated
             "forward_query": forward_query,
@@ -2128,6 +2143,12 @@ def parse(
             "gcs_secret": assign_var("gcs_hmac_secret"),
             "gcs_hmac_access_id": assign_var("gcs_hmac_access_id"),
             "gcs_hmac_secret": assign_var("gcs_hmac_secret"),
+            "pg_host": assign_var("pg_host"),
+            "pg_port": assign_var("pg_port"),
+            "pg_database": assign_var("pg_database"),
+            "pg_user": assign_var("pg_user"),
+            "pg_password": assign_var("pg_password"),
+            "pg_ssl_mode": assign_var("pg_ssl_mode"),
             "include": include,
         },
     }

{tinybird-4.6.16.dev0 → tinybird-4.6.18.dev0}/tinybird/service_datasources.py

@@ -48,6 +48,7 @@ def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "client_ip", "type": "Nullable(String)"},
                 {"name": "resource_tags", "type": "Array(String)"},
                 {"name": "memory_usage", "type": "UInt64"},
+                {"name": "replica", "type": "String"},
             ],
         },
         {
@@ -333,6 +334,25 @@ def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "job_metadata", "type": "String"},
             ],
         },
+        {
+            "name": "tinybird.rate_limits_log",
+            "description": "Endpoint concurrency limiter transitions in your workspace.",
+            "dateColumn": "timestamp",
+            "engine": {
+                "engine": "MergeTree",
+                "sorting_key": "timestamp, pipe_name",
+                "partition_key": "toYYYYMM(timestamp)",
+            },
+            "columns": [
+                {"name": "timestamp", "type": "DateTime64(3)"},
+                {"name": "workspace_id", "type": "String"},
+                {"name": "workspace_name", "type": "String"},
+                {"name": "pipe_name", "type": "String"},
+                {"name": "rate_limit_type", "type": "LowCardinality(String)"},
+                {"name": "action", "type": "LowCardinality(String)"},
+                {"name": "limit_metadata", "type": "String"},
+            ],
+        },
         {
             "name": "tinybird.hook_log",
             "description": "Log of hook executions and their results.",
@@ -518,22 +538,6 @@ def get_organization_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "origin", "type": "String"},
             ],
         },
-        {
-            "name": "organization.processed_data",
-            "description": "Information related to all processed data per day per workspace.",
-            "dateColumn": "date",
-            "engine": {
-                "engine": "SummingMergeTree",
-                "sorting_key": "database, date",
-                "partition_key": "toYYYYMM(date)",
-            },
-            "columns": [
-                {"name": "date", "type": "Date"},
-                {"name": "database", "type": "String"},
-                {"name": "read_bytes", "type": "UInt64"},
-                {"name": "written_bytes", "type": "UInt64"},
-            ],
-        },
         {
             "name": "organization.datasources_storage",
             "description": "Similar to tinybird.datasources_storage but with data for all Organization Workspaces.",
@@ -683,6 +687,7 @@ def get_organization_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "user_agent", "type": "Nullable(String)"},
                 {"name": "resource_tags", "type": "Array(String)"},
                 {"name": "memory_usage", "type": "UInt64"},
+                {"name": "replica", "type": "String"},
             ],
         },
         {
@@ -726,6 +731,25 @@ def get_organization_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "job_metadata", "type": "String"},
             ],
         },
+        {
+            "name": "organization.rate_limits_log",
+            "description": "Endpoint concurrency limiter transitions across the organization.",
+            "dateColumn": "timestamp",
+            "engine": {
+                "engine": "MergeTree",
+                "sorting_key": "workspace_id, timestamp, pipe_name",
+                "partition_key": "toYYYYMM(timestamp)",
+            },
+            "columns": [
+                {"name": "timestamp", "type": "DateTime64(3)"},
+                {"name": "workspace_id", "type": "String"},
+                {"name": "workspace_name", "type": "String"},
+                {"name": "pipe_name", "type": "String"},
+                {"name": "rate_limit_type", "type": "LowCardinality(String)"},
+                {"name": "action", "type": "LowCardinality(String)"},
+                {"name": "limit_metadata", "type": "String"},
+            ],
+        },
         {
             "name": "organization.sinks_ops_log",
             "description": "Historic Logs for all Sink job executions across the organization",

{tinybird-4.6.16.dev0 → tinybird-4.6.18.dev0}/tinybird/sql_toolset.py

@@ -493,7 +493,6 @@ def replace_tables_chquery_cached(
     sorted_replacements: Optional[tuple] = None,
     default_database: str = "",
     output_one_line: bool = False,
-    timestamp: Optional[datetime] = None,
     function_allow_list: Optional[FrozenSet[str]] = None,
     settings_allow_list: Optional[FrozenSet[str]] = None,
 ) -> str:
@@ -536,7 +535,6 @@ def replace_tables(
             sql,
             None,
             output_one_line=output_one_line,
-            timestamp=timestamp,
             function_allow_list=hashable_list,
             settings_allow_list=hashable_settings_list,
         )
@@ -610,7 +608,6 @@ def replace_tables(
             sql,
             None,
             output_one_line=output_one_line,
-            timestamp=timestamp,
             function_allow_list=hashable_list,
             settings_allow_list=hashable_settings_list,
         )
@@ -649,7 +646,6 @@ def replace_tables(
                 r,
                 default_database=default_database,
                 output_one_line=output_one_line,
-                timestamp=timestamp,
                 function_allow_list=hashable_list,
                 settings_allow_list=hashable_settings_list,
             )
@@ -658,7 +654,6 @@ def replace_tables(
                 sql,
                 None,
                 output_one_line=output_one_line,
-                timestamp=timestamp,
                 function_allow_list=hashable_list,
                 settings_allow_list=hashable_settings_list,
             )

{tinybird-4.6.16.dev0 → tinybird-4.6.18.dev0}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '4.6.16.dev0'
-__revision__ = '532ee82'
+__version__ = '4.6.18.dev0'
+__revision__ = '0bbef5c'

{tinybird-4.6.16.dev0 → tinybird-4.6.18.dev0}/tinybird/tb/client.py

@@ -833,8 +833,8 @@ class TinyB:
     def user_workspace_branches(self, version: str = "v0"):
         return self._req(f"/{version}/user/workspaces/?with_environments=true&only_environments=true")
 
-    def branches(self):
-        return self._req("/v1/environments")
+    def branches(self, version: str = "v1"):
+        return self._req(f"/{version}/environments")
 
     def releases(self, workspace_id):
         return self._req(f"/v0/workspaces/{workspace_id}/releases")

{tinybird-4.6.16.dev0 → tinybird-4.6.18.dev0}/tinybird/tb/modules/common.py

@@ -18,7 +18,7 @@ from copy import deepcopy
 from enum import Enum
 from os import getcwd, getenv
 from pathlib import Path
-from typing import Any, Callable, Dict, Iterable, List, Literal, Optional, Set, Tuple, TypedDict, Union
+from typing import Any, Callable, Dict, Iterable, List, Literal, NoReturn, Optional, Set, Tuple, TypedDict, Union
 from urllib.parse import urlparse
 
 import click
@@ -2370,7 +2370,7 @@ def ask_for_organization(
 event_error_separator = "__error__"
 
 
-def sys_exit(event: str, msg: str) -> None:
+def sys_exit(event: str, msg: str) -> NoReturn:
     sys.exit(f"{event}{event_error_separator}{msg}")
 
 

{tinybird-4.6.16.dev0 → tinybird-4.6.18.dev0}/tinybird/tb/modules/deployment.py

@@ -155,16 +155,13 @@ def api_fetch(url: str, headers: dict, request_from: Optional[str] = None) -> di
     return {}
 
 
-def _get_classic_workspace_branches(client: TinyB, workspace_id: str) -> list[dict[str, Any]]:
-    branches: list[dict[str, Any]] = client.user_workspace_branches(version="v0").get("workspaces", [])
-    return [branch for branch in branches if str(branch.get("main")) == workspace_id]
+def _get_classic_workspace_branches(client: TinyB) -> list[dict[str, Any]]:
+    return client.branches(version="v0").get("environments", [])
 
 
-def _cleanup_classic_migration_blockers(client: TinyB, config: Dict[str, Any]) -> None:
-    workspace_id = str(config["id"])
-
+def _cleanup_classic_migration_blockers(client: TinyB) -> None:
     try:
-        branches = _get_classic_workspace_branches(client, workspace_id)
+        branches = _get_classic_workspace_branches(client)
         if not branches:
             return
 
@@ -488,7 +485,7 @@ def migrate_to_forward(ctx: click.Context, allow_destructive_operations: bool) -
         click.echo(FeedbackManager.info(message="Migration cancelled."))
         return
 
-    _cleanup_classic_migration_blockers(client, config)
+    _cleanup_classic_migration_blockers(client)
     _persist_migrate_to_forward_config(project)
     migrate_to_forward_workspace(client=client, output=output, dry_run=False)
     create_deployment(

{tinybird-4.6.16.dev0 → tinybird-4.6.18.dev0}/tinybird/tb/modules/deployment_common.py

@@ -118,23 +118,57 @@ def _get_migrate_to_forward_error_message(result: dict[str, Any]) -> str:
 def _is_first_deployment_with_seed_live(host: Optional[str], headers: dict) -> bool:
     """Best-effort check for first real deployment when seed deployment (id=0) is still live."""
     try:
-        response = requests.get(f"{host}/v1/deployments", headers=headers)
-        if response.status_code >= 300:
-            return False
-
-        result = response.json()
-        logging.debug(json.dumps(result, indent=2))
-        deployments = result.get("deployments") or []
-        if not deployments:
-            return False
-
-        live_deployment = next((deployment for deployment in deployments if deployment.get("live")), deployments[0])
-        return str(live_deployment.get("id")) == "0"
+        return _get_current_live_deployment_id(host, headers) == "0"
     except Exception:
-        # Hint calculation must never break deployment flow.
+        logging.exception("Error checking for live seed deployment while deciding first-deployment hint")
         return False
 
 
+def _list_deployments(host: Optional[str], headers: dict) -> list[dict]:
+    """Fetch the current deployments. Raises if the lookup fails."""
+    response = requests.get(f"{host}/v1/deployments", headers=headers)
+    response.raise_for_status()
+
+    result = response.json()
+    logging.debug(json.dumps(result, indent=2))
+    return result.get("deployments") or []
+
+
+def _get_current_live_deployment_id(host: Optional[str], headers: dict) -> Optional[str]:
+    deployments = _list_deployments(host, headers)
+    if not deployments:
+        return None
+
+    live_deployment = next((deployment for deployment in deployments if deployment.get("live")), deployments[0])
+    live_id = live_deployment.get("id")
+    return str(live_id) if live_id is not None else None
+
+
+def _get_lingering_old_deployment_id(host: Optional[str], headers: dict, new_deployment_id: str) -> Optional[str]:
+    """Look up a still-active deployment other than `new_deployment_id`, if any."""
+    deployments = _list_deployments(host, headers)
+    other_deployment = next((d for d in deployments if str(d.get("id")) != new_deployment_id), None)
+    return str(other_deployment["id"]) if other_deployment else None
+
+
+def _wait_for_deployment_deletion(
+    host: Optional[str],
+    headers: dict,
+    deployment_id: str,
+    request_from: Optional[str] = None,
+    poll_interval: int = 5,
+) -> bool:  # whether the deployment has been deleted successfully
+    url = f"{host}/v1/deployments/{deployment_id}"
+    while True:
+        result = api_fetch(url, headers, request_from=request_from)
+        old_deployment = result.get("deployment")
+        if not old_deployment or old_deployment.get("status") == "deleted":
+            return True
+        if old_deployment.get("status") == "failed":
+            return False
+        time.sleep(poll_interval)
+
+
 def _get_deployment_job(client: TinyB, deployment_id: Optional[Union[str, int]]) -> Optional[Dict[str, Any]]:
     if deployment_id is None:
         return None
@@ -264,12 +298,21 @@ def promote_deployment(
                 click.echo(FeedbackManager.error(message="Error parsing deployment from response"))
                 sys_exit("deployment_error", "Error parsing deployment from response")
 
-            if last_deployment and last_deployment.get("status") == "deleted":
+            last_deployment_status = last_deployment.get("status")
+
+            if last_deployment_status == "deleted":
                 if not is_first_deployment:
                     click.echo(FeedbackManager.info(message="✓ Old deployment removed"))
                 click.echo(FeedbackManager.success(message=f"✓ Deployment #{candidate_deployment.get('id')} is live!"))
                 break
 
+            if last_deployment_status == "failed":
+                click.echo(FeedbackManager.error(message="Removing the old deployment failed"))
+                deploy_errors = last_deployment.get("errors", [])
+                for deploy_error in deploy_errors:
+                    click.echo(FeedbackManager.error(message=f"* {deploy_error}"))
+                sys_exit("deployment_error", "Removing the old deployment failed: " + str(deploy_errors))
+
             time.sleep(5)
     if is_first_deployment and ingest_hint and len(candidate_deployment.get("new_data_connector_ids", [])) == 0:
         # This is the first deployment, so we prompt the user to ingest data
@@ -326,9 +369,23 @@ def discard_deployment(host: Optional[str], headers: dict, wait: bool, request_f
             result = api_fetch(TINYBIRD_API_URL, headers, request_from=request_from)
 
             deployment_to_discard = result.get("deployment")
-            if deployment_to_discard and deployment_to_discard.get("status") == "deleted":
+            if not deployment_to_discard:
+                click.echo(FeedbackManager.error(message="Error parsing deployment from response"))
+                sys_exit("deployment_error", "Error parsing deployment from response")
+
+            discard_status = deployment_to_discard.get("status")
+
+            if discard_status == "deleted":
                 click.echo(FeedbackManager.success(message="Discard process successfully completed"))
                 break
+
+            if discard_status == "failed":
+                click.echo(FeedbackManager.error(message="Discarding the deployment failed"))
+                deploy_errors = deployment_to_discard.get("errors", [])
+                for deploy_error in deploy_errors:
+                    click.echo(FeedbackManager.error(message=f"* {deploy_error}"))
+                sys_exit("deployment_error", "Discarding the deployment failed: " + str(deploy_errors))
+
             time.sleep(5)
 
 
@@ -635,6 +692,42 @@ def create_deployment(
             click.echo(FeedbackManager.highlight(message="» Waiting for deployment to be promoted..."))
         click.echo(FeedbackManager.info(message="✓ Deployment promoted"))
         click.echo(FeedbackManager.success(message=f"✓ Deployment #{deployment.get('id')} is live!"))
+
+        old_deployment_id: Optional[str] = None
+        try:
+            old_deployment_id = _get_lingering_old_deployment_id(client.host, HEADERS, str(deployment.get("id")))
+        except Exception:
+            click.echo(
+                FeedbackManager.warning(
+                    message="Could not confirm if there's a previous deployment to remove. "
+                    f"If a follow-up `tb deploy` reports one already in progress, check `tb --{env} deployment ls`."
+                )
+            )
+
+        if old_deployment_id:
+            click.echo(FeedbackManager.highlight(message="» Removing old deployment..."))
+            try:
+                deleted = _wait_for_deployment_deletion(
+                    client.host, HEADERS, old_deployment_id, request_from=request_from
+                )
+            except KeyboardInterrupt:
+                click.echo(
+                    FeedbackManager.warning(
+                        message=f"\nDeployment is live, but the old deployment is still being removed. "
+                        f"Check its status with `tb --{env} deployment ls`."
+                    )
+                )
+                raise click.Abort()
+            if deleted:
+                click.echo(FeedbackManager.info(message="✓ Old deployment removed"))
+            else:
+                click.echo(
+                    FeedbackManager.warning(
+                        message="Deployment is live, but the old deployment failed to be removed. "
+                        f"Check its status with `tb --{env} deployment ls`."
+                    )
+                )
+
         if ingest_hint and len(deployment.get("new_data_connector_ids", [])) == 0 and is_first_deployment:
             click.echo(
                 FeedbackManager.info(

{tinybird-4.6.16.dev0 → tinybird-4.6.18.dev0}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 4.6.16.dev0
+Version: 4.6.18.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -51,11 +51,27 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 
 Changelog
 ----------
+
+4.6.16
+********
+
+- `Fixed` ``tb deployment promote --wait`` and ``tb deployment discard --wait`` no longer hang forever when the server reports the old/discarded deployment removal as ``failed``; the CLI now detects the failure, prints the deployment errors, and exits non-zero.
+
+4.6.15
+********
+
+- `Fixed` ``tb deploy`` waits for the old deployment to delete to avoid clashing with a new deployment submission.
+
 4.6.14
 ********
 
 - `Changed` ``tb datasource append`` and ``tb datasource replace`` submit local CSV, NDJSON, and Parquet files through the v1 import endpoint by default. ``tb datasource append --events`` also submits its request body through v1. These asynchronous imports print the queued job ID; pass ``--wait`` to wait for completion. Remote URL imports continue to use their existing route.
 
+4.6.14
+********
+
+- `Fixed` `tb migrate-to-forward` correctly finds and offers to delete all branches, not just those created by the user running the migration.
+
 4.6.13
 ********