tinybird 0.0.1.dev306__py3-none-any.whl → 1.0.5__py3-none-any.whl

tinybird/datafile/common.py

@@ -2127,6 +2127,9 @@ def parse(
     return ParseResult(datafile=doc, warnings=warnings)
 
 
+# TODO: This class is duplicated in tinybird/datafile_common.py with a slightly different
+# _REPLACEMENTS tuple. The duplication happened during the CLI/server code split (commit
+# f86d02cdd7). Consider extracting shared code into a common module that both files can import.
 class ImportReplacements:
     _REPLACEMENTS: Tuple[Tuple[str, str, Optional[str]], ...] = (
         ("import_strategy", "mode", "replace"),
@@ -2146,7 +2149,7 @@ class ImportReplacements:
         return [x[0] for x in ImportReplacements._REPLACEMENTS]
 
     @staticmethod
-    def get_api_param_for_datafile_param(connector_service: str, key: str) -> Tuple[Optional[str], Optional[str]]:
+    def get_api_param_for_datafile_param(key: str) -> Tuple[Optional[str], Optional[str]]:
         """Returns the API parameter name and default value for a given
         datafile parameter.
         """

tinybird/feedback_manager.py

@@ -506,6 +506,9 @@ Ready? """
     prompt_init_git_release_force = prompt_message(
         "You are going to manually update workspace commit reference manually, this is just for special occasions. Do you want to update current commit reference '{current_commit}' to '{new_commit}'?"
     )
+    prompt_init_git_release_new = prompt_message(
+        "This workspace does not have any release yet. Do you want to create one with commit '{commit}'? This will enable 'tb deploy' to work."
+    )
 
     warning_exchange = warning_message(
         "Warning: Do you want to exchange Data Source {datasource_a} by Data Source {datasource_b}?"

tinybird/service_datasources.py

@@ -5,6 +5,7 @@ This module provides access to predefined service datasources and their schemas
 for both Tinybird and Organization scopes.
 """
 
+from functools import lru_cache
 from typing import Any, Dict, List, Optional
 
 
@@ -490,6 +491,7 @@ def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
                 "engine": "ReplacingMergeTree",
                 "sorting_key": "event_time, organization_id, query_id",
                 "partition_key": "toStartOfDay(event_time)",
+                "ttl": "toDate(event_time) + INTERVAL 30 DAY",
             },
             "columns": [
                 {"name": "event_time", "type": "DateTime"},
@@ -516,18 +518,46 @@ def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
         {
             "name": "tinybird.vcpu_time",
             "description": "vCPU time metrics from your workspace.",
-            "dateColumn": "minute_slot",
+            "dateColumn": "second_slot",
             "engine": {
                 "engine": "AggregatingMergeTree",
-                "sorting_key": "organization_id, minute_slot",
-                "partition_key": "toStartOfDay(minute_slot)",
+                "sorting_key": "organization_id, second_slot",
+                "partition_key": "toStartOfDay(second_slot)",
+                "ttl": "toDate(second_slot) + INTERVAL 30 DAY",
             },
             "columns": [
-                {"name": "minute_slot", "type": "DateTime"},
+                {"name": "second_slot", "type": "DateTime"},
                 {"name": "organization_id", "type": "String"},
                 {"name": "vcpu_time", "type": "Float64"},
             ],
         },
+        {
+            "name": "tinybird.query_validator_log",
+            "description": "Log of failed queries executions in the next available ClickHouse version and their results.",
+            "dateColumn": "run_validation",
+            "engine": {
+                "engine": "MergeTree",
+                "sorting_key": "database, host, run_validation, query_hash",
+                "partition_key": "toYYYYMM(run_validation)",
+            },
+            "columns": [
+                {"name": "host", "type": "LowCardinality(String)"},
+                {"name": "version", "type": "LowCardinality(String)"},
+                {"name": "stable_version", "type": "LowCardinality(String)"},
+                {"name": "query_hash", "type": "UInt64"},
+                {"name": "query_last_execution", "type": "DateTime"},
+                {"name": "region", "type": "String"},
+                {"name": "workspace", "type": "String"},
+                {"name": "database", "type": "String"},
+                {"name": "pipe_name", "type": "String"},
+                {"name": "query_id", "type": "String"},
+                {"name": "query", "type": "String"},
+                {"name": "error_code", "type": "Int16"},
+                {"name": "error", "type": "String"},
+                {"name": "fix_suggestion", "type": "String"},
+                {"name": "run_validation", "type": "DateTime"},
+            ],
+        },
     ]
 
 
@@ -906,6 +936,22 @@ def get_organization_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "active_minutes", "type": "Float64"},
             ],
         },
+        {
+            "name": "organization.shared_infra_active_seconds",
+            "description": "Contains information about vCPU active seconds consumption aggregated by second for all Organization workspaces. Only available for Developer and Enterprise plans in shared infrastructure.",
+            "dateColumn": "second",
+            "engine": {
+                "engine": "AggregatingMergeTree",
+                "sorting_key": "second_slot, organization_id",
+                "partition_key": "toYYYYMM(second_slot)",
+            },
+            "columns": [
+                {"name": "second_slot", "type": "DateTime"},
+                {"name": "organization_id", "type": "String"},
+                {"name": "organization_name", "type": "String"},
+                {"name": "cpu_time", "type": "Float64"},
+            ],
+        },
         {
             "name": "organization.shared_infra_qps_overages",
             "description": "Contains information about QPS consumption and overages aggregated by second for all Organization workspaces. Only available for Developer and Enterprise plans in shared infrastructure.",
@@ -1016,6 +1062,7 @@ def get_organization_service_datasources() -> List[Dict[str, Any]]:
                 "engine": "ReplacingMergeTree",
                 "sorting_key": "event_time, organization_id, query_id",
                 "partition_key": "toStartOfDay(event_time)",
+                "ttl": "toDate(event_time) + INTERVAL 30 DAY",
             },
             "columns": [
                 {"name": "event_time", "type": "DateTime"},
@@ -1042,14 +1089,15 @@ def get_organization_service_datasources() -> List[Dict[str, Any]]:
         {
             "name": "organization.vcpu_time",
             "description": "vCPU time metrics from your workspace.",
-            "dateColumn": "minute_slot",
+            "dateColumn": "second_slot",
             "engine": {
                 "engine": "AggregatingMergeTree",
-                "sorting_key": "organization_id, minute_slot",
-                "partition_key": "toStartOfDay(minute_slot)",
+                "sorting_key": "organization_id, second_slot",
+                "partition_key": "toStartOfDay(second_slot)",
+                "ttl": "toDate(second_slot) + INTERVAL 30 DAY",
             },
             "columns": [
-                {"name": "minute_slot", "type": "DateTime"},
+                {"name": "second_slot", "type": "DateTime"},
                 {"name": "organization_id", "type": "String"},
                 {"name": "workspace_id", "type": "String"},
                 {"name": "vcpu_time", "type": "Float64"},
@@ -1058,6 +1106,7 @@ def get_organization_service_datasources() -> List[Dict[str, Any]]:
     ]
 
 
+@lru_cache(maxsize=1)
 def get_service_datasources() -> List[Dict[str, Any]]:
     """
     Get the list of all Tinybird and Organization service datasources.

tinybird/sql_template.py

@@ -1841,7 +1841,7 @@ def get_var_names_and_types(t, node_id=None):
         raise SQLTemplateException(e)
 
 
-@lru_cache(maxsize=256)
+@lru_cache(maxsize=512)
 def get_var_names_and_types_cached(t: Template):
     return get_var_names_and_types(t)
 

tinybird/sql_template_fmt.py

@@ -203,13 +203,15 @@ class TinybirdDialect(ClickHouse):
                 Rule(
                     name="jinja_if_block_end",
                     priority=203,
-                    pattern=group(r"\{%-?\s*end\s*-?%\}"),
+                    # Accept both Tornado-style {% end %} and {% end if %}
+                    pattern=group(r"\{%-?\s*end(\s+if)?\s*-?%\}"),
                     action=actions.raise_sqlfmt_bracket_error,
                 ),
                 Rule(
                     name="jinja_for_block_end",
                     priority=211,
-                    pattern=group(r"\{%-?\s*end\s*-?%\}"),
+                    # Accept both Tornado-style {% end %} and {% end for %}
+                    pattern=group(r"\{%-?\s*end(\s+for)?\s*-?%\}"),
                     action=actions.raise_sqlfmt_bracket_error,
                 ),
             ],
@@ -261,7 +263,13 @@ def _calc_str(self) -> str:
 Comment._calc_str = property(_calc_str)
 
 
-def format_sql_template(sql: str, line_length: Optional[int] = None, lower_keywords: bool = False) -> str:
+def format_sql_template(
+    sql: str,
+    line_length: Optional[int] = None,
+    lower_keywords: bool = False,
+    resource_name: Optional[str] = None,
+    resource_source: Optional[str] = None,
+) -> str:
     try:
         # https://github.com/tconbeer/sqlfmt/blob/c11775b92d8a45f0e91d871b81a88a894d620bec/src/sqlfmt/mode.py#L16-L29
         config: Dict[str, Any] = {
@@ -277,5 +285,7 @@ def format_sql_template(sql: str, line_length: Optional[int] = None, lower_keywo
             else api.format_string(sql, mode=mode).strip()
         )
     except Exception as e:
-        logging.warning(f"sqlfmt error: {str(e)}")
+        resource_info = f" in '{resource_name}'" if resource_name else ""
+        source_info = f" ({resource_source})" if resource_source else ""
+        logging.warning(f"sqlfmt error{resource_info}{source_info}: {str(e)}")
         return sql

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev306'
-__revision__ = 'dcef65b'
+__version__ = '1.0.5'
+__revision__ = '60ae688'

tinybird/tb/cli.py

@@ -1,4 +1,5 @@
 import tinybird.tb.modules.agent
+import tinybird.tb.modules.branch
 import tinybird.tb.modules.build
 import tinybird.tb.modules.cli
 import tinybird.tb.modules.common

tinybird/tb/client.py

@@ -300,33 +300,33 @@ class TinyB:
         response = self._req(f"/v0/connectors?{urlencode(params)}")
         return response["connectors"]
 
-    def connections(self, connector: Optional[str] = None):
+    def connections(self, connector: Optional[str] = None, datasources: Optional[List[Dict[str, Any]]] = None):
         response = self._req("/v0/connectors")
         connectors = response["connectors"]
+        connectors_to_return = []
+        for c in connectors:
+            if connector and c["service"] != connector:
+                continue
+            if connector == "gcscheduler":
+                continue
+            if datasources and len(datasources) > 0:
+                datasource_ids = [linker["datasource_id"] for linker in c["linkers"]]
+                datasource_names = [ds["name"] for ds in datasources if ds["id"] in datasource_ids]
+                connected_datasources = ", ".join(datasource_names) if len(datasource_names) > 0 else ""
+            else:
+                connected_datasources = str(len(c["linkers"]))
 
-        if connector:
-            return [
+            connectors_to_return.append(
                 {
                     "id": c["id"],
                     "service": c["service"],
                     "name": c["name"],
-                    "connected_datasources": len(c["linkers"]),
+                    "connected_datasources": connected_datasources,
                     **c["settings"],
                 }
-                for c in connectors
-                if c["service"] == connector
-            ]
-        return [
-            {
-                "id": c["id"],
-                "service": c["service"],
-                "name": c["name"],
-                "connected_datasources": len(c["linkers"]),
-                **c["settings"],
-            }
-            for c in connectors
-            if c["service"] != "gcscheduler"
-        ]
+            )
+
+        return connectors_to_return
 
     def get_datasource(self, ds_name: str, used_by: bool = False) -> Dict[str, Any]:
         params = {
@@ -711,7 +711,7 @@ class TinyB:
         return self._req(f"/{version}/user/workspaces/?with_environments=true&only_environments=true")
 
     def branches(self):
-        return self._req("/v0/environments")
+        return self._req("/v1/environments")
 
     def releases(self, workspace_id):
         return self._req(f"/v0/workspaces/{workspace_id}/releases")
@@ -740,7 +740,7 @@ class TinyB:
         }
         if ignore_datasources:
             params["ignore_datasources"] = ",".join(ignore_datasources)
-        return self._req(f"/v0/environments?{urlencode(params)}", method="POST", data=b"")
+        return self._req(f"/v1/environments?{urlencode(params)}", method="POST", data=b"")
 
     def branch_workspace_data(
         self,
@@ -996,10 +996,92 @@ class TinyB:
             data=json.dumps(connection_params),
         )
 
-    def kafka_list_topics(self, connection_id: str, timeout=5):
-        resp = self._req(f"/v0/connectors/{connection_id}/preview?preview_activity=false", timeout=timeout)
+    def kafka_list_topics(self, connection_id: str, timeout=10, retries=3):
+        resp = self._req(
+            f"/v0/connectors/{connection_id}/preview?preview_activity=false",
+            timeout=timeout,
+            retries=retries,
+        )
         return [x["topic"] for x in resp["preview"]]
 
+    def kafka_preview_group(self, connection_id: str, topic: str, group_id: str, timeout=30):
+        params = {
+            "log": "previewGroup",
+            "kafka_group_id": group_id,
+            "kafka_topic": topic,
+            "preview_group": "true",
+        }
+        return self._req(f"/v0/connectors/{connection_id}/preview?{urlencode(params)}", method="GET", timeout=timeout)
+
+    def kafka_preview_topic(self, connection_id: str, topic: str, group_id: str, timeout: int = 30) -> Dict[str, Any]:
+        """Preview a Kafka topic and return structured preview data.
+
+        Args:
+            connection_id: The ID of the Kafka connection
+            topic: The Kafka topic name to preview
+            group_id: The Kafka consumer group ID
+            timeout: Request timeout in seconds
+
+        Returns:
+            A dictionary containing:
+                - analysis: Dictionary with columns information
+                - preview: Dictionary with data and meta arrays
+                - earliestTimestamp: The earliest message timestamp (if available)
+        """
+        params = {
+            "max_records": "12",
+            "preview_activity": "true",
+            "preview_earliest_timestamp": "true",
+            "kafka_topic": topic,
+            "kafka_group_id": group_id,
+            "log": "previewTopic",
+        }
+        response = self._req(
+            f"/v0/connectors/{connection_id}/preview?{urlencode(params)}", method="GET", timeout=timeout
+        )
+
+        if not response:
+            return {
+                "analysis": {"columns": []},
+                "preview": {"data": [], "meta": []},
+                "earliestTimestamp": "",
+            }
+
+        # Extract preview data (similar to TypeScript previewKafkaTopic)
+        preview_data = response.get("preview", [])
+        if not preview_data:
+            return {
+                "analysis": {"columns": []},
+                "preview": {"data": [], "meta": []},
+                "earliestTimestamp": "",
+            }
+
+        topic_preview = preview_data[0]
+        analysis = topic_preview.get("analysis", {})
+        deserialized = topic_preview.get("deserialized", {})
+
+        # Extract columns from analysis
+        columns = analysis.get("columns", []) if analysis else []
+
+        # Extract data and meta from deserialized
+        base_data = deserialized.get("data", []) if deserialized else []
+        base_meta = deserialized.get("meta", []) if deserialized else []
+
+        # Extract earliest timestamp
+        earliest = response.get("earliest", [])
+        earliest_timestamp = earliest[0].get("timestamp", "") if earliest else ""
+
+        return {
+            "analysis": {
+                "columns": columns,
+            },
+            "preview": {
+                "data": base_data,
+                "meta": base_meta,
+            },
+            "earliestTimestamp": earliest_timestamp,
+        }
+
     def get_gcp_service_account_details(self) -> Dict[str, Any]:
         return self._req("/v0/datasources-bigquery-credentials")
 

tinybird/tb/modules/agent/tools/execute_query.py

@@ -148,7 +148,7 @@ def execute_query(
         ctx.deps.thinking_animation.stop()
         click.echo(FeedbackManager.error(message=error))
         ctx.deps.thinking_animation.start()
-        if "not found" in error.lower() and cloud:
+        if "not found" in error.lower() and cloud_or_local == "cloud":
             return f"Error executing query: {error}. Please run the query against Tinybird local instead of cloud."
         else:
             return f"Error executing query: {error}. Please try again."

tinybird/tb/modules/agent/tools/request_endpoint.py

@@ -87,7 +87,7 @@ def request_endpoint(
         click.echo(FeedbackManager.error(message=error))
         ctx.deps.thinking_animation.start()
         not_found_errors = ["not found", "does not exist"]
-        if any(not_found_error in error.lower() for not_found_error in not_found_errors) and cloud:
+        if any(not_found_error in error.lower() for not_found_error in not_found_errors) and cloud_or_local == "cloud":
             return f"Error executing query: {error}. Please run the query against Tinybird local instead of cloud."
         else:
             return f"Error executing query: {error}. Please try again."

tinybird/tb/modules/branch.py

@@ -0,0 +1,150 @@
+# This is a command file for our CLI. Please keep it clean.
+#
+# - If it makes sense and only when strictly necessary, you can create utility functions in this file.
+# - But please, **do not** interleave utility functions and command definitions.
+
+from typing import List, Optional, Tuple
+
+import click
+
+from tinybird.tb.modules.cli import cli
+from tinybird.tb.modules.common import (
+    MAIN_BRANCH,
+    create_workspace_branch,
+    echo_safe_humanfriendly_tables_format_smart_table,
+    get_current_main_workspace,
+    get_current_workspace_branches,
+    get_workspace_member_email,
+    switch_to_workspace_by_user_workspace_data,
+    try_update_config_with_remote,
+)
+from tinybird.tb.modules.config import CLIConfig
+from tinybird.tb.modules.exceptions import CLIBranchException, CLIException
+from tinybird.tb.modules.feedback_manager import FeedbackManager
+
+
+@cli.group()
+def branch() -> None:
+    """Branch commands. Custom branches is an experimental feature in beta."""
+    pass
+
+
+@branch.command(name="ls")
+@click.option("--sort/--no-sort", default=False, help="Sort the table rows by name")
+def branch_ls(sort: bool) -> None:
+    """List all the branches available in the current workspace"""
+
+    config = CLIConfig.get_project_config()
+    _ = try_update_config_with_remote(config, only_if_needed=True)
+
+    client = config.get_client()
+
+    current_main_workspace = get_current_main_workspace(config)
+    assert isinstance(current_main_workspace, dict)
+
+    if current_main_workspace["id"] != config["id"]:
+        client = config.get_client(token=current_main_workspace["token"])
+
+    response = client.branches()
+
+    columns = ["name", "id", "created_at", "owner", "current"]
+
+    table: List[Tuple[str, str, str, str, bool]] = []
+
+    for branch in response["environments"]:
+        branch_owner_email = get_workspace_member_email(branch, branch["owner"])
+
+        table.append(
+            (branch["name"], branch["id"], branch["created_at"], branch_owner_email, config["id"] == branch["id"])
+        )
+
+    current_branch = [row for row in table if row[4]]
+    other_branches = [row for row in table if not row[4]]
+
+    if sort:
+        other_branches.sort(key=lambda x: x[0])
+
+    sorted_table = current_branch + other_branches
+
+    click.echo(FeedbackManager.info(message="\n** Branches:"))
+    echo_safe_humanfriendly_tables_format_smart_table(sorted_table, column_names=columns)
+
+
+@branch.command(name="create", short_help="Create a new branch in the current Workspace")
+@click.argument("branch_name", required=False)
+@click.option(
+    "--last-partition",
+    is_flag=True,
+    default=False,
+    help="Attach the last modified partition from the current workspace to the new branch",
+)
+@click.option(
+    "-i",
+    "--ignore-datasource",
+    "ignore_datasources",
+    type=str,
+    multiple=True,
+    help="Ignore specified data source partitions",
+)
+@click.option(
+    "--wait/--no-wait",
+    is_flag=True,
+    default=True,
+    help="Wait for data branch jobs to finish, showing a progress bar. Disabled by default.",
+)
+def create_branch(branch_name: Optional[str], last_partition: bool, ignore_datasources: List[str], wait: bool) -> None:
+    create_workspace_branch(branch_name, last_partition, False, list(ignore_datasources), wait)
+
+
+@branch.command(name="rm", short_help="Removes an branch from the workspace. It can't be recovered.")
+@click.argument("branch_name_or_id")
+@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
+def delete_branch(branch_name_or_id: str, yes: bool) -> None:
+    """Remove an branch"""
+
+    config = CLIConfig.get_project_config()
+    _ = try_update_config_with_remote(config)
+
+    client = config.get_client()
+
+    if branch_name_or_id == MAIN_BRANCH:
+        raise CLIException(FeedbackManager.error_not_allowed_in_main_branch())
+
+    try:
+        workspace_branches = get_current_workspace_branches(config)
+        workspace_to_delete = next(
+            (
+                workspace
+                for workspace in workspace_branches
+                if workspace["name"] == branch_name_or_id or workspace["id"] == branch_name_or_id
+            ),
+            None,
+        )
+    except Exception as e:
+        raise CLIBranchException(FeedbackManager.error_exception(error=str(e)))
+
+    if not workspace_to_delete:
+        raise CLIBranchException(FeedbackManager.error_branch(branch=branch_name_or_id))
+
+    if yes or click.confirm(FeedbackManager.warning_confirm_delete_branch(branch=workspace_to_delete["name"])):
+        need_to_switch_to_main = workspace_to_delete.get("main") and config["id"] == workspace_to_delete["id"]
+        # get origin workspace if deleting current branch
+        if need_to_switch_to_main:
+            try:
+                workspaces = (client.user_workspaces()).get("workspaces", [])
+                workspace_main = next(
+                    (workspace for workspace in workspaces if workspace["id"] == workspace_to_delete["main"]), None
+                )
+            except Exception:
+                workspace_main = None
+        try:
+            client.delete_branch(workspace_to_delete["id"])
+            click.echo(FeedbackManager.success_branch_deleted(branch_name=workspace_to_delete["name"]))
+        except Exception as e:
+            raise CLIBranchException(FeedbackManager.error_exception(error=str(e)))
+        else:
+            if need_to_switch_to_main:
+                if workspace_main:
+                    switch_to_workspace_by_user_workspace_data(config, workspace_main)
+                else:
+                    raise CLIException(FeedbackManager.error_switching_to_main())