tinybird 0.0.1.dev291__py3-none-any.whl → 1.0.5__py3-none-any.whl

tinybird/ch_utils/constants.py

@@ -258,4 +258,9 @@ VALID_QUERY_FORMATS = (
     "RowBinaryWithNamesAndTypes",
     "TabSeparated",
     "JSONCompactEachRowWithNamesAndTypes",
+    "TabSeparatedWithNamesAndTypes",
+    "JSONCompactEachRow",
+    "JSONCompact",
+    "JSONStringsEachRowWithProgress",
+    "ODBCDriver2",
 )

tinybird/connectors.py

@@ -369,13 +369,7 @@ class Snowflake(Connector):
             the_type = "String"
             if t.startswith("NUMBER"):
                 the_type = "Int32"
-            if (
-                t.startswith("FLOAT")
-                or t.startswith("DOUBLE")
-                or t.startswith("REAL")
-                or t.startswith("NUMERIC")
-                or t.startswith("DECIMAL")
-            ):
+            if t.startswith(("FLOAT", "DOUBLE", "REAL", "NUMERIC", "DECIMAL")):
                 the_type = "Float32"
             if t == "DATE":
                 the_type = "Date"

tinybird/context.py

@@ -3,15 +3,15 @@ from typing import TYPE_CHECKING
 
 # Avoid circular import error
 if TYPE_CHECKING:
-    from tinybird.user import User
+    from hfi.hfi_workspace_data import HfiWorkspaceData
+
 
 workspace_id: ContextVar[str] = ContextVar("workspace_id")
-workspace: ContextVar["User"] = ContextVar("workspace")
+hfi_workspace_data: ContextVar["HfiWorkspaceData"] = ContextVar("hfi_workspace_data")
 table_id: ContextVar[str] = ContextVar("table_id")
 hfi_frequency: ContextVar[float] = ContextVar("hfi_frequency")
 hfi_frequency_gatherer: ContextVar[float] = ContextVar("hfi_frequency_gatherer")
 use_gatherer: ContextVar[bool] = ContextVar("use_gatherer")
-allow_gatherer_fallback: ContextVar[bool] = ContextVar("allow_gatherer_fallback")
 gatherer_allow_s3_backup_on_user_errors: ContextVar[bool] = ContextVar("gatherer_allow_s3_backup_on_user_errors")
 disable_template_security_validation: ContextVar[bool] = ContextVar("disable_template_security_validation")
 origin: ContextVar[str] = ContextVar("origin")

tinybird/datafile/common.py

@@ -630,7 +630,10 @@ class Datafile:
     def _validate_single_column(self, col_name: str, column_info: Dict[str, Any]) -> None:
         """Validate a single column for use in sorting keys."""
         col_type = column_info.get("type", "").lower()
-        is_nullable = column_info.get("nullable", False)
+
+        # we need to check any presence of Nullable in the column type
+        is_nullable = column_info.get("nullable", False) or "Nullable(" in column_info.get("type", "")
+
         if is_nullable:
             raise DatafileValidationError(
                 f"Sorting key contains nullable column '{col_name}'. Nullable columns cannot be used in sorting keys."
@@ -2048,11 +2051,7 @@ def parse(
                 lexer = list(sa)
             if lexer:
                 cmd, args = lexer[0], lexer[1:]
-                if (
-                    parser_state.multiline
-                    and cmd.lower() in cmds
-                    and not (line.startswith(" ") or line.startswith("\t"))
-                ):
+                if parser_state.multiline and cmd.lower() in cmds and not line.startswith((" ", "\t")):
                     cmds[parser_state.command](
                         parser_state.multiline_string,
                         lineno=lineno,
@@ -2128,6 +2127,9 @@ def parse(
     return ParseResult(datafile=doc, warnings=warnings)
 
 
+# TODO: This class is duplicated in tinybird/datafile_common.py with a slightly different
+# _REPLACEMENTS tuple. The duplication happened during the CLI/server code split (commit
+# f86d02cdd7). Consider extracting shared code into a common module that both files can import.
 class ImportReplacements:
     _REPLACEMENTS: Tuple[Tuple[str, str, Optional[str]], ...] = (
         ("import_strategy", "mode", "replace"),
@@ -2147,7 +2149,7 @@ class ImportReplacements:
         return [x[0] for x in ImportReplacements._REPLACEMENTS]
 
     @staticmethod
-    def get_api_param_for_datafile_param(connector_service: str, key: str) -> Tuple[Optional[str], Optional[str]]:
+    def get_api_param_for_datafile_param(key: str) -> Tuple[Optional[str], Optional[str]]:
         """Returns the API parameter name and default value for a given
         datafile parameter.
         """
@@ -2505,7 +2507,7 @@ def is_file_a_datasource(filename: str) -> bool:
 
         for line in lines:
             trimmed_line = line.strip().lower()
-            if trimmed_line.startswith("schema") or trimmed_line.startswith("engine"):
+            if trimmed_line.startswith(("schema", "engine")):
                 return True
 
     return False

tinybird/datafile/parse_pipe.py

@@ -11,7 +11,7 @@ from tinybird.datafile.common import (
     parse,
 )
 from tinybird.datafile.exceptions import IncludeFileNotFoundException, ParseException
-from tinybird.sql_template import get_template_and_variables, render_sql_template, secret_template_key
+from tinybird.sql_template import get_template_and_variables, render_sql_template
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tornado_template import UnClosedIfError
 
@@ -54,7 +54,7 @@ def parse_pipe(
             if sql.strip()[0] == "%":
                 secrets_list: Optional[List[str]] = None
                 if secrets:
-                    secrets_list = [f"{secret_template_key(secret)}" for secret in secrets.keys()]
+                    secrets_list = list(secrets.keys())
                 # Setting test_mode=True to ignore errors on required parameters and
                 # secrets_in_test_mode=False to raise errors on missing secrets
                 sql, _, variable_warnings = render_sql_template(

tinybird/feedback_manager.py

@@ -506,6 +506,9 @@ Ready? """
     prompt_init_git_release_force = prompt_message(
         "You are going to manually update workspace commit reference manually, this is just for special occasions. Do you want to update current commit reference '{current_commit}' to '{new_commit}'?"
     )
+    prompt_init_git_release_new = prompt_message(
+        "This workspace does not have any release yet. Do you want to create one with commit '{commit}'? This will enable 'tb deploy' to work."
+    )
 
     warning_exchange = warning_message(
         "Warning: Do you want to exchange Data Source {datasource_a} by Data Source {datasource_b}?"

tinybird/prompts.py

@@ -776,6 +776,7 @@ datasource_instructions = """
     - Use always json paths to define the schema. Example: `user_id` String `json:$.user_id`,
     - Array columns are supported with a special syntax. Example: `items` Array(String) `json:$.items[:]`
     - If the datasource is using an S3 or GCS connection, they need to set IMPORT_CONNECTION_NAME, IMPORT_BUCKET_URI and IMPORT_SCHEDULE (GCS @on-demand only, S3 supports @auto too)
+    - If the datasource is using a Kafka connection, they need to set KAFKA_CONNECTION_NAME as the name of the .connection file, KAFKA_TOPIC topic_name and KAFKA_GROUP_ID as the group id for the datasource
     - Unless the user asks for them, do not include ENGINE_PARTITION_KEY and ENGINE_PRIMARY_KEY.
     - DateTime64 type without precision is not supported. Use DateTime64(3) instead.
 </datasource_file_instructions>

tinybird/service_datasources.py

@@ -5,6 +5,7 @@ This module provides access to predefined service datasources and their schemas
 for both Tinybird and Organization scopes.
 """
 
+from functools import lru_cache
 from typing import Any, Dict, List, Optional
 
 
@@ -428,6 +429,135 @@ def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "total_cpu_time_seconds", "type": "Float64"},
             ],
         },
+        {
+            "name": "tinybird.llm_usage",
+            "description": "LLM usage metrics from Tinybird AI features including token consumption, costs, and model usage for each request in the workspace.",
+            "dateColumn": "start_time",
+            "engine": {
+                "engine": "MergeTree",
+                "sorting_key": "workspace_id, start_time, user_email, request_id",
+                "partition_key": "toYYYYMM(start_time)",
+            },
+            "columns": [
+                {"name": "start_time", "type": "DateTime"},
+                {"name": "end_time", "type": "DateTime"},
+                {"name": "organization_id", "type": "String"},
+                {"name": "organization_name", "type": "String"},
+                {"name": "workspace_id", "type": "String"},
+                {"name": "workspace_name", "type": "String"},
+                {"name": "user_email", "type": "String"},
+                {"name": "request_id", "type": "String"},
+                {"name": "prompt_tokens", "type": "UInt32"},
+                {"name": "completion_tokens", "type": "UInt32"},
+                {"name": "total_tokens", "type": "UInt32"},
+                {"name": "duration", "type": "Float32"},
+                {"name": "cost", "type": "Float32"},
+                {"name": "origin", "type": "String"},
+                {"name": "feature", "type": "String"},
+            ],
+        },
+        {
+            "name": "tinybird.llm_usage",
+            "description": "LLM usage metrics from Tinybird AI features including token consumption, costs, and model usage for each request in the workspace.",
+            "dateColumn": "start_time",
+            "engine": {
+                "engine": "MergeTree",
+                "sorting_key": "workspace_id, start_time, user_email, request_id",
+                "partition_key": "toYYYYMM(start_time)",
+            },
+            "columns": [
+                {"name": "start_time", "type": "DateTime"},
+                {"name": "end_time", "type": "DateTime"},
+                {"name": "organization_id", "type": "String"},
+                {"name": "organization_name", "type": "String"},
+                {"name": "workspace_id", "type": "String"},
+                {"name": "workspace_name", "type": "String"},
+                {"name": "user_email", "type": "String"},
+                {"name": "request_id", "type": "String"},
+                {"name": "prompt_tokens", "type": "UInt32"},
+                {"name": "completion_tokens", "type": "UInt32"},
+                {"name": "total_tokens", "type": "UInt32"},
+                {"name": "duration", "type": "Float32"},
+                {"name": "cost", "type": "Float32"},
+                {"name": "origin", "type": "String"},
+                {"name": "feature", "type": "String"},
+            ],
+        },
+        {
+            "name": "tinybird.query_metrics",
+            "description": "Query stats metrics from your workspace.",
+            "dateColumn": "event_time",
+            "engine": {
+                "engine": "ReplacingMergeTree",
+                "sorting_key": "event_time, organization_id, query_id",
+                "partition_key": "toStartOfDay(event_time)",
+                "ttl": "toDate(event_time) + INTERVAL 30 DAY",
+            },
+            "columns": [
+                {"name": "event_time", "type": "DateTime"},
+                {"name": "organization_id", "type": "String"},
+                {"name": "workspace_id", "type": "String"},
+                {"name": "query", "type": "String"},
+                {"name": "query_id", "type": "String"},
+                {"name": "query_type", "type": "String"},
+                {"name": "query_start_time", "type": "DateTime"},
+                {"name": "query_duration_ms", "type": "Int32"},
+                {"name": "pipe_id", "type": "String"},
+                {"name": "job_id", "type": "String"},
+                {"name": "job_kind", "type": "String"},
+                {"name": "read_rows", "type": "Int32"},
+                {"name": "read_bytes", "type": "Int32"},
+                {"name": "written_rows", "type": "Int32"},
+                {"name": "written_bytes", "type": "Int32"},
+                {"name": "memory_usage", "type": "Int32"},
+                {"name": "vcpu_time", "type": "Float32"},
+                {"name": "exception_code", "type": "Int32"},
+                {"name": "exception", "type": "String"},
+            ],
+        },
+        {
+            "name": "tinybird.vcpu_time",
+            "description": "vCPU time metrics from your workspace.",
+            "dateColumn": "second_slot",
+            "engine": {
+                "engine": "AggregatingMergeTree",
+                "sorting_key": "organization_id, second_slot",
+                "partition_key": "toStartOfDay(second_slot)",
+                "ttl": "toDate(second_slot) + INTERVAL 30 DAY",
+            },
+            "columns": [
+                {"name": "second_slot", "type": "DateTime"},
+                {"name": "organization_id", "type": "String"},
+                {"name": "vcpu_time", "type": "Float64"},
+            ],
+        },
+        {
+            "name": "tinybird.query_validator_log",
+            "description": "Log of failed queries executions in the next available ClickHouse version and their results.",
+            "dateColumn": "run_validation",
+            "engine": {
+                "engine": "MergeTree",
+                "sorting_key": "database, host, run_validation, query_hash",
+                "partition_key": "toYYYYMM(run_validation)",
+            },
+            "columns": [
+                {"name": "host", "type": "LowCardinality(String)"},
+                {"name": "version", "type": "LowCardinality(String)"},
+                {"name": "stable_version", "type": "LowCardinality(String)"},
+                {"name": "query_hash", "type": "UInt64"},
+                {"name": "query_last_execution", "type": "DateTime"},
+                {"name": "region", "type": "String"},
+                {"name": "workspace", "type": "String"},
+                {"name": "database", "type": "String"},
+                {"name": "pipe_name", "type": "String"},
+                {"name": "query_id", "type": "String"},
+                {"name": "query", "type": "String"},
+                {"name": "error_code", "type": "Int16"},
+                {"name": "error", "type": "String"},
+                {"name": "fix_suggestion", "type": "String"},
+                {"name": "run_validation", "type": "DateTime"},
+            ],
+        },
     ]
 
 
@@ -806,6 +936,22 @@ def get_organization_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "active_minutes", "type": "Float64"},
             ],
         },
+        {
+            "name": "organization.shared_infra_active_seconds",
+            "description": "Contains information about vCPU active seconds consumption aggregated by second for all Organization workspaces. Only available for Developer and Enterprise plans in shared infrastructure.",
+            "dateColumn": "second",
+            "engine": {
+                "engine": "AggregatingMergeTree",
+                "sorting_key": "second_slot, organization_id",
+                "partition_key": "toYYYYMM(second_slot)",
+            },
+            "columns": [
+                {"name": "second_slot", "type": "DateTime"},
+                {"name": "organization_id", "type": "String"},
+                {"name": "organization_name", "type": "String"},
+                {"name": "cpu_time", "type": "Float64"},
+            ],
+        },
         {
             "name": "organization.shared_infra_qps_overages",
             "description": "Contains information about QPS consumption and overages aggregated by second for all Organization workspaces. Only available for Developer and Enterprise plans in shared infrastructure.",
@@ -881,9 +1027,86 @@ def get_organization_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "total_written_bytes", "type": "UInt64"},
             ],
         },
+        {
+            "name": "organization.llm_usage",
+            "description": "LLM usage metrics from Tinybird AI features including token consumption, costs, and model usage for each request in the organization.",
+            "dateColumn": "start_time",
+            "engine": {
+                "engine": "MergeTree",
+                "sorting_key": "workspace_id, start_time, user_email, request_id",
+                "partition_key": "toYYYYMM(start_time)",
+            },
+            "columns": [
+                {"name": "start_time", "type": "DateTime"},
+                {"name": "end_time", "type": "DateTime"},
+                {"name": "organization_id", "type": "String"},
+                {"name": "organization_name", "type": "String"},
+                {"name": "workspace_id", "type": "String"},
+                {"name": "workspace_name", "type": "String"},
+                {"name": "user_email", "type": "String"},
+                {"name": "request_id", "type": "String"},
+                {"name": "prompt_tokens", "type": "UInt32"},
+                {"name": "completion_tokens", "type": "UInt32"},
+                {"name": "total_tokens", "type": "UInt32"},
+                {"name": "duration", "type": "Float32"},
+                {"name": "cost", "type": "Float32"},
+                {"name": "origin", "type": "String"},
+                {"name": "feature", "type": "String"},
+            ],
+        },
+        {
+            "name": "organization.query_metrics",
+            "description": "Query stats metrics from your workspace.",
+            "dateColumn": "event_time",
+            "engine": {
+                "engine": "ReplacingMergeTree",
+                "sorting_key": "event_time, organization_id, query_id",
+                "partition_key": "toStartOfDay(event_time)",
+                "ttl": "toDate(event_time) + INTERVAL 30 DAY",
+            },
+            "columns": [
+                {"name": "event_time", "type": "DateTime"},
+                {"name": "organization_id", "type": "String"},
+                {"name": "workspace_id", "type": "String"},
+                {"name": "query", "type": "String"},
+                {"name": "query_id", "type": "String"},
+                {"name": "query_type", "type": "String"},
+                {"name": "query_start_time", "type": "DateTime"},
+                {"name": "query_duration_ms", "type": "Int32"},
+                {"name": "pipe_id", "type": "String"},
+                {"name": "job_id", "type": "String"},
+                {"name": "job_kind", "type": "String"},
+                {"name": "read_rows", "type": "Int32"},
+                {"name": "read_bytes", "type": "Int32"},
+                {"name": "written_rows", "type": "Int32"},
+                {"name": "written_bytes", "type": "Int32"},
+                {"name": "memory_usage", "type": "Int32"},
+                {"name": "vcpu_time", "type": "Float32"},
+                {"name": "exception_code", "type": "Int32"},
+                {"name": "exception", "type": "String"},
+            ],
+        },
+        {
+            "name": "organization.vcpu_time",
+            "description": "vCPU time metrics from your workspace.",
+            "dateColumn": "second_slot",
+            "engine": {
+                "engine": "AggregatingMergeTree",
+                "sorting_key": "organization_id, second_slot",
+                "partition_key": "toStartOfDay(second_slot)",
+                "ttl": "toDate(second_slot) + INTERVAL 30 DAY",
+            },
+            "columns": [
+                {"name": "second_slot", "type": "DateTime"},
+                {"name": "organization_id", "type": "String"},
+                {"name": "workspace_id", "type": "String"},
+                {"name": "vcpu_time", "type": "Float64"},
+            ],
+        },
     ]
 
 
+@lru_cache(maxsize=1)
 def get_service_datasources() -> List[Dict[str, Any]]:
     """
     Get the list of all Tinybird and Organization service datasources.

tinybird/sql_template.py

@@ -384,14 +384,13 @@ def array_type(types):
             if isinstance(x, Placeholder):
                 if default:
                     x = default
-                else:
-                    if _type and _type in types:
-                        if _type == "String":
-                            x = ""
-                        else:
-                            x = ",".join(map(str, [types[_type](x) for _ in range(2)]))
-                    else:
+                elif _type and _type in types:
+                    if _type == "String":
                         x = ""
+                    else:
+                        x = ",".join(map(str, [types[_type](x) for _ in range(2)]))
+                else:
+                    x = ""
             elif x is None:
                 x = default
                 if x is None:
@@ -1405,8 +1404,11 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
     namespace = {}
     template_execution_results = TemplateExecutionResults()
     for key in kwargs.get("tb_secrets", []):
+        # Avoid double-prefixing if the key already has the tb_secret_ prefix
         if is_secret_template_key(key):
             template_execution_results.add_template_param(key)
+        else:
+            template_execution_results.add_template_param(secret_template_key(key))
 
     if TB_SECRET_IN_TEST_MODE in kwargs:
         template_execution_results[TB_SECRET_IN_TEST_MODE] = None
@@ -1415,15 +1417,20 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
         try:
             key = secret_template_key(x)
             if key in template_execution_results.template_params:
+                # secret available: Always use workspace secret regardless of test mode
                 template_execution_results.add_ch_param(x)
                 return Symbol("{" + sqlescape(x) + ": String}")
             else:
+                # secret not available: Check test mode and defaults
                 is_test_mode = TB_SECRET_IN_TEST_MODE in template_execution_results
-                if is_test_mode and default is None:
-                    return Symbol("{" + sqlescape(x) + ": String}")
-                elif default is not None:
+                if default is not None:
+                    # Use provided default value
                     return default
+                elif is_test_mode:
+                    # In test mode without default - return placeholder
+                    return Symbol("{" + sqlescape(x) + ": String}")
                 else:
+                    # Not in test mode, no secret, no default - raise error
                     raise SQLTemplateException(
                         f"Cannot access secret '{x}'. Check the secret exists in the Workspace and the token has the required scope."
                     )
@@ -1834,7 +1841,7 @@ def get_var_names_and_types(t, node_id=None):
         raise SQLTemplateException(e)
 
 
-@lru_cache(maxsize=256)
+@lru_cache(maxsize=512)
 def get_var_names_and_types_cached(t: Template):
     return get_var_names_and_types(t)
 
@@ -2378,6 +2385,14 @@ def render_sql_template(
                 documentation="/cli/advanced-templates.html",
             )
         raise SQLTemplateException(str(e), documentation="/cli/advanced-templates.html")
+    except IndexError as e:
+        # This happens when trying to access string indices on empty strings
+        if "string index out of range" in str(e):
+            raise SQLTemplateException(
+                "String index out of range. Check that string parameters have values before accessing specific characters (e.g., param[0]). Provide default values or add length checks in your template.",
+                documentation="/cli/advanced-templates.html",
+            )
+        raise SQLTemplateException(str(e), documentation="/cli/advanced-templates.html")
     except Exception as e:
         # errors might vary here, we need to support as much as possible
         # https://gitlab.com/tinybird/analytics/-/issues/943

tinybird/sql_template_fmt.py

@@ -203,13 +203,15 @@ class TinybirdDialect(ClickHouse):
                 Rule(
                     name="jinja_if_block_end",
                     priority=203,
-                    pattern=group(r"\{%-?\s*end\s*-?%\}"),
+                    # Accept both Tornado-style {% end %} and {% end if %}
+                    pattern=group(r"\{%-?\s*end(\s+if)?\s*-?%\}"),
                     action=actions.raise_sqlfmt_bracket_error,
                 ),
                 Rule(
                     name="jinja_for_block_end",
                     priority=211,
-                    pattern=group(r"\{%-?\s*end\s*-?%\}"),
+                    # Accept both Tornado-style {% end %} and {% end for %}
+                    pattern=group(r"\{%-?\s*end(\s+for)?\s*-?%\}"),
                     action=actions.raise_sqlfmt_bracket_error,
                 ),
             ],
@@ -261,7 +263,13 @@ def _calc_str(self) -> str:
 Comment._calc_str = property(_calc_str)
 
 
-def format_sql_template(sql: str, line_length: Optional[int] = None, lower_keywords: bool = False) -> str:
+def format_sql_template(
+    sql: str,
+    line_length: Optional[int] = None,
+    lower_keywords: bool = False,
+    resource_name: Optional[str] = None,
+    resource_source: Optional[str] = None,
+) -> str:
     try:
         # https://github.com/tconbeer/sqlfmt/blob/c11775b92d8a45f0e91d871b81a88a894d620bec/src/sqlfmt/mode.py#L16-L29
         config: Dict[str, Any] = {
@@ -277,5 +285,7 @@ def format_sql_template(sql: str, line_length: Optional[int] = None, lower_keywo
             else api.format_string(sql, mode=mode).strip()
         )
     except Exception as e:
-        logging.warning(f"sqlfmt error: {str(e)}")
+        resource_info = f" in '{resource_name}'" if resource_name else ""
+        source_info = f" ({resource_source})" if resource_source else ""
+        logging.warning(f"sqlfmt error{resource_info}{source_info}: {str(e)}")
         return sql

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev291'
-__revision__ = '87da698'
+__version__ = '1.0.5'
+__revision__ = '60ae688'

tinybird/tb/cli.py

@@ -1,4 +1,5 @@
 import tinybird.tb.modules.agent
+import tinybird.tb.modules.branch
 import tinybird.tb.modules.build
 import tinybird.tb.modules.cli
 import tinybird.tb.modules.common