tinybird 3.0.2.dev0__tar.gz → 3.1.0.dev0__tar.gz

{tinybird-3.0.2.dev0 → tinybird-3.1.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 3.0.2.dev0
+Version: 3.1.0.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-3.0.2.dev0 → tinybird-3.1.0.dev0}/tinybird/service_datasources.py

@@ -541,6 +541,7 @@ def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
                 "partition_key": "toYYYYMM(run_validation)",
             },
             "columns": [
+                {"name": "cluster", "type": "LowCardinality(String)"},
                 {"name": "host", "type": "LowCardinality(String)"},
                 {"name": "version", "type": "LowCardinality(String)"},
                 {"name": "stable_version", "type": "LowCardinality(String)"},

{tinybird-3.0.2.dev0 → tinybird-3.1.0.dev0}/tinybird/sql_template.py

@@ -6,9 +6,8 @@ import re
 from collections import deque
 from datetime import datetime
 from functools import lru_cache
-from io import StringIO
 from json import loads
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 from tornado import escape
 from tornado.util import ObjectDict, exec_in, unicode_type
@@ -29,6 +28,7 @@ REQUIRED_PARAM_NOT_DEFINED = "Required parameter is not defined"
 # Pre-compiled regex patterns for performance
 _STRING_LINE_NUMBER_RE = re.compile(r"\<string\>:(\d*)")
 _ARRAY_TYPE_RE = re.compile(r"Array\((\w+)\)")
+_EMBEDDED_TEMPLATE_EXPRESSION_RE = re.compile(r"\{\{(.*?)\}\}")
 
 
 def secret_template_key(secret_name: str) -> str:
@@ -2159,12 +2159,32 @@ def get_var_names_and_types(t, node_id=None):
     [{'name': 'with_value', 'type': 'Float32', 'default': 0.1}]
     >>> get_var_names_and_types(Template("SELECT * FROM filter_value WHERE description = {{String(d, 'test_1')}} AND value = {{Int8(v, 3)}}"))
     [{'name': 'd', 'type': 'String', 'default': 'test_1'}, {'name': 'v', 'type': 'Int8', 'default': 3}]
+    >>> get_var_names_and_types(Template("select * from test {% if defined(number_variable) %} where number_variable = {{UInt64(number_variable)}} {% end %}"))
+    [{'name': 'number_variable', 'type': 'UInt64', 'default': None}]
     >>> get_var_names_and_types(Template("select * from test {% if defined({{UInt64(number_variable)}}) %} where 1 {% end %}"))
     [{'name': 'number_variable', 'type': 'UInt64', 'default': None}]
     >>> get_var_names_and_types(Template("select * from test {% if defined(testing) and defined(testing2) %} where 1 {%end %}"))
     [{'name': 'testing', 'type': 'String', 'default': None, 'used_in': 'function_call'}, {'name': 'testing2', 'type': 'String', 'default': None, 'used_in': 'function_call'}]
     >>> get_var_names_and_types(Template("select * from test {% if defined({{UInt64(number_variable)}}) %} where 1 {% end %}"))
     [{'name': 'number_variable', 'type': 'UInt64', 'default': None}]
+    >>> get_var_names_and_types(Template("select * from test {% if defined({{UInt64(x)}}) and defined(y) %} where 1 {% end %}"))
+    [{'name': 'x', 'type': 'UInt64', 'default': None}, {'name': 'y', 'type': 'String', 'default': None, 'used_in': 'function_call'}]
+    >>> get_var_names_and_types(Template("select * from test {% if defined(y) and x > 0 %} where x = {{UInt64(x)}} {% end %}"))
+    [{'name': 'x', 'type': 'UInt64', 'default': None}, {'name': 'y', 'type': 'String', 'default': None, 'used_in': 'function_call'}]
+    >>> get_var_names_and_types(Template("select * from test {% if defined(y) and defined(z) and x > 0 %} {{UInt64(x)}} {% end %}"))
+    [{'name': 'x', 'type': 'UInt64', 'default': None}, {'name': 'y', 'type': 'String', 'default': None, 'used_in': 'function_call'}, {'name': 'z', 'type': 'String', 'default': None, 'used_in': 'function_call'}]
+    >>> get_var_names_and_types(Template("{% if '{{' in marker %}{{UInt64(x)}}{% end %}"))
+    [{'name': 'x', 'type': 'UInt64', 'default': None}, {'name': 'marker', 'type': 'String', 'default': None}]
+    >>> get_var_names_and_types(Template("{% if a %}1{% elif defined(y) %}{{UInt64(x)}}{% end %}"))
+    [{'name': 'y', 'type': 'String', 'default': None, 'used_in': 'function_call'}, {'name': 'x', 'type': 'UInt64', 'default': None}, {'name': 'a', 'type': 'String', 'default': None}]
+    >>> get_var_names_and_types(Template("{% for x in items %}{{Int32(x, 0)}}{% end %}"))
+    [{'name': 'x', 'type': 'Int32', 'default': 0}, {'name': 'items', 'type': 'String', 'default': None}]
+    >>> get_var_names_and_types(Template("{% while more %}{{UInt64(x)}}{% end %}"))
+    [{'name': 'x', 'type': 'UInt64', 'default': None}, {'name': 'more', 'type': 'String', 'default': None}]
+    >>> get_var_names_and_types(Template("{% set a = Int32(x, 0) %}"))
+    [{'name': 'a', 'type': 'String', 'default': None}, {'name': 'x', 'type': 'Int32', 'default': 0}]
+    >>> get_var_names_and_types(Template("{% try %}{{UInt64(x)}}{% except E as e %}{{e}}{% end %}"))
+    [{'name': 'x', 'type': 'UInt64', 'default': None}, {'name': 'E', 'type': 'String', 'default': None}, {'name': 'e', 'type': 'String', 'default': None}]
     >>> get_var_names_and_types(Template("select {{Array(cod_stock_source_type,'Int16', defined=False)}}"))
     [{'name': 'cod_stock_source_type', 'type': 'Array(Int16)', 'defined': False, 'default': None}]
     >>> get_var_names_and_types(Template("select {{Array(cod_stock_source_type, defined=False)}}"))
@@ -2194,7 +2214,7 @@ def get_var_names_and_types(t, node_id=None):
     >>> get_var_names_and_types(Template("SELECT * FROM filter_value WHERE description = {{Float32(with_value, -0.1)}} AND description = {{Float32(zero, 0)}} AND value = {{Float32(no_default)}}"))
     [{'name': 'with_value', 'type': 'Float32', 'default': -0.1}, {'name': 'zero', 'type': 'Float32', 'default': 0}, {'name': 'no_default', 'type': 'Float32', 'default': None}]
     >>> get_var_names_and_types(Template('''SELECT * FROM abcd WHERE hotel_id <> 0 {% if defined(date_from) %} AND script_created_at > {{DateTime(date_from, '2020-09-09 10:10:10', description="This is a description", required=True)(date_from, '2020-09-09', description="Filter script alert creation date", required=False)}} {% end %}'''))
-    [{'name': 'date_from', 'type': 'DateTime', 'description': 'This is a description', 'required': True, 'default': '2020-09-09 10:10:10'}, {'name': 'date_from', 'type': 'DateTime', 'description': 'This is a description', 'required': True, 'default': '2020-09-09 10:10:10'}]
+    [{'name': 'date_from', 'type': 'DateTime', 'description': 'This is a description', 'required': True, 'default': '2020-09-09 10:10:10'}]
     >>> get_var_names_and_types(Template("SELECT * FROM filter_value WHERE symbol = {{Int128(symbol_id, 11111, description='Symbol Id', required=True)}} AND user = {{Int256(user_id, 3555, description='User Id')}}"))
     [{'name': 'symbol_id', 'type': 'Int128', 'description': 'Symbol Id', 'required': True, 'default': 11111}, {'name': 'user_id', 'type': 'Int256', 'description': 'User Id', 'default': 3555}]
     >>> get_var_names_and_types(Template("SELECT now() > {{DateTime64(timestamp, '2020-09-09 10:10:10.000')}}"))
@@ -2205,32 +2225,118 @@ def get_var_names_and_types(t, node_id=None):
     [{'name': 'symbol_id', 'type': 'Int64', 'default': '9223372036854775807'}]
     """
     try:
+        # Recursive helper that traverses the template's parsed chunks and collects
+        # variable data including types and defaults.
+        #
+        # Optimization: Instead of calling x.generate(writer) which generates full
+        # Python code (expensive for large templates), we parse just the statement
+        # and recurse into the body separately.
+        #
+        # Backward compatibility: The original implementation called get_var_data on
+        # full generated code, where type functions (Array, Int32, etc.) would overwrite
+        # variables from defined(). To maintain this behavior, we process body expressions
+        # FIRST to get types, then only add variables from control statements if they
+        # weren't already found in body expressions.
+
+        # Track variable names seen from EXPRESSIONS (not control statements)
+        # Used to prevent control statements from adding variables with wrong types
+        typed_names: set[str] = set()
+
+        statement_wraps: dict[str, Callable[[str], str]] = {
+            "control": lambda statement: f"{statement}: pass",
+            "elif": lambda statement: "if False: pass\n" + statement + ": pass",
+            "except": lambda statement: "try: pass\n" + statement + ": pass",
+        }
 
-        def _n(chunks, v):
-            for x in chunks:
-                if type(x).__name__ == "_ChunkList":
-                    _n(x.chunks, v)
-                elif type(x).__name__ == "_Expression":
-                    var_data = get_var_data(x.expression, node_id=node_id)
-                    if var_data:
-                        v += var_data
-                elif type(x).__name__ == "_ControlBlock":
-                    buffer = StringIO()
-                    writer = CodeWriter(buffer, t)
-                    x.generate(writer)
-                    var_data = get_var_data(buffer.getvalue(), node_id=node_id)
-                    if var_data:
-                        v += var_data
-                    _n(x.body.chunks, v)
+        def parse_statement_code_if_new_vars(statement_code: str, *, skip_names: set[str]) -> list[dict[str, Any]]:
+            """Parse statement code and return variables not already typed by expressions."""
+            try:
+                # `{{...}}` inside `{% ... %}` is not template syntax in Tornado templates.
+                # It becomes Python braces (e.g. set literals), which can hide unrelated vars
+                # in the statement. Strip them so we still parse the rest of the statement.
+                if "{{" in statement_code and "}}" in statement_code:
+                    statement_code = _EMBEDDED_TEMPLATE_EXPRESSION_RE.sub("None", statement_code)
+                var_data = get_var_data(statement_code, node_id=node_id)
+            except Exception:
+                return []
 
-        var = []
-        _n(t.file.body.chunks, var)
-        return var
+            if not var_data:
+                return []
+
+            return [vd for vd in var_data if vd["name"] not in skip_names]
+
+        def parse_statement(statement: str, *, wrap: Callable[[str], str] | None) -> list[dict[str, Any]]:
+            """Parse a statement and return variables not already typed by expressions."""
+            statement_expr_names: set[str] = set()
+            vars_out: list[dict[str, Any]] = []
+
+            if "{{" in statement and "}}" in statement:
+                for match in _EMBEDDED_TEMPLATE_EXPRESSION_RE.finditer(statement):
+                    expr = match.group(1).strip()
+                    if not expr:
+                        continue
+                    try:
+                        var_data = get_var_data(expr, node_id=node_id)
+                    except Exception:
+                        continue
+
+                    if not var_data:
+                        continue
+
+                    for vd in var_data:
+                        # Body expressions win for types; do not add statement-derived typed vars.
+                        if vd["name"] in typed_names:
+                            continue
+                        statement_expr_names.add(vd["name"])
+                        vars_out.append(vd)
+
+            statement_code = wrap(statement) if wrap else statement
+            vars_out.extend(
+                parse_statement_code_if_new_vars(statement_code, skip_names=typed_names | statement_expr_names)
+            )
+            return vars_out
+
+        def _n(chunks: list, vars_out: list[dict[str, Any]]) -> None:
+            for x in chunks:
+                kind = type(x).__name__
+                match kind:
+                    case "_ChunkList":
+                        _n(x.chunks, vars_out)
+
+                    case "_Expression":
+                        # Template expression like {{Int32(num_val, 0)}} - extract type info
+                        var_data = get_var_data(x.expression, node_id=node_id)
+                        if var_data:
+                            typed_names.update(vd["name"] for vd in var_data)
+                            vars_out.extend(var_data)
+
+                    case "_ControlBlock":
+                        # Process body FIRST to get type functions
+                        _n(x.body.chunks, vars_out)
+                        # Then parse statement, but only add vars NOT already typed by expressions
+                        if x.statement != "try":
+                            vars_out.extend(parse_statement(x.statement, wrap=statement_wraps["control"]))
+
+                    case "_IntermediateControlBlock":
+                        # elif/else/except/finally - only add vars not typed by expressions
+                        if x.statement.startswith("elif "):
+                            vars_out.extend(parse_statement(x.statement, wrap=statement_wraps["elif"]))
+                        elif x.statement.startswith("except "):
+                            vars_out.extend(parse_statement(x.statement, wrap=statement_wraps["except"]))
+
+                    case "_Statement":
+                        # {% set x = ... %}, {% break %}, etc.
+                        if x.statement not in ("break", "continue") and "{{" not in x.statement:
+                            vars_out.extend(parse_statement(x.statement, wrap=None))
+
+        vars_out: list[dict[str, Any]] = []
+        _n(t.file.body.chunks, vars_out)
+        return vars_out
     except SecurityException as e:
         raise SQLTemplateException(e)
 
 
-@lru_cache(maxsize=512)
+@lru_cache(maxsize=2**10)
 def get_var_names_and_types_cached(t: Template):
     return get_var_names_and_types(t)
 
@@ -2700,6 +2806,8 @@ def render_sql_template(
     t, template_variables, variable_warnings = get_template_and_variables(
         sql, name, escape_arrays=escape_split_to_array
     )
+
+    ## TODO: Could we skip running this unless we need it for some variable preprocessing?
     template_variables_with_types = get_var_names_and_types_cached(t)
 
     if variables is not None:

{tinybird-3.0.2.dev0 → tinybird-3.1.0.dev0}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '3.0.2.dev0'
-__revision__ = '5975d81'
+__version__ = '3.1.0.dev0'
+__revision__ = 'a987e5a'

{tinybird-3.0.2.dev0 → tinybird-3.1.0.dev0}/tinybird/tb/cli.py

@@ -21,6 +21,7 @@ import tinybird.tb.modules.materialization
 import tinybird.tb.modules.mock
 import tinybird.tb.modules.open
 import tinybird.tb.modules.pipe
+import tinybird.tb.modules.project_commands
 import tinybird.tb.modules.secret
 import tinybird.tb.modules.sink
 import tinybird.tb.modules.test

{tinybird-3.0.2.dev0 → tinybird-3.1.0.dev0}/tinybird/tb/modules/cli.py

@@ -5,6 +5,7 @@
 import json
 import logging
 import os
+import re
 import shutil
 import sys
 from os import environ, getcwd
@@ -455,8 +456,6 @@ def ch(ctx: Context, query: str, user: Optional[str], password: Optional[str], m
 
 
 def __patch_click_output():
-    import re
-
     CUSTOM_PATTERNS: List[str] = []
 
     _env_patterns = os.getenv("OBFUSCATE_REGEX_PATTERN", None)
@@ -534,6 +533,7 @@ def create_ctx_client(
         "diff",
         "fmt",
         "init",
+        "project",
     ]
     command = ctx.invoked_subcommand
     if not command or command in commands_without_ctx_client:

tinybird-3.1.0.dev0/tinybird/tb/modules/project_commands.py

@@ -0,0 +1,53 @@
+import re
+import tarfile
+from pathlib import Path
+from typing import Optional
+
+import click
+
+from tinybird.tb.modules.cli import cli
+from tinybird.tb.modules.feedback_manager import FeedbackManager
+from tinybird.tb.modules.project import Project
+
+
+@cli.group(hidden=True)
+@click.pass_context
+def project(ctx: click.Context) -> None:
+    """Project commands."""
+
+
+@project.command(name="archive")
+@click.option("-o", "--output", type=click.Path(), default=None, help="Output file path")
+@click.pass_context
+def project_archive(ctx: click.Context, output: Optional[str]) -> None:
+    """Create a tar.gz archive of all datafiles for support purposes."""
+    project_obj: Project = ctx.ensure_object(dict)["project"]
+    config = ctx.ensure_object(dict)["config"]
+
+    all_files = project_obj.get_project_files() + project_obj.get_vendored_files()
+
+    if not all_files:
+        click.echo(FeedbackManager.warning(message="No datafiles found in the project"))
+        return
+
+    if output:
+        output_path = Path(output)
+        if not output_path.is_absolute():
+            output_path = Path(project_obj.folder) / output_path
+    else:
+        workspace_name = config.get("name", "") or project_obj.workspace_name or "workspace"
+        workspace_name = re.sub(r"[^a-zA-Z0-9_-]", "_", workspace_name)
+        output_filename = f"{workspace_name}_datafiles.tar.gz"
+        output_path = Path(project_obj.folder) / output_filename
+
+    if output_path.exists() and not click.confirm(f"{output_path} already exists. Overwrite?"):
+        click.echo("Aborted.")
+        return
+
+    with tarfile.open(output_path, "w:gz") as tar:
+        for f in all_files:
+            rel_path = Path(f).relative_to(project_obj.path)
+            tar.add(f, arcname=str(rel_path))
+            click.echo(f"a {rel_path}")
+
+    click.echo(FeedbackManager.success(message=f"Created {output_path}"))

{tinybird-3.0.2.dev0 → tinybird-3.1.0.dev0}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 3.0.2.dev0
+Version: 3.1.0.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-3.0.2.dev0 → tinybird-3.1.0.dev0}/tinybird.egg-info/SOURCES.txt

@@ -72,6 +72,7 @@ tinybird/tb/modules/mock_common.py
 tinybird/tb/modules/open.py
 tinybird/tb/modules/pipe.py
 tinybird/tb/modules/project.py
+tinybird/tb/modules/project_commands.py
 tinybird/tb/modules/regions.py
 tinybird/tb/modules/secret.py
 tinybird/tb/modules/secret_common.py