tinybird 4.6.22.dev0__py3-none-any.whl → 4.6.24.dev0__py3-none-any.whl

tinybird/sql_template.py

@@ -1,5 +1,4 @@
 import ast
-import builtins
 import linecache
 import logging
 import re
@@ -1336,8 +1335,10 @@ sqlescape_for_string_expression = sqlescape_generator(
 )
 
 
-def escape_single_quote_str(s):
-    return "'" + s.replace("'", "''") + "'"
+def escape_single_quote_str(s: str) -> str:
+    # ClickHouse string literals treat backslash as an escape char, so it must be doubled too:
+    # otherwise a trailing backslash escapes the closing quote and the following input becomes SQL
+    return "'" + s.replace("\\", "\\\\").replace("'", "''") + "'"
 
 
 def expression_wrapper(x, name, escape_arrays: bool = False):
@@ -1385,16 +1386,16 @@ _namespace = {
     "close": None,
     "print": None,
     "input": None,
-    # Keep customer parameters named `type`/`dir` usable without restoring the unsafe builtins.
-    "type": None,
-    "dir": None,
 }
 
 
-reserved_vars = {"_tt_tmp", "_tt_append", "isinstance", "str", "error", "custom_error", *list(vars(builtins))}
+# Built from the builtins templates can actually resolve: names missing from SAFE_BUILTINS (`type`, `dir`,
+# `property`, `object`, ...) can only be template parameters, so they must be discovered as such to get a
+# value (None or a test-mode placeholder) instead of raising NameError.
+reserved_vars = {"_tt_tmp", "_tt_append", "isinstance", "str", "error", "custom_error", *SAFE_BUILTINS}
 for p in DEFAULT_PARAM_NAMES:  # we handle these in an specific manner
     reserved_vars.discard(p)  # `format` is part of builtins
-# Allow 'id' to be used as a template parameter - https://gitlab.com/tinybird/analytics/-/issues/19119
+# Allow 'id' as a template parameter name despite being a builtin - https://gitlab.com/tinybird/analytics/-/issues/19119
 reserved_vars.discard("id")
 error_vars = ["error", "custom_error"]
 
@@ -1471,8 +1472,7 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
         return Expression(f"-- disable {feature}\n")
 
     namespace.update(_namespace)
-    # This is to fix the issue https://gitlab.com/tinybird/analytics/-/issues/19119
-    # We need to do the override here because if we modify the _namespace, we would filter out parameters from the users
+    # Default it here, not in _namespace, so it stays visible to variable extraction.
     namespace.update({"id": None})
     namespace.update(kwargs)
     namespace.update(
@@ -1822,7 +1822,7 @@ def get_var_names(t: Template):
     >>> [v['name'] for v in get_var_names(Template("{% for name in secret_names %}{{tb_secret(name)}}{% end %}"))]
     ['secret_names', 'name', 'tb_secret', 'name']
 
-    Type casting with date function in elif chain (type is in reserved_vars):
+    Type casting with date function in elif chain:
     >>> t = Template("{% if kind == 'int' %}{{Int32(val, 0)}}{% elif kind == 'date' %}{{DateTime(val)}}{% elif kind == 'diff' %}{{date_diff_in_days(val, now)}}{% end %}")
     >>> [v['name'] for v in get_var_names(t)]
     ['kind', 'Int32', 'val', 'kind', 'DateTime', 'val', 'kind', 'date_diff_in_days', 'val', 'now']
@@ -2374,7 +2374,24 @@ def wrap_vars(t, escape_arrays: bool = False):
     return var
 
 
-def get_used_tables_in_template(sql):
+def _constant_table_name(node: ast.expr) -> Optional[str]:
+    """Value of a constant table() argument: a literal or a concatenation of string literals."""
+    if isinstance(node, ast.Constant) and isinstance(node.value, (int, float)):
+        return str(node.value)
+    return _string_literal(node)
+
+
+def _string_literal(node: ast.expr) -> Optional[str]:
+    if isinstance(node, ast.Constant) and isinstance(node.value, str):
+        return node.value
+    if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
+        left, right = _string_literal(node.left), _string_literal(node.right)
+        if left is not None and right is not None:
+            return left + right
+    return None
+
+
+def get_used_tables_in_template(sql: str) -> List[str]:
     """
     >>> get_used_tables_in_template("select * from {{table('test')}}")
     ['test']
@@ -2388,30 +2405,42 @@ def get_used_tables_in_template(sql):
     []
     >>> get_used_tables_in_template("select * from {{table('my.test')}}, {{table('another.one')}}")
     ['my.test', 'another.one']
+
+    Only constant table names can be known without rendering, so anything else is skipped
+    >>> get_used_tables_in_template("select * from {{table(tbl)}}, {{TABLE('b')}}, {{table('my.' + 'c')}}")
+    ['b', 'my.c']
     """
+    return list(_get_used_tables_in_template_cached(sql))
+
+
+# Building the Template dominates the cost and the result only depends on the SQL, which deployments and
+# /v0/sql pipelines scan repeatedly
+@lru_cache(maxsize=2**10)
+def _get_used_tables_in_template_cached(sql: str) -> Tuple[str, ...]:
     try:
         t = Template(sql)
 
         def _n(chunks, tables):
             for x in chunks:
                 if type(x).__name__ == "_Expression":
-                    c = compile(x.expression, "<string>", "exec", dont_inherit=True)
-                    v = [x.lower() for x in c.co_names if x not in _namespace and x not in reserved_vars]
-                    if "table" in v:
-
-                        def _t(*args, **kwargs):
-                            return str(args[0])
-
-                        n = {"table": _t, "TABLE": _t}
-                        e = "_tt_tmp = %s" % x.expression
-                        exec_in(e, n)
-                        tables += [n["_tt_tmp"]]
+                    # Read the names from the AST instead of executing the expression: this runs outside
+                    # generate(), without the restricted builtins, on every pipe, query and deployment.
+                    for node in ast.walk(ast.parse(x.expression)):
+                        if (
+                            isinstance(node, ast.Call)
+                            and isinstance(node.func, ast.Name)
+                            and node.func.id in ("table", "TABLE")
+                            and node.args
+                        ):
+                            name = _constant_table_name(node.args[0])
+                            if name is not None:
+                                tables.append(name)
                 elif type(x).__name__ == "_ControlBlock":
                     _n(x.body.chunks, tables)
 
-        tables = []
+        tables: List[str] = []
         _n(t.file.body.chunks, tables)
-        return tables
+        return tuple(tables)
     except SecurityException as e:
         raise SQLTemplateException(e)
 

tinybird/sql_template_fmt.py

@@ -214,8 +214,8 @@ class TinybirdDialect(ClickHouse):
             ],
         }
 
-        for section in override_rules:
-            for rule in override_rules[section]:
+        for section, rules in override_rules.items():
+            for rule in rules:
                 for rr in self.RULES[section]:
                     if rr.name == rule.name:
                         self.RULES[section].remove(rr)

tinybird/syncasync.py

@@ -260,49 +260,6 @@ def _iscoroutinefunction_or_partial(func: Any) -> bool:
     return asyncio.iscoroutinefunction(func)
 
 
-class ThreadSensitiveContext:
-    """Async context manager to manage context for thread sensitive mode
-    This context manager controls which thread pool executor is used when in
-    thread sensitive mode. By default, a single thread pool executor is shared
-    within a process.
-    In Python 3.7+, the ThreadSensitiveContext() context manager may be used to
-    specify a thread pool per context.
-    In Python 3.6, usage of this context manager has no effect.
-    This context manager is re-entrant, so only the outer-most call to
-    ThreadSensitiveContext will set the context.
-    """
-
-    def __init__(self):
-        self.token = None
-
-    if contextvars:
-
-        async def __aenter__(self):
-            try:
-                SyncToAsync.thread_sensitive_context.get()
-            except LookupError:
-                self.token = SyncToAsync.thread_sensitive_context.set(self)
-
-            return self
-
-        async def __aexit__(self, exc, value, tb):
-            if not self.token:
-                return
-
-            executor = SyncToAsync.context_to_thread_executor.pop(self, None)
-            if executor:
-                executor.shutdown()
-            SyncToAsync.thread_sensitive_context.reset(self.token)
-
-    else:
-
-        async def __aenter__(self):
-            return self
-
-        async def __aexit__(self, exc, value, tb):
-            pass
-
-
 class AsyncToSync:
     """
     Utility class which turns an awaitable that only works on the thread with

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '4.6.22.dev0'
-__revision__ = '3a75e0ef'
+__version__ = '4.6.24.dev0'
+__revision__ = '5269fe22'

tinybird/tb/client.py

@@ -58,14 +58,6 @@ class TimeoutException(Exception):
     pass
 
 
-class ReachRetryLimit(Exception):
-    pass
-
-
-class ConnectorNothingToLoad(Exception):
-    pass
-
-
 class JobException(Exception):
     pass
 

tinybird/tb/modules/common.py

@@ -1277,11 +1277,11 @@ class ConnectionReplacements:
         """Maps prompt parameters to API parameters."""
 
         api_params = {}
-        for key in params.keys():
+        for key, value in params.items():
             try:
-                api_params[ConnectionReplacements._PARAMS_REPLACEMENTS[service][key]] = params[key]
+                api_params[ConnectionReplacements._PARAMS_REPLACEMENTS[service][key]] = value
             except KeyError:
-                api_params[key] = params[key]
+                api_params[key] = value
 
         api_params["service"] = service
         return api_params

tinybird/tb/modules/create.py

@@ -215,14 +215,12 @@ def create_project_structure(folder: str):
         "connections       →": "Connect to and ingest data from popular sources: Kafka, S3 or GCS.",
     }
 
-    for x in PROJECT_PATHS_DESCRIPTIONS.keys():
+    for x, description in PROJECT_PATHS_DESCRIPTIONS.items():
         try:
             path = x.split("→")[0].strip()
             f = folder_path / path
             f.mkdir()
-            click.echo(
-                FeedbackManager.info(message=f"./{x} ") + FeedbackManager.gray(message=PROJECT_PATHS_DESCRIPTIONS[x])
-            )
+            click.echo(FeedbackManager.info(message=f"./{x} ") + FeedbackManager.gray(message=description))
         except FileExistsError:
             pass
 

tinybird/tb/modules/datafile/pull.py

@@ -1,8 +1,11 @@
+from datetime import datetime
 from pathlib import Path
 from typing import Any, Optional
 
 import click
 
+from tinybird.datafile.common import PipeNodeTypes
+from tinybird.datafile.parse_pipe import parse_pipe
 from tinybird.tb.client import AuthNoTokenException, TinyB
 from tinybird.tb.modules.datafile.format_datasource import format_datasource
 from tinybird.tb.modules.datafile.format_pipe import format_pipe
@@ -10,6 +13,52 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 PULL_EXISTING_FILE_MAX_DEPTH = 5
 
+DEMOTABLE_NODE_TYPES = {
+    PipeNodeTypes.DATA_SINK: ("sink_pipes", "Sink"),
+    PipeNodeTypes.COPY: ("copy_pipes", "Copy"),
+}
+
+
+def _parse_created_at(value: Any) -> datetime:
+    try:
+        return datetime.fromisoformat(str(value))
+    except ValueError:
+        return datetime.min
+
+
+def pipes_over_node_type_limit(pipes: list[dict[str, Any]], node_type: str, limit: Optional[int]) -> set[str]:
+    if limit is None:
+        return set()
+
+    matching_pipes = [p for p in pipes if p.get("type") == node_type]
+    if len(matching_pipes) <= limit:
+        return set()
+
+    matching_pipes.sort(key=lambda p: _parse_created_at(p.get("created_at")))
+    return {p["name"] for p in matching_pipes[limit:]}
+
+
+def pipes_over_plan_limits(pipes: list[dict[str, Any]], limits: dict[str, Any]) -> dict[str, str]:
+    demoted: dict[str, str] = {}
+    for node_type, (limits_key, _label) in DEMOTABLE_NODE_TYPES.items():
+        limit = limits.get(limits_key, {}).get("max")
+        for name in pipes_over_node_type_limit(pipes, node_type, limit):
+            demoted[name] = node_type
+    return demoted
+
+
+def demote_pipe_content(name: str, content: str, node_type: str) -> str:
+    datafile = parse_pipe(name, content=content, replace_includes=False).datafile
+    demoted = False
+    for node in datafile.nodes:
+        if (node.get("type") or "").lower() == node_type:
+            node.pop("type", None)
+            demoted = True
+
+    if not demoted:
+        return content
+    return format_pipe(name, datafile=datafile)
+
 
 def folder_pull(
     client: TinyB,
@@ -61,6 +110,7 @@ def folder_pull(
         get_resource_function: str,
         progress_bar: bool = False,
         fmt: bool = False,
+        demoted_pipe_types: Optional[dict[str, str]] = None,
     ) -> list[Path]:
         def write_resource(k: dict[str, Any]) -> Optional[Path]:
             name = f"{k['name']}.{extension}"
@@ -73,8 +123,19 @@ def folder_pull(
                 else:
                     resource = getattr(client, get_resource_function)(k["name"])
                 resource_to_write = resource
-
-                if fmt:
+                resource_type = k.get("type")
+
+                demoted_node_type = demoted_pipe_types.get(k["name"]) if demoted_pipe_types else None
+                if demoted_node_type:
+                    resource_to_write = demote_pipe_content(name, resource, demoted_node_type)
+                    resource_type = None
+                    label = DEMOTABLE_NODE_TYPES[demoted_node_type][1]
+                    warnings.append(
+                        f"{label} pipe '{k['name']}' exceeds the {demoted_node_type} pipe limit of your current "
+                        f"plan and was pulled as a plain pipe without its {demoted_node_type} configuration. "
+                        "Upgrade your plan and reconfigure it to restore it."
+                    )
+                elif fmt:
                     if extension == "datasource":
                         resource_to_write = format_datasource(name, content=resource)
                     elif extension == "pipe":
@@ -85,7 +146,7 @@ def folder_pull(
                     dest_folder = Path(folder) / "vendor" / k["name"].split(".", 1)[0]
                     name = f"{k['name'].split('.', 1)[1]}.{extension}"
 
-                file_folder = get_file_folder(extension, k.get("type"))
+                file_folder = get_file_folder(extension, resource_type)
                 f = Path(dest_folder) / file_folder if file_folder is not None else Path(dest_folder)
 
                 existing_file = find_existing_resource_file(Path(dest_folder), name)
@@ -128,6 +189,14 @@ def folder_pull(
         pipes = client.pipes()
         connections = client.connections()
 
+        demoted_pipe_types: dict[str, str] = {}
+        if not only_vendored:
+            try:
+                limits = client.workspace_info().get("limits", {})
+            except Exception:
+                limits = {}
+            demoted_pipe_types = pipes_over_plan_limits(pipes, limits)
+
         if only_vendored:
             written_files = write_files(
                 resources=[ds for ds in datasources if "shared_from" in ds],
@@ -154,6 +223,7 @@ def folder_pull(
                     get_resource_function="pipe_file",
                     progress_bar=progress_bar,
                     fmt=fmt,
+                    demoted_pipe_types=demoted_pipe_types,
                 )
             )
             written_files.extend(

tinybird/tb/modules/datasource.py

@@ -677,7 +677,7 @@ def datasource_start(ctx: Context, datasource_name: str) -> None:
 
 @datasource.command(name="delete")
 @click.argument("datasource_name")
-@click.option("--sql-condition", default=None, help="SQL WHERE condition to remove rows", hidden=True, required=True)
+@click.option("--sql-condition", help="SQL WHERE condition to remove rows", hidden=True, required=True)
 @click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
 @click.option(
     "--wait/--no-wait",

tinybird/tb/modules/deployment_common.py

@@ -335,13 +335,12 @@ def discard_deployment(host: Optional[str], headers: dict, wait: bool, request_f
         click.echo(FeedbackManager.error(message="Only one deployment found. Cannot discard the only deployment."))
         return
 
-    current_deployment, deployment_to_discard = deployments[0], deployments[1]
+    live_deployment = next((deployment for deployment in deployments if deployment.get("live")), deployments[0])
+    deployment_to_discard = next(deployment for deployment in deployments if deployment is not live_deployment)
 
-    # NOTE(eclbg): we never get here. We wrote this code when we though we'd enable promoting back and forth between
-    # staging and live, but the current CLI commands don't allow getting in that state
-    if current_deployment.get("status") != "data_ready":
+    if live_deployment.get("status") != "data_ready":
         click.echo(FeedbackManager.error(message="Previous deployment is not ready"))
-        deploy_errors = current_deployment.get("errors", [])
+        deploy_errors = live_deployment.get("errors", [])
         for deploy_error in deploy_errors:
             click.echo(FeedbackManager.error(message=f"* {deploy_error}"))
         return
@@ -359,7 +358,6 @@ def discard_deployment(host: Optional[str], headers: dict, wait: bool, request_f
         click.echo(FeedbackManager.error(message=result.get("error")))
         sys_exit("deployment_error", result.get("error", "Unknown error"))
 
-    deployment_to_discard = deployments[1]
     click.echo(FeedbackManager.success(message="Discard process successfully started"))
 
     if wait:

tinybird/tb/modules/exceptions.py

@@ -92,20 +92,6 @@ class CLITokenException(CLIException):
         super().__init__(message, "token_error", **kw_telemetry_event_data)
 
 
-class CLIBuildException(CLIException):
-    """Exceptions generated by the build commands"""
-
-    def __init__(self, message: str, **kw_telemetry_event_data: Any) -> None:
-        super().__init__(message, "build_error", **kw_telemetry_event_data)
-
-
-class CLIDeploymentException(CLIException):
-    """Exceptions generated by the deployment commands"""
-
-    def __init__(self, message: str, **kw_telemetry_event_data: Any) -> None:
-        super().__init__(message, "deployment_error", **kw_telemetry_event_data)
-
-
 class CLITestException(CLIException):
     """Exceptions generated by the test commands"""
 
@@ -127,13 +113,6 @@ class CLIMockException(CLIException):
         super().__init__(message, "mock_error", **kw_telemetry_event_data)
 
 
-class CLIAgentException(CLIException):
-    """Exceptions generated by the agent commands"""
-
-    def __init__(self, message: str, **kw_telemetry_event_data: Any) -> None:
-        super().__init__(message, "agent_error", **kw_telemetry_event_data)
-
-
 class CLILoginException(CLIException):
     """Exceptions generated by the login commands"""
 

tinybird/tb_cli_modules/common.py

@@ -1304,11 +1304,11 @@ class ConnectionReplacements:
         """Maps prompt parameters to API parameters."""
 
         api_params = {}
-        for key in params.keys():
+        for key, value in params.items():
             try:
-                api_params[ConnectionReplacements._PARAMS_REPLACEMENTS[service][key]] = params[key]
+                api_params[ConnectionReplacements._PARAMS_REPLACEMENTS[service][key]] = value
             except KeyError:
-                api_params[key] = params[key]
+                api_params[key] = value
 
         api_params["service"] = service
         return api_params

tinybird/tornado_template.py

@@ -200,7 +200,6 @@ import builtins
 import datetime
 import linecache
 import os.path
-import posixpath
 import re
 import threading
 from io import StringIO
@@ -465,28 +464,6 @@ class Loader(BaseLoader):
             return Template(f.read(), name=name, loader=self)
 
 
-class DictLoader(BaseLoader):
-    """A template loader that loads from a dictionary."""
-
-    def __init__(self, dict, **kwargs):
-        super().__init__(**kwargs)
-        self.dict = dict
-
-    def resolve_path(self, name, parent_path=None):
-        if (
-            parent_path
-            and not parent_path.startswith("<")
-            and not parent_path.startswith("/")
-            and not name.startswith("/")
-        ):
-            file_dir = posixpath.dirname(parent_path)
-            name = posixpath.normpath(posixpath.join(file_dir, name))
-        return name
-
-    def _create_template(self, name):
-        return Template(self.dict[name], name=name, loader=self)
-
-
 class _Node:
     def each_child(self):
         return ()
@@ -554,43 +531,6 @@ class _ExtendsBlock(_Node):
         self.name = name
 
 
-class _IncludeBlock(_Node):
-    def __init__(self, name, reader, line):
-        self.name = name
-        self.template_name = reader.name
-        self.line = line
-
-    def find_named_blocks(self, loader, named_blocks):
-        included = loader.load(self.name, self.template_name)
-        included.file.find_named_blocks(loader, named_blocks)
-
-    def generate(self, writer):
-        included = writer.loader.load(self.name, self.template_name)
-        with writer.include(included, self.line):
-            included.file.body.generate(writer)
-
-
-class _ApplyBlock(_Node):
-    def __init__(self, method, line, body=None):
-        self.method = method
-        self.line = line
-        self.body = body
-
-    def each_child(self):
-        return (self.body,)
-
-    def generate(self, writer):
-        method_name = "_tt_apply%d" % writer.apply_counter
-        writer.apply_counter += 1
-        writer.write_line("def %s():" % method_name, self.line)
-        with writer.indent():
-            writer.write_line("_tt_buffer = []", self.line)
-            writer.write_line("_tt_append = _tt_buffer.append", self.line)
-            self.body.generate(writer)
-            writer.write_line("return _tt_utf8('').join(_tt_buffer)", self.line)
-        writer.write_line("_tt_append(_tt_utf8(%s(%s())))" % (self.method, method_name), self.line)
-
-
 class _ControlBlock(_Node):
     def __init__(self, statement, line, body=None):
         if not disable_template_security_validation.get(False):
@@ -654,11 +594,6 @@ class _Expression(_Node):
         writer.write_line("_tt_append(_tt_tmp)", self.line)
 
 
-class _Module(_Expression):
-    def __init__(self, expression, line):
-        super().__init__("_tt_modules." + expression, line, raw=True)
-
-
 class _Text(_Node):
     def __init__(self, value, line, whitespace):
         self.value = value
@@ -1125,6 +1060,10 @@ def check_valid_code(code):
     Traceback (most recent call last):
     ...
     tinybird.tornado_template.SecurityException: Invalid expression type: Attribute(value=Name(id='len', ctx=Load()), attr='__self__', ctx=Load())
+    >>> check_valid_code('''String(__builtins__)''')
+    Traceback (most recent call last):
+    ...
+    tinybird.tornado_template.SecurityException: Invalid name: __builtins__
     """
     body = ast.parse(code).body
     for expr in body:
@@ -1148,9 +1087,11 @@ def check_valid_expr(expr):
             raise SecurityException(f"Invalid function: {ast.dump(expr)}")
         for subexpr in expr.args:
             check_valid_expr(subexpr)
-        # Keyword values are evaluated before the call, so they need the same checks as positional args
+        # Keyword values never went through the allowlist and customers call functions it doesn't list
+        # there, so they only get the checks that block reaching interpreter internals
         for keyword in expr.keywords:
-            check_valid_expr(keyword.value)
+            for node in ast.walk(keyword.value):
+                _check_node_is_safe(node)
     elif isinstance(expr, ast.Constant):
         if isinstance(expr.value, int):
             return
@@ -1162,7 +1103,9 @@ def check_valid_expr(expr):
             return
         raise SecurityException(f"Invalid Constant: {ast.dump(expr)}")
     elif isinstance(expr, ast.Name):
-        return
+        # Same rule as control blocks: dunder names such as __builtins__ only reach interpreter internals
+        if _is_dunder(expr.id):
+            raise SecurityException(f"Invalid name: {expr.id}")
     elif isinstance(expr, ast.UnaryOp):
         if not isinstance(expr.op, ast.USub) and not isinstance(expr.op, ast.Not):
             raise SecurityException(f"Invalid UnaryOp: {ast.dump(expr.op)}")
@@ -1182,6 +1125,8 @@ def check_valid_expr(expr):
                 check_valid_expr(expr.slice.lower)
             if expr.slice.upper is not None:
                 check_valid_expr(expr.slice.upper)
+            if expr.slice.step is not None:
+                check_valid_expr(expr.slice.step)
         elif isinstance(expr.slice, (ast.Constant, ast.Subscript)):
             check_valid_expr(expr.slice)
         else:
@@ -1199,6 +1144,9 @@ def check_valid_expr(expr):
             check_valid_expr(x)
     elif isinstance(expr, ast.FormattedValue):
         check_valid_expr(expr.value)
+        # f"{x:{expr}}" nests expressions in the format spec, which JoinedStr.values doesn't include
+        if expr.format_spec is not None:
+            check_valid_expr(expr.format_spec)
     elif isinstance(expr, ast.IfExp):
         check_valid_expr(expr.test)
         check_valid_expr(expr.body)
@@ -1208,7 +1156,9 @@ def check_valid_expr(expr):
 
 
 # Names that allow evaluating code, reflection or reading attributes by string. They are not part of
-# SAFE_BUILTINS either; rejecting them while parsing gives a clear error instead of a NameError.
+# SAFE_BUILTINS, so they can only resolve to template parameters (customers use `type`, `dir`, `property`
+# or `object` as parameter names). Reading them is allowed; calling them is rejected while parsing to give
+# a clear error instead of a NameError or "'str' object is not callable".
 UNSAFE_NAMES = frozenset(
     {
         "eval",
@@ -1235,11 +1185,6 @@ UNSAFE_NAMES = frozenset(
     }
 )
 
-# Common customer parameter names that collide with unsafe builtins (e.g. a `dir` sort-direction
-# parameter, or a `type` filter). Reads are allowed so `defined(dir)` keeps working; only calling the
-# builtin itself (`dir(...)`) is rejected.
-NAMES_ALLOWED_AS_PARAMETERS = frozenset({"type", "dir"})
-
 # Attributes that reach interpreter internals without a leading underscore (frames and code objects of
 # generators, coroutines and tracebacks) or that read arbitrary attributes through a format string
 # ("{0.__class__}".format(x)). Any attribute starting with "_" is rejected too. Same idea as jinja2.sandbox.
@@ -1298,29 +1243,28 @@ def _is_dunder(name: str) -> bool:
     return len(name) > 4 and name.startswith("__") and name.endswith("__")
 
 
+def _check_node_is_safe(node: ast.AST) -> None:
+    if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id in UNSAFE_NAMES:
+        raise SecurityException(f"Invalid function name: {node.func.id}")
+    if isinstance(node, ast.Attribute):
+        if node.attr.startswith("_") or node.attr in UNSAFE_ATTRIBUTE_NAMES:
+            raise SecurityException(f"Invalid attribute: {node.attr}")
+    elif isinstance(node, ast.Name):
+        if _is_dunder(node.id):
+            raise SecurityException(f"Invalid name: {node.id}")
+    elif isinstance(node, ast.ExceptHandler):
+        if node.name and _is_dunder(node.name):
+            raise SecurityException(f"Invalid name: {node.name}")
+    elif isinstance(node, (ast.Yield, ast.YieldFrom, ast.Await)):
+        raise SecurityException(f"Invalid expression type: {ast.dump(node)}")
+
+
 def _check_valid_statement_tree(tree: ast.Module, expected_statements: int, statement: str) -> None:
     statements = 0
     for node in ast.walk(tree):
         if isinstance(node, ast.stmt):
             statements += 1
-        # Customer pipes use `type`/`dir` as parameters; allow reads but forbid calls to the unsafe builtin.
-        elif (
-            isinstance(node, ast.Call)
-            and isinstance(node.func, ast.Name)
-            and node.func.id in NAMES_ALLOWED_AS_PARAMETERS
-        ):
-            raise SecurityException(f"Invalid function name: {node.func.id}")
-        elif isinstance(node, ast.Attribute):
-            if node.attr.startswith("_") or node.attr in UNSAFE_ATTRIBUTE_NAMES:
-                raise SecurityException(f"Invalid attribute: {node.attr}")
-        elif isinstance(node, ast.Name):
-            if _is_dunder(node.id) or (node.id in UNSAFE_NAMES and node.id not in NAMES_ALLOWED_AS_PARAMETERS):
-                raise SecurityException(f"Invalid name: {node.id}")
-        elif isinstance(node, ast.ExceptHandler):
-            if node.name and _is_dunder(node.name):
-                raise SecurityException(f"Invalid name: {node.name}")
-        elif isinstance(node, (ast.Yield, ast.YieldFrom, ast.Await)):
-            raise SecurityException(f"Invalid expression type: {ast.dump(node)}")
+        _check_node_is_safe(node)
     if statements != expected_statements:
         raise SecurityException(f"Invalid control block: {statement}")
 
@@ -1342,7 +1286,8 @@ def check_valid_control_statement(statement: str) -> None:
     >>> check_valid_control_statement("elif eval('1')")
     Traceback (most recent call last):
     ...
-    tinybird.tornado_template.SecurityException: Invalid name: eval
+    tinybird.tornado_template.SecurityException: Invalid function name: eval
+    >>> check_valid_control_statement("if defined(property) and object != 'x'")
     >>> check_valid_control_statement("for x in (i for i in [1]).gi_frame.f_builtins")
     Traceback (most recent call last):
     ...

{tinybird-4.6.22.dev0.dist-info → tinybird-4.6.24.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 4.6.22.dev0
+Version: 4.6.24.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -52,6 +52,16 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+4.6.22
+********
+
+- `Added` ``tb pull`` demotes sink and copy pipes that exceed the workspace's sink/copy pipe plan limit to plain pipes instead of failing, stripping their ``TYPE``/``EXPORT_*``/``TARGET_DATASOURCE``/``COPY_SCHEDULE`` settings and warning that the plan must be upgraded to restore them.
+
+4.6.21
+********
+
+- `Fixed` ``tb deployment discard`` now discards the deployment that is not live, instead of assuming the live deployment is listed first. Previously, if removing the old deployment failed after a promotion, the command refused to discard it.
+
 4.6.20
 ********
 

{tinybird-4.6.22.dev0.dist-info → tinybird-4.6.24.dev0.dist-info}/RECORD

@@ -5,11 +5,11 @@ tinybird/git_settings.py,sha256=ETrcrvnhi3I9W-cGOwILXTBBNVFm5lYbeDdgTFaxT48,3898
 tinybird/prompts.py,sha256=FNYpphDYlrXpm5AQM3yfQPbj4gd_TVnwoaoeswtx0OE,32699
 tinybird/service_datasources.py,sha256=kNTlnSV6dngANVVDhFQsY77mlD5jKExy_YRC4Cw5rsQ,57555
 tinybird/sql.py,sha256=V807-z7tlZWMowr2ezYhsxjFWm-rWejaQ1mlr0ZXtPU,47173
-tinybird/sql_template.py,sha256=mQy_b9iLg8Tfoabl3lhUMyc3_XgTVk5TAeynvbjS6aw,132009
-tinybird/sql_template_fmt.py,sha256=UZbCNmFNdSBTC1joEu-1C11vBbyWgG_sCbYJFqSKNwo,10528
+tinybird/sql_template.py,sha256=T9dcuB6vdYnw9BvpptIqo4MsWXbzn3O_uS6PoH0xmAM,133695
+tinybird/sql_template_fmt.py,sha256=cOhhkA9h_M8II-06Qx5pt3E7IXUgzvrzfWGMtWdtkUM,10525
 tinybird/sql_toolset.py,sha256=gB6ZkQww8pU-7Zu2H1l7GHttPHkgqGhYBt6MfuhduDc,29926
-tinybird/syncasync.py,sha256=VV2yn9TKc-aEr0hzQQWIwUcklWZG8AHv7fj6V5LnQSU,27718
-tinybird/tornado_template.py,sha256=uPfjB-Q558g9C0sDkeYdrYzkCODxWm2ihqsTmADNtnw,50708
+tinybird/syncasync.py,sha256=fFQ5GPPnUlTJ-J3VTw9AZCNXtp5IUiP9Q22sZXcixU4,26352
+tinybird/tornado_template.py,sha256=mTQMTywuqTfk9xX737mT5Fv_o-PNHJxpgTa4LF-edCE,49079
 tinybird/ch_utils/constants.py,sha256=8q5SH-0uBpYsBzmPpYsyhy-wcqvLgK1hmPr_NsKKoi8,4349
 tinybird/ch_utils/engine.py,sha256=XxBpukaupup8QEKWB7CPZu8c3r1NQuliA2ao85oWw8Y,37883
 tinybird/datafile/common.py,sha256=QZlhZhzRuVV1UAMQRzyogPjxA6YzgD1SGwrQdEJO6QE,112481
@@ -19,30 +19,30 @@ tinybird/datafile/parse_datasource.py,sha256=yd58HrUF4yNJXLn6OsvKGpZJpvrcjLGAeJG
 tinybird/datafile/parse_pipe.py,sha256=StMwF0Y1ny94hQxgPgcyx6-Jr46gCYBuPGpHF4bFiSQ,4956
 tinybird/iterating/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tinybird/iterating/data_branch_modes.py,sha256=5YuDa-gr8mKwmES8Xro6TRKbQtaOtIw4GbC606Qhu3o,184
-tinybird/tb/__cli__.py,sha256=SvGe_cLMTLnOZVvc16OOiDPBbFvqvZ09nw_SxCSXi9w,247
+tinybird/tb/__cli__.py,sha256=cSOtgXlDSfy7RTi_BRGXUDdv5zrVJUtpKIQ8hGf8tro,247
 tinybird/tb/check_pypi.py,sha256=Gp0HkHHDFMSDL6nxKlOY51z7z1Uv-2LRexNTZSHHGmM,552
 tinybird/tb/cli.py,sha256=IjiGfNIpxSxi1odK1kMj9s8lEhx3sAUgGA263XdmyR0,1119
-tinybird/tb/client.py,sha256=NGlqdw2FvBMU8F7r_sYYhwp2VqhrhTyEfh9jvuex9qM,59963
+tinybird/tb/client.py,sha256=gWlh7v_tB7n2VC-sNjVQqMByWIqaR9dLYRJe7pzpb-k,59866
 tinybird/tb/config.py,sha256=l6NtYvSPZDVOtt1W4G4ExXXDIf0hm6lvRnRoDMbdrsY,5429
 tinybird/tb/modules/branch.py,sha256=U50nj2kZllCkOHQfJWQ-YQ260pvlcssgStMSovmqMiU,10157
 tinybird/tb/modules/build.py,sha256=q7-aVM7q1tApiax_y6vYSCZq4cJ9ATSAm9D-elpNp7I,6543
 tinybird/tb/modules/build_common.py,sha256=Ks_fUbS376x6KaSihUl8MMvAk4pXYIMZauVL_SOpVEA,25533
 tinybird/tb/modules/cicd.py,sha256=xKa7XBQ1wgkYeG9lBLJQcwM7P18cY0c_QeF-KFO9_HI,8287
 tinybird/tb/modules/cli.py,sha256=6leKHypUjieVS9DbEXizFmUZq2lbJFlBDXYyvy_v8G4,44230
-tinybird/tb/modules/common.py,sha256=tfyYO7kzGPlfCoNwl-CPctTCSSJEc83jt2IhahZ3qiw,91041
+tinybird/tb/modules/common.py,sha256=pRWBwL2O8SgdGTdd8kcAPTSG9mEFXgzZGXyQZMi72HY,91037
 tinybird/tb/modules/config.py,sha256=U13HYGCILZjtWeP-LG1g63GDGR0CbqT0QE_cBZeNeOs,11157
 tinybird/tb/modules/connection.py,sha256=HwHn0YgwqcKEYCLr2OqDzJzAUFd65Wv8MZiSUw_TLII,18452
 tinybird/tb/modules/connection_dynamodb.py,sha256=vaB28d6t8nOd6kcUQGMUkl3IZin5DYSBN2TBPm2968Q,9490
 tinybird/tb/modules/connection_kafka.py,sha256=AwGRrHqkV8mb5xSsFEEZ1HuamOPRukFRr5HsUJLfnbE,33502
 tinybird/tb/modules/connection_s3.py,sha256=PeXZ4NHKhvAz0pUpXLPc7V1A8s9fllbyZJdyUFIC5Pw,16393
 tinybird/tb/modules/copy.py,sha256=Rh4R38KLsYnNlSLIFLExqjP3J6Oy4SRJktWKOOmC8fA,7440
-tinybird/tb/modules/create.py,sha256=kPeDZb60cfEyA8nIv_8JiMRAaZp4_03YsJfx4gQAgjs,22869
-tinybird/tb/modules/datasource.py,sha256=6QGnjy5RDggyXq3GJ3ejeNo0Rx8itsn6zGvLKmz-_ek,79999
+tinybird/tb/modules/create.py,sha256=K4rzbQEtMyu2wqoTEHzYoBSGkMzxOfPSjgxKn0EY8io,22835
+tinybird/tb/modules/datasource.py,sha256=bxmZGaoAyuk6ev9SfqJPn8kOjyk5g1RQwwaStoz5dT0,79985
 tinybird/tb/modules/deployment.py,sha256=vB4QSMsN4-DH-yDfSESQ5KKXgzC1t1fdCsqJ9lh8ej0,20956
-tinybird/tb/modules/deployment_common.py,sha256=vU3MsZu4FGvgx9P2i1ulPElhc6LlwA1c_KiNb1nsEBk,36137
+tinybird/tb/modules/deployment_common.py,sha256=kqBGO1Psdb9eP8fxKuO7z6_rWOc4DFvO0Ra35NFP-20,36030
 tinybird/tb/modules/deprecations.py,sha256=XXekrzPO9v12F1ToQDUGzLYJJ2wrEUlGKOkLCSdfHiM,4935
 tinybird/tb/modules/endpoint.py,sha256=yRTh6rQFJ98LA0hSC8rPD3EcDaJj41gk9oCsgcZPu_c,12112
-tinybird/tb/modules/exceptions.py,sha256=_1BHy0OixEkeF0fOCu1_1Pj8pJS4BNfUyucqCViJGTw,5958
+tinybird/tb/modules/exceptions.py,sha256=vCEaVa13BhKRSBeICVYwazdaL8VSPIDD55oTs68h-e0,5196
 tinybird/tb/modules/feedback_manager.py,sha256=B6iGI15HN2IQiJJ5J5FrwrAWsfBwPXsY6MYlTopMEgA,78236
 tinybird/tb/modules/fmt.py,sha256=ejQC1-2mK42saA2R9DA-CENYgu06SUoiCoX4bCtRXT8,3734
 tinybird/tb/modules/info.py,sha256=oO2DpPT-hpQ2Gf_TBAJHQ_xIfc1WCBpiGWhiIeM_K4w,13042
@@ -91,18 +91,18 @@ tinybird/tb/modules/datafile/format_datasource.py,sha256=_w5Y4tgl3Sejf0T8146F_ow
 tinybird/tb/modules/datafile/format_pipe.py,sha256=Y4XBrJtOlQyDiRTHAfu7jo66Ud52TayNtJ55eevOb8g,7938
 tinybird/tb/modules/datafile/pipe_checker.py,sha256=jpHfaql5sjYPKb9jSYMwV0pWSVncTZIokijKZAgWweY,24929
 tinybird/tb/modules/datafile/playground.py,sha256=LeQj8udkolUZIrSGlwiB0WKk-x4eAyWt15HusV4hxRg,41035
-tinybird/tb/modules/datafile/pull.py,sha256=MBLT9qO51Bz2BQwj2h1jTwlOVokFKZyfbxehRQOcdeQ,6441
+tinybird/tb/modules/datafile/pull.py,sha256=OwYsa11uT7s1ALJL6EuF_gOppHR9O8Fr7kh4azVbFsM,9262
 tinybird/tb/modules/tinyunit/tinyunit.py,sha256=JZhIgyVv-S1H_gu0Tuv1U7U4OzFfVM_r7twzPM_JKeU,11047
 tinybird/tb_cli_modules/cicd.py,sha256=T5GBMRmxLeblNKV9XICGNA37mJVtUHx2sex463-QLsk,13845
-tinybird/tb_cli_modules/common.py,sha256=Tl_zKec6SNVUjZ2uaLEPe03k8CPoJlcU-1sjx0Vl074,76019
+tinybird/tb_cli_modules/common.py,sha256=7PCdcyK1pEyDwTUjNk7-DmSaitTIqAkVx_XYC-nIqLA,76015
 tinybird/tb_cli_modules/config.py,sha256=xF4L95Flpfg-TzGkfgpYmiiDbDjxoSWJLfYt1dCdFNQ,11164
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=W098H6jmS4kpE7hN3tadaREBTf7oMocel-lkKWN0pU8,10466
 tinybird/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tinybird/utils/bools.py,sha256=Rwuh_R6j62JVkcxKskLQao7DE2Jj6TQzsbgvtCsLHP4,587
-tinybird-4.6.22.dev0.dist-info/METADATA,sha256=vsK-j0g0Ofjw4mE6nLG5-11nDMBJikfbHVsHq9LRAlk,18280
-tinybird-4.6.22.dev0.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
-tinybird-4.6.22.dev0.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-4.6.22.dev0.dist-info/top_level.txt,sha256=ZIQJTPCzMqnfDzM_hEGZrJqDSEcKnIK_49T86DGWpyQ,78
-tinybird-4.6.22.dev0.dist-info/RECORD,,
+tinybird-4.6.24.dev0.dist-info/METADATA,sha256=kXHsAaCfmemek9Dj_XvDXDfgPZdb5Vy1fmf6qUcaBUY,18845
+tinybird-4.6.24.dev0.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
+tinybird-4.6.24.dev0.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-4.6.24.dev0.dist-info/top_level.txt,sha256=ZIQJTPCzMqnfDzM_hEGZrJqDSEcKnIK_49T86DGWpyQ,78
+tinybird-4.6.24.dev0.dist-info/RECORD,,