tinybird 4.6.21.dev0__tar.gz → 4.6.22.dev0__tar.gz

{tinybird-4.6.21.dev0 → tinybird-4.6.22.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 4.6.21.dev0
+Version: 4.6.22.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -52,6 +52,11 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+4.6.20
+********
+
+- `Added` Copy pipes accept ``ON_DEMAND_COMPUTE true`` so scheduled runs use on-demand compute. Manual ``tb copy run`` still requires ``--on-demand-compute``, and asks for confirmation when the local datafile does not enable it (skip it with ``--yes``).
+
 4.6.19
 ********
 

{tinybird-4.6.21.dev0 → tinybird-4.6.22.dev0}/pyproject.toml

@@ -150,6 +150,8 @@ doc = [
 ]
 lint = [
     "botocore-stubs==1.36.23",
+    "flake8~=7.3",
+    "flake8-async~=26.8",
     "mypy==2.3.1",
     "ruff==0.16.8",
     "types-Markdown",
@@ -184,6 +186,12 @@ test = [
     "rangehttpserver==1.4.0",
     "yappi==1.7.6",
 ]
+# Synced automatically by plain `uv run`/`uv sync`, no `--group` needed.
+dev = [
+    { include-group = "test" },
+    { include-group = "deploy" },
+    { include-group = "lint" },
+]
 
 [project.scripts]
 tinybird_server = "tinybird.app:run"
@@ -193,7 +201,10 @@ tb = "tinybird.tb.cli:cli"
 colibri = "tinybird.colibri.cli:cli"
 
 [tool.uv.workspace]
-members = ["data_connector"]
+members = [
+    "chbackups",
+    "data_connector",
+]
 
 [tool.setuptools]
 include-package-data = true

{tinybird-4.6.21.dev0 → tinybird-4.6.22.dev0}/tinybird/client.py

@@ -691,6 +691,7 @@ class TinyB:
         target_datasource: Optional[str] = None,
         schedule_cron: Optional[str] = None,
         mode: Optional[str] = None,
+        on_demand_compute: Optional[bool] = None,
     ):
         data = {"schedule_cron": schedule_cron}
 
@@ -700,6 +701,9 @@ class TinyB:
         if mode:
             data["mode"] = mode
 
+        if on_demand_compute is not None:
+            data["on_demand_compute"] = "true" if on_demand_compute else "false"
+
         return await self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/copy", method="PUT", data=data)
 
     async def pipe_remove_copy(self, pipe_name_or_id: str, node_id: str):

{tinybird-4.6.21.dev0 → tinybird-4.6.22.dev0}/tinybird/datafile/common.py

@@ -28,6 +28,7 @@ from tinybird.ch_utils.engine import VALID_ENGINE_PARAMS, EngineParam
 from tinybird.datafile.exceptions import IncludeFileNotFoundException, ParseException, ValidationException
 from tinybird.tb.modules.exceptions import CLIPipeException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
+from tinybird.utils.bools import parse_optional_bool
 
 # Code from sql.py has been duplicated so I can change it without breaking absolutely everything in the app
 # I'll try not to make logic changes, just error reporting changes
@@ -179,9 +180,10 @@ class CopyParameters(Parameters):
     COPY_SCHEDULE = "copy_schedule"
     COPY_MODE = "copy_mode"
     COPY_MODE_ALIAS = "mode"  # we need this because bot MODE and COPY_MODE go to `mode` variable inside the node
+    ON_DEMAND_COMPUTE = "on_demand_compute"
     MANDATORY_ATTRIBUTES = PipeParameters.MANDATORY_ATTRIBUTES.union({TARGET_DATASOURCE})
     ACCEPTED_ATTRIBUTES = PipeParameters.ACCEPTED_ATTRIBUTES.union(MANDATORY_ATTRIBUTES).union(
-        {COPY_SCHEDULE, COPY_MODE_ALIAS}
+        {COPY_SCHEDULE, COPY_MODE_ALIAS, ON_DEMAND_COMPUTE}
     )
 
 
@@ -416,6 +418,11 @@ class Datafile:
                 )
             if not croniter.is_valid(copy_schedule):
                 raise DatafileValidationError("COPY node schedule must be @on-demand or a valid cron expression.")
+        if CopyParameters.ON_DEMAND_COMPUTE in node:
+            try:
+                node[CopyParameters.ON_DEMAND_COMPUTE] = parse_optional_bool(node[CopyParameters.ON_DEMAND_COMPUTE])
+            except ValueError:
+                raise DatafileValidationError("ON_DEMAND_COMPUTE must be true or false")
         for key in node.keys():
             if key not in CopyParameters.valid_params():
                 raise DatafileValidationError(
@@ -2097,6 +2104,7 @@ def parse(
             "copy_schedule": assign_node_var(CopyParameters.COPY_SCHEDULE),
             "copy_mode": assign_node_var("mode"),
             "mode": assign_node_var("mode"),
+            "on_demand_compute": assign_node_var(CopyParameters.ON_DEMAND_COMPUTE),
             "filter": assign_node_var("filter"),
             "token": add_token,
             "include": include,

{tinybird-4.6.21.dev0 → tinybird-4.6.22.dev0}/tinybird/datatypes.py

@@ -120,52 +120,74 @@ def datetime_test(x: str) -> bool:
     return any(p.match(x) for p in datetime_patterns)
 
 
+def bounded_integer(x: str, pattern: re.Pattern[str]) -> Optional[int]:
+    if pattern.match(x) is None:
+        return None
+    digits = x.lstrip("+-").lstrip("0")
+    # 256-bit unsigned integers have at most 78 decimal digits.
+    if len(digits) > 78:
+        return None
+    return int(x) if len(x) <= 78 else int(digits or "0") * (-1 if x.startswith("-") else 1)
+
+
 def int_8_test(x: str) -> bool:
-    return intx_re.match(x) is not None and -int_8_max <= int(x) < int_8_max
+    value = bounded_integer(x, intx_re)
+    return value is not None and -int_8_max <= value < int_8_max
 
 
 def int16_test(x: str) -> bool:
-    return intx_re.match(x) is not None and -int16_max <= int(x) < int16_max
+    value = bounded_integer(x, intx_re)
+    return value is not None and -int16_max <= value < int16_max
 
 
 def int32_test(x: str) -> bool:
-    return intx_re.match(x) is not None and -int32_max <= int(x) < int32_max
+    value = bounded_integer(x, intx_re)
+    return value is not None and -int32_max <= value < int32_max
 
 
 def int64_test(x: str) -> bool:
-    return intx_re.match(x) is not None and -int64_max <= int(x) < int64_max
+    value = bounded_integer(x, intx_re)
+    return value is not None and -int64_max <= value < int64_max
 
 
 def int128_test(x: str) -> bool:
-    return intx_re.match(x) is not None and -int128_max <= int(x) < int128_max
+    value = bounded_integer(x, intx_re)
+    return value is not None and -int128_max <= value < int128_max
 
 
 def int256_test(x: str) -> bool:
-    return intx_re.match(x) is not None and -int256_max <= int(x) < int256_max
+    value = bounded_integer(x, intx_re)
+    return value is not None and -int256_max <= value < int256_max
 
 
 def uint_8_test(x: str) -> bool:
-    return uintx_re.match(x) is not None and 0 <= int(x) < uint_8_max
+    value = bounded_integer(x, uintx_re)
+    return value is not None and 0 <= value < uint_8_max
 
 
 def uint16_test(x: str) -> bool:
-    return uintx_re.match(x) is not None and 0 <= int(x) < uint16_max
+    value = bounded_integer(x, uintx_re)
+    return value is not None and 0 <= value < uint16_max
 
 
 def uint32_test(x: str) -> bool:
-    return uintx_re.match(x) is not None and 0 <= int(x) < uint32_max
+    value = bounded_integer(x, uintx_re)
+    return value is not None and 0 <= value < uint32_max
 
 
 def uint64_test(x: str) -> bool:
-    return uintx_re.match(x) is not None and 0 <= int(x) < uint64_max
+    value = bounded_integer(x, uintx_re)
+    return value is not None and 0 <= value < uint64_max
 
 
 def uint128_test(x: str) -> bool:
-    return intx_re.match(x) is not None and 0 <= int(x) < uint128_max
+    value = bounded_integer(x, intx_re)
+    return value is not None and 0 <= value < uint128_max
 
 
 def uint256_test(x: str) -> bool:
-    return intx_re.match(x) is not None and 0 <= int(x) < uint256_max
+    value = bounded_integer(x, intx_re)
+    return value is not None and 0 <= value < uint256_max
 
 
 def float_test(x: str) -> bool:

{tinybird-4.6.21.dev0 → tinybird-4.6.22.dev0}/tinybird/feedback_manager.py

@@ -572,6 +572,11 @@ class FeedbackManager:
         "You can read more about the pricing at https://www.tinybird.co/docs/classic/work-with-data/process-and-copy/materialized-views#compute-compute-separation-for-populates\n\n"
         "Do you want to proceed with on-demand compute?"
     )
+    warning_confirm_on_demand_compute_differs_from_datafile = warning_message(
+        "The local datafile of pipe '{pipe}' does not set ON_DEMAND_COMPUTE true, but --on-demand-compute was passed.\n"
+        "On-demand compute uses dedicated resources that are billed separately from your regular usage.\n\n"
+        "Do you want to proceed?"
+    )
 
     warning_confirm_delete_token = prompt_message("Do you want to delete Token {token}?")
     warning_confirm_refresh_token = prompt_message("Do you want to refresh Token {token}?")

{tinybird-4.6.21.dev0 → tinybird-4.6.22.dev0}/tinybird/sql_template.py

@@ -18,7 +18,7 @@ from tinybird.context import (
 )
 
 from .datatypes import testers
-from .tornado_template import VALID_CUSTOM_FUNCTION_NAMES, SecurityException, Template
+from .tornado_template import SAFE_BUILTINS, VALID_CUSTOM_FUNCTION_NAMES, SecurityException, Template
 
 VALID_ACTIVATE_FEATURES = frozenset(["analyzer", "parallel_replicas", "optimize_aggregation_in_order"])
 
@@ -1375,7 +1375,7 @@ _namespace = {
     # they raise a pretty non understandable error but if someone
     # is using them they know what they are trying to do
     # read https://anee.me/escaping-python-jails-849c65cf306e on how to escape from python jails
-    "__buildins__": {},
+    "__builtins__": SAFE_BUILTINS,
     "__import__": {},
     "__debug__": {},
     "__doc__": {},
@@ -1385,6 +1385,9 @@ _namespace = {
     "close": None,
     "print": None,
     "input": None,
+    # Keep customer parameters named `type`/`dir` usable without restoring the unsafe builtins.
+    "type": None,
+    "dir": None,
 }
 
 
@@ -1478,6 +1481,8 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
             # the generated source code.
             "__name__": self.name.replace(".", "_"),
             "__loader__": ObjectDict(get_source=lambda name: self.code),
+            # Set again after kwargs so request parameters can't override it
+            "__builtins__": SAFE_BUILTINS,
             "max_threads": set_max_threads,
             "tb_secret": set_tb_secret,
             "tb_var": set_tb_secret,

{tinybird-4.6.21.dev0 → tinybird-4.6.22.dev0}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '4.6.21.dev0'
-__revision__ = '965c47ce'
+__version__ = '4.6.22.dev0'
+__revision__ = '3a75e0ef'

{tinybird-4.6.21.dev0 → tinybird-4.6.22.dev0}/tinybird/tb/client.py

@@ -743,6 +743,7 @@ class TinyB:
         target_datasource: Optional[str] = None,
         schedule_cron: Optional[str] = None,
         mode: Optional[str] = None,
+        on_demand_compute: Optional[bool] = None,
     ):
         data = {"schedule_cron": schedule_cron}
 
@@ -752,6 +753,9 @@ class TinyB:
         if mode:
             data["mode"] = mode
 
+        if on_demand_compute is not None:
+            data["on_demand_compute"] = "true" if on_demand_compute else "false"
+
         return self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/copy", method="PUT", data=data)
 
     def pipe_remove_copy(self, pipe_name_or_id: str, node_id: str):

{tinybird-4.6.21.dev0 → tinybird-4.6.22.dev0}/tinybird/tb/modules/copy.py

@@ -10,13 +10,29 @@ from typing import Optional, Tuple
 import click
 from click import Context
 
-from tinybird.datafile.common import get_name_version
+from tinybird.datafile.common import CopyParameters, PipeNodeTypes, get_name_version
 from tinybird.tb.client import AuthNoTokenException, TinyB
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import echo_safe_humanfriendly_tables_format_smart_table, wait_job
 from tinybird.tb.modules.exceptions import CLIPipeException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.job_common import echo_job_url
+from tinybird.tb.modules.project import Project
+from tinybird.utils.bools import parse_optional_bool
+
+
+def local_copy_node_uses_on_demand_compute(project: Project, pipe_name: str) -> Optional[bool]:
+    """Reads ON_DEMAND_COMPUTE from the local datafile. Returns None when there is no parseable copy pipe to compare"""
+    pipe_path = project.get_resource_path(pipe_name, "pipe")
+    if not pipe_path:
+        return None
+    datafile = project.get_pipe_datafile(str(project.path / pipe_path))
+    if datafile is None:
+        return None
+    copy_node = next((node for node in datafile.nodes if node.get("type", "").lower() == PipeNodeTypes.COPY), None)
+    if copy_node is None:
+        return None
+    return parse_optional_bool(copy_node.get(CopyParameters.ON_DEMAND_COMPUTE))
 
 
 @cli.group()
@@ -89,6 +105,7 @@ def copy_ls(ctx: Context, match: str, format_: str):
     default=False,
     help="Use on-demand compute instances for the copy job.",
 )
+@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
 @click.pass_context
 def copy_run(
     ctx: click.Context,
@@ -97,9 +114,24 @@ def copy_run(
     mode: str,
     param: Optional[Tuple[str]],
     on_demand_compute: bool,
+    yes: bool,
 ):
     """Run an on-demand copy pipe"""
 
+    project: Optional[Project] = ctx.ensure_object(dict).get("project")
+    if (
+        on_demand_compute
+        and not yes
+        and project
+        and local_copy_node_uses_on_demand_compute(project, pipe_name_or_id) is False
+        and not click.confirm(
+            FeedbackManager.warning_confirm_on_demand_compute_differs_from_datafile(pipe=pipe_name_or_id),
+            default=False,
+        )
+    ):
+        click.echo(FeedbackManager.info_cancelled_by_user())
+        return
+
     params = dict(key_value.split("=") for key_value in param) if param else {}
     click.echo(FeedbackManager.highlight(message=f"\n» Running on-demand copy '{pipe_name_or_id}'"))
     client: TinyB = ctx.ensure_object(dict)["client"]

{tinybird-4.6.21.dev0 → tinybird-4.6.22.dev0}/tinybird/tb/modules/datafile/build.py

@@ -881,6 +881,7 @@ def process_file(
                 "target_datasource": node.get("target_datasource", None),
                 "copy_schedule": node.get(CopyParameters.COPY_SCHEDULE, None),
                 "mode": node.get("mode", CopyModes.APPEND),
+                "on_demand_compute": node.get(CopyParameters.ON_DEMAND_COMPUTE, None),
             }
 
             is_export_node = ExportReplacements.is_export_node(node)

{tinybird-4.6.21.dev0 → tinybird-4.6.22.dev0}/tinybird/tb/modules/datafile/format_pipe.py

@@ -23,6 +23,7 @@ from tinybird.tb.modules.datafile.format_common import (
     format_tags,
     format_tokens,
 )
+from tinybird.utils.bools import parse_optional_bool
 
 
 def format_node_sql(
@@ -81,6 +82,9 @@ def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[str]:
             file_parts.append(
                 f"{CopyParameters.COPY_SCHEDULE.upper()} {ON_DEMAND if is_ondemand else node[CopyParameters.COPY_SCHEDULE]}"
             )
+        if parse_optional_bool(node.get(CopyParameters.ON_DEMAND_COMPUTE)):
+            file_parts.append(DATAFILE_NEW_LINE)
+            file_parts.append("ON_DEMAND_COMPUTE true")
         file_parts.append(DATAFILE_NEW_LINE)
 
     # Sink or Stream pipe

{tinybird-4.6.21.dev0 → tinybird-4.6.22.dev0}/tinybird/tb/modules/datafile/playground.py

@@ -922,6 +922,7 @@ def process_file(
                 "target_datasource": node.get("target_datasource", None),
                 "copy_schedule": node.get(CopyParameters.COPY_SCHEDULE, None),
                 "mode": node.get("mode", CopyModes.APPEND),
+                "on_demand_compute": node.get(CopyParameters.ON_DEMAND_COMPUTE, None),
             }
 
             is_export_node = ExportReplacements.is_export_node(node)

{tinybird-4.6.21.dev0 → tinybird-4.6.22.dev0}/tinybird/tb/modules/feedback_manager.py

@@ -749,6 +749,11 @@ STEP 3: ADD KEY TO SERVICE ACCOUNT
     )
     warning_confirm_delete_pipe = prompt_message('Do you want to remove the pipe "{pipe}"?')
     warning_confirm_copy_pipe = prompt_message('Do you want to run a copy job from the pipe "{pipe}"?')
+    warning_confirm_on_demand_compute_differs_from_datafile = warning_message(
+        "The local datafile of pipe '{pipe}' does not set ON_DEMAND_COMPUTE true, but --on-demand-compute was passed.\n"
+        "On-demand compute uses dedicated resources that are billed separately from your regular usage.\n\n"
+        "Do you want to proceed?"
+    )
     warning_confirm_sink_job = prompt_message('Do you want to run a sink job from the pipe "{pipe}"?')
     warning_confirm_clear_workspace = prompt_message(
         "Do you want to remove all pipes and Data Sources from this workspace?"

{tinybird-4.6.21.dev0 → tinybird-4.6.22.dev0}/tinybird/tb_cli_modules/pipe.py

@@ -16,7 +16,15 @@ from click import Context
 from tinybird import context
 from tinybird.client import AuthNoTokenException, DoesNotExistException, TinyB
 from tinybird.config import DEFAULT_API_HOST, FeatureFlags
-from tinybird.datafile_common import PipeNodeTypes, PipeTypes, folder_push, get_name_version, process_file, wait_job
+from tinybird.datafile_common import (
+    PipeNodeTypes,
+    PipeTypes,
+    folder_push,
+    get_name_version,
+    local_copy_node_uses_on_demand_compute,
+    process_file,
+    wait_job,
+)
 from tinybird.feedback_manager import FeedbackManager
 from tinybird.tb_cli_modules.branch import warn_if_in_live
 from tinybird.tb_cli_modules.cli import cli
@@ -749,30 +757,41 @@ async def pipe_copy_run(
 
     params = dict(key_value.split("=") for key_value in param) if param else {}
 
-    if yes or click.confirm(FeedbackManager.warning_confirm_copy_pipe(pipe=pipe_name_or_id)):
-        click.echo(FeedbackManager.info_copy_job_running(pipe=pipe_name_or_id))
-        client: TinyB = ctx.ensure_object(dict)["client"]
+    if not yes and not click.confirm(FeedbackManager.warning_confirm_copy_pipe(pipe=pipe_name_or_id)):
+        return
 
-        try:
-            response = await client.pipe_run_copy(pipe_name_or_id, params, mode, on_demand_compute=on_demand_compute)
+    if (
+        on_demand_compute
+        and not yes
+        and local_copy_node_uses_on_demand_compute(".", pipe_name_or_id) is False
+        and not click.confirm(
+            FeedbackManager.warning_confirm_on_demand_compute_differs_from_datafile(pipe=pipe_name_or_id),
+            default=False,
+        )
+    ):
+        return
 
-            job_id = response["job"]["job_id"]
-            job_url = response["job"]["job_url"]
-            target_datasource_id = response["tags"]["copy_target_datasource"]
-            target_datasource = await client.get_datasource(target_datasource_id)
-            target_datasource_name = target_datasource["name"]
-            click.echo(
-                FeedbackManager.success_copy_job_created(target_datasource=target_datasource_name, job_url=job_url)
-            )
+    click.echo(FeedbackManager.info_copy_job_running(pipe=pipe_name_or_id))
+    client: TinyB = ctx.ensure_object(dict)["client"]
 
-            if wait:
-                await wait_job(client, job_id, job_url, "** Copying data")
-                click.echo(FeedbackManager.success_data_copied_to_ds(target_datasource=target_datasource_name))
+    try:
+        response = await client.pipe_run_copy(pipe_name_or_id, params, mode, on_demand_compute=on_demand_compute)
 
-        except AuthNoTokenException:
-            raise
-        except Exception as e:
-            raise CLIPipeException(FeedbackManager.error_creating_copy_job(error=e))
+        job_id = response["job"]["job_id"]
+        job_url = response["job"]["job_url"]
+        target_datasource_id = response["tags"]["copy_target_datasource"]
+        target_datasource = await client.get_datasource(target_datasource_id)
+        target_datasource_name = target_datasource["name"]
+        click.echo(FeedbackManager.success_copy_job_created(target_datasource=target_datasource_name, job_url=job_url))
+
+        if wait:
+            await wait_job(client, job_id, job_url, "** Copying data")
+            click.echo(FeedbackManager.success_data_copied_to_ds(target_datasource=target_datasource_name))
+
+    except AuthNoTokenException:
+        raise
+    except Exception as e:
+        raise CLIPipeException(FeedbackManager.error_creating_copy_job(error=e))
 
 
 @pipe_copy.command(name="resume", short_help="Resume a paused copy pipe")