tinybird 4.6.12.dev0__py3-none-any.whl → 4.6.15.dev0__py3-none-any.whl

tinybird/datafile/common.py

@@ -401,12 +401,15 @@ class Datafile:
         if node.get("mode") and node["mode"] not in ["append", "replace"]:
             raise DatafileValidationError("COPY node mode must be append or replace")
         # copy schedule must be @on-demand or a cron-expression
-        if (
-            node.get("copy_schedule")
-            and node["copy_schedule"] != ON_DEMAND
-            and not croniter.is_valid(node["copy_schedule"])
-        ):
-            raise DatafileValidationError("COPY node schedule must be @on-demand or a valid cron expression.")
+        copy_schedule = node.get("copy_schedule")
+        if copy_schedule and copy_schedule != ON_DEMAND:
+            if isinstance(copy_schedule, str) and "tb_secret" in copy_schedule:
+                raise DatafileValidationError(
+                    "COPY_SCHEDULE does not support { tb_secret(...) }. "
+                    'Use a literal cron expression (for example "0 * * * *") or @on-demand.'
+                )
+            if not croniter.is_valid(copy_schedule):
+                raise DatafileValidationError("COPY node schedule must be @on-demand or a valid cron expression.")
         for key in node.keys():
             if key not in CopyParameters.valid_params():
                 raise DatafileValidationError(
@@ -503,10 +506,16 @@ class Datafile:
 
         # Validate schedule format (common for both Kafka and S3/GCS)
         export_schedule = node.get("export_schedule")
-        if export_schedule and export_schedule != ON_DEMAND and not croniter.is_valid(export_schedule):
-            raise DatafileValidationError(
-                f"Sink node {repr(node['name'])} has invalid export_schedule '{export_schedule}'. Must be @on-demand or a valid cron expression."
-            )
+        if export_schedule and export_schedule != ON_DEMAND:
+            if isinstance(export_schedule, str) and "tb_secret" in export_schedule:
+                raise DatafileValidationError(
+                    f"Sink node {repr(node['name'])}: EXPORT_SCHEDULE does not support {{ tb_secret(...) }}. "
+                    'Use a literal cron expression (for example "0 * * * *") or @on-demand.'
+                )
+            if not croniter.is_valid(export_schedule):
+                raise DatafileValidationError(
+                    f"Sink node {repr(node['name'])} has invalid export_schedule '{export_schedule}'. Must be @on-demand or a valid cron expression."
+                )
 
     def validate(self):
         if self.kind == DatafileKind.pipe:
@@ -2082,6 +2091,7 @@ def parse(
             "sql": sql("sql"),
             "version": version,
             "deployment_method": assign_var("deployment_method", allowed_values={"alter"}),
+            "export_service": export_service,  # Deprecated
             "export_connection_name": assign_var("export_connection_name"),
             "export_schedule": assign_var("export_schedule"),
             "export_bucket_uri": assign_var("export_bucket_uri"),

tinybird/datafile/parse_pipe.py

@@ -52,20 +52,35 @@ def parse_pipe(
             if "type" in node:
                 node["type"] = node["type"].lower()
             sql = node.get("sql", "")
-            if sql.strip()[0] == "%":
+            source_sql = sql.strip()
+            if not source_sql:
+                raise click.ClickException(
+                    FeedbackManager.error_parsing_node(node=node["name"], pipe=filename, error="Empty query")
+                )
+            if source_sql.startswith("%"):
                 secrets_list: Optional[List[str]] = None
                 if secrets:
                     secrets_list = list(secrets.keys())
                 # Setting test_mode=True to ignore errors on required parameters and
                 # secrets_in_test_mode=False to raise errors on missing secrets
                 sql, _, variable_warnings = render_sql_template(
-                    sql[1:],
+                    source_sql[1:],
                     name=node["name"],
                     secrets=secrets_list,
                     test_mode=True,
                     secrets_in_test_mode=ignore_secrets,
                 )
                 doc.warnings = variable_warnings
+                if not sql.strip():
+                    raise click.ClickException(
+                        FeedbackManager.error_parsing_node(
+                            node=node["name"],
+                            pipe=filename,
+                            error="Template renders to an empty query without parameters",
+                        )
+                    )
+            else:
+                sql = source_sql
             # it'll fail with a ModuleNotFoundError when the toolset is not available but it returns the parsed doc
             from tinybird.sql_toolset import format_sql as toolset_format_sql
 
@@ -79,13 +94,15 @@ def parse_pipe(
             )
         )
     except ValueError as e:
-        t, template_variables, _ = get_template_and_variables(sql, name=node["name"])
+        source_sql = node.get("sql", "")
+        source_sql_stripped = source_sql.strip()
+        t, template_variables, _ = get_template_and_variables(source_sql, name=node["name"])
 
-        if sql.strip()[0] != "%" and len(template_variables) > 0:
+        if not source_sql_stripped.startswith("%") and len(template_variables) > 0:
             raise click.ClickException(FeedbackManager.error_template_start(filename=filename))
         raise click.ClickException(
             FeedbackManager.error_parsing_file(
-                filename=filename, lineno="", error=f"{str(e)} + SQL(value error): {sql}"
+                filename=filename, lineno="", error=f"{str(e)} + SQL(value error): {source_sql}"
             )
         )
     except UnClosedIfError as e:

tinybird/feedback_manager.py

@@ -864,6 +864,9 @@ class FeedbackManager:
     success_total_rows = success_message("** Total rows in {datasource}: {total_rows}")
     success_appended_datasource = success_message("** Data appended to Data Source '{datasource}' successfully!")
     success_replaced_datasource = success_message("** Data replaced in Data Source '{datasource}' successfully!")
+    success_import_job_queued = success_message("** {operation} import job queued: {job_id}")
+    success_import_job_completed = success_message("** {operation} import completed: {job_id}")
+    info_import_job_status = info_message("Check status: tb job details {job_id}")
     success_auth = success_message(
         "** Auth successful! \n** Configuration written to .tinyb file, consider adding it to .gitignore"
     )

tinybird/sql_template.py

@@ -1396,7 +1396,7 @@ _namespace = {
 }
 
 
-reserved_vars = set(["_tt_tmp", "_tt_append", "isinstance", "str", "error", "custom_error", *list(vars(builtins))])
+reserved_vars = {"_tt_tmp", "_tt_append", "isinstance", "str", "error", "custom_error", *list(vars(builtins))}
 for p in DEFAULT_PARAM_NAMES:  # we handle these in an specific manner
     reserved_vars.discard(p)  # `format` is part of builtins
 # Allow 'id' to be used as a template parameter - https://gitlab.com/tinybird/analytics/-/issues/19119
@@ -2911,13 +2911,21 @@ def extract_variables_from_sql(sql: str, params: List[Dict[str, Any]]) -> Dict[s
     return defaults
 
 
-def render_template_with_secrets(name: str, content: str, secrets: Optional[Dict[str, str]] = None) -> str:
+def render_template_with_secrets(
+    name: str,
+    content: str,
+    secrets: Optional[Dict[str, str]] = None,
+    empty_secret_raises: bool = False,
+) -> str:
     """Renders a template with secrets, allowing for default values.
 
     Args:
         name: The name of the template
         content: The template content
         secrets: A dictionary mapping secret names to their values
+        empty_secret_raises: When True, empty secret values or empty defaults raise
+            instead of returning '""'. Used for sink metadata (Forward deploy) where
+            an empty bucket path must not be baked as s3://"".
 
     Returns:
         The rendered template
@@ -2959,6 +2967,26 @@ def render_template_with_secrets(name: str, content: str, secrets: Optional[Dict
         Traceback (most recent call last):
         ...
         tinybird.sql_template.SQLTemplateException: Template Syntax Error: Cannot access secret 'MISSING_SECRET'. Check the secret exists in the Workspace and the token has the required scope.
+
+        >>> render_template_with_secrets(
+        ...     "sink_bucket",
+        ...     "s3://{{ tb_secret('MISSING_SECRET', '') }}",
+        ...     secrets = {},
+        ...     empty_secret_raises=True,
+        ... )
+        Traceback (most recent call last):
+        ...
+        tinybird.sql_template.SQLTemplateException: Template Syntax Error: Secret 'MISSING_SECRET' resolves to an empty value and cannot be used in sink metadata.
+
+        >>> render_template_with_secrets(
+        ...     "sink_bucket",
+        ...     "s3://{{ tb_secret('EMPTY_SECRET', 'fallback') }}",
+        ...     secrets = {'EMPTY_SECRET': ''},
+        ...     empty_secret_raises=True,
+        ... )
+        Traceback (most recent call last):
+        ...
+        tinybird.sql_template.SQLTemplateException: Template Syntax Error: Secret 'EMPTY_SECRET' resolves to an empty value and cannot be used in sink metadata.
     """
     if not secrets:
         secrets = {}
@@ -2974,15 +3002,24 @@ def render_template_with_secrets(name: str, content: str, secrets: Optional[Dict
             The secret value or default
 
         Raises:
-            SQLTemplateException: If the secret is not found and no default is provided
+            SQLTemplateException: If the secret is not found and no default is provided,
+                or if empty_secret_raises is True and the resolved value is empty
         """
         if secret_name in secrets:
             value = secrets[secret_name]
             if isinstance(value, str) and len(value) == 0:
+                if empty_secret_raises:
+                    raise SQLTemplateException(
+                        f"Secret '{secret_name}' resolves to an empty value and cannot be used in sink metadata."
+                    )
                 return '""'
             return value
         elif default is not None:
             if isinstance(default, str) and len(default) == 0:
+                if empty_secret_raises:
+                    raise SQLTemplateException(
+                        f"Secret '{secret_name}' resolves to an empty value and cannot be used in sink metadata."
+                    )
                 return '""'
             return default
         else:

tinybird/sql_toolset.py

@@ -158,7 +158,20 @@ def has_unoptimized_join(sql: str, left_table: Optional[Union[Tuple[str, str], T
             raise UnoptimizedJoinException(sql)
 
 
-def format_where_for_mutation_command(where_clause: str, lightweight: bool = False) -> str:
+def _format_fragment_for_mutation_command(fragment: str) -> str:
+    """Normalize a SQL fragment the way CH stores it in the
+    `system.mutations.command` column: format the expression with CH's own
+    formatter, then escape it as it appears inside the serialized command
+    (string literals keep backslash-escaped quotes)."""
+    formatted = chquery.format(f"""SELECT {fragment}""").split("SELECT ")[1]
+    formatted = formatted.replace("\\", "\\\\").replace("'", "''")
+    quoted = chquery.format(f"SELECT '{formatted}'").split("SELECT ")[1]
+    return quoted[1:-1]
+
+
+def format_where_for_mutation_command(
+    where_clause: str, lightweight: bool = False, partition: Optional[str] = None
+) -> str:
     """
     >>> format_where_for_mutation_command("numnights = 99")
     'DELETE WHERE numnights = 99'
@@ -172,12 +185,18 @@ def format_where_for_mutation_command(where_clause: str, lightweight: bool = Fal
     "DELETE WHERE reservationid = \\\\'\\\\\\\\\\\\'foo\\\\'"
     >>> format_where_for_mutation_command("number < 3", lightweight=True)
     'UPDATE _row_exists = 0 WHERE number < 3'
+    >>> format_where_for_mutation_command("number < 3", lightweight=True, partition="201901")
+    'UPDATE _row_exists = 0 IN PARTITION 201901 WHERE number < 3'
+    >>> format_where_for_mutation_command("number < 3", lightweight=True, partition="'2019-01-01'")
+    "UPDATE _row_exists = 0 IN PARTITION \\\\'2019-01-01\\\\' WHERE number < 3"
     """
-    formatted_condition = chquery.format(f"""SELECT {where_clause}""").split("SELECT ")[1]
-    formatted_condition = formatted_condition.replace("\\", "\\\\").replace("'", "''")
-    quoted_condition = chquery.format(f"SELECT '{formatted_condition}'").split("SELECT ")[1]
-    prefix = "UPDATE _row_exists = 0 WHERE" if lightweight else "DELETE WHERE"
-    return f"{prefix} {quoted_condition[1:-1]}"
+    quoted_condition = _format_fragment_for_mutation_command(where_clause)
+    prefix = "UPDATE _row_exists = 0" if lightweight else "DELETE"
+    # Partition-scoped mutations store the partition expression in the
+    # command (`… IN PARTITION <expr> WHERE <cond>`), normalized by the same
+    # CH formatter, so it must be part of the match.
+    partition_clause = f" IN PARTITION {_format_fragment_for_mutation_command(partition)}" if partition else ""
+    return f"{prefix}{partition_clause} WHERE {quoted_condition}"
 
 
 # Functions that take table/dictionary names as string literal arguments.
@@ -624,7 +643,7 @@ def replace_tables(
 
         if current_replacements:
             # We need to transform the dictionary into something cacheable, so a sorted tuple of tuples it is
-            r = tuple(sorted([(k, v) for k, v in current_replacements.items()]))
+            r = tuple(sorted(current_replacements.items()))
             sql = replace_tables_chquery_cached(
                 sql,
                 r,

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '4.6.12.dev0'
-__revision__ = '7bdea3d'
+__version__ = '4.6.15.dev0'
+__revision__ = '8311ae9'

tinybird/tb/client.py

@@ -465,6 +465,7 @@ class TinyB:
         sql_condition: Optional[str] = None,
         format: str = "csv",
         replace_options: Optional[Set[str]] = None,
+        use_v1: bool = False,
     ):
         params = {"name": datasource_name, "mode": mode, "format": format, "debug": "blocks_block_log"}
 
@@ -476,6 +477,30 @@ class TinyB:
 
         with open(file, "rb") as content:
             file_content = content.read()
+        content_types = {
+            "csv": "text/csv",
+            "ndjson": "application/x-ndjson",
+            "parquet": "application/vnd.apache.parquet",
+        }
+        headers = {"Content-Type": content_types[format]}
+        if str(file).endswith(".gz"):
+            headers["Content-Encoding"] = "gzip"
+        if use_v1:
+            v1_params = {"format": format}
+            if sql_condition:
+                v1_params["replace_condition"] = sql_condition
+            if replace_options:
+                for option in list(replace_options):
+                    v1_params[option] = "true"
+            res = self._req(
+                f"/v1/datasources/{quote(datasource_name, safe='')}/{mode}?{urlencode(v1_params, safe='')}",
+                data=file_content,
+                headers=headers,
+                method="POST",
+            )
+            if "error" in res:
+                raise Exception(res["error"])
+            return res
         if format == "csv":
             files = {"csv": ("csv", file_content)}
         else:

tinybird/tb/modules/common.py

@@ -925,7 +925,8 @@ def push_data(
     replace_options=None,
     concurrency: int = 1,
     silent: bool = False,
-):
+    use_v1: bool = False,
+) -> Optional[list[str]]:
     if url and type(url) is tuple:
         url = url[0]
 
@@ -962,6 +963,8 @@ def push_data(
         datasource_name: str, url: str, mode: str, sql_condition: Optional[str], replace_options: Optional[Set[str]]
     ):
         parsed = urlparse(url)
+        if use_v1 and parsed.scheme in ("http", "https"):
+            raise CLIException("--experimental=use_v1 only supports local files.")
         # poor man's format detection
         _format = get_format_from_filename_or_url(url)
         if parsed.scheme in ("http", "https"):
@@ -982,8 +985,15 @@ def push_data(
                 sql_condition=sql_condition,
                 format=_format,
                 replace_options=replace_options,
+                use_v1=use_v1,
             )
 
+        if use_v1:
+            job_id = res.get("id") or res.get("import_id")
+            if not isinstance(job_id, str):
+                raise CLIException("The v1 import response did not include a job ID.")
+            return job_id
+
         datasource_name = res["datasource"]["name"]
         try:
             datasource = client.get_datasource(datasource_name)
@@ -1014,6 +1024,8 @@ def push_data(
     try:
         tasks = [process_url(datasource_name, url, mode, sql_condition, replace_options) for url in urls]
         output = gather_with_concurrency(concurrency, *tasks)
+        if use_v1:
+            return list(output)
         parser, total_rows, appended_rows = list(output)[-1]
     except AuthNoTokenException:
         raise
@@ -1031,6 +1043,8 @@ def push_data(
 
             click.echo(FeedbackManager.success_progress_blocks())
 
+    return None
+
 
 def sync_data(ctx, datasource_name: str, yes: bool):
     client: TinyB = ctx.obj["client"]

tinybird/tb/modules/config.py

@@ -85,7 +85,7 @@ class CLIConfig:
     def to_dict(self) -> Dict[str, Any]:
         """Helper to ease"""
         result: Dict[str, Any] = {}
-        result.update(dict((v.name, deepcopy(v.value)) for v in self._values.values()))
+        result.update({v.name: deepcopy(v.value) for v in self._values.values()})
         return result
 
     def __getitem__(self, key) -> Any:

tinybird/tb/modules/datafile/build.py

@@ -951,7 +951,7 @@ def process_file(
                 "filename": filename,
                 "name": name + version,
                 "nodes": nodes,
-                "deps": [x for x in set(deps)],
+                "deps": list(set(deps)),
                 "tokens": doc.tokens,
                 "description": description,
                 "warnings": doc.warnings,

tinybird/tb/modules/datafile/pipe_checker.py

@@ -162,12 +162,12 @@ class PipeChecker(unittest.TestCase):
 
         if self.ignore_order:
             current_data = (
-                sorted(normalize_array(current_data), key=itemgetter(*[k for k in current_data[0].keys()]))
+                sorted(normalize_array(current_data), key=itemgetter(*list(current_data[0].keys())))
                 if len(current_data) > 0
                 else current_data
             )
             checker_data = (
-                sorted(normalize_array(checker_data), key=itemgetter(*[k for k in checker_data[0].keys()]))
+                sorted(normalize_array(checker_data), key=itemgetter(*list(checker_data[0].keys())))
                 if len(checker_data) > 0
                 else checker_data
             )

tinybird/tb/modules/datafile/playground.py

@@ -992,7 +992,7 @@ def process_file(
                 "filename": filename,
                 "name": name + version,
                 "nodes": nodes,
-                "deps": [x for x in set(deps)],
+                "deps": list(set(deps)),
                 "tokens": doc.tokens,
                 "description": description,
                 "warnings": doc.warnings,

tinybird/tb/modules/datasource.py

@@ -35,6 +35,7 @@ from tinybird.tb.modules.common import (
     get_format_from_filename_or_url,
     normalize_datasource_name,
     push_data,
+    wait_job,
 )
 from tinybird.tb.modules.config import CLIConfig
 from tinybird.tb.modules.connection_dynamodb import connection_create_dynamodb, validate_dynamodb_table
@@ -58,7 +59,7 @@ from tinybird.tb.modules.create import (
     generate_gcs_connection_file_with_secrets,
 )
 from tinybird.tb.modules.datafile.fixture import persist_fixture
-from tinybird.tb.modules.exceptions import CLIDatasourceException
+from tinybird.tb.modules.exceptions import CLIDatasourceException, CLIException
 from tinybird.tb.modules.feedback_manager import FeedbackManager, get_cli_name
 from tinybird.tb.modules.llm import LLM
 from tinybird.tb.modules.llm_utils import extract_xml
@@ -66,6 +67,36 @@ from tinybird.tb.modules.project import Project
 from tinybird.tb.modules.secret import save_secret_to_env_file
 from tinybird.tb.modules.telemetry import add_telemetry_event
 
+EXPERIMENTAL_FEATURE_USE_V1 = "use_v1"
+
+
+def _echo_v1_import_jobs_queued(job_ids: list[str], operation: str) -> None:
+    for job_id in job_ids:
+        click.echo(FeedbackManager.success(message=f"✓ {operation} import job queued: {job_id}"))
+        click.echo(FeedbackManager.gray(message=f"Check status: tb job details {job_id}"))
+
+
+def _wait_for_v1_import_jobs(client: TinyB, job_ids: list[str], operation: str) -> None:
+    for job_id in job_ids:
+        try:
+            wait_job(client, job_id, f"/v0/jobs/{job_id}", f"{operation} import")
+        except CLIException:
+            if _echo_v1_import_job_details(client, job_id):
+                raise click.exceptions.Exit(1)
+            raise
+        click.echo(FeedbackManager.success(message=f"✓ {operation} import completed: {job_id}"))
+
+
+def _echo_v1_import_job_details(client: TinyB, job_id: str) -> bool:
+    try:
+        job = client.job(job_id)
+    except Exception:
+        return False
+    click.echo(FeedbackManager.info_job(job=job_id))
+    echo_safe_humanfriendly_tables_format_smart_table([job.values()], column_names=job.keys())
+    click.echo("\n")
+    return True
+
 
 def _dynamodb_key_schema_sort_key(key_schema: dict[str, str]) -> int:
     if key_schema.get("key_type") == "HASH":
@@ -213,7 +244,14 @@ def datasource_ls(ctx: Context, match: Optional[str], format_: str):
 @click.option("--url", type=str, help="URL to append data from")
 @click.option("--file", type=str, help="Local file to append data from")
 @click.option("--events", type=str, help="Events to append data from")
+@click.option(
+    "--experimental",
+    type=click.Choice([EXPERIMENTAL_FEATURE_USE_V1]),
+    multiple=True,
+    help="Enable an experimental feature. May be specified multiple times.",
+)
 @click.option("--concurrency", help="How many files to submit concurrently", default=1, hidden=True)
+@click.option("--wait", is_flag=True, default=False, help="Wait for a v1 import job to finish.")
 @click.pass_context
 def datasource_append(
     ctx: Context,
@@ -222,7 +260,9 @@ def datasource_append(
     url: str,
     file: str,
     events: str,
+    experimental: tuple[str, ...],
     concurrency: int,
+    wait: bool,
 ):
     """
     Appends data to an existing data source from URL, local file  or a connector
@@ -237,6 +277,9 @@ def datasource_append(
     env: str = ctx.ensure_object(dict)["env"]
     client: TinyB = ctx.obj["client"]
     project: Project = ctx.ensure_object(dict)["project"]
+    use_v1 = EXPERIMENTAL_FEATURE_USE_V1 in experimental
+    if wait and not use_v1:
+        raise CLIDatasourceException("--wait requires --experimental=use_v1.")
 
     # If data is passed as argument, we detect if it's a JSON object, a URL or a file
     if data:
@@ -346,6 +389,8 @@ def datasource_append(
             raise CLIDatasourceException(FeedbackManager.error(message="Invalid ingestion option"))
 
     if events:
+        if use_v1:
+            raise CLIDatasourceException("--experimental=use_v1 only supports local files.")
         click.echo(FeedbackManager.highlight(message=f"\n» Sending events to {datasource_name}"))
         events_params = {"name": datasource_name}
         request_from = getattr(client, "request_from", None)
@@ -382,13 +427,14 @@ def datasource_append(
     else:
         click.echo(FeedbackManager.highlight(message=f"\n» Appending data to {datasource_name}"))
         try:
-            push_data(
+            job_ids = push_data(
                 client,
                 datasource_name,
                 data,
                 mode="append",
                 concurrency=concurrency,
                 silent=True,
+                use_v1=use_v1,
             )
         except Exception as e:
             is_quarantined = "quarantine" in str(e)
@@ -398,7 +444,12 @@ def datasource_append(
                 return
             else:
                 raise e
-        click.echo(FeedbackManager.success(message="✓ Rows appended!"))
+        if use_v1:
+            _echo_v1_import_jobs_queued(job_ids or [], "Append")
+            if wait:
+                _wait_for_v1_import_jobs(client, job_ids or [], "Append")
+        else:
+            click.echo(FeedbackManager.success(message="✓ Rows appended!"))
 
 
 @datasource.command(name="replace")
@@ -406,6 +457,13 @@ def datasource_append(
 @click.argument("url", nargs=-1, required=True)
 @click.option("--sql-condition", default=None, help="SQL WHERE condition to replace data", hidden=True)
 @click.option("--skip-incompatible-partition-key", is_flag=True, default=False, hidden=True)
+@click.option("--wait", is_flag=True, default=False, help="Wait for a v1 import job to finish.")
+@click.option(
+    "--experimental",
+    type=click.Choice([EXPERIMENTAL_FEATURE_USE_V1]),
+    multiple=True,
+    help="Enable an experimental feature. May be specified multiple times.",
+)
 @click.pass_context
 def datasource_replace(
     ctx: Context,
@@ -413,6 +471,8 @@ def datasource_replace(
     url,
     sql_condition,
     skip_incompatible_partition_key,
+    experimental: tuple[str, ...],
+    wait: bool,
 ):
     """
     Replaces the data in a data source from a URL, local file or a connector
@@ -426,14 +486,22 @@ def datasource_replace(
     if skip_incompatible_partition_key:
         replace_options.add("skip_incompatible_partition_key")
     client: TinyB = ctx.obj["client"]
-    push_data(
+    use_v1 = EXPERIMENTAL_FEATURE_USE_V1 in experimental
+    if wait and not use_v1:
+        raise CLIDatasourceException("--wait requires --experimental=use_v1.")
+    job_ids = push_data(
         client,
         datasource_name,
         url,
         mode="replace",
         sql_condition=sql_condition,
         replace_options=replace_options,
+        use_v1=use_v1,
     )
+    if use_v1:
+        _echo_v1_import_jobs_queued(job_ids or [], "Replace")
+        if wait:
+            _wait_for_v1_import_jobs(client, job_ids or [], "Replace")
 
 
 @datasource.command(name="analyze")

tinybird/tb_cli_modules/common.py

@@ -963,7 +963,8 @@ async def push_data(
     sql_condition: Optional[str] = None,
     replace_options=None,
     concurrency: int = 1,
-):
+    use_v1: bool = False,
+) -> Optional[list[str]]:
     if url and type(url) is tuple:
         url = url[0]
     client: TinyB = ctx.obj["client"]
@@ -1000,6 +1001,8 @@ async def push_data(
         datasource_name: str, url: str, mode: str, sql_condition: Optional[str], replace_options: Optional[Set[str]]
     ):
         parsed = urlparse(url)
+        if use_v1 and parsed.scheme in ("http", "https"):
+            raise CLIException("--experimental=use_v1 only supports local files.")
         # poor man's format detection
         _format = get_format_from_filename_or_url(url)
         if parsed.scheme in ("http", "https"):
@@ -1020,8 +1023,15 @@ async def push_data(
                 sql_condition=sql_condition,
                 format=_format,
                 replace_options=replace_options,
+                use_v1=use_v1,
             )
 
+        if use_v1:
+            job_id = res.get("id") or res.get("import_id")
+            if not isinstance(job_id, str):
+                raise CLIException("The v1 import response did not include a job ID.")
+            return job_id
+
         datasource_name = res["datasource"]["name"]
         try:
             datasource = await client.get_datasource(datasource_name)
@@ -1052,6 +1062,8 @@ async def push_data(
     try:
         tasks = [process_url(datasource_name, url, mode, sql_condition, replace_options) for url in urls]
         output = await gather_with_concurrency(concurrency, *tasks)
+        if use_v1:
+            return list(output)
         parser, total_rows, appended_rows = list(output)[-1]
     except AuthNoTokenException:
         raise
@@ -1072,6 +1084,8 @@ async def push_data(
             click.echo(FeedbackManager.success_appended_datasource(datasource=datasource_name))
         click.echo(FeedbackManager.info_data_pushed(datasource=datasource_name))
 
+    return None
+
 
 async def sync_data(ctx, datasource_name: str, yes: bool):
     client: TinyB = ctx.obj["client"]

tinybird/tb_cli_modules/config.py

@@ -88,7 +88,7 @@ class CLIConfig:
     def to_dict(self) -> Dict[str, Any]:
         """Helper to ease"""
         result: Dict[str, Any] = self._parent.to_dict() if self._parent else {}
-        result.update(dict((v.name, deepcopy(v.value)) for v in self._values.values()))
+        result.update({v.name: deepcopy(v.value) for v in self._values.values()})
         return result
 
     def __getitem__(self, key) -> Any:

tinybird/tornado_template.py

@@ -916,10 +916,10 @@ def _parse(reader: _TemplateReader, template, in_block=None, in_loop=None):
 
         # Intermediate ("else", "elif", etc) blocks
         intermediate_blocks = {
-            "else": set(["if", "for", "while", "try"]),
-            "elif": set(["if"]),
-            "except": set(["try"]),
-            "finally": set(["try"]),
+            "else": {"if", "for", "while", "try"},
+            "elif": {"if"},
+            "except": {"try"},
+            "finally": {"try"},
         }
         allowed_parents = intermediate_blocks.get(operator)
 
@@ -997,7 +997,7 @@ def _parse(reader: _TemplateReader, template, in_block=None, in_loop=None):
 
         elif operator in ("break", "continue"):
             if not in_loop:
-                reader.raise_parse_error("%s outside %s block" % (operator, set(["for", "while"])))
+                reader.raise_parse_error("%s outside %s block" % (operator, {"for", "while"}))
             body.chunks.append(_Statement(contents, line))
             continue
 

{tinybird-4.6.12.dev0.dist-info → tinybird-4.6.15.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 4.6.12.dev0
+Version: 4.6.15.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -52,6 +52,18 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+4.6.13
+********
+
+- `Added` ``tb datasource append`` and ``tb datasource replace`` support the experimental ``--experimental=use_v1`` option for local CSV, NDJSON, and Parquet files. It submits the import through the v1 endpoint and prints the queued job ID; pass ``--wait`` to wait for completion.
+
+4.6.12
+********
+
+- `Added` Forward sink pipes support ``{{ tb_secret(...) }}`` in ``EXPORT_*`` metadata (for example ``EXPORT_BUCKET_URI``). Secrets are resolved at deploy time and baked into the sink settings; changing a secret requires a redeploy.
+- `Fixed` Forward deploy now raises an error when a secret used in sink metadata resolves to an empty value.
+- `Fixed` Forward deploy now rejects ``{{ tb_secret(...) }}`` in ``EXPORT_SCHEDULE`` and ``COPY_SCHEDULE``. Use a literal cron expression (for example ``0 * * * *``) or ``@on-demand``.
+
 4.6.11
 ********
 

{tinybird-4.6.12.dev0.dist-info → tinybird-4.6.15.dev0.dist-info}/RECORD

@@ -1,43 +1,43 @@
 tinybird/context.py,sha256=o4yvlXPkMLmdh-XJl3wpmqPAMeRRz5ScKzKlHHKn_I8,1201
 tinybird/datatypes.py,sha256=DRScTI1-lgYjbC7u_4qxOLFleWL-vief_eBg_9WU37U,11304
-tinybird/feedback_manager.py,sha256=ZnRsExCkEZKuslmSxn3m706-0UIZ_qOpdek9ar16KOI,68001
+tinybird/feedback_manager.py,sha256=hbO4o16rleDxzfkv04fTeJIjAxlpn477qTK0QCuleTA,68274
 tinybird/git_settings.py,sha256=mqWgeboOlOFsSo97qyv595UCR2R1QCAqT4GTawBNPBg,3935
 tinybird/prompts.py,sha256=FNYpphDYlrXpm5AQM3yfQPbj4gd_TVnwoaoeswtx0OE,32699
 tinybird/service_datasources.py,sha256=asBTXONMv7sf7LFBx1a4eTcZCHPUq-I3r030AtorRzY,53643
 tinybird/sql.py,sha256=L3tTmm62iTrf6WQdcV8mmwrCMitLFUvJ3OfDzyRTMGw,47385
-tinybird/sql_template.py,sha256=8A5Jfzej7OHXzbfFMGpbq-YUzSEOoDHy5u7uy0vLZo4,127947
+tinybird/sql_template.py,sha256=dhRiQH6o-ndPkrq73rAaSU6LEqmVShefgNEilwphM2A,129661
 tinybird/sql_template_fmt.py,sha256=Ma4qcs-2r8ZXQC4GUmrCqYz34DsnGF8k5lE2Jwnr314,10638
-tinybird/sql_toolset.py,sha256=TvWeH3DsB13SfJ97V_t_ZaFTJ1IAtsrxQ8E3FLL8_tE,27442
+tinybird/sql_toolset.py,sha256=qNKHPGKTR4MrMHKkj1qHqmlB1kPqnbT-SozyzUBeD6s,28498
 tinybird/syncasync.py,sha256=rIPmCvygWSFqfnlVqhZH4N9gVVTvD6DEPsfoxGizYrI,27776
-tinybird/tornado_template.py,sha256=1_0nYFk_xJh_TMHh6AKkJILvnNY6xYmaM-uJ3Ofv7e8,42085
+tinybird/tornado_template.py,sha256=8e0ipnBQI4-RO5sFokybDNwkd1CBxGqvnnyJ5kjs5XU,42060
 tinybird/ch_utils/constants.py,sha256=yTNizMzgYNBzUc2EV3moBfdrDIggOe9hiuAgWF7sv2c,4333
 tinybird/ch_utils/engine.py,sha256=OkR86FOOUPRpCsRxlR-kunykg-K4Daqo3uS06X1uvYQ,36887
-tinybird/datafile/common.py,sha256=ASO118MEypTFiPcL9Kgr9VSje9iMbMjrUiz0CvdRDPw,110449
+tinybird/datafile/common.py,sha256=Cuj6vhmZDQUXFOU2GelkOJqVpyakZ-7bSBdAtHIBMW0,111188
 tinybird/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
 tinybird/datafile/parse_connection.py,sha256=GxmGp_XnWbDZPDbh_PBxitlIMqZRYfDwxMBw-JQBp1g,1890
 tinybird/datafile/parse_datasource.py,sha256=yd58HrUF4yNJXLn6OsvKGpZJpvrcjLGAeJG1lgBe_zk,1891
-tinybird/datafile/parse_pipe.py,sha256=-9bbgVuiWRyDYydrLVflDBt8GstZotMy6dklsrc6MUY,3859
+tinybird/datafile/parse_pipe.py,sha256=FXtMUmAxqpgvIMziUNdAyCg5tUsSFdd83p8ZGZS3uws,4657
 tinybird/iterating/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tinybird/iterating/data_branch_modes.py,sha256=5YuDa-gr8mKwmES8Xro6TRKbQtaOtIw4GbC606Qhu3o,184
-tinybird/tb/__cli__.py,sha256=9p6_Q_WXJ9kGn9dcnhQ0iyQ0NJRGtsvjS_OZwZyBsYo,246
+tinybird/tb/__cli__.py,sha256=ioZh5NOhCrtp051ruK5RvBOJQkWu-AmQpiEJV5kL69Y,246
 tinybird/tb/check_pypi.py,sha256=Gp0HkHHDFMSDL6nxKlOY51z7z1Uv-2LRexNTZSHHGmM,552
 tinybird/tb/cli.py,sha256=IjiGfNIpxSxi1odK1kMj9s8lEhx3sAUgGA263XdmyR0,1119
-tinybird/tb/client.py,sha256=LQ470JkU9NQcmM_rjElD_aV2l8nlWHTKuqzC3Yx1IzM,57605
+tinybird/tb/client.py,sha256=_Izo7DrhaAtH_MgMQsL1c3mjQzTNYNk09oSoftsOlyo,58588
 tinybird/tb/config.py,sha256=l6NtYvSPZDVOtt1W4G4ExXXDIf0hm6lvRnRoDMbdrsY,5429
 tinybird/tb/modules/branch.py,sha256=U50nj2kZllCkOHQfJWQ-YQ260pvlcssgStMSovmqMiU,10157
 tinybird/tb/modules/build.py,sha256=q7-aVM7q1tApiax_y6vYSCZq4cJ9ATSAm9D-elpNp7I,6543
 tinybird/tb/modules/build_common.py,sha256=o04aeaoyGTnwhR0cEXAgQzc7SJya97YECoEihsi4SyU,24979
 tinybird/tb/modules/cicd.py,sha256=IO4qqsoLRXcubALb7vx_QnRpg3zIIxfaVO9bGomlESY,8267
 tinybird/tb/modules/cli.py,sha256=oerpcsZ3vLsLZdCjgtVa6F9D3OFj-cqWFJfX-M40N2c,44038
-tinybird/tb/modules/common.py,sha256=XWuEIpMo3-JkplFqPzgxQqJDKAZD7rujY2VOXerGPPQ,93355
-tinybird/tb/modules/config.py,sha256=kZsMBf_qrGXPvhx1GW46DXS62ClVIKFFPjDXEZyGeyE,11181
+tinybird/tb/modules/common.py,sha256=H74TVe8Bd7LX5NC2pUbkW1FP1vhxmTE65uJwkRG7r5w,93880
+tinybird/tb/modules/config.py,sha256=AHBnVIO25JuOsygaU0gh0Iiw5jUgt6lXX5yrets_Y28,11175
 tinybird/tb/modules/connection.py,sha256=HwHn0YgwqcKEYCLr2OqDzJzAUFd65Wv8MZiSUw_TLII,18452
 tinybird/tb/modules/connection_dynamodb.py,sha256=vaB28d6t8nOd6kcUQGMUkl3IZin5DYSBN2TBPm2968Q,9490
 tinybird/tb/modules/connection_kafka.py,sha256=Qc8uUNl5SvUcUGb_aG3AirYKw6673FoX1Sfdlzgz6Y8,33530
 tinybird/tb/modules/connection_s3.py,sha256=eR7lsrCYbVWZ8LI7F5MUBxm8_DAvuaVSKqUDq9CUUks,16499
 tinybird/tb/modules/copy.py,sha256=Apzjxiyinp6KmgamypPKEe3BAnpxG7MwYcSYkHiG8sA,6033
 tinybird/tb/modules/create.py,sha256=G3_wCUjNVzBiowo70NgwzmUUXXOxYO2LWjF9eEfcffc,22947
-tinybird/tb/modules/datasource.py,sha256=baxfUQd7vDyEDh65fyOa2X5Xwf4OS3ligyznbbpAo6Q,76671
+tinybird/tb/modules/datasource.py,sha256=UZ2wTy7GtoLL3uyPyt6Qti9yVakmhUNDv50NWfvCT4I,79393
 tinybird/tb/modules/deployment.py,sha256=TNDlvaYmk-zouvaAdMXlyFPbgbnMKQOt76xpZBggJI0,21184
 tinybird/tb/modules/deployment_common.py,sha256=fLPWNNs8ZwSkWHJjLeqQUqTbKqnnhW1y8yKffBdDCo0,31948
 tinybird/tb/modules/deprecations.py,sha256=XXekrzPO9v12F1ToQDUGzLYJJ2wrEUlGKOkLCSdfHiM,4935
@@ -79,7 +79,7 @@ tinybird/tb/modules/ts_project.py,sha256=XeDN-3gthT7GZYIFn0l-y5RJaofB_2T-I8gq6dL
 tinybird/tb/modules/watch.py,sha256=HTZwqlGWDLJPBxBh04J0tAZDP_6BhqLupHBDx4uZDYo,5218
 tinybird/tb/modules/workspace.py,sha256=7shWyyZD9IT32EeP0aEgsZGk0U1ZdT6hWnVfctRoIJk,8939
 tinybird/tb/modules/workspace_members.py,sha256=8oQLTczEh9cIdD3iF-N3SJuqLtKdF-_g-YzeVaqKGP0,9486
-tinybird/tb/modules/datafile/build.py,sha256=WtxPakSjnpQhs2SxyF8Ckm_BdOVOUMlRdhPjVFli2SE,40910
+tinybird/tb/modules/datafile/build.py,sha256=fkbh_JD9hcslF3lekkY9YUnTFYWqaGA6Y6stz7pNsm0,40903
 tinybird/tb/modules/datafile/build_common.py,sha256=2yNdxe49IMA9wNvl25NemY2Iaz8L66snjOdT64dm1is,4511
 tinybird/tb/modules/datafile/build_datasource.py,sha256=keAx2lchYoxEDXQfTBtAe94Oyr4zfw3kWMumBFNtYso,16914
 tinybird/tb/modules/datafile/build_pipe.py,sha256=jQYyIw4imqx25-5lN8IrXTecSmxXo2xgIm7gPuibq9c,11269
@@ -89,18 +89,18 @@ tinybird/tb/modules/datafile/format_common.py,sha256=pLB508Akui1G1-X2mNirv2WvjZ_
 tinybird/tb/modules/datafile/format_connection.py,sha256=ieD9Jb6In6PmZqIysZ1tD1JBwfI5tV1RG1trweMR9F4,5089
 tinybird/tb/modules/datafile/format_datasource.py,sha256=rqA0Mr_3x5vaXe5vXWs7Fe2XmgVWn4niSlpvLnZh4vg,5325
 tinybird/tb/modules/datafile/format_pipe.py,sha256=9UuUsl5DjV4tJIRr6Whuv-AtIM9Gx7R3396AbKjHaJ4,7724
-tinybird/tb/modules/datafile/pipe_checker.py,sha256=dxsCQoA6ruxg1fvF6sMLFowpjaqws8lUQcM7XyhgZPE,24609
-tinybird/tb/modules/datafile/playground.py,sha256=5AUFNi2Oh6VQdaoCbQ4JeZAXKD0xzCOEUkEk5IXOj00,40973
+tinybird/tb/modules/datafile/pipe_checker.py,sha256=0hg6x2kW4wekEfRZra3bS4mNuBPQC5ZPoCG8V5L_WOE,24595
+tinybird/tb/modules/datafile/playground.py,sha256=lri033He_-HgzrKkH6nV3wOJqIqY_y9Fz49hi61zOio,40966
 tinybird/tb/modules/datafile/pull.py,sha256=C-px80jgA6pwPaGi5mcF5HjqpgwC0grso-F9SDDnRT0,6447
 tinybird/tb/modules/tinyunit/tinyunit.py,sha256=ZhKSwk5uIRggxDAPaqYZrcMMI0hLN1qGNIf-eRoJJWc,11055
 tinybird/tb_cli_modules/cicd.py,sha256=i2Mw8AbmEVNBcEPYdio7liy3PGqh1ezVFZ0OmJ9ww5o,13809
-tinybird/tb_cli_modules/common.py,sha256=rp1IAfDhbPSwnoMQyW4ZniYDqNZepo1cBgwL8naAa8U,76711
-tinybird/tb_cli_modules/config.py,sha256=Ey9yqM27C4Irglm5-63RXte8K0bFh8ConKdIoAaQv-k,11188
+tinybird/tb_cli_modules/common.py,sha256=OzXkQGTnlUtOfx3EOdSLreewFRu_zaTXLSYDG-MT1_I,77236
+tinybird/tb_cli_modules/config.py,sha256=lukiJeihI09blGeeS0FWeqVbKWhqzZmyoaJDIL0NkyQ,11182
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=W098H6jmS4kpE7hN3tadaREBTf7oMocel-lkKWN0pU8,10466
-tinybird-4.6.12.dev0.dist-info/METADATA,sha256=P0J5QYbTdd-3FkyYfdK6VCcNhgXhzcTEA7RAAOufl8U,15164
-tinybird-4.6.12.dev0.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
-tinybird-4.6.12.dev0.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-4.6.12.dev0.dist-info/top_level.txt,sha256=ZIQJTPCzMqnfDzM_hEGZrJqDSEcKnIK_49T86DGWpyQ,78
-tinybird-4.6.12.dev0.dist-info/RECORD,,
+tinybird-4.6.15.dev0.dist-info/METADATA,sha256=dlQkiLbkh-i9TK1RHXR-nSOok3f_NY_O1Q13N7oJ0N0,16008
+tinybird-4.6.15.dev0.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
+tinybird-4.6.15.dev0.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-4.6.15.dev0.dist-info/top_level.txt,sha256=ZIQJTPCzMqnfDzM_hEGZrJqDSEcKnIK_49T86DGWpyQ,78
+tinybird-4.6.15.dev0.dist-info/RECORD,,