tinybird 4.6.15.dev0__tar.gz → 4.6.16.dev0__tar.gz

{tinybird-4.6.15.dev0 → tinybird-4.6.16.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 4.6.15.dev0
+Version: 4.6.16.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -51,11 +51,17 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 
 Changelog
 ----------
+4.6.14
+********
+
+- `Changed` ``tb datasource append`` and ``tb datasource replace`` submit local CSV, NDJSON, and Parquet files through the v1 import endpoint by default. ``tb datasource append --events`` also submits its request body through v1. These asynchronous imports print the queued job ID; pass ``--wait`` to wait for completion. Remote URL imports continue to use their existing route.
 
 4.6.13
 ********
 
+- `Fixed` `tb deploy` and `tb build` now reject pipes that declare a parameter without a valid name (e.g. a numeric-literal cast like ``{{Float32(1682892000.0)}}``), instead of creating an unusable parameter.
 - `Added` ``tb datasource append`` and ``tb datasource replace`` support the experimental ``--experimental=use_v1`` option for local CSV, NDJSON, and Parquet files. It submits the import through the v1 endpoint and prints the queued job ID; pass ``--wait`` to wait for completion.
+- `Fixed` Forward builds branch with vendored files without incorrectly detecting drift.
 
 4.6.12
 ********
@@ -74,6 +80,7 @@ Changelog
 
 - `Removed` `tinybird.bi_stats_rt` from `tb logs` sources. The BI Connector `bi_stats` / `bi_stats_rt` service datasources are deprecated.
 
+
 4.6.9
 *******
 
@@ -82,6 +89,7 @@ Changelog
 4.6.8
 *******
 
+- `Changed` Upgraded Tornado to 6.5.5.
 - `Fixed` Tinybird Local CLI commands to respect Git branch local workspaces correctly.
 
 4.6.7

{tinybird-4.6.15.dev0 → tinybird-4.6.16.dev0}/tinybird/client.py

@@ -411,7 +411,7 @@ class TinyB:
         sql_condition: Optional[str] = None,
         format: str = "csv",
         replace_options: Optional[Set[str]] = None,
-    ):
+    ) -> Dict[str, Any]:
         params = {"name": table_name, "url": url, "mode": mode, "debug": "blocks_block_log", "format": format}
 
         if sql_condition:
@@ -437,58 +437,37 @@ class TinyB:
         datasource_name: str,
         file: Union[str, Path],
         mode: str = "append",
-        status_callback=None,
+        status_callback=None,  # noqa: ARG002 - kept for v0 caller compatibility
         sql_condition: Optional[str] = None,
         format: str = "csv",
         replace_options: Optional[Set[str]] = None,
-        use_v1: bool = False,
     ):
-        params = {"name": datasource_name, "mode": mode, "format": format, "debug": "blocks_block_log"}
-
+        # V1 returns the import job immediately, so waiting and progress reporting
+        # belong to the caller.
+        async with aiofiles.open(file, "rb") as content:
+            file_content = await content.read()
+        v1_params = {"format": format}
         if sql_condition:
-            params["replace_condition"] = sql_condition
+            v1_params["replace_condition"] = sql_condition
         if replace_options:
             for option in list(replace_options):
-                params[option] = "true"
-
-        async with aiofiles.open(file, "rb") as content:
-            file_content = await content.read()
-
-        if use_v1:
-            v1_params = {"format": format}
-            if sql_condition:
-                v1_params["replace_condition"] = sql_condition
-            if replace_options:
-                for option in list(replace_options):
-                    v1_params[option] = "true"
-            content_types = {
-                "csv": "text/csv",
-                "ndjson": "application/x-ndjson",
-                "parquet": "application/vnd.apache.parquet",
-            }
-            headers = {"Content-Type": content_types[format]}
-            if str(file).endswith(".gz"):
-                headers["Content-Encoding"] = "gzip"
-            return await self._req(
-                f"/v1/datasources/{quote(datasource_name, safe='')}/{mode}?{urlencode(v1_params, safe='')}",
-                data=file_content,
-                headers=headers,
-                method="POST",
-            )
-
-        if format == "csv":
-            files = {"csv": ("csv", file_content)}
-        else:
-            files = {"ndjson": ("ndjson", file_content)}
-
+                v1_params[option] = "true"
+        content_types = {
+            "csv": "text/csv",
+            "ndjson": "application/x-ndjson",
+            "parquet": "application/vnd.apache.parquet",
+        }
+        headers = {"Content-Type": content_types[format]}
+        if str(file).endswith(".gz"):
+            headers["Content-Encoding"] = "gzip"
         res = await self._req(
-            f"v0/datasources?{urlencode(params, safe='')}",
-            files=files,
+            f"/v1/datasources/{quote(datasource_name, safe='')}/{mode}?{urlencode(v1_params, safe='')}",
+            data=file_content,
+            headers=headers,
             method="POST",
         )
-        if status_callback:
-            status_callback(res)
-
+        if not isinstance(res, dict):
+            raise RuntimeError("We couldn't confirm that your import started. Please try again.")
         return res
 
     async def datasource_truncate(self, datasource_name: str):

{tinybird-4.6.15.dev0 → tinybird-4.6.16.dev0}/tinybird/datafile/parse_pipe.py

@@ -11,7 +11,11 @@ from tinybird.datafile.common import (
     parse,
 )
 from tinybird.datafile.exceptions import IncludeFileNotFoundException, ParseException
-from tinybird.sql_template import get_template_and_variables, render_sql_template
+from tinybird.sql_template import (
+    get_template_and_variables,
+    render_sql_template,
+    validate_template_parameter_names,
+)
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tornado_template import UnClosedIfError
 
@@ -61,6 +65,9 @@ def parse_pipe(
                 secrets_list: Optional[List[str]] = None
                 if secrets:
                     secrets_list = list(secrets.keys())
+                # Reject parameters declared without a usable name (e.g. a numeric
+                # literal cast like `{{Float32(1682892000.0)}}`) before rendering.
+                validate_template_parameter_names(sql[1:], node_id=node["name"])
                 # Setting test_mode=True to ignore errors on required parameters and
                 # secrets_in_test_mode=False to raise errors on missing secrets
                 sql, _, variable_warnings = render_sql_template(

{tinybird-4.6.15.dev0 → tinybird-4.6.16.dev0}/tinybird/feedback_manager.py

@@ -242,10 +242,11 @@ class FeedbackManager:
     error_organization_index = error_message(
         "Error selecting organization '{organization_index}'. Select a valid index or 0 to cancel"
     )
-    warning_none_organization = warning_message(
-        "Tinybird is now based on organizations. Please, go to the UI ({ui_host}) to follow the migration process. \nYour workspace will be created any way."
-    )
     error_while_fetching_orgs = error_message("Error while fetching organizations: {error}")
+    error_classic_workspace_creation_deprecated = error_message(
+        "The Classic experience is deprecated. New workspaces should be created in Forward. "
+        "Migration guide: https://www.tinybird.co/docs/forward/guides/migrate-from-classic"
+    )
     error_deleted_include = error_message(
         "Related include file {include_file} was deleted and it's used in {filename}. Delete or remove dependency from {filename}."
     )

{tinybird-4.6.15.dev0 → tinybird-4.6.16.dev0}/tinybird/service_datasources.py

@@ -19,12 +19,12 @@ def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
     return [
         {
             "name": "tinybird.pipe_stats_rt",
-            "description": "Contains information about all requests made to your API endpoints in real time. This data source has a TTL of 7 days. If you need to query data older than 7 days you must use the aggregated by day data available at tinybird.pipe_stats.",
+            "description": "Contains information about all requests made to your API endpoints in real time. This data source has a TTL of 3 months. If you need to query data older than 3 months you must use the aggregated by day data available at tinybird.pipe_stats.",
             "dateColumn": "start_datetime",
             "engine": {
                 "engine": "MergeTree",
                 "sorting_key": "pipe_id, start_datetime",
-                "ttl": "start_datetime + toIntervalDay(7)",
+                "ttl": "start_datetime + toIntervalMonth(3)",
             },
             "columns": [
                 {"name": "start_datetime", "type": "DateTime"},
@@ -659,7 +659,7 @@ def get_organization_service_datasources() -> List[Dict[str, Any]]:
             "engine": {
                 "engine": "MergeTree",
                 "sorting_key": "workspace_id, pipe_id, start_datetime",
-                "ttl": "start_datetime + toIntervalDay(7)",
+                "ttl": "start_datetime + toIntervalMonth(3)",
             },
             "columns": [
                 {"name": "workspace_id", "type": "String"},

{tinybird-4.6.15.dev0 → tinybird-4.6.16.dev0}/tinybird/sql_template.py

@@ -2263,7 +2263,12 @@ def get_var_names_and_types(t: Template, node_id: Optional[str] = None) -> List[
             vars_out.extend(
                 parse_statement_code_if_new_vars(statement_code, skip_names=typed_names | statement_expr_names)
             )
-            return vars_out
+            # Inside a {% %} statement a type function with a literal first arg
+            # (e.g. {% set x = Float32(1.5) %}) is a plain value cast: the
+            # assigned variable is the parameter, the literal is not. Drop
+            # these artifacts so they don't surface as bogus params nor get
+            # rejected by validate_template_parameter_names.
+            return [vd for vd in vars_out if isinstance(vd["name"], str) and vd["name"]]
 
         def _n(chunks: list, vars_out: list[dict[str, Any]]) -> None:
             for x in chunks:
@@ -2310,6 +2315,43 @@ def get_var_names_and_types_cached(t: Template):
     return get_var_names_and_types(t)
 
 
+def validate_template_parameter_names(sql: str, node_id: Optional[str] = None) -> None:
+    """Reject template parameters that don't have a usable name.
+
+    Type functions such as ``{{Float32(x)}}`` double as casts, so a literal
+    first argument (e.g. ``{{Float32(1682892000.0)}}`` or ``{{String('')}}``)
+    is parsed as a parameter whose name is a number or an empty string instead
+    of an identifier. Such a "parameter" can never be supplied by a caller and
+    breaks parameter metadata consumers, so it must be rejected at write time.
+
+    Raises ``ValueError`` with the same wording used elsewhere for invalid names.
+
+    Templates that fail to parse are left untouched: there are no parameters to
+    validate, and reporting/deferring those errors is the job of the existing
+    render/validation flow (some paths, e.g. ``ignore_sql_errors``, tolerate
+    unparseable SQL on purpose). This keeps the check strictly additive. The
+    only exception is the invalid-name ``ValueError`` that extraction itself
+    raises (e.g. for a list literal used as first argument), which is exactly
+    what this validator exists to report.
+    """
+    try:
+        params = get_var_names_and_types(Template(sql), node_id=node_id)
+    except SQLTemplateException:
+        # Template-level problem (e.g. a SecurityException converted by
+        # get_var_names_and_types), not an invalid parameter name.
+        return
+    except ValueError:
+        # Extraction itself rejects some invalid names (e.g. a list literal
+        # as first argument) with the same ValueError this function raises.
+        raise
+    except Exception:
+        return
+    for param in params:
+        name = param.get("name")
+        if not isinstance(name, str) or not name:
+            raise ValueError(f'"{name}" can not be used as a variable name')
+
+
 def wrap_vars(t, escape_arrays: bool = False):
     def _n(chunks, v):
         for x in chunks:
@@ -2851,6 +2893,13 @@ def render_sql_template(
                 documentation="/cli/advanced-templates.html",
             )
         raise SQLTemplateException(str(e), documentation="/cli/advanced-templates.html")
+    except KeyError as e:
+        key = e.args[0] if e.args else "unknown"
+        raise SQLTemplateException(
+            f"Missing key '{key}' while rendering the template. "
+            "Ensure every object accessed in the template has this key.",
+            documentation="/cli/advanced-templates.html",
+        )
     except IndexError as e:
         # This happens when trying to access string indices on empty strings
         if "string index out of range" in str(e):

{tinybird-4.6.15.dev0 → tinybird-4.6.16.dev0}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '4.6.15.dev0'
-__revision__ = '8311ae9'
+__version__ = '4.6.16.dev0'
+__revision__ = '532ee82'

{tinybird-4.6.15.dev0 → tinybird-4.6.16.dev0}/tinybird/tb/client.py

@@ -435,7 +435,7 @@ class TinyB:
         sql_condition: Optional[str] = None,
         format: str = "csv",
         replace_options: Optional[Set[str]] = None,
-    ):
+    ) -> Dict[str, Any]:
         params = {"name": table_name, "url": url, "mode": mode, "debug": "blocks_block_log", "format": format}
 
         if sql_condition:
@@ -465,16 +465,9 @@ class TinyB:
         sql_condition: Optional[str] = None,
         format: str = "csv",
         replace_options: Optional[Set[str]] = None,
-        use_v1: bool = False,
     ):
-        params = {"name": datasource_name, "mode": mode, "format": format, "debug": "blocks_block_log"}
-
-        if sql_condition:
-            params["replace_condition"] = sql_condition
-        if replace_options:
-            for option in list(replace_options):
-                params[option] = "true"
-
+        # Keep this argument for SDK compatibility; v1 returns before import completion.
+        del status_callback
         with open(file, "rb") as content:
             file_content = content.read()
         content_types = {
@@ -485,29 +478,34 @@ class TinyB:
         headers = {"Content-Type": content_types[format]}
         if str(file).endswith(".gz"):
             headers["Content-Encoding"] = "gzip"
-        if use_v1:
-            v1_params = {"format": format}
-            if sql_condition:
-                v1_params["replace_condition"] = sql_condition
-            if replace_options:
-                for option in list(replace_options):
-                    v1_params[option] = "true"
-            res = self._req(
-                f"/v1/datasources/{quote(datasource_name, safe='')}/{mode}?{urlencode(v1_params, safe='')}",
-                data=file_content,
-                headers=headers,
-                method="POST",
-            )
-            if "error" in res:
-                raise Exception(res["error"])
-            return res
-        if format == "csv":
-            files = {"csv": ("csv", file_content)}
-        else:
-            files = {"ndjson": ("ndjson", file_content)}
-        res = self._req(f"v0/datasources?{urlencode(params, safe='')}", files=files, method="POST")
-        if status_callback:
-            status_callback(res)
+        v1_params = {"format": format}
+        if sql_condition:
+            v1_params["replace_condition"] = sql_condition
+        if replace_options:
+            for option in list(replace_options):
+                v1_params[option] = "true"
+        res = self._req(
+            f"/v1/datasources/{quote(datasource_name, safe='')}/{mode}?{urlencode(v1_params, safe='')}",
+            data=file_content,
+            headers=headers,
+            method="POST",
+        )
+        if not isinstance(res, dict):
+            raise RuntimeError("We couldn't confirm that your import started. Please try again.")
+        if "error" in res:
+            raise Exception(res["error"])
+        return res
+
+    def datasource_append_events(self, datasource_name: str, events: str) -> dict:
+        """Submit NDJSON event data through the asynchronous v1 append route."""
+        res = self._req(
+            f"/v1/datasources/{quote(datasource_name, safe='')}/append?format=ndjson",
+            data=events.encode(),
+            headers={"Content-Type": "application/x-ndjson"},
+            method="POST",
+        )
+        if "error" in res:
+            raise Exception(res["error"])
         return res
 
     def datasource_truncate(self, datasource_name: str):

{tinybird-4.6.15.dev0 → tinybird-4.6.16.dev0}/tinybird/tb/modules/build_common.py

@@ -300,7 +300,8 @@ def build_project_with_deploy_api(
     if build_result == "no_changes":
         return False
     if build_result != "success":
-        deployment_errors = deployment.get("errors", []) if deployment else result.get("errors", [])
+        deployment_errors = list(deployment.get("errors", []) if deployment else result.get("errors", []))
+        deployment_errors.extend(item for item in deployment.get("feedback", []) if item.get("level") == "ERROR")
         raise click.ClickException(result.get("error") or format_build_errors(deployment_errors))
     if not deployment:
         raise click.ClickException("Couldn't parse deployment response from server")
@@ -328,6 +329,7 @@ def get_build_request_files(
     project_with_vendors: Optional[Project] = None,
 ) -> tuple[list[tuple[str, tuple[str, str, str]]], list[str]]:
     multipart_boundary_data_project = "data_project://"
+    multipart_boundary_data_project_vendored = "data_project_vendored://"
     datafile_type_to_content_type = {
         ".datasource": "text/plain",
         ".pipe": "text/plain",
@@ -350,6 +352,13 @@ def get_build_request_files(
 
             files.append((multipart_boundary_data_project, (relative_path, content, content_type)))
 
+    for file_path in project.get_vendored_files():
+        relative_path = Path(file_path).relative_to(project_path).as_posix()
+        with open(file_path, "rb") as fd:
+            content_type = datafile_type_to_content_type.get(Path(file_path).suffix, "application/unknown")
+            content = fd.read().decode("utf-8")
+            files.append((multipart_boundary_data_project_vendored, (relative_path, content, content_type)))
+
     return files, project_files
 
 

{tinybird-4.6.15.dev0 → tinybird-4.6.16.dev0}/tinybird/tb/modules/common.py

@@ -925,7 +925,7 @@ def push_data(
     replace_options=None,
     concurrency: int = 1,
     silent: bool = False,
-    use_v1: bool = False,
+    wait: bool = True,
 ) -> Optional[list[str]]:
     if url and type(url) is tuple:
         url = url[0]
@@ -963,11 +963,11 @@ def push_data(
         datasource_name: str, url: str, mode: str, sql_condition: Optional[str], replace_options: Optional[Set[str]]
     ):
         parsed = urlparse(url)
-        if use_v1 and parsed.scheme in ("http", "https"):
-            raise CLIException("--experimental=use_v1 only supports local files.")
+        # Remote URL imports retain the v0 API; local files use v1 body imports.
+        is_remote_url = parsed.scheme in ("http", "https")
         # poor man's format detection
         _format = get_format_from_filename_or_url(url)
-        if parsed.scheme in ("http", "https"):
+        if is_remote_url:
             res = client.datasource_create_from_url(
                 datasource_name,
                 url,
@@ -985,13 +985,12 @@ def push_data(
                 sql_condition=sql_condition,
                 format=_format,
                 replace_options=replace_options,
-                use_v1=use_v1,
             )
 
-        if use_v1:
+        if not is_remote_url:
             job_id = res.get("id") or res.get("import_id")
             if not isinstance(job_id, str):
-                raise CLIException("The v1 import response did not include a job ID.")
+                raise CLIException("We couldn't confirm that your import started. Please try again.")
             return job_id
 
         datasource_name = res["datasource"]["name"]
@@ -1024,8 +1023,17 @@ def push_data(
     try:
         tasks = [process_url(datasource_name, url, mode, sql_condition, replace_options) for url in urls]
         output = gather_with_concurrency(concurrency, *tasks)
-        if use_v1:
-            return list(output)
+        v1_job_ids = [result for result in output if isinstance(result, str)]
+        if v1_job_ids:
+            if not wait:
+                return v1_job_ids
+            for job_id in v1_job_ids:
+                wait_job_no_ui(client, job_id)
+            if not silent:
+                if mode == "replace":
+                    click.echo(FeedbackManager.success_replaced_datasource(datasource=datasource_name))
+                click.echo(FeedbackManager.success_progress_blocks())
+            return None
         parser, total_rows, appended_rows = list(output)[-1]
     except AuthNoTokenException:
         raise
@@ -2380,7 +2388,7 @@ def force_echo(string: str) -> None:
     click.echo(string, force_output=True)  # type: ignore
 
 
-def echo_json(data: Dict[str, Any], indent: Union[None, int, str] = None) -> None:
+def echo_json(data: Dict[str, Any], indent: Union[int, str, None] = None) -> None:
     force_echo(json.dumps(data, indent=indent))
 
 

{tinybird-4.6.15.dev0 → tinybird-4.6.16.dev0}/tinybird/tb/modules/datafile/pipe_checker.py

@@ -282,6 +282,8 @@ class PipeCheckerRunner:
                         FROM {pipe_stats_rt}
                         WHERE
                             pipe_name = '{self.pipe_name}'
+                            -- pipe_stats_rt retains 3 months; recent requests are enough for checks
+                            AND start_datetime >= now() - INTERVAL 7 DAY
                             AND url IS NOT NULL
                             AND extractURLParameter(assumeNotNull(url), 'from') <> 'ui'
                             AND extractURLParameter(assumeNotNull(url), 'pipe_checker') <> 'true'
@@ -309,6 +311,8 @@ class PipeCheckerRunner:
                                     FROM {pipe_stats_rt}
                                     WHERE
                                         pipe_name = '{self.pipe_name}'
+                                        -- pipe_stats_rt retains 3 months; recent requests are enough for checks
+                                        AND start_datetime >= now() - INTERVAL 7 DAY
                                         AND url IS NOT NULL
                                         AND extractURLParameter(assumeNotNull(url), 'from') <> 'ui'
                                         AND extractURLParameter(assumeNotNull(url), 'pipe_checker') <> 'true'

{tinybird-4.6.15.dev0 → tinybird-4.6.16.dev0}/tinybird/tb/modules/datafile/pull.py

@@ -53,7 +53,7 @@ def folder_pull(
         if not candidates:
             return None
 
-        return sorted(candidates, key=lambda path: (len(path.parent.relative_to(dest_folder).parts), str(path)))[0]
+        return min(candidates, key=lambda path: (len(path.parent.relative_to(dest_folder).parts), str(path)))
 
     def write_files(
         resources: list[dict[str, Any]],