tinybird-cli 3.3.1.dev9__tar.gz → 3.3.1.dev11__tar.gz

{tinybird-cli-3.3.1.dev9 → tinybird-cli-3.3.1.dev11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 3.3.1.dev9
+Version: 3.3.1.dev11
 Summary: Tinybird Command Line Tool
 Home-page: https://docs.tinybird.co/cli.html
 Author: Tinybird
@@ -19,6 +19,16 @@ Changelog
 
 ---------
 
+3.3.1.dev11
+************
+
+- `Add` Check backfill on preview is required when `tb deploy` modified datasources. Disable check with `TB_CHECK_BACKFILL_REQUIRED=0`.
+
+3.3.1.dev10
+************
+
+- `Fixed` Fixed bug when a Data Source was created using an INDEX
+
 3.3.1.dev9
 ************
 

{tinybird-cli-3.3.1.dev9 → tinybird-cli-3.3.1.dev11}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://docs.tinybird.co/cli.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '3.3.1.dev9'
-__revision__ = 'fbdd4be'
+__version__ = '3.3.1.dev11'
+__revision__ = '6db530d'

{tinybird-cli-3.3.1.dev9 → tinybird-cli-3.3.1.dev11}/tinybird/datafile.py

@@ -3745,6 +3745,8 @@ async def folder_push(
     fork: Optional[bool] = False,
     is_internal: Optional[bool] = False,
     release_created: Optional[bool] = False,
+    auto_promote: Optional[bool] = False,
+    check_backfill_required: bool = False,
 ):  # noqa: C901
     workspaces: List[Dict[str, Any]] = (await tb_client.user_workspaces_and_branches()).get("workspaces", [])
     current_ws: Dict[str, Any] = next(
@@ -3879,6 +3881,7 @@ async def folder_push(
         dry_run: bool,
         fork_downstream: Optional[bool] = False,
         fork: Optional[bool] = False,
+        check_backfill_required: bool = False,
     ):
         if name in to_run:
             if not dry_run:
@@ -3970,10 +3973,23 @@ async def folder_push(
                                 latest_version=resource_versions.get(name),
                             )
                         )
+                    if (
+                        check_backfill_required
+                        and auto_promote
+                        and release_created
+                        and "resource" in to_run[name]
+                        and to_run[name]["resource"] == "datasources"
+                    ):
+                        error_msg = FeedbackManager.error_check_backfill_required(resource_name=name)
+                        raise click.ClickException(error_msg)
                 else:
                     click.echo(FeedbackManager.warning_dry_name_already_exists(name=name))
 
-    async def push_files(dependency_graph: GraphDependencies, dry_run: bool = False):
+    async def push_files(
+        dependency_graph: GraphDependencies,
+        dry_run: bool = False,
+        check_backfill_required: bool = False,
+    ):
         endpoints_dep_map = dict()
         processed = set()
 
@@ -4057,6 +4073,7 @@ async def folder_push(
                             dry_run,
                             fork_downstream,
                             fork,
+                            check_backfill_required=False,  # keep it always to false for new resources
                         )
                     else:
                         await push(
@@ -4067,6 +4084,7 @@ async def folder_push(
                             dry_run,
                             fork_downstream,
                             fork,
+                            check_backfill_required=check_backfill_required,
                         )
                     processed.add(name)
 
@@ -4104,6 +4122,7 @@ async def folder_push(
                         dry_run,
                         fork_downstream,
                         fork,
+                        check_backfill_required=check_backfill_required,
                     )
                     processed.add(name)
 
@@ -4124,6 +4143,7 @@ async def folder_push(
                         dry_run,
                         fork_downstream,
                         fork,
+                        check_backfill_required=check_backfill_required,
                     )
                     processed.add(name)
 
@@ -4144,12 +4164,13 @@ async def folder_push(
                         dry_run,
                         fork_downstream,
                         fork,
+                        check_backfill_required=check_backfill_required,
                     )
                     processed.add(name)
 
     if deployment.is_git_release:
         deployment.deploying_dry_run()
-        await push_files(dependencies_graph, dry_run=True)
+        await push_files(dependencies_graph, dry_run=True, check_backfill_required=check_backfill_required)
 
         await deployment.delete_resources(deleted, pipes, dry_run=True)
         if not deployment.dry_run:

{tinybird-cli-3.3.1.dev9 → tinybird-cli-3.3.1.dev11}/tinybird/feedback_manager.py

@@ -326,6 +326,10 @@ class FeedbackManager:
     error_unsupported_diff = error_message(
         "There are resources renamed. `tb deploy` can't deploy renamed resources, create new resources instead."
     )
+    error_check_backfill_required = error_message(
+        "Not safe to deploy to live resource '{resource_name}', backfill might be required. Consider to deploy to preview bumping semver to major or disable 'TB_AUTO_PROMOTE'. In case you want to disable this check use 'TB_CHECK_BACKFILL_REQUIRED=0'"
+    )
+
     info_ignoring_incl_file = info_message(
         "** Ignoring file {filename}. .incl files are not checked independently. They are checked as part of the file that includes them. Please check the file that includes this .incl file."
     )

{tinybird-cli-3.3.1.dev9 → tinybird-cli-3.3.1.dev11}/tinybird/git_settings.py

@@ -70,6 +70,9 @@ VERSION=0.0.0
 # Deploy a new Release in preview status (default is 1)
 # TB_AUTO_PROMOTE=0
 
+# Check if deploy requires backfilling on preview (default is 1)
+# TB_CHECK_BACKFILL_REQUIRED=0
+
 # Force old Releases deletion on promote (default is 0)
 # Setting it to 1 will remove oldest rollback Releases even when some resource is still in use
 # TB_FORCE_REMOVE_OLDEST_ROLLBACK=0

{tinybird-cli-3.3.1.dev9 → tinybird-cli-3.3.1.dev11}/tinybird/sql.py

@@ -1,8 +1,41 @@
 import re
 import string
 from collections import namedtuple
+from dataclasses import dataclass
 from typing import Any, Dict, Iterable, List, Optional
 
+valid_chars_name: str = string.ascii_letters + string.digits + "._`*<>+-'"
+valid_chars_fn: str = valid_chars_name + "[](),=!?:/ \n\t\r"
+
+
+@dataclass
+class TableIndex:
+    """Defines a CH table INDEX"""
+
+    name: str
+    expr: str
+    type_full: str
+    granularity: Optional[str] = None
+
+    def to_datafile(self):
+        granularity_expr = f"GRANULARITY {self.granularity}" if self.granularity else ""
+        return f"{self.name} {self.expr} TYPE {self.type_full} {granularity_expr}"
+
+    def to_sql(self):
+        return f"INDEX {self.to_datafile()}"
+
+    def add_index_sql(self):
+        return f"ADD {self.to_sql()}"
+
+    def drop_index_sql(self):
+        return f"DROP INDEX IF EXISTS {self.name}"
+
+    def materialize_index_sql(self):
+        return f"MATERIALIZE INDEX IF EXISTS {self.name}"
+
+    def clear_index_sql(self):
+        return f"CLEAR INDEX IF EXISTS {self.name}"
+
 
 def as_subquery(sql: str) -> str:
     return f"""(\n{sql}\n)"""
@@ -177,6 +210,49 @@ def format_parse_error(
     return message
 
 
+def parse_indices_structure(indices: List[str]) -> List[TableIndex]:
+    """
+    >>> parse_indices_structure(["index_name a TYPE set(100) GRANULARITY 100", "index_name_bf mapValues(d) TYPE bloom_filter(0.001) GRANULARITY 16"])
+    [TableIndex(name='index_name', expr='a', type_full='set(100)', granularity='100'), TableIndex(name='index_name_bf', expr='mapValues(d)', type_full='bloom_filter(0.001)', granularity='16')]
+    >>> parse_indices_structure(["INDEX index_name a TYPE set(100) GRANULARITY 100", " INDEX  index_name_bf mapValues(d) TYPE bloom_filter(0.001) GRANULARITY 16"])
+    [TableIndex(name='index_name', expr='a', type_full='set(100)', granularity='100'), TableIndex(name='index_name_bf', expr='mapValues(d)', type_full='bloom_filter(0.001)', granularity='16')]
+    >>> parse_indices_structure(["index_name type TYPE set(100) GRANULARITY 100", "index_name_bf mapValues(d) TYPE bloom_filter(0.001) GRANULARITY 16"])
+    [TableIndex(name='index_name', expr='type', type_full='set(100)', granularity='100'), TableIndex(name='index_name_bf', expr='mapValues(d)', type_full='bloom_filter(0.001)', granularity='16')]
+    >>> parse_indices_structure(["index_name a TYPE set(100) GRANULARITY 100,", "index_name_bf mapValues(d) TYPE bloom_filter(0.001) GRANULARITY 16"])
+    [TableIndex(name='index_name', expr='a', type_full='set(100)', granularity='100'), TableIndex(name='index_name_bf', expr='mapValues(d)', type_full='bloom_filter(0.001)', granularity='16')]
+    >>> parse_indices_structure(["index_name a TYPE set(100)", "index_name_bf mapValues(d) TYPE bloom_filter(0.001)"])
+    [TableIndex(name='index_name', expr='a', type_full='set(100)', granularity=None), TableIndex(name='index_name_bf', expr='mapValues(d)', type_full='bloom_filter(0.001)', granularity=None)]
+    >>> parse_indices_structure(["index_name u64 * length(s) TYPE set(100)", "index_name_bf mapValues(d) TYPE bloom_filter"])
+    [TableIndex(name='index_name', expr='u64 * length(s)', type_full='set(100)', granularity=None), TableIndex(name='index_name_bf', expr='mapValues(d)', type_full='bloom_filter', granularity=None)]
+    >>> parse_indices_structure(["index_name u64 * length(s)"])
+    Traceback (most recent call last):
+    ...
+    ValueError: invalid INDEX format. Usage: `name expr TYPE type_full GRANULARITY granularity`
+    >>> parse_indices_structure(["index_name a TYPE set(100) GRANULARITY 100, index_name_bf mapValues(d) TYPE bloom_filter(0.001) GRANULARITY 16"])
+    Traceback (most recent call last):
+    ...
+    ValueError: invalid INDEX format. Usage: `name expr TYPE type_full GRANULARITY granularity`
+    """
+    parsed_indices: List[TableIndex] = []
+    if not indices:
+        return parsed_indices
+
+    for index in indices:
+        index = index.strip().rstrip(",")
+        index = index.lstrip("INDEX").strip()
+        if index.count("TYPE") != 1:
+            raise ValueError("invalid INDEX format. Usage: `name expr TYPE type_full GRANULARITY granularity`")
+
+        match = re.match(r"(\w+)\s+([\w\s*()]+)\s+TYPE\s+(\w+)(?:\(([\w.]+)\))?(?:\s+GRANULARITY\s+(\d+))?", index)
+        if match:
+            index_name, a, index_type, value, granularity = match.groups()
+            index_expr = f"{index_type}({value})" if value else index_type
+            parsed_indices.append(TableIndex(index_name, a.strip(), f"{index_expr}", granularity))
+        else:
+            raise ValueError("invalid INDEX format. Usage: `name expr TYPE type_full GRANULARITY granularity`")
+    return parsed_indices
+
+
 def parse_table_structure(schema: str) -> List[Dict[str, Any]]:
     """This parses the SQL schema for a CREATE TABLE
     Columns follow the syntax: name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1][,]
@@ -308,6 +384,12 @@ def parse_table_structure(schema: str) -> List[Dict[str, Any]]:
     [{'name': 'arr', 'type': 'Array(String)', 'codec': None, 'default_value': "DEFAULT array('-')", 'jsonpath': None, 'nullable': False, 'normalized_name': 'arr'}]
     >>> parse_table_structure('`a2` Float32 CODEC(Delta, ZSTD(4)) `json:$.a2`, `a3` String `json:$.a3`\\n')
     [{'name': 'a2', 'type': 'Float32', 'codec': 'CODEC(Delta, ZSTD(4))', 'default_value': None, 'jsonpath': '$.a2', 'nullable': False, 'normalized_name': 'a2'}, {'name': 'a3', 'type': 'String', 'codec': None, 'default_value': None, 'jsonpath': '$.a3', 'nullable': False, 'normalized_name': 'a3'}]
+    >>> parse_table_structure('`a` String, INDEX index_name a TYPE set(100) GRANULARITY 100')
+    [{'name': 'a', 'type': 'String', 'codec': None, 'default_value': None, 'jsonpath': None, 'nullable': False, 'normalized_name': 'a'}]
+    >>> parse_table_structure('`a` String, INDEX index_name a TYPE set(100, 1) GRANULARITY 100')
+    [{'name': 'a', 'type': 'String', 'codec': None, 'default_value': None, 'jsonpath': None, 'nullable': False, 'normalized_name': 'a'}]
+    >>> parse_table_structure('`index` String, INDEX index_name a TYPE set(100, 1) GRANULARITY 100')
+    [{'name': 'index', 'type': 'String', 'codec': None, 'default_value': None, 'jsonpath': None, 'nullable': False, 'normalized_name': 'index'}]
     """
     return _parse_table_structure(schema)
 
@@ -367,9 +449,6 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:  # noqa: C901
             for _ in s:
                 advance_single_char()
 
-    valid_chars_name: str = string.ascii_letters + string.digits + "._`*<>+-'"
-    valid_chars_fn: str = valid_chars_name + "[](),=!?:/ \n\t\r"
-
     def get_backticked() -> str:
         begin = i
         while i < len(schema):
@@ -447,6 +526,8 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:  # noqa: C901
 
     def add_column(found: str) -> None:
         nonlocal name, _type, default, materialized, codec, jsonpath
+        if name == "INDEX":
+            return
         if not name:
             raise ValueError(
                 format_parse_error(schema, i, pos, f"Syntax error: expecting NAME, found {found}", line=line)
@@ -479,6 +560,8 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:  # noqa: C901
             continue
         found = lookahead_matches(
             [NULL, NOTNULL, DEFAULT, MATERIALIZED, ALIAS, CODEC, TTL, JSONPATH, COMMA, NEW_LINE, TYPE]
+            if name != "INDEX"
+            else [COMMA, NEW_LINE]
         )
         if found and found not in valid_next:
             after = f" after {last.name}" if last else ""
@@ -531,11 +614,16 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:  # noqa: C901
             advance("`json:")
             jsonpath = get_backticked()
         elif found == COMMA:
+            if name == "INDEX":
+                advance(",")
+                continue
             advance(",")
             valid_next = []
             add_column("COMMA")
         elif found == NEW_LINE:
             i += 1
+        elif name == "INDEX" and not found:
+            i += 1
         else:
             raise ValueError(
                 format_parse_error(

{tinybird-cli-3.3.1.dev9 → tinybird-cli-3.3.1.dev11}/tinybird/tb_cli_modules/cli.py

@@ -1494,6 +1494,7 @@ async def deploy(
 
         release_created = False
         new_release = False
+        check_backfill_required = False
         if semver and current_semver:
             new_version = version.parse(semver.split("-snapshot")[0])
             current_version = version.parse(current_semver.split("-snapshot")[0])
@@ -1531,6 +1532,8 @@ async def deploy(
                     raise CLIException(FeedbackManager.error_exception(error=str(e)))
             release_created = True
             fork_downstream = True
+            # allows TB_CHECK_BACKFILL_REQUIRED=0 so it is not checked
+            check_backfill_required = getenv_bool("TB_CHECK_BACKFILL_REQUIRED", True)
         try:
             await folder_push(
                 tb_client=create_tb_client(ctx),
@@ -1560,6 +1563,8 @@ async def deploy(
                 fork=fork,
                 is_internal=is_internal,
                 release_created=release_created,
+                auto_promote=auto_promote,
+                check_backfill_required=check_backfill_required,
             )
         except Exception as e:
             if release_created and not dry_run:

{tinybird-cli-3.3.1.dev9 → tinybird-cli-3.3.1.dev11}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 3.3.1.dev9
+Version: 3.3.1.dev11
 Summary: Tinybird Command Line Tool
 Home-page: https://docs.tinybird.co/cli.html
 Author: Tinybird
@@ -19,6 +19,16 @@ Changelog
 
 ---------
 
+3.3.1.dev11
+************
+
+- `Add` Check backfill on preview is required when `tb deploy` modified datasources. Disable check with `TB_CHECK_BACKFILL_REQUIRED=0`.
+
+3.3.1.dev10
+************
+
+- `Fixed` Fixed bug when a Data Source was created using an INDEX
+
 3.3.1.dev9
 ************