tinybird-cli 5.9.1.dev1__tar.gz → 5.9.1.dev3__tar.gz

{tinybird-cli-5.9.1.dev1 → tinybird-cli-5.9.1.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.9.1.dev1
+Version: 5.9.1.dev3
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,10 +18,16 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
-5.9.1.dev1
+5.9.1.dev3
+***********
+
+- `Fixed` Correctly parse lambda expressions in indexes
+
+5.9.1.dev2
 ***********
 
 - `Changed` Upgrade clickhouse-toolset to 0.32.dev0
+- `Added` new "File not found" error to `tb check` when including files from missing paths.
 
 5.9.0
 ***********

{tinybird-cli-5.9.1.dev1 → tinybird-cli-5.9.1.dev3}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '5.9.1.dev1'
-__revision__ = '738cd9e'
+__version__ = '5.9.1.dev3'
+__revision__ = '6e69bc3'

{tinybird-cli-5.9.1.dev1 → tinybird-cli-5.9.1.dev3}/tinybird/ch_utils/engine.py

@@ -4,7 +4,14 @@ from collections import defaultdict
 from dataclasses import asdict
 from typing import Any, Callable, Dict, Iterable, List, Optional
 
-from ..sql import TableIndex, col_name, engine_replicated_to_local, parse_indexes_structure, parse_table_structure
+from ..sql import (
+    TableIndex,
+    TableProjection,
+    col_name,
+    engine_replicated_to_local,
+    parse_indexes_structure,
+    parse_table_structure,
+)
 
 DEFAULT_EMPTY_PARAMETERS = ["ttl", "partition_key", "sorting_key"]
 DEFAULT_JOIN_EMPTY_PARAMETERS = ["join_strictness", "join_type", "key_columns"]
@@ -237,6 +244,10 @@ class TableDetails:
     def indexes(self) -> List[TableIndex]:
         return _parse_indexes(str(self.details.get("create_table_query", "")))
 
+    @property
+    def projections(self) -> List[TableProjection]:
+        return _parse_projections(self.details.get("create_table_query", ""))
+
     def to_json(self, exclude: Optional[List[str]] = None, include_empty_details: bool = False):
         # name, database are not exported since they are not part of the engine
         d: Dict[str, Any] = {
@@ -833,3 +844,12 @@ def _parse_indexes(create_table_query_expr: str) -> List[TableIndex]:
         return []
 
     return parse_indexes_structure(indexes)
+
+
+def _parse_projections(create_table_query_expr: str) -> List[TableProjection]:
+    return [
+        TableProjection(name, expr)
+        for name, expr in re.findall(
+            r"PROJECTION\s+(\w+)\s*\(((?:[^()]|\((?:[^()]|\([^()]*\))*\))*)\)", create_table_query_expr
+        )
+    ]

{tinybird-cli-5.9.1.dev1 → tinybird-cli-5.9.1.dev3}/tinybird/datafile.py

@@ -278,7 +278,9 @@ class ParseException(Exception):
 
 
 class IncludeFileNotFoundException(Exception):
-    pass
+    def __init__(self, err: str, lineno: int = -1):
+        self.lineno: int = lineno
+        super().__init__(err)
 
 
 class ValidationException(Exception):
@@ -584,7 +586,7 @@ class Deployment:
                 self.cli_git_release.validate_local_for_release(self.current_release, check_outdated=check_outdated)
             click.echo(FeedbackManager.info_deployment_detecting_changes_header())
             commit = self.cli_git_release.get_main_branch_commit() if use_main else self.current_release["commit"]
-            diffs = self.cli_git_release.diff_datafiles(commit)
+            diffs = self.cli_git_release.diff_datafiles(commit)  # type: ignore
             click.echo(
                 FeedbackManager.info_git_release_diffs(
                     workspace=self.current_ws["name"],
@@ -863,6 +865,8 @@ def parse_pipe(
         raise click.ClickException(
             FeedbackManager.error_parsing_node_with_unclosed_if(node=e.node, pipe=filename, lineno=e.lineno, sql=e.sql)
         )
+    except IncludeFileNotFoundException as e:
+        raise click.ClickException(FeedbackManager.error_not_found_include(filename=e, lineno=e.lineno))
     except ModuleNotFoundError:
         pass
     return doc
@@ -1110,7 +1114,7 @@ def parse(
                     StringIO(Template(file.read()).safe_substitute(attrs), newline=None)
                 )
         except FileNotFoundError:
-            raise IncludeFileNotFoundException(f)
+            raise IncludeFileNotFoundException(f, lineno)
 
     def version(*args: str, **kwargs: Any) -> None:
         if len(args) < 1:
@@ -1298,7 +1302,7 @@ def parse(
         else:
             raise ValidationException(f"Validation error, found {line} in line {str(lineno)}: {str(e)}", lineno=lineno)
     except IncludeFileNotFoundException as e:
-        raise e
+        raise IncludeFileNotFoundException(str(e), lineno=lineno)
     except Exception as e:
         traceback.print_tb(e.__traceback__)
         raise ParseException(f"Unexpected error: {e}", lineno=lineno)
@@ -2114,7 +2118,7 @@ class PipeCheckerRunnerResponse:
 
 
 class PipeCheckerRunner:
-    checker_stream_result_class = unittest.runner._WritelnDecorator  # type: ignore
+    checker_stream_result_class = unittest.runner._WritelnDecorator
 
     def __init__(self, pipe_name: str, host: str):
         self.pipe_name = pipe_name
@@ -2277,7 +2281,7 @@ class PipeCheckerRunner:
             )
 
         result = PipeCheckerTextTestResult(
-            self.checker_stream_result_class(sys.stdout), descriptions=True, verbosity=2, custom_output=custom_output
+            self.checker_stream_result_class(sys.stdout), descriptions=True, verbosity=2, custom_output=custom_output  # type: ignore
         )
         result.failfast = failfast
         suite.run(result)
@@ -3985,7 +3989,7 @@ async def build_graph(
                     mapped_workspaces.append(
                         workspace_map.get(shared_with)
                         if workspace_map.get(shared_with, None) is not None
-                        else shared_with
+                        else shared_with  # type: ignore
                     )
                 r["shared_with"] = mapped_workspaces
 

{tinybird-cli-5.9.1.dev1 → tinybird-cli-5.9.1.dev3}/tinybird/feedback_manager.py

@@ -242,6 +242,9 @@ class FeedbackManager:
     error_deleted_include = error_message(
         "Related include file {include_file} was deleted and it's used in {filename}. Delete or remove dependency from {filename}."
     )
+    error_not_found_include = error_message(
+        "Included file {filename} at line {lineno} not found. Check if the file exists and the path is correct."
+    )
     error_branch = error_message(
         "Branch {branch} not found. use 'tb branch ls' to list your Branches, make sure you are authenticated using the right workspace token"
     )

{tinybird-cli-5.9.1.dev1 → tinybird-cli-5.9.1.dev3}/tinybird/sql.py

@@ -38,11 +38,37 @@ class TableIndex:
         return f"CLEAR INDEX IF EXISTS {self.name}"
 
 
+@dataclass
+class TableProjection:
+    """Defines a CH table PROJECTION"""
+
+    name: str
+    expr: str
+
+    def to_datafile(self):
+        return f"{self.name} ({self.expr})"
+
+    def to_sql(self):
+        return f"PROJECTION {self.to_datafile()}"
+
+    def add_index_sql(self):
+        return f"ADD {self.to_sql()}"
+
+    def drop_index_sql(self):
+        return f"DROP PROJECTION IF EXISTS {self.name}"
+
+    def materialize_index_sql(self):
+        return f"MATERIALIZE PROJECTION IF EXISTS {self.name}"
+
+    def clear_index_sql(self):
+        return f"CLEAR PROJECTION IF EXISTS {self.name}"
+
+
 def as_subquery(sql: str) -> str:
     return f"""(\n{sql}\n)"""
 
 
-def get_format(sql: str) -> str:
+def get_format(sql: str) -> Optional[str]:
     """
     retrieves FORMAT from CH sql
     >>> get_format('select * from test')
@@ -211,7 +237,7 @@ def format_parse_error(
     return message
 
 
-def parse_indexes_structure(indexes: List[str]) -> List[TableIndex]:
+def parse_indexes_structure(indexes: Optional[List[str]]) -> List[TableIndex]:
     """
     >>> parse_indexes_structure(["index_name a TYPE set(100) GRANULARITY 100", "index_name_bf mapValues(d) TYPE bloom_filter(0.001) GRANULARITY 16"])
     [TableIndex(name='index_name', expr='a', type_full='set(100)', granularity='100'), TableIndex(name='index_name_bf', expr='mapValues(d)', type_full='bloom_filter(0.001)', granularity='16')]
@@ -239,6 +265,8 @@ def parse_indexes_structure(indexes: List[str]) -> List[TableIndex]:
     ValueError: invalid INDEX format. Usage: `name expr TYPE type_full GRANULARITY granularity`
     >>> parse_indexes_structure(["my_index m['key'] TYPE ngrambf_v1(1, 1024, 1, 42) GRANULARITY 1"])
     [TableIndex(name='my_index', expr="m['key']", type_full='ngrambf_v1(1, 1024, 1, 42)', granularity='1')]
+    >>> parse_indexes_structure(["my_index_lambda arrayMap(x -> tupleElement(x,'message'), column_name) TYPE ngrambf_v1(1, 1024, 1, 42) GRANULARITY 1"])
+    [TableIndex(name='my_index_lambda', expr="arrayMap(x -> tupleElement(x,'message'), column_name)", type_full='ngrambf_v1(1, 1024, 1, 42)', granularity='1')]
     >>> parse_indexes_structure(["ip_range_minmax_idx (toIPv6(ip_range_start), toIPv6(ip_range_end)) TYPE minmax GRANULARITY 1"])
     [TableIndex(name='ip_range_minmax_idx', expr='(toIPv6(ip_range_start), toIPv6(ip_range_end))', type_full='minmax', granularity='1')]
     """
@@ -253,7 +281,7 @@ def parse_indexes_structure(indexes: List[str]) -> List[TableIndex]:
             raise ValueError("invalid INDEX format. Usage: `name expr TYPE type_full GRANULARITY granularity`")
 
         match = re.match(
-            r"(\w+)\s+([\w\s*\[\]\*\(\),\'\".]+)\s+TYPE\s+(\w+)(?:\(([\w\s*.,]+)\))?(?:\s+GRANULARITY\s+(\d+))?",
+            r"(\w+)\s+([\w\s*\[\]\*\(\),\'\"-><.]+)\s+TYPE\s+(\w+)(?:\(([\w\s*.,]+)\))?(?:\s+GRANULARITY\s+(\d+))?",
             index,
         )
         if match:

{tinybird-cli-5.9.1.dev1 → tinybird-cli-5.9.1.dev3}/tinybird/sql_template.py

@@ -1619,8 +1619,10 @@ def get_var_data(content, node_id=None):  # noqa: C901
 
                     kwargs = {}
                     for x in node.keywords:
-                        kwargs[x.arg] = node_to_value(x.value)
-
+                        value = node_to_value(x.value)
+                        kwargs[x.arg] = value
+                        if x.arg == "default":
+                            kwargs["default"] = check_default_value(value)
                     if func in VALID_CUSTOM_FUNCTION_NAMES:
                         # Type definition here is set to 'String' because it comes from a
                         # `defined(variable)` expression that does not contain any type hint.
@@ -1628,11 +1630,15 @@ def get_var_data(content, node_id=None):  # noqa: C901
                         # args[0] check is used to avoid adding unnamed parameters found in
                         # templates like: `split_to_array('')`
                         if len(args) > 0 and args[0] not in vars and args[0]:
-                            vars[args[0]] = {"type": "String", "default": None, "used_in": "function_call"}
+                            vars[args[0]] = {
+                                "type": "String",
+                                "default": None,
+                                "used_in": "function_call",
+                            }
                     elif func == "Array":
                         if "default" not in kwargs:
                             default = kwargs.get("default", args[2] if len(args) > 2 and args[2] else None)
-                            kwargs["default"] = default
+                            kwargs["default"] = check_default_value(default)
                         if len(args):
                             vars[args[0]] = {
                                 "type": f"Array({args[1]})" if len(args) > 1 else "Array(String)",
@@ -1644,9 +1650,11 @@ def get_var_data(content, node_id=None):  # noqa: C901
                             # if this is a cast use the function name to get the type
                             if "default" not in kwargs:
                                 default = kwargs.get("default", args[1] if len(args) > 1 else None)
-                                kwargs["default"] = default
+                                kwargs["default"] = check_default_value(default)
                             try:
                                 vars[args[0]] = {"type": func, **kwargs}
+                                if "default" in kwargs:
+                                    kwargs["default"] = check_default_value(kwargs["default"])
                             except TypeError as e:
                                 logging.exception(f"pipe parsing problem {content} (node '{node_id}'):  {e}")
                 except Exception as e:
@@ -1665,6 +1673,13 @@ def get_var_data(content, node_id=None):  # noqa: C901
 
         return vars
 
+    def check_default_value(value):
+        if isinstance(value, int):
+            MAX_SAFE_INTEGER = 9007199254740991
+            if value > MAX_SAFE_INTEGER:
+                return str(value)
+        return value
+
     def parse_content(content, retries=0):
         try:
             parsed = ast.parse(content)
@@ -1744,6 +1759,8 @@ def get_var_names_and_types(t, node_id=None):  # noqa: C901
     [{'name': 'symbol_id', 'type': 'Int128', 'description': 'Symbol Id', 'required': True, 'default': 11111}, {'name': 'user_id', 'type': 'Int256', 'description': 'User Id', 'default': 3555}]
     >>> get_var_names_and_types(Template("SELECT now() > {{DateTime64(timestamp, '2020-09-09 10:10:10.000')}}"))
     [{'name': 'timestamp', 'type': 'DateTime64', 'default': '2020-09-09 10:10:10.000'}]
+    >>> get_var_names_and_types(Template("SELECT * FROM filter_value WHERE symbol = {{Int64(symbol_id, 9223372036854775807)}}"))
+    [{'name': 'symbol_id', 'type': 'Int64', 'default': '9223372036854775807'}]
     """
     try:
 

{tinybird-cli-5.9.1.dev1 → tinybird-cli-5.9.1.dev3}/tinybird/tb_cli_modules/common.py

@@ -1214,12 +1214,12 @@ def autocomplete_topics(ctx: Context, args, incomplete):
 
 
 def validate_datasource_name(name):
-    if not isinstance(name, str) or str == "":
+    if not isinstance(name, str) or name == "":
         raise CLIException(FeedbackManager.error_datasource_name())
 
 
 def validate_connection_id(connection_id):
-    if not isinstance(connection_id, str) or str == "":
+    if not isinstance(connection_id, str) or connection_id == "":
         raise CLIException(FeedbackManager.error_datasource_connection_id())
 
 
@@ -1235,7 +1235,7 @@ def validate_kafka_group(group):
 
 def validate_kafka_auto_offset_reset(auto_offset_reset):
     valid_values = {"latest", "earliest", "none"}
-    if not (auto_offset_reset in valid_values):
+    if auto_offset_reset not in valid_values:
         raise CLIException(FeedbackManager.error_kafka_auto_offset_reset())
 
 
@@ -1588,7 +1588,7 @@ async def try_update_config_with_remote(
 def ask_for_admin_token_interactively(ui_host: str, default_token: Optional[str]) -> str:
     return (
         click.prompt(
-            f"\nCopy the \"admin your@email\" token from {ui_host}/tokens and paste it here { f'OR press enter to use the token from .tinyb file' if default_token else ''}",
+            f"\nCopy the \"admin your@email\" token from {ui_host}/tokens and paste it here { 'OR press enter to use the token from .tinyb file' if default_token else ''}",
             hide_input=True,
             show_default=False,
             default=default_token,

{tinybird-cli-5.9.1.dev1 → tinybird-cli-5.9.1.dev3}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.9.1.dev1
+Version: 5.9.1.dev3
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,10 +18,16 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
-5.9.1.dev1
+5.9.1.dev3
+***********
+
+- `Fixed` Correctly parse lambda expressions in indexes
+
+5.9.1.dev2
 ***********
 
 - `Changed` Upgrade clickhouse-toolset to 0.32.dev0
+- `Added` new "File not found" error to `tb check` when including files from missing paths.
 
 5.9.0
 ***********