tinybird-cli 5.20.1.dev0__tar.gz → 5.20.1.dev1__tar.gz

{tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird_cli
-Version: 5.20.1.dev0
+Version: 5.20.1.dev1
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli
 Author: Tinybird
@@ -61,6 +61,11 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.20.1.dev1
+***********
+
+- `Added` When trying to push a materialized view with an unoptimized join, the CLI will now warn the user.
+
 5.19.0
 ***********
 
@@ -198,6 +203,7 @@ Changelog
 - `Added` `tb tag` commands to manage tags from the CLI.
 - `Added` support to `TAGS` in `tb fmt`.
 - `Added` support to `TAGS` in `tb pull` and `tb push`. Allows tagging resources for filtering in the UI.
+- `Changed` Improved message when creating a DynamoDB connection
 
 5.7.0
 **********

{tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev1}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '5.20.1.dev0'
-__revision__ = '4f9495f'
+__version__ = '5.20.1.dev1'
+__revision__ = '6ec3b53'

{tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev1}/tinybird/ch_utils/engine.py

@@ -146,10 +146,9 @@ class TableDetails:
     def is_replacing_engine(self) -> bool:
         if self.engine:
             engine_lower = self.engine.lower()
-            is_aggregating = "aggregatingmergetree" in engine_lower
             is_replacing = "replacingmergetree" in engine_lower
             is_collapsing = "collapsingmergetree" in engine_lower
-            return is_aggregating or is_replacing or is_collapsing
+            return is_replacing or is_collapsing
         return False
 
     def diff_ttl(self, new_ttl: str) -> bool:
@@ -169,6 +168,10 @@ class TableDetails:
     @property
     def sorting_key(self) -> Optional[str]:
         _sorting_key = self.details.get("sorting_key", None)
+        # TODO: This should use ENABLED_ENGINES to guess if the sorting key is required or not
+        # Also checking this and raising an error in a getter is a bit of an anti-pattern,
+        # a data source could have a "wrong" sorting key and we won't be able to even show it in the API.
+        # All these checks be performed only on creation time.
         if self.is_replacing_engine() and not _sorting_key:
             raise ValueError(f"SORTING_KEY must be defined for the {self.engine} engine")
         if self.is_mergetree_family():

{tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev1}/tinybird/datafile.py

@@ -1872,8 +1872,7 @@ def drop_token(url: str) -> str:
     parsed = urlparse(url)
     qs = parse_qs(parsed.query)
     qs_simplify = {k: v[0] for k, v in qs.items()}  # change several arguments to single one
-    if "token" in qs_simplify:
-        del qs_simplify["token"]
+    qs_simplify.pop("token", None)
     return f"{parsed.scheme}://{parsed.netloc}{parsed.path}?{urlencode(qs_simplify)}"
 
 
@@ -4849,7 +4848,7 @@ def format_include(file_parts: List[str], doc: Datafile, unroll_includes: bool =
     assert doc.raw is not None
 
     include = [line for line in doc.raw if "INCLUDE" in line and ".incl" in line]
-    if len(include):
+    if include:
         file_parts.append(include[0])
         file_parts.append(DATAFILE_NEW_LINE)
     return file_parts

{tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev1}/tinybird/feedback_manager.py

@@ -491,7 +491,7 @@ Ready? """
         """\n[2] Go to IAM > Policies. Create a new policy with the following permissions. Please, replace {replacements}:\n\n{access_policy}\n\n(The policy has been copied to your clipboard)\n\n"""
     )
     prompt_s3_iamrole_connection_policy_not_copied = prompt_message(
-        """\n[2] Go to IAM > Policies. Create a new policy with the following permissions. Please, copy this policy and replace <bucket> with your bucket name:\n\n{access_policy}\n\n"""
+        """\n[2] Go to IAM > Policies. Create a new policy with the following permissions. Please, copy this policy and replace <bucket> with your bucket name and <table_name> with your table name:\n\n{access_policy}\n\n"""
     )
     prompt_s3_iamrole_connection_role = prompt_message(
         """\n[3] Go to IAM > Roles. Create a new IAM Role using the following custom trust policy and attach the access policy you just created in the previous step:\n\n{trust_policy}\n\n(The policy has been copied to your clipboard)\n\n"""

{tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev1}/tinybird/sql_template.py

@@ -1657,7 +1657,7 @@ def get_var_data(content, node_id=None):
                         # It will be overriden in later definitions or left as is otherwise.
                         # args[0] check is used to avoid adding unnamed parameters found in
                         # templates like: `split_to_array('')`
-                        if len(args) and isinstance(args[0], list):
+                        if args and isinstance(args[0], list):
                             raise ValueError(f'"{args[0]}" can not be used as a variable name')
                         if len(args) > 0 and args[0] not in vars and args[0]:
                             vars[args[0]] = {
@@ -1669,7 +1669,7 @@ def get_var_data(content, node_id=None):
                         if "default" not in kwargs:
                             default = kwargs.get("default", args[2] if len(args) > 2 and args[2] else None)
                             kwargs["default"] = check_default_value(default)
-                        if len(args):
+                        if args:
                             if isinstance(args[0], list):
                                 raise ValueError(f'"{args[0]}" can not be used as a variable name')
                             vars[args[0]] = {
@@ -1678,7 +1678,7 @@ def get_var_data(content, node_id=None):
                             }
                     elif func in parameter_types:
                         # avoid variable names to be None
-                        if len(args) and args[0] is not None:
+                        if args and args[0] is not None:
                             # if this is a cast use the function name to get the type
                             if "default" not in kwargs:
                                 default = kwargs.get("default", args[1] if len(args) > 1 else None)

{tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev1}/tinybird/sql_toolset.py

@@ -1,5 +1,6 @@
 import copy
 import logging
+import re
 from collections import defaultdict
 from datetime import datetime
 from functools import lru_cache
@@ -42,10 +43,88 @@ class InvalidResource(ValueError):
         self.table = table
 
 
+class UnoptimizedJoinException(Exception):
+    def __init__(self, sql: str):
+        self.sql = sql
+        self.msg = f"Materialized node SQL contains a join that is not optimized: {sql}"
+        self.documentation = (
+            "/docs/work-with-data/optimization/opt201-fix-mistakes#understanding-the-materialized-join-issue"
+        )
+        super().__init__(self.msg)
+
+
+ChQueryTable = Tuple[Optional[str], Optional[str], Optional[str]]
+
+
+def get_left_table(sql: str, default_database: Optional[str] = None) -> ChQueryTable:
+    if default_database is None:
+        left_table = chquery.get_left_table(sql)
+    else:
+        left_table = chquery.get_left_table(sql, default_database=default_database)
+    return left_table
+
+
 def format_sql(sql: str) -> str:
     return chquery.format(sql)
 
 
+def explain_plan(sql: str) -> str:
+    return chquery.explain_ast(sql)
+
+
+def has_join(sql: str) -> bool:
+    return any(line.rstrip().startswith("TableJoin") for line in explain_plan(sql).split())
+
+
+def has_unoptimized_join(sql: str, left_table: Optional[Union[Tuple[str, str], Tuple[str, str, str]]] = None) -> None:
+    """
+    Check if a SQL query contains an unoptimized join.
+    A join is considered optimized if the right table is filtered by the left table's data.
+
+    Args:
+        sql: The SQL query to check
+        left_table: Optional tuple of (database, table) for the left table
+
+    Raises:
+        UnoptimizedJoin: If an unoptimized join is found
+    """
+    # TODO: We should check that we are filtering the right table by the left table's data
+    # TODO: We should check if using EXPLAIN AST is better than using regex
+
+    number_of_joins = sum(1 for line in explain_plan(sql).split() if line.rstrip().startswith("TableJoin"))
+    if number_of_joins == 0:
+        return
+
+    if not left_table:
+        left_table = chquery.get_left_table(sql)
+        if not left_table:
+            return
+
+    # Find all JOIN clauses with subqueries
+    # This pattern matches anything between JOIN and ON/USING
+    join_pattern = r"(?:LEFT\s+|RIGHT\s+|INNER\s+|FULL\s+OUTER\s+)?JOIN\s*\((.*?)\)\s+(?:AS\s+\w+)?\s*(?:ON|USING)"
+
+    # Find all joins with subqueries
+    join_matches = list(re.finditer(join_pattern, sql, re.IGNORECASE | re.DOTALL))
+
+    if number_of_joins != len(join_matches):
+        logging.debug(f"number_of_joins: {number_of_joins}, join_matches: {join_matches}")
+        raise UnoptimizedJoinException(sql)
+
+    # If no joins with subqueries found, probably is an unoptimized join
+    if not join_matches:
+        raise UnoptimizedJoinException(sql)
+
+    # Check if the left table is referenced in the subquery
+    left_table_ref = f"{left_table[0]}.{left_table[1]}"
+
+    for match in join_matches:
+        subquery = match.group(1)  # Get the captured subquery
+        logging.debug(f"subquery: {subquery} left_table_ref: {left_table_ref}")
+        if left_table_ref not in subquery:
+            raise UnoptimizedJoinException(sql)
+
+
 def format_where_for_mutation_command(where_clause: str) -> str:
     """
     >>> format_where_for_mutation_command("numnights = 99")

{tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev1}/tinybird/tb_cli_modules/connection.py

@@ -293,7 +293,7 @@ async def connection_create_snowflake(
     if not warehouse:
         warehouses = await client.get_snowflake_warehouses(account, username, password, role) or []
         warehouses_names = [w["name"] for w in warehouses]
-        default_warehouse = warehouses_names[0] if len(warehouses_names) else ""
+        default_warehouse = warehouses_names[0] if warehouses_names else ""
         warehouse = click.prompt(
             "Warehouse (optional)",
             type=click.types.Choice(warehouses_names, case_sensitive=False),

{tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev1}/tinybird/tb_cli_modules/fmt.py

@@ -83,7 +83,7 @@ async def fmt(
 
                 click.echo(FeedbackManager.success_generated_local_file(file=filename))
 
-    if len(failed):
+    if failed:
         click.echo(FeedbackManager.error_failed_to_format_files(number=len(failed)))
         for f in failed:
             click.echo(f"tb fmt {f} --yes")

{tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev1}/tinybird/tb_cli_modules/tinyunit/tinyunit.py

@@ -317,7 +317,7 @@ def test_run_summary(results: List[TestSummaryResults], only_fail: bool = False,
             if test.error:
                 click.secho(test.error, fg=test.status.color, bold=True, nl=True, err=True)
 
-    if len(total_counts):
+    if total_counts:
         click.echo("\nTotals:")
         for key_status, value_total in total_counts.items():
             code_summary = f"Total {key_status.description}: {value_total}"

{tinybird_cli-5.20.1.dev0 → tinybird_cli-5.20.1.dev1}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird_cli
-Version: 5.20.1.dev0
+Version: 5.20.1.dev1
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli
 Author: Tinybird
@@ -61,6 +61,11 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.20.1.dev1
+***********
+
+- `Added` When trying to push a materialized view with an unoptimized join, the CLI will now warn the user.
+
 5.19.0
 ***********
 
@@ -198,6 +203,7 @@ Changelog
 - `Added` `tb tag` commands to manage tags from the CLI.
 - `Added` support to `TAGS` in `tb fmt`.
 - `Added` support to `TAGS` in `tb pull` and `tb push`. Allows tagging resources for filtering in the UI.
+- `Changed` Improved message when creating a DynamoDB connection
 
 5.7.0
 **********