tinybird-cli 5.9.1.dev0__tar.gz → 5.9.1.dev2__tar.gz

{tinybird-cli-5.9.1.dev0 → tinybird-cli-5.9.1.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.9.1.dev0
+Version: 5.9.1.dev2
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,6 +18,12 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.9.1.dev2
+***********
+
+- `Changed` Upgrade clickhouse-toolset to 0.32.dev0
+- `Added` new "File not found" error to `tb check` when including files from missing paths.
+
 5.9.0
 ***********
 

{tinybird-cli-5.9.1.dev0 → tinybird-cli-5.9.1.dev2}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '5.9.1.dev0'
-__revision__ = '9718298'
+__version__ = '5.9.1.dev2'
+__revision__ = '4f458c3'

{tinybird-cli-5.9.1.dev0 → tinybird-cli-5.9.1.dev2}/tinybird/ch_utils/constants.py

@@ -6,7 +6,7 @@ SNAPSHOT_WS_NAME = "snapshot"
 ENABLED_TABLE_FUNCTIONS = {"generateRandom", "null", "numbers", "numbers_mt", "values", "zeros", "zeros_mt"}
 # there's a workspace limit allowed_table_functions used in the APIs limit in cheriff
 COPY_ENABLED_TABLE_FUNCTIONS = frozenset(
-    ["postgresql", "mysql", "mongodb", "url", "azureBlobStorage", "gcs", "iceberg", "s3"]
+    ["postgresql", "mysql", "mongodb", "url", "azureBlobStorage", "gcs", "iceberg", "s3", "deltaLake"]
 )
 
 ENABLED_SYSTEM_TABLES = {

{tinybird-cli-5.9.1.dev0 → tinybird-cli-5.9.1.dev2}/tinybird/datafile.py

@@ -278,7 +278,9 @@ class ParseException(Exception):
 
 
 class IncludeFileNotFoundException(Exception):
-    pass
+    def __init__(self, err: str, lineno: int = -1):
+        self.lineno: int = lineno
+        super().__init__(err)
 
 
 class ValidationException(Exception):
@@ -307,6 +309,10 @@ def is_shared_datasource(ds_name: str) -> bool:
     return "." in ds_name
 
 
+def format_filename(filename: str, hide_folders: bool = False):
+    return os.path.basename(filename) if hide_folders else filename
+
+
 class CLIGitRelease:
     NO_DATAFILES_PATHS = ["vendor/", "tests/", "scripts/", ".diff_tmp/"]
     DATAFILES_SUFFIXES = [".datasource", ".pipe", ".incl", ".token"]
@@ -783,7 +789,11 @@ class Datafile:
 
 
 def parse_datasource(
-    filename: str, replace_includes: bool = True, content: Optional[str] = None, skip_eval: bool = False
+    filename: str,
+    replace_includes: bool = True,
+    content: Optional[str] = None,
+    skip_eval: bool = False,
+    hide_folders: bool = False,
 ) -> Datafile:
     basepath = ""
     if not content:
@@ -793,6 +803,7 @@ def parse_datasource(
     else:
         s = content
 
+    filename = format_filename(filename, hide_folders)
     try:
         doc = parse(s, "default", basepath, replace_includes=replace_includes, skip_eval=skip_eval)
     except ParseException as e:
@@ -807,7 +818,11 @@ def parse_datasource(
 
 
 def parse_pipe(
-    filename: str, replace_includes: bool = True, content: Optional[str] = None, skip_eval: bool = False
+    filename: str,
+    replace_includes: bool = True,
+    content: Optional[str] = None,
+    skip_eval: bool = False,
+    hide_folders: bool = False,
 ) -> Datafile:
     basepath = ""
     if not content:
@@ -817,6 +832,7 @@ def parse_pipe(
     else:
         s = content
 
+    filename = format_filename(filename, hide_folders)
     try:
         sql = ""
         doc = parse(s, basepath=basepath, replace_includes=replace_includes, skip_eval=skip_eval)
@@ -849,13 +865,19 @@ def parse_pipe(
         raise click.ClickException(
             FeedbackManager.error_parsing_node_with_unclosed_if(node=e.node, pipe=filename, lineno=e.lineno, sql=e.sql)
         )
+    except IncludeFileNotFoundException as e:
+        raise click.ClickException(FeedbackManager.error_not_found_include(filename=e, lineno=e.lineno))
     except ModuleNotFoundError:
         pass
     return doc
 
 
 def parse_token(
-    filename: str, replace_includes: bool = True, content: Optional[str] = None, skip_eval: bool = False
+    filename: str,
+    replace_includes: bool = True,
+    content: Optional[str] = None,
+    skip_eval: bool = False,
+    hide_folders: bool = False,
 ) -> Datafile:
     if not content:
         with open(filename) as file:
@@ -864,6 +886,7 @@ def parse_token(
     else:
         s = content
 
+    filename = format_filename(filename, hide_folders)
     try:
         sql = ""
         doc = parse(s, basepath=basepath, replace_includes=replace_includes, skip_eval=skip_eval)
@@ -1091,7 +1114,7 @@ def parse(
                     StringIO(Template(file.read()).safe_substitute(attrs), newline=None)
                 )
         except FileNotFoundError:
-            raise IncludeFileNotFoundException(f)
+            raise IncludeFileNotFoundException(f, lineno)
 
     def version(*args: str, **kwargs: Any) -> None:
         if len(args) < 1:
@@ -1279,7 +1302,7 @@ def parse(
         else:
             raise ValidationException(f"Validation error, found {line} in line {str(lineno)}: {str(e)}", lineno=lineno)
     except IncludeFileNotFoundException as e:
-        raise e
+        raise IncludeFileNotFoundException(str(e), lineno=lineno)
     except Exception as e:
         traceback.print_tb(e.__traceback__)
         raise ParseException(f"Unexpected error: {e}", lineno=lineno)
@@ -4165,6 +4188,7 @@ async def folder_push(
     check_backfill_required: bool = False,
     use_main: bool = False,
     check_outdated: bool = True,
+    hide_folders: bool = False,
 ):  # noqa: C901
     workspaces: List[Dict[str, Any]] = (await tb_client.user_workspaces_and_branches()).get("workspaces", [])
     current_ws: Dict[str, Any] = next(
@@ -4361,8 +4385,12 @@ async def folder_push(
                                 )
                             )
                     except Exception as e:
+                        filename = (
+                            os.path.basename(to_run[name]["filename"]) if hide_folders else to_run[name]["filename"]
+                        )
                         exception = FeedbackManager.error_push_file_exception(
-                            filename=to_run[name]["filename"], error=e
+                            filename=filename,
+                            error=e,
                         )
                         raise click.ClickException(exception)
                 else:

{tinybird-cli-5.9.1.dev0 → tinybird-cli-5.9.1.dev2}/tinybird/feedback_manager.py

@@ -242,6 +242,9 @@ class FeedbackManager:
     error_deleted_include = error_message(
         "Related include file {include_file} was deleted and it's used in {filename}. Delete or remove dependency from {filename}."
     )
+    error_not_found_include = error_message(
+        "Included file {filename} at line {lineno} not found. Check if the file exists and the path is correct."
+    )
     error_branch = error_message(
         "Branch {branch} not found. use 'tb branch ls' to list your Branches, make sure you are authenticated using the right workspace token"
     )

{tinybird-cli-5.9.1.dev0 → tinybird-cli-5.9.1.dev2}/tinybird/sql.py

@@ -1,3 +1,4 @@
+import logging
 import re
 import string
 from collections import namedtuple
@@ -401,10 +402,84 @@ def parse_table_structure(schema: str) -> List[Dict[str, Any]]:
     [{'name': 'a', 'type': 'String', 'codec': None, 'default_value': None, 'jsonpath': None, 'nullable': False, 'normalized_name': 'a'}]
     >>> parse_table_structure('`index` String, INDEX index_name a TYPE set(100, 1) GRANULARITY 100')
     [{'name': 'index', 'type': 'String', 'codec': None, 'default_value': None, 'jsonpath': None, 'nullable': False, 'normalized_name': 'index'}]
+    >>> parse_table_structure('`a2` String `json:$.a--2`, `a3` String `json:$.a3`\\n')
+    [{'name': 'a2', 'type': 'String', 'codec': None, 'default_value': None, 'jsonpath': '$.a--2', 'nullable': False, 'normalized_name': 'a2'}, {'name': 'a3', 'type': 'String', 'codec': None, 'default_value': None, 'jsonpath': '$.a3', 'nullable': False, 'normalized_name': 'a3'}]
     """
     return _parse_table_structure(schema)
 
 
+def clean_comments(schema_to_clean: str) -> str:
+    """Remove the comments from the schema
+    if the comments are between backticks, they will not be removed
+    >>> clean_comments(None) is None
+    True
+    >>> clean_comments('')
+    ''
+    >>> clean_comments('    ')
+    ''
+    >>> clean_comments('\\n')
+    ''
+    >>> clean_comments('\\n\\n\\n\\n')
+    ''
+    >>> clean_comments('c Float32')
+    'c Float32'
+    >>> clean_comments('c Float32\\n')
+    'c Float32'
+    >>> clean_comments('c Float32\\n--this is a comment')
+    'c Float32'
+    >>> clean_comments('c Float32\\n--this is a comment\\n')
+    'c Float32'
+    >>> clean_comments('c Float32\\t-- this is a comment\\t\\n')
+    'c Float32'
+    >>> clean_comments('c Float32\\n--this is a comment\\r\\n')
+    'c Float32'
+    >>> clean_comments('c Float32\\n--this is a comment\\n--this is a comment2\\n')
+    'c Float32'
+    >>> clean_comments('c Float32\\n--this is a ```comment\\n')
+    'c Float32'
+    >>> clean_comments('c Float32\\n--this is a ```comment\\n')
+    'c Float32'
+    >>> clean_comments('c Float32, -- comment\\nd Float32 -- comment2')
+    'c Float32,\\nd Float32'
+    >>> clean_comments('c Float32, -- comment\\n   -- comment \\nd Float32 -- comment2')
+    'c Float32,\\nd Float32'
+    >>> clean_comments('c Float32 `json:$.aa--aa`\\n--this is a ```comment\\n')
+    'c Float32 `json:$.aa--aa`'
+    >>> clean_comments('c Float32 `json:$.cc--cc`\\nd Float32 `json:$.dd--dd`\\n--this is a ```comment\\n')
+    'c Float32 `json:$.cc--cc`\\nd Float32 `json:$.dd--dd`'
+    >>> clean_comments('c--c Float32 `json:$.cc--cc`\\n')
+    'c'
+    >>> clean_comments('`c--c` Float32 `json:$.cc--cc`\\n')
+    '`c'
+    """
+
+    def clean_line_comments(line: str) -> str:
+        if not line:
+            return line
+        i = 0
+        inside_json_path = False
+        while i < len(line):
+            if i + 1 < len(line) and line[i] == "-" and line[i + 1] == "-" and not inside_json_path:
+                return line[:i].strip()
+
+            if not inside_json_path and line[i:].startswith("`json:"):
+                inside_json_path = True
+            elif inside_json_path and line[i] == "`":
+                inside_json_path = False
+            i += 1
+        return line
+
+    if schema_to_clean is None:
+        return schema_to_clean
+
+    cleaned_schema = ""
+    for line in schema_to_clean.splitlines():
+        cleaned_line = clean_line_comments(line)
+        if cleaned_line:
+            cleaned_schema += cleaned_line + "\n"
+    return cleaned_schema.strip()
+
+
 SyntaxExpr = namedtuple("SyntaxExpr", ["name", "regex"])
 
 NULL = SyntaxExpr("NULL", re.compile(r"\s+NULL([^a-z0-9_]|$)", re.IGNORECASE))
@@ -426,7 +501,11 @@ REGEX_COMMENT = re.compile(r"\-\-[^\n\r]*[\n\r]")
 def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:  # noqa: C901
     # CH syntax from https://clickhouse.com/docs/en/sql-reference/statements/create/table/
     # name1 [type1] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1]
-    schema = REGEX_COMMENT.sub(" ", schema + "\n").strip()
+    try:
+        schema = clean_comments(schema + "\n")
+    except Exception as e:
+        logging.exception(f"Error cleaning comments: {e}")
+        schema = REGEX_COMMENT.sub(" ", schema + "\n").strip()
 
     if REGEX_WHITESPACE.fullmatch(schema):
         return []

{tinybird-cli-5.9.1.dev0 → tinybird-cli-5.9.1.dev2}/tinybird/sql_template.py

@@ -750,6 +750,20 @@ def _parse_datetime(date_string, date_format, backup_date_format=None):
 
 
 def json_type(x, default=None):
+    """
+    >>> json_type(None, '[]')
+    []
+    >>> json_type(None)
+    {}
+    >>> json_type('{"a": 1}')
+    {'a': 1}
+    >>> json_type('[{"a": 1}]')
+    [{'a': 1}]
+    >>> json_type({"a": 1})
+    {'a': 1}
+    >>> json_type([{"a": 1}])
+    [{'a': 1}]
+    """
     if isinstance(x, Placeholder):
         if default:
             x = default
@@ -764,15 +778,14 @@ def json_type(x, default=None):
                 x = "{}"
 
         value = ""  # used for exception message
-        if isinstance(x, str):
+        if isinstance(x, (str, bytes, bytearray)):
             if len(x) > 16:
                 value = x[:16] + "..."
             else:
                 value = x
 
-        parsed = loads(x)
-        x = parsed
-
+            parsed = loads(x)
+            x = parsed
     except Exception as e:
         msg = f"Error parsing JSON: '{value}' - {str(e)}"
         raise SQLTemplateException(msg)

{tinybird-cli-5.9.1.dev0 → tinybird-cli-5.9.1.dev2}/tinybird/tb_cli_modules/branch.py

@@ -47,7 +47,6 @@ def release() -> None:
 @coro
 async def release_ls() -> None:
     """List current available Releases in the Workspace"""
-    click.echo(FeedbackManager.warning_deprecated_releases())
     config = CLIConfig.get_project_config()
     _ = await try_update_config_with_remote(config, only_if_needed=True)
 

{tinybird-cli-5.9.1.dev0 → tinybird-cli-5.9.1.dev2}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.9.1.dev0
+Version: 5.9.1.dev2
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,6 +18,12 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.9.1.dev2
+***********
+
+- `Changed` Upgrade clickhouse-toolset to 0.32.dev0
+- `Added` new "File not found" error to `tb check` when including files from missing paths.
+
 5.9.0
 ***********
 

{tinybird-cli-5.9.1.dev0 → tinybird-cli-5.9.1.dev2}/tinybird_cli.egg-info/requires.txt

@@ -1,5 +1,5 @@
 click<8.2,>=8.1.6
-clickhouse-toolset==0.31.dev1
+clickhouse-toolset==0.32.dev0
 colorama==0.4.6
 cryptography>=3.4.8
 croniter==1.3.8