tinybird 0.0.1.dev290__tar.gz → 0.0.1.dev291__tar.gz

{tinybird-0.0.1.dev290 → tinybird-0.0.1.dev291}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev290
+Version: 0.0.1.dev291
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev290 → tinybird-0.0.1.dev291}/tinybird/datafile/common.py

@@ -1870,7 +1870,7 @@ def parse(
 
         for entries in args:
             # In case they specify multiple workspaces, handle both line-separated and comma-separated values
-            lines = entries.splitlines()
+            lines = _unquote(entries).splitlines()
             for line in lines:
                 # Split by comma and strip whitespace from each workspace name
                 workspaces = [workspace.strip().rstrip(",") for workspace in line.split(",") if workspace.strip()]
@@ -1957,7 +1957,7 @@ def parse(
             "import_query": assign_var("import_query"),  # Deprecated, BQ and SFK
             "import_table_arn": assign_var("import_table_arn"),  # Only for DynamoDB
             "import_export_bucket": assign_var("import_export_bucket"),  # For DynamoDB
-            "shared_with": shared_with,  # Not supported yet
+            "shared_with": shared_with,
             "export_service": export_service,  # Deprecated
             "forward_query": sql("forward_query"),
             "backfill": assign_var("backfill", allowed_values={"skip"}),

{tinybird-0.0.1.dev290 → tinybird-0.0.1.dev291}/tinybird/sql.py

@@ -7,6 +7,9 @@ from typing import Any, Dict, Iterable, List, Optional
 
 valid_chars_name: str = string.ascii_letters + string.digits + "._`*<>+-'"
 valid_chars_fn: str = valid_chars_name + "[](),=!?:/ \n\t\r"
+# Use sets for O(1) membership checks in hot loops
+_VALID_CHARS_NAME_SET = set(valid_chars_name)
+_VALID_CHARS_FN_SET = set(valid_chars_fn)
 
 INDEX_WHITELIST = ["minmax", "set", "bloom_filter", "ngrambf_v1", "tokenbf_v1"]
 INDEX_SUPPORTED_TYPES = {
@@ -28,6 +31,15 @@ INDEX_SUPPORTED_TYPES = {
     "ngrambf_v1": ["String", "FixedString", "Map"],
 }
 
+# Precompiled regexes used across functions to avoid recompilation overhead
+_RE_FORMAT = re.compile(r"\s+format\s+(\w+)\s*$", re.IGNORECASE)
+_RE_REMOVE_FORMAT = re.compile(r"\s+(format)\s+(\w+)\s*$", re.IGNORECASE)
+_RE_TRY_FIX_NULLABLE_SAF = re.compile(r"SimpleAggregateFunction\((\w+),\s*(?!(?:Nullable))([\w,.()]+)\)")
+_RE_INDEX_ENTRY = re.compile(
+    r"(\w+)\s+([\w\s*\[\]\*\(\),\'\"-><.]+)\s+TYPE\s+(\w+)(?:\(([\w\s*.,]+)\))?(?:\s+GRANULARITY\s+(\d+))?"
+)
+_RE_REPLICATED_MT = re.compile(r"Replicated(.*)MergeTree\(([^\)]*)\)(.*)")
+
 
 @dataclass
 class TableIndex:
@@ -137,10 +149,9 @@ def get_format(sql: str) -> Optional[str]:
     >>> get_format('select * from test formAt JSON')
     'JSON'
     """
-    FORMAT_RE = r"\s+format\s+(\w+)\s*$"
     sql = sql.strip()
-    format = re.findall(FORMAT_RE, sql, re.IGNORECASE)
-    return format[0] if format else None
+    match = _RE_FORMAT.findall(sql)
+    return match[0] if match else None
 
 
 def get_format_group(sql: str) -> str:
@@ -151,10 +162,9 @@ def get_format_group(sql: str) -> str:
     >>> get_format_group('select * from test formAt JSON')
     ' formAt JSON'
     """
-    FORMAT_RE = r"\s+format\s+(\w+)\s*$"
     sql = sql.strip()
-    format = re.search(FORMAT_RE, sql, re.IGNORECASE)
-    return format.group() if format else ""
+    match = _RE_FORMAT.search(sql)
+    return match.group() if match else ""
 
 
 def wrap_finalize_aggregation(sql: str, describe_result: Dict[str, Any], fm_group: Optional[str] = None) -> str:
@@ -186,9 +196,8 @@ def remove_format(sql: str) -> str:
     >>> remove_format('select * from test formAt JSON')
     'select * from test'
     """
-    FORMAT_RE = r"\s+(format)\s+(\w+)\s*$"
     sql = sql.strip()
-    return re.sub(FORMAT_RE, "", sql, flags=re.IGNORECASE)
+    return _RE_REMOVE_FORMAT.sub("", sql)
 
 
 def col_name(name: str, backquotes: bool = True) -> str:
@@ -220,7 +229,7 @@ def try_to_fix_nullable_in_simple_aggregating_function(t: str) -> Optional[str]:
     # as it is done with other aggregate functions.
     # If not, the aggregation could return incorrect results.
     result = None
-    if match := re.search(r"SimpleAggregateFunction\((\w+),\s*(?!(?:Nullable))([\w,.()]+)\)", t):
+    if match := _RE_TRY_FIX_NULLABLE_SAF.search(t):
         fn = match.group(1)
         inner_type = match.group(2)
         result = f"SimpleAggregateFunction({fn}, Nullable({inner_type}))"
@@ -342,10 +351,7 @@ def parse_indexes_structure(indexes: Optional[List[str]]) -> List[TableIndex]:
         if index.count("TYPE") != 1:
             raise ValueError("invalid INDEX format. Usage: `name expr TYPE type_full GRANULARITY granularity`")
 
-        match = re.match(
-            r"(\w+)\s+([\w\s*\[\]\*\(\),\'\"-><.]+)\s+TYPE\s+(\w+)(?:\(([\w\s*.,]+)\))?(?:\s+GRANULARITY\s+(\d+))?",
-            index,
-        )
+        match = _RE_INDEX_ENTRY.match(index)
         if match:
             index_name, a, index_type, value, granularity = match.groups()
             index_expr = f"{index_type}({value})" if value else index_type
@@ -552,7 +558,7 @@ def clean_comments(schema_to_clean: str) -> str:
             if i + 1 < len(line) and line[i] == "-" and line[i + 1] == "-" and not inside_json_path:
                 return line[:i].strip()
 
-            if not inside_json_path and line[i:].startswith("`json:"):
+            if not inside_json_path and line.startswith("`json:", i):
                 inside_json_path = True
             elif inside_json_path and line[i] == "`":
                 inside_json_path = False
@@ -562,12 +568,12 @@ def clean_comments(schema_to_clean: str) -> str:
     if schema_to_clean is None:
         return schema_to_clean
 
-    cleaned_schema = ""
+    out_lines: List[str] = []
     for line in schema_to_clean.splitlines():
         cleaned_line = clean_line_comments(line)
         if cleaned_line:
-            cleaned_schema += cleaned_line + "\n"
-    return cleaned_schema.strip()
+            out_lines.append(cleaned_line)
+    return "\n".join(out_lines).strip()
 
 
 SyntaxExpr = namedtuple("SyntaxExpr", ["name", "regex"])
@@ -608,9 +614,9 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
 
     # Find the first SyntaxExpr in lookup that matches the schema at the current offset
     def lookahead_matches(lookup: Iterable) -> Optional[SyntaxExpr]:
+        # Use substring to preserve semantics of patterns anchored to end ($)
         s = schema[i:]
-        match = next((x for x in lookup if x.regex.match(s)), None)
-        return match
+        return next((x for x in lookup if x.regex.match(s)), None)
 
     def advance_single_char() -> None:
         nonlocal i, line, pos
@@ -647,7 +653,7 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
                 c = schema[i]
                 if c in " \t\r\n":
                     return schema[begin:i]
-                if c not in valid_chars_name:
+                if c not in _VALID_CHARS_NAME_SET:
                     raise ValueError(
                         format_parse_error(schema, i, pos, "wrong value, please check the schema syntax", line=line)
                     )
@@ -677,7 +683,9 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
                 context_stack.append("(")
             elif context is None and lookahead_matches(lookup):
                 return schema[begin:i].strip(" \t\r\n")
-            elif (context is None and c not in valid_chars_fn) or (context == "(" and c not in valid_chars_fn):
+            elif (context is None and c not in _VALID_CHARS_FN_SET) or (
+                context == "(" and c not in _VALID_CHARS_FN_SET
+            ):
                 raise ValueError(
                     format_parse_error(schema, i, pos, "wrong value, please check the schema syntax", line=line)
                 )
@@ -884,7 +892,7 @@ def engine_replicated_to_local(engine: str) -> str:
     if "Replicated" not in engine:
         return engine
 
-    return re.sub(r"Replicated(.*)MergeTree\(([^\)]*)\)(.*)", _replace, engine.strip())
+    return _RE_REPLICATED_MT.sub(_replace, engine.strip())
 
 
 def engine_patch_replicated_engine(engine: str, engine_full: Optional[str], new_table_name: str) -> Optional[str]:

{tinybird-0.0.1.dev290 → tinybird-0.0.1.dev291}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev290'
-__revision__ = '4d1cca4'
+__version__ = '0.0.1.dev291'
+__revision__ = '87da698'

{tinybird-0.0.1.dev290 → tinybird-0.0.1.dev291}/tinybird/tb/modules/cli.py

@@ -212,16 +212,39 @@ def cli(
 
 
 @cli.command(hidden=True)
+@click.option("--only-vendored", is_flag=True, default=False, help="Only update vendored files")
 @click.option("-f", "--force", is_flag=True, default=False, help="Override existing files")
 @click.option("--fmt", is_flag=True, default=False, help="Format files before saving")
 @click.pass_context
-def pull(ctx: Context, force: bool, fmt: bool) -> None:
+def pull(ctx: Context, only_vendored: bool, force: bool, fmt: bool) -> None:
     """Retrieve latest versions for project files from Tinybird."""
 
     client = ctx.ensure_object(dict)["client"]
     project = ctx.ensure_object(dict)["project"]
 
-    return folder_pull(client, project.path, force, fmt=fmt)
+    if only_vendored:
+        force = True
+
+    written_files = folder_pull(client, project.path, force, only_vendored=only_vendored, fmt=fmt)
+
+    if only_vendored:
+        for_user_to_delete = set(project.get_vendored_files()) - set(written_files)
+        if for_user_to_delete:
+            # TODO(eclbg): this prints the full path of the files. Let's print the relative path from the project root
+            display_paths = []
+            for full_path in for_user_to_delete:
+                try:
+                    display_paths.append(str(Path(full_path).relative_to(project.path)))
+                except:
+                    display_paths.append(full_path)
+            click.echo(
+                FeedbackManager.warning(
+                    message=(
+                        f"This workspace no longer has access to the following files: {display_paths}. "
+                        "Please remove them manually to be able to deploy."
+                    )
+                )
+            )
 
 
 @cli.command()
@@ -392,7 +415,7 @@ def create_ctx_client(
     if not command or command in commands_without_ctx_client:
         return None
 
-    commands_always_cloud = ["pull", "infra"]
+    commands_always_cloud = ["infra"]
     commands_always_local = ["build", "dev"]
     command_always_test = ["test"]
 

{tinybird-0.0.1.dev290 → tinybird-0.0.1.dev291}/tinybird/tb/modules/datafile/diff.py

@@ -121,12 +121,12 @@ def diff_command(
     if filenames:
         if len(filenames) == 1:
             filenames = [filenames[0], *get_project_filenames(filenames[0])]
-        folder_pull(client, target_dir, True, verbose=False)
+        folder_pull(client, target_dir, True, only_vendored=False, verbose=False)
     else:
         filenames = get_project_filenames(".")
         if verbose:
             click.echo("Saving remote resources in .diff_tmp folder.\n")
-        folder_pull(client, target_dir, True, verbose=verbose, progress_bar=progress_bar)
+        folder_pull(client, target_dir, True, verbose=verbose, only_vendored=False, progress_bar=progress_bar)
 
     remote_datasources: List[Dict[str, Any]] = client.datasources()
     remote_pipes: List[Dict[str, Any]] = client.pipes()

{tinybird-0.0.1.dev290 → tinybird-0.0.1.dev291}/tinybird/tb/modules/datafile/pull.py

@@ -13,10 +13,11 @@ def folder_pull(
     client: TinyB,
     folder: str,
     force: bool,
+    only_vendored: bool,
     verbose: bool = True,
     progress_bar: bool = False,
     fmt: bool = False,
-):
+) -> list[str]:
     def get_file_folder(extension: str, resource_type: Optional[str]):
         if extension == "datasource":
             return "datasources"
@@ -40,8 +41,8 @@ def folder_pull(
         get_resource_function: str,
         progress_bar: bool = False,
         fmt: bool = False,
-    ):
-        def write_resource(k: dict[str, Any]):
+    ) -> list[Path]:
+        def write_resource(k: dict[str, Any]) -> Optional[Path]:
             name = f"{k['name']}.{extension}"
             try:
                 resource = getattr(client, get_resource_function)(k["name"])
@@ -72,59 +73,71 @@ def folder_pull(
                     with open(f, "w") as fd:
                         if resource_to_write:
                             fd.write(resource_to_write)
+                            return f
                 else:
                     if verbose:
                         click.echo(FeedbackManager.info_skip_already_exists())
+                return None
             except Exception as e:
                 raise click.ClickException(FeedbackManager.error_exception(error=e))
 
         if progress_bar:
+            written_files = []
             with click.progressbar(resources, label=f"Pulling {extension}s") as resources:  # type: ignore
                 for k in resources:
-                    write_resource(k)
+                    written_file = write_resource(k)
+                    if written_file:
+                        written_files.append(written_file)
         else:
-            tasks = [write_resource(k) for k in resources]
-            _gather_with_concurrency(5, *tasks)
+            written_files = list(filter(None, (write_resource(k) for k in resources)))
+
+        return written_files
 
     try:
         datasources = client.datasources()
         pipes = client.pipes()
         connections = client.connections()
 
-        write_files(
-            resources=datasources,
-            extension="datasource",
-            get_resource_function="datasource_file",
-            progress_bar=progress_bar,
-            fmt=fmt,
-        )
-        write_files(
-            resources=pipes,
-            extension="pipe",
-            get_resource_function="pipe_file",
-            progress_bar=progress_bar,
-            fmt=fmt,
-        )
-        write_files(
-            resources=connections,
-            extension="connection",
-            get_resource_function="connection_file",
-            progress_bar=progress_bar,
-            fmt=fmt,
-        )
-        return
+        if only_vendored:
+            written_files = write_files(
+                resources=[ds for ds in datasources if "shared_from" in ds],
+                extension="datasource",
+                get_resource_function="datasource_file",
+                progress_bar=progress_bar,
+                fmt=fmt,
+            )
+        else:
+            written_files = []
+            written_files.extend(
+                write_files(
+                    resources=datasources,
+                    extension="datasource",
+                    get_resource_function="datasource_file",
+                    progress_bar=progress_bar,
+                    fmt=fmt,
+                )
+            )
+            written_files.extend(
+                write_files(
+                    resources=pipes,
+                    extension="pipe",
+                    get_resource_function="pipe_file",
+                    progress_bar=progress_bar,
+                    fmt=fmt,
+                )
+            )
+            written_files.extend(
+                write_files(
+                    resources=connections,
+                    extension="connection",
+                    get_resource_function="connection_file",
+                    progress_bar=progress_bar,
+                    fmt=fmt,
+                )
+            )
+        return [str(wf) for wf in written_files]
 
     except AuthNoTokenException:
         raise
     except Exception as e:
         raise click.ClickException(FeedbackManager.error_pull(error=str(e)))
-
-
-def _gather_with_concurrency(n, *tasks):
-    results = []
-    for task in tasks:
-        if callable(task):
-            results.append(task())
-        else:
-            results.append(task)
-    return results

{tinybird-0.0.1.dev290 → tinybird-0.0.1.dev291}/tinybird/tb/modules/deployment_common.py

@@ -222,6 +222,7 @@ def create_deployment(
     # TODO: This code is duplicated in build_server.py
     # Should be refactored to be shared
     MULTIPART_BOUNDARY_DATA_PROJECT = "data_project://"
+    MULTIPART_BOUNDARY_DATA_PROJECT_VENDORED = "data_project_vendored://"
     DATAFILE_TYPE_TO_CONTENT_TYPE = {
         ".datasource": "text/plain",
         ".pipe": "text/plain",
@@ -248,6 +249,13 @@ def create_deployment(
         with open(file_path, "rb") as fd:
             content_type = DATAFILE_TYPE_TO_CONTENT_TYPE.get(Path(file_path).suffix, "application/unknown")
             files.append((MULTIPART_BOUNDARY_DATA_PROJECT, (relative_path, fd.read().decode("utf-8"), content_type)))
+    for file_path in project.get_vendored_files():
+        relative_path = Path(file_path).relative_to(project.path).as_posix()
+        with open(file_path, "rb") as fd:
+            content_type = DATAFILE_TYPE_TO_CONTENT_TYPE.get(Path(file_path).suffix, "application/unknown")
+            files.append(
+                (MULTIPART_BOUNDARY_DATA_PROJECT_VENDORED, (relative_path, fd.read().decode("utf-8"), content_type))
+            )
 
     deployment = None
     try:

{tinybird-0.0.1.dev290 → tinybird-0.0.1.dev291}/tinybird/tb/modules/project.py

@@ -37,11 +37,13 @@ class Project:
     def has_deeper_level(self) -> bool:
         """Check if there are folders with depth greater than max_depth in project path.
 
+        Does not consider the vendor directory.
+
         Returns:
             bool: True if there are folders deeper than max_depth, False otherwise
         """
         for obj in glob.glob(f"{self.path}{'/*' * (self.max_depth - 1)}/*", recursive=False):
-            if Path(obj).is_dir():
+            if Path(obj).is_dir() and self.vendor_path not in obj:
                 return True
         return False
 
@@ -54,6 +56,13 @@ class Project:
                 project_files.append(project_file)
         return project_files
 
+    def get_vendored_files(self) -> List[str]:
+        vendored_files: List[str] = []
+        for extension in self.extensions:
+            for level in range(3):
+                vendored_files.extend(glob.glob(f"{self.vendor_path}{'/*' * level}/*.{extension}", recursive=True))
+        return vendored_files
+
     def get_fixture_files(self) -> List[str]:
         fixture_files: List[str] = []
         for extension in [

{tinybird-0.0.1.dev290 → tinybird-0.0.1.dev291}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev290
+Version: 0.0.1.dev291
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird