tinybird-cli 6.5.7.dev0__py3-none-any.whl → 6.5.8.dev0__py3-none-any.whl

tinybird/datafile_common.py

@@ -70,6 +70,7 @@ from tinybird.tb_cli_modules.common import (
 )
 from tinybird.tb_cli_modules.config import CLIConfig
 from tinybird.tb_cli_modules.exceptions import CLIGitReleaseException, CLIPipeException
+from tinybird.utils.bools import parse_bool, parse_optional_bool
 
 from .ch_utils.engine import ENABLED_ENGINES
 from .client import AuthException, AuthNoTokenException, CanNotBeDeletedException, DoesNotExistException, TinyB
@@ -130,6 +131,7 @@ class CopyParameters:
     TARGET_DATASOURCE = "target_datasource"
     COPY_SCHEDULE = "copy_schedule"
     COPY_MODE = "copy_mode"
+    ON_DEMAND_COMPUTE = "on_demand_compute"
 
 
 class CopyModes:
@@ -424,8 +426,7 @@ class CLIGitRelease:
     def is_dirty_to_release(self, use_include_dir: bool = True) -> bool:
         if use_include_dir:
             return any(self.repo.is_dirty(path=p) for p in self.paths) or self.has_untracked_files()
-        else:
-            return self.repo.is_dirty(path=self.path) or self.has_untracked_files()
+        return self.repo.is_dirty(path=self.path) or self.has_untracked_files()
 
     def is_dirty_to_init(self) -> bool:
         return self.is_dirty_to_release(use_include_dir=False) or self.is_head_empty()
@@ -474,8 +475,7 @@ class CLIGitRelease:
             if not self.is_head_outdated(current_release["commit"]):
                 if check_outdated:
                     raise CLIGitReleaseException(FeedbackManager.error_head_outdated(commit=self.head_commit()))
-                else:
-                    click.echo(FeedbackManager.warning_head_outdated(commit=self.head_commit()))
+                click.echo(FeedbackManager.warning_head_outdated(commit=self.head_commit()))
         except CLIGitReleaseException as e:
             if check_outdated:
                 raise e
@@ -533,8 +533,7 @@ class CLIGitRelease:
     async def update_release(
         self, tb_client: TinyB, current_ws: Dict[str, Any], commit: Optional[str] = None
     ) -> Dict[str, Any]:
-        release = await tb_client.workspace_commit_update(current_ws["id"], commit or self.head().commit.hexsha)
-        return release
+        return await tb_client.workspace_commit_update(current_ws["id"], commit or self.head().commit.hexsha)
 
 
 class Deployment:
@@ -851,6 +850,11 @@ def parse_pipe(
         sql = ""
         doc = parse(s, basepath=basepath, replace_includes=replace_includes, skip_eval=skip_eval)
         for node in doc.nodes:
+            if node.get("type", "").lower() == PipeNodeTypes.COPY and CopyParameters.ON_DEMAND_COMPUTE in node:
+                try:
+                    node[CopyParameters.ON_DEMAND_COMPUTE] = parse_optional_bool(node[CopyParameters.ON_DEMAND_COMPUTE])
+                except ValueError:
+                    raise click.ClickException("ON_DEMAND_COMPUTE must be true or false")
             sql = node.get("sql", "")
             source_sql = sql.strip()
             if not source_sql:
@@ -1229,6 +1233,7 @@ def parse(
         "copy_schedule": assign_node_var(CopyParameters.COPY_SCHEDULE),
         "copy_mode": assign_node_var("mode"),
         "mode": assign_node_var("mode"),
+        "on_demand_compute": assign_node_var(CopyParameters.ON_DEMAND_COMPUTE),
         "resource": assign_node_var("resource"),
         "filter": assign_node_var("filter"),
         "token": add_token,
@@ -1345,12 +1350,11 @@ def parse(
     except IndexError as e:
         if "node" in line.lower():
             raise click.ClickException(FeedbackManager.error_missing_node_name())
-        elif "sql" in line.lower():
+        if "sql" in line.lower():
             raise click.ClickException(FeedbackManager.error_missing_sql_command())
-        elif "datasource" in line.lower():
+        if "datasource" in line.lower():
             raise click.ClickException(FeedbackManager.error_missing_datasource_name())
-        else:
-            raise ValidationException(f"Validation error, found {line} in line {str(lineno)}: {str(e)}", lineno=lineno)
+        raise ValidationException(f"Validation error, found {line} in line {str(lineno)}: {str(e)}", lineno=lineno)
     except IncludeFileNotFoundException as e:
         raise IncludeFileNotFoundException(str(e), lineno=lineno)
     except Exception as e:
@@ -1671,7 +1675,7 @@ async def process_file(
 
         return resources
 
-    elif DataFileExtensions.PIPE in filename:
+    if DataFileExtensions.PIPE in filename:
         doc = parse_pipe(filename)
         version = f"__v{doc.version}" if doc.version is not None else ""
         name = os.path.basename(filename).split(".")[0]
@@ -1691,6 +1695,7 @@ async def process_file(
                 "target_datasource": node.get("target_datasource", None),
                 "copy_schedule": node.get(CopyParameters.COPY_SCHEDULE, None),
                 "mode": node.get("mode", CopyModes.APPEND),
+                "on_demand_compute": node.get(CopyParameters.ON_DEMAND_COMPUTE, None),
             }
 
             is_export_node = ExportReplacements.is_export_node(node)
@@ -1777,7 +1782,7 @@ async def process_file(
                 "filtering_tags": doc.filtering_tags,
             }
         ]
-    elif DataFileExtensions.TOKEN in filename:
+    if DataFileExtensions.TOKEN in filename:
         doc = parse_token(filename)
         name = os.path.basename(filename).split(DataFileExtensions.TOKEN)[0]
         description = doc.description or ""
@@ -1794,8 +1799,7 @@ async def process_file(
                 "description": description,
             }
         ]
-    else:
-        raise click.ClickException(FeedbackManager.error_file_extension(filename=filename))
+    raise click.ClickException(FeedbackManager.error_file_extension(filename=filename))
 
 
 def full_path_by_name(
@@ -1831,6 +1835,21 @@ def full_path_by_name(
     return None
 
 
+def local_copy_node_uses_on_demand_compute(folder: str, pipe_name: str) -> Optional[bool]:
+    """Reads ON_DEMAND_COMPUTE from the local datafile. Returns None when there is no parseable copy pipe to compare"""
+    pipe_path = next((f for f in get_project_filenames(folder) if f.endswith(f"/{pipe_name}.pipe")), None)
+    if pipe_path is None:
+        return None
+    try:
+        datafile = parse_pipe(pipe_path)
+    except Exception:
+        return None
+    copy_node = next((node for node in datafile.nodes if node.get("type", "").lower() == PipeNodeTypes.COPY), None)
+    if copy_node is None:
+        return None
+    return parse_optional_bool(copy_node.get(CopyParameters.ON_DEMAND_COMPUTE))
+
+
 def find_file_by_name(
     folder: str,
     name: str,
@@ -2045,13 +2064,12 @@ class PipeChecker(unittest.TestCase):
         headers = {"Authorization": f"Bearer {self.token}"}
         if self.http_method == "GET":
             return requests.get(pipe_url, headers=headers, verify=not getenv_bool("TB_DISABLE_SSL_CHECKS", False))
-        else:
-            return requests.post(
-                pipe_url,
-                headers=headers,
-                verify=not getenv_bool("TB_DISABLE_SSL_CHECKS", False),
-                data=self.pipe_request_params,
-            )
+        return requests.post(
+            pipe_url,
+            headers=headers,
+            verify=not getenv_bool("TB_DISABLE_SSL_CHECKS", False),
+            data=self.pipe_request_params,
+        )
 
     def _write_performance(self):
         return ""
@@ -2706,6 +2724,13 @@ async def check_copy_pipe(pipe, copy_node, tb_client: TinyB):
     if not is_valid_mode:
         raise CLIPipeException(FeedbackManager.error_creating_copy_pipe_invalid_mode(mode=mode))
 
+    on_demand_compute = copy_node["params"].get(CopyParameters.ON_DEMAND_COMPUTE)
+    if on_demand_compute not in (None, ""):
+        try:
+            parse_bool(on_demand_compute)
+        except ValueError:
+            raise CLIPipeException("ON_DEMAND_COMPUTE must be true or false")
+
     if not pipe:
         return
 
@@ -3150,6 +3175,7 @@ async def new_pipe(
                 schedule_cron_should_be_removed = current_schedule_cron and not schedule_cron
                 copy_params["schedule_cron"] = "None" if schedule_cron_should_be_removed else schedule_cron
                 copy_params["mode"] = mode
+                copy_params["on_demand_compute"] = parse_optional_bool(copy_node.get(CopyParameters.ON_DEMAND_COMPUTE))
                 await tb_client.pipe_update_copy(**copy_params)
             except Exception as e:
                 raise Exception(
@@ -3798,12 +3824,11 @@ def get_name_version(ds: str) -> Dict[str, Any]:
     tk = ds.rsplit("__", 2)
     if len(tk) == 1:
         return {"name": tk[0], "version": None}
-    elif len(tk) == 2:
+    if len(tk) == 2:
         if len(tk[1]):
             if tk[1][0] == "v" and _PATTERN_VERSION_NUMBER.match(tk[1][1:]):
                 return {"name": tk[0], "version": int(tk[1][1:])}
-            else:
-                return {"name": tk[0] + "__" + tk[1], "version": None}
+            return {"name": tk[0] + "__" + tk[1], "version": None}
     elif len(tk) == 3 and len(tk[2]):
         if tk[2] == "checker":
             return {"name": tk[0] + "__" + tk[1] + "__" + tk[2], "version": None}
@@ -4667,7 +4692,7 @@ async def folder_push(
                             fork,
                         )
                     else:
-                        # If we are trying to modify a Kafka or CDK datasource, we need to inform the user that the resource needs to be post-released
+                        # Connected datasources require post-release changes to avoid modifying the live connection.
                         kafka_connection_name = (
                             resources_to_run_fork_downstream[name].get("params", {}).get("kafka_connection_name")
                         )
@@ -5007,8 +5032,7 @@ async def format_datasource(
         format_import_settings(file_parts, doc.nodes[0])
         format_shared_with(file_parts, doc)
     result = "".join(file_parts)
-    result = result.rstrip("\n") + "\n"
-    return result
+    return result.rstrip("\n") + "\n"
 
 
 def format_version(file_parts: List[str], doc: Datafile) -> List[str]:
@@ -5120,17 +5144,16 @@ async def format_engine(
                     file_parts.append(DATAFILE_NEW_LINE)
             file_parts.append(DATAFILE_NEW_LINE)
         return file_parts
-    else:
-        if node.get("engine", None):
-            empty = '""'
-            file_parts.append(f"ENGINE {node['engine']['type']}" if node.get("engine", {}).get("type") else empty)
-            file_parts.append(DATAFILE_NEW_LINE)
-            for arg in sorted(node["engine"].get("args", [])):
-                elem = ", ".join([x.strip() for x in arg[1].split(",")])
-                file_parts.append(f"ENGINE_{arg[0].upper()} {elem or empty}")
-                file_parts.append(DATAFILE_NEW_LINE)
+    if node.get("engine", None):
+        empty = '""'
+        file_parts.append(f"ENGINE {node['engine']['type']}" if node.get("engine", {}).get("type") else empty)
+        file_parts.append(DATAFILE_NEW_LINE)
+        for arg in sorted(node["engine"].get("args", [])):
+            elem = ", ".join([x.strip() for x in arg[1].split(",")])
+            file_parts.append(f"ENGINE_{arg[0].upper()} {elem or empty}")
             file_parts.append(DATAFILE_NEW_LINE)
-        return file_parts
+        file_parts.append(DATAFILE_NEW_LINE)
+    return file_parts
 
 
 async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[str]:
@@ -5162,6 +5185,9 @@ async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[
         else:
             file_parts.append(DATAFILE_NEW_LINE)
             file_parts.append(f"{CopyParameters.COPY_SCHEDULE.upper()} {ON_DEMAND}")
+        if parse_optional_bool(node.get(CopyParameters.ON_DEMAND_COMPUTE)):
+            file_parts.append(DATAFILE_NEW_LINE)
+            file_parts.append("ON_DEMAND_COMPUTE true")
         file_parts.append(DATAFILE_NEW_LINE)
 
     # Sink or Stream pipe
@@ -5296,8 +5322,7 @@ async def format_pipe(
             file_parts.append(DATAFILE_NEW_LINE)
 
     result = "".join(file_parts)
-    result = result.rstrip("\n") + "\n"
-    return result
+    return result.rstrip("\n") + "\n"
 
 
 def format_sql(
@@ -5632,6 +5657,7 @@ async def diff_files(
             click.echo(f"Unsupported file type: {filename}")
         if lines:
             return [f"{l}\n" for l in lines.split("\n")] if with_format else lines  # noqa: E741
+        return None
 
     try:
         lines1 = await parse(from_file, with_format, resource_source="remote")
@@ -5641,7 +5667,7 @@ async def diff_files(
         raise click.ClickException(FeedbackManager.error_diff_file(filename=filename))
 
     if not lines1 or not lines2:
-        return
+        return None
 
     diff = difflib.unified_diff(
         lines1, lines2, fromfile=f"{Path(from_file).name} {from_file_suffix}", tofile=f"{to_file} {to_file_suffix}"
@@ -5661,10 +5687,7 @@ def is_materialized(resource: Optional[Dict[str, Any]]) -> bool:
     if not resource:
         return False
 
-    is_materialized = any(
-        node.get("params", {}).get("type", None) == "materialized" for node in resource.get("nodes", []) or []
-    )
-    return is_materialized
+    return any(node.get("params", {}).get("type", None) == "materialized" for node in resource.get("nodes", []) or [])
 
 
 def is_endpoint_with_no_dependencies(

tinybird/datatypes.py

@@ -120,52 +120,74 @@ def datetime_test(x: str) -> bool:
     return any(p.match(x) for p in datetime_patterns)
 
 
+def bounded_integer(x: str, pattern: re.Pattern[str]) -> Optional[int]:
+    if pattern.match(x) is None:
+        return None
+    digits = x.lstrip("+-").lstrip("0")
+    # 256-bit unsigned integers have at most 78 decimal digits.
+    if len(digits) > 78:
+        return None
+    return int(x) if len(x) <= 78 else int(digits or "0") * (-1 if x.startswith("-") else 1)
+
+
 def int_8_test(x: str) -> bool:
-    return intx_re.match(x) is not None and -int_8_max <= int(x) < int_8_max
+    value = bounded_integer(x, intx_re)
+    return value is not None and -int_8_max <= value < int_8_max
 
 
 def int16_test(x: str) -> bool:
-    return intx_re.match(x) is not None and -int16_max <= int(x) < int16_max
+    value = bounded_integer(x, intx_re)
+    return value is not None and -int16_max <= value < int16_max
 
 
 def int32_test(x: str) -> bool:
-    return intx_re.match(x) is not None and -int32_max <= int(x) < int32_max
+    value = bounded_integer(x, intx_re)
+    return value is not None and -int32_max <= value < int32_max
 
 
 def int64_test(x: str) -> bool:
-    return intx_re.match(x) is not None and -int64_max <= int(x) < int64_max
+    value = bounded_integer(x, intx_re)
+    return value is not None and -int64_max <= value < int64_max
 
 
 def int128_test(x: str) -> bool:
-    return intx_re.match(x) is not None and -int128_max <= int(x) < int128_max
+    value = bounded_integer(x, intx_re)
+    return value is not None and -int128_max <= value < int128_max
 
 
 def int256_test(x: str) -> bool:
-    return intx_re.match(x) is not None and -int256_max <= int(x) < int256_max
+    value = bounded_integer(x, intx_re)
+    return value is not None and -int256_max <= value < int256_max
 
 
 def uint_8_test(x: str) -> bool:
-    return uintx_re.match(x) is not None and 0 <= int(x) < uint_8_max
+    value = bounded_integer(x, uintx_re)
+    return value is not None and 0 <= value < uint_8_max
 
 
 def uint16_test(x: str) -> bool:
-    return uintx_re.match(x) is not None and 0 <= int(x) < uint16_max
+    value = bounded_integer(x, uintx_re)
+    return value is not None and 0 <= value < uint16_max
 
 
 def uint32_test(x: str) -> bool:
-    return uintx_re.match(x) is not None and 0 <= int(x) < uint32_max
+    value = bounded_integer(x, uintx_re)
+    return value is not None and 0 <= value < uint32_max
 
 
 def uint64_test(x: str) -> bool:
-    return uintx_re.match(x) is not None and 0 <= int(x) < uint64_max
+    value = bounded_integer(x, uintx_re)
+    return value is not None and 0 <= value < uint64_max
 
 
 def uint128_test(x: str) -> bool:
-    return intx_re.match(x) is not None and 0 <= int(x) < uint128_max
+    value = bounded_integer(x, intx_re)
+    return value is not None and 0 <= value < uint128_max
 
 
 def uint256_test(x: str) -> bool:
-    return intx_re.match(x) is not None and 0 <= int(x) < uint256_max
+    value = bounded_integer(x, intx_re)
+    return value is not None and 0 <= value < uint256_max
 
 
 def float_test(x: str) -> bool:

tinybird/feedback_manager.py

@@ -572,6 +572,11 @@ class FeedbackManager:
         "You can read more about the pricing at https://www.tinybird.co/docs/classic/work-with-data/process-and-copy/materialized-views#compute-compute-separation-for-populates\n\n"
         "Do you want to proceed with on-demand compute?"
     )
+    warning_confirm_on_demand_compute_differs_from_datafile = warning_message(
+        "The local datafile of pipe '{pipe}' does not set ON_DEMAND_COMPUTE true, but --on-demand-compute was passed.\n"
+        "On-demand compute uses dedicated resources that are billed separately from your regular usage.\n\n"
+        "Do you want to proceed?"
+    )
 
     warning_confirm_delete_token = prompt_message("Do you want to delete Token {token}?")
     warning_confirm_refresh_token = prompt_message("Do you want to refresh Token {token}?")

tinybird/git_settings.py

@@ -137,9 +137,7 @@ def bump_version(version: str, next_version: Optional[str]) -> str:
 
 
 def get_default_init_files_deploy(workspace_name: str) -> Dict[str, str]:
-    modified_files = {
+    return {
         path.replace("tinybird_", f"tinybird_{workspace_name.lower()}_"): content
         for path, content in DEFAULT_INIT_FILES_DEPLOY.items()
     }
-
-    return modified_files

tinybird/sql.py

@@ -129,15 +129,9 @@ class TableProjection:
     def add_projection_sql(self):
         return f"ADD {self.to_sql()}"
 
-    def drop_projection_sql(self):
-        return f"DROP PROJECTION IF EXISTS {self.name}"
-
     def materialize_projection_sql(self):
         return f"MATERIALIZE PROJECTION IF EXISTS {self.name}"
 
-    def clear_projection_sql(self):
-        return f"CLEAR PROJECTION IF EXISTS {self.name}"
-
 
 def as_subquery(sql: str) -> str:
     return f"""(\n{sql}\n)"""
@@ -675,10 +669,9 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
                     )
                 advance_single_char()
             return schema[begin:i]
-        else:
-            # backticked name
-            advance_single_char()
-            return get_backticked()
+        # backticked name
+        advance_single_char()
+        return get_backticked()
 
     def parse_expr(lookup: Iterable[SyntaxExpr]) -> str:
         nonlocal i, line, pos