tinybird-cli 6.5.3.dev0__py3-none-any.whl → 6.5.5__py3-none-any.whl

tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '6.5.3.dev0'
-__revision__ = '78e5916'
+__version__ = '6.5.5'
+__revision__ = '5a6cd1d'

tinybird/ch_utils/engine.py

@@ -121,6 +121,12 @@ class TableDetails:
     def is_mergetree_family(self) -> bool:
         return self.engine is not None and "mergetree" in self.engine.lower()
 
+    def is_view(self) -> bool:
+        return self.original_engine == "View"
+
+    def is_merge_final_engine(self) -> bool:
+        return self.original_engine == "MergeFinal"
+
     def supports_alter_add_column(self) -> bool:
         return self.is_mergetree_family() or (self.engine is not None and self.engine.lower() == "null")
 
@@ -354,31 +360,6 @@ def sorting_key_is_valid(columns: List[Dict[str, Any]], value: Optional[str]) ->
     return value
 
 
-def case_insensitive_check(valid_values: List[str]) -> Callable[[List[Dict[str, Any]], str], Optional[str]]:
-    """
-    >>> valid_values = ['ANY', 'ALL']
-    >>> checker = case_insensitive_check(valid_values)
-    >>> checker([],'ALL')
-
-    >>> valid_values = ['ANY', 'ALL']
-    >>> checker = case_insensitive_check(valid_values)
-    >>> checker([],'any')
-
-    >>> valid_values = ['ANY', 'ALL']
-    >>> checker = case_insensitive_check(valid_values)
-    >>> checker([],'foo')
-    Traceback (most recent call last):
-    ...
-    ValueError: valid values are ANY, ALL
-    """
-
-    def checker(columns: List[Dict[str, Any]], value: str):
-        if value.upper() not in valid_values:
-            raise ValueError(f"valid values are {', '.join(valid_values)}")
-
-    return checker
-
-
 # [PARTITION BY expr]
 # [ORDER BY expr]
 # [PRIMARY KEY expr]

tinybird/client.py

@@ -441,6 +441,7 @@ class TinyB:
         sql_condition: Optional[str] = None,
         format: str = "csv",
         replace_options: Optional[Set[str]] = None,
+        use_v1: bool = False,
     ):
         params = {"name": datasource_name, "mode": mode, "format": format, "debug": "blocks_block_log"}
 
@@ -452,28 +453,67 @@ class TinyB:
 
         async with aiofiles.open(file, "rb") as content:
             file_content = await content.read()
-            if format == "csv":
-                files = {"csv": ("csv", file_content)}
-            else:
-                files = {"ndjson": ("ndjson", file_content)}
 
-            res = await self._req(
-                f"v0/datasources?{urlencode(params, safe='')}",
-                files=files,
+        if use_v1:
+            v1_params = {"format": format}
+            if sql_condition:
+                v1_params["replace_condition"] = sql_condition
+            if replace_options:
+                for option in list(replace_options):
+                    v1_params[option] = "true"
+            content_types = {
+                "csv": "text/csv",
+                "ndjson": "application/x-ndjson",
+                "parquet": "application/vnd.apache.parquet",
+            }
+            headers = {"Content-Type": content_types[format]}
+            if str(file).endswith(".gz"):
+                headers["Content-Encoding"] = "gzip"
+            return await self._req(
+                f"/v1/datasources/{quote(datasource_name, safe='')}/{mode}?{urlencode(v1_params, safe='')}",
+                data=file_content,
+                headers=headers,
                 method="POST",
             )
-            if status_callback:
-                status_callback(res)
 
-            return res
+        if format == "csv":
+            files = {"csv": ("csv", file_content)}
+        else:
+            files = {"ndjson": ("ndjson", file_content)}
+
+        res = await self._req(
+            f"v0/datasources?{urlencode(params, safe='')}",
+            files=files,
+            method="POST",
+        )
+        if status_callback:
+            status_callback(res)
+
+        return res
 
     async def datasource_truncate(self, datasource_name: str):
         return await self._req(f"/v0/datasources/{datasource_name}/truncate", method="POST", data="")
 
-    async def datasource_delete_rows(self, datasource_name: str, delete_condition: str, dry_run: bool = False):
-        params = {"delete_condition": delete_condition}
+    async def datasource_delete_rows(
+        self,
+        datasource_name: str,
+        delete_condition: str,
+        dry_run: bool = False,
+        lightweight: bool = False,
+        wait: bool = True,
+        partition: Optional[str] = None,
+        projection_mode: Optional[str] = None,
+    ):
+        params: Dict[str, Any] = {"delete_condition": delete_condition}
+        if lightweight:
+            params["wait"] = "true" if wait else "false"
+            if partition:
+                params["partition"] = partition
+            if projection_mode:
+                params["projection_mode"] = projection_mode
+            return await self._req(f"/v1/datasources/{datasource_name}/delete", method="POST", data=params)
         if dry_run:
-            params.update({"dry_run": "true"})
+            params["dry_run"] = "true"
         return await self._req(f"/v0/datasources/{datasource_name}/delete", method="POST", data=params)
 
     async def datasource_dependencies(

tinybird/config.py

@@ -112,12 +112,6 @@ async def get_config(
     return config
 
 
-async def write_config(config: Dict[str, Any], dest_file: str = ".tinyb"):
-    config_file = Path(getcwd()) / dest_file
-    async with aiofiles.open(config_file, "w") as file:
-        await file.write(json.dumps(config, indent=4, sort_keys=True))
-
-
 def get_display_host(ui_host: str):
     return LEGACY_HOSTS.get(ui_host, ui_host)
 

tinybird/datafile_common.py

@@ -9,7 +9,7 @@ import aiofiles
 from requests import Response
 
 try:
-    from colorama import Back, Fore, Style, init
+    from colorama import Fore, init
 
     init()
 except ImportError:  # fallback so that the imported classes always exist
@@ -18,7 +18,7 @@ except ImportError:  # fallback so that the imported classes always exist
         def __getattr__(self, name):
             return ""
 
-    Fore = Back = Style = ColorFallback()
+    Fore = ColorFallback()
 
 import difflib
 import glob
@@ -94,8 +94,6 @@ INTERNAL_TABLES: Tuple[str, ...] = (
     "kafka_ops_log",
     "datasources_storage",
     "endpoint_errors",
-    "bi_stats_rt",
-    "bi_stats",
 )
 
 
@@ -306,10 +304,10 @@ class ValidationException(Exception):
         super().__init__(err)
 
 
-def sizeof_fmt(num: Union[int, float], suffix: str = "b") -> str:
+def sizeof_fmt(num: float, suffix: str = "b") -> str:
     """Readable file size
     :param num: Bytes value
-    :type num: int
+    :type num: float
     :param suffix: Unit suffix (optionnal) default = o
     :type suffix: str
     :rtype: str
@@ -854,9 +852,24 @@ def parse_pipe(
         doc = parse(s, basepath=basepath, replace_includes=replace_includes, skip_eval=skip_eval)
         for node in doc.nodes:
             sql = node.get("sql", "")
-            if sql.strip()[0] == "%":
-                sql, _, variable_warnings = render_sql_template(sql[1:], test_mode=True, name=node["name"])
+            source_sql = sql.strip()
+            if not source_sql:
+                raise click.ClickException(
+                    FeedbackManager.error_parsing_node(node=node["name"], pipe=filename, error="Empty query")
+                )
+            if source_sql.startswith("%"):
+                sql, _, variable_warnings = render_sql_template(source_sql[1:], test_mode=True, name=node["name"])
                 doc.warnings = variable_warnings
+                if not sql.strip():
+                    raise click.ClickException(
+                        FeedbackManager.error_parsing_node(
+                            node=node["name"],
+                            pipe=filename,
+                            error="Template renders to an empty query without parameters",
+                        )
+                    )
+            else:
+                sql = source_sql
             # it'll fail with a ModuleNotFoundError when the toolset is not available but it returns the parsed doc
             from tinybird.sql_toolset import format_sql as toolset_format_sql
 
@@ -868,13 +881,15 @@ def parse_pipe(
             )
         )
     except ValueError as e:
-        t, template_variables, _ = get_template_and_variables(sql, name=node["name"])
+        source_sql = node.get("sql", "")
+        source_sql_stripped = source_sql.strip()
+        t, template_variables, _ = get_template_and_variables(source_sql, name=node["name"])
 
-        if sql.strip()[0] != "%" and len(template_variables) > 0:
+        if not source_sql_stripped.startswith("%") and len(template_variables) > 0:
             raise click.ClickException(FeedbackManager.error_template_start(filename=filename))
         raise click.ClickException(
             FeedbackManager.error_parsing_file(
-                filename=filename, lineno="", error=f"{str(e)} + SQL(value error): {sql}"
+                filename=filename, lineno="", error=f"{str(e)} + SQL(value error): {source_sql}"
             )
         )
     except UnClosedIfError as e:
@@ -1028,10 +1043,13 @@ def parse(
 
         parser_state.current_node["indexes"] = indexes
 
-    def assign_var(v: str) -> Callable[[VarArg(str), KwArg(Any)], None]:
+    def assign_var(v: str, lowercase: bool = False) -> Callable[[VarArg(str), KwArg(Any)], None]:
         def _f(*args: str, **kwargs: Any):
             s = _unquote((" ".join(args)).strip())
-            parser_state.current_node[v.lower()] = eval_var(s, skip=skip_eval)
+            val = eval_var(s, skip=skip_eval)
+            if lowercase:
+                val = val.lower()
+            parser_state.current_node[v.lower()] = val
 
         return _f
 
@@ -1246,7 +1264,9 @@ def parse(
         "import_external_datasource": assign_var("import_external_datasource"),
         "import_bucket_uri": assign_var("import_bucket_uri"),
         "import_from_timestamp": assign_var("import_from_timestamp"),
-        "import_format": assign_var("import_format"),
+        # Lowercase: format checks and executor routing downstream expect
+        # lowercase values (ANALYTICS-A8M)
+        "import_format": assign_var("import_format", lowercase=True),
         "import_query": assign_var("import_query"),
         "import_table_arn": assign_var("import_table_arn"),
         "import_export_bucket": assign_var("import_export_bucket"),
@@ -1750,7 +1770,7 @@ async def process_file(
                 "filename": filename,
                 "name": name + version,
                 "nodes": nodes,
-                "deps": [x for x in set(deps)],
+                "deps": list(set(deps)),
                 "tokens": doc.tokens,
                 "description": description,
                 "warnings": doc.warnings,
@@ -2082,12 +2102,12 @@ class PipeChecker(unittest.TestCase):
 
         if self.ignore_order:
             current_data = (
-                sorted(normalize_array(current_data), key=itemgetter(*[k for k in current_data[0].keys()]))
+                sorted(normalize_array(current_data), key=itemgetter(*list(current_data[0].keys())))
                 if len(current_data) > 0
                 else current_data
             )
             checker_data = (
-                sorted(normalize_array(checker_data), key=itemgetter(*[k for k in checker_data[0].keys()]))
+                sorted(normalize_array(checker_data), key=itemgetter(*list(checker_data[0].keys())))
                 if len(checker_data) > 0
                 else checker_data
             )
@@ -2202,6 +2222,8 @@ class PipeCheckerRunner:
                         FROM {pipe_stats_rt}
                         WHERE
                             pipe_name = '{self.pipe_name}'
+                            -- pipe_stats_rt retains 3 months; recent requests are enough for checks
+                            AND start_datetime >= now() - INTERVAL 7 DAY
                             AND url IS NOT NULL
                             AND extractURLParameter(assumeNotNull(url), 'from') <> 'ui'
                             AND extractURLParameter(assumeNotNull(url), 'pipe_checker') <> 'true'
@@ -2229,6 +2251,8 @@ class PipeCheckerRunner:
                                     FROM {pipe_stats_rt}
                                     WHERE
                                         pipe_name = '{self.pipe_name}'
+                                        -- pipe_stats_rt retains 3 months; recent requests are enough for checks
+                                        AND start_datetime >= now() - INTERVAL 7 DAY
                                         AND url IS NOT NULL
                                         AND extractURLParameter(assumeNotNull(url), 'from') <> 'ui'
                                         AND extractURLParameter(assumeNotNull(url), 'pipe_checker') <> 'true'
@@ -4141,7 +4165,7 @@ async def build_graph(
                     elif len(warnings) > 1:
                         click.echo(
                             FeedbackManager.warning_pipe_restricted_params(
-                                words=", ".join(["'{}'".format(param) for param in warnings[:-1]]),
+                                words=", ".join([f"'{param}'" for param in warnings[:-1]]),
                                 last_word=warnings[-1],
                             )
                         )
@@ -4328,7 +4352,7 @@ async def folder_push(
 
     workspace_lib_paths = list(workspace_lib_paths)
     # include vendor libs without overriding user ones
-    existing_workspaces = set(x[1] for x in workspace_lib_paths)
+    existing_workspaces = {x[1] for x in workspace_lib_paths}
     vendor_path = Path("vendor")
     if vendor_path.exists():
         for x in vendor_path.iterdir():
@@ -4566,7 +4590,7 @@ async def folder_push(
 
         if not fork_downstream:
             # First, we will deploy the all the resources following the dependency graph except for the endpoints
-            groups = [group for group in toposort(dependencies_graph)]
+            groups = list(toposort(dependencies_graph))
             for group in groups:
                 for name in group:
                     if name in processed:
@@ -4592,7 +4616,7 @@ async def folder_push(
                     processed.add(name)
 
             # Then, we will deploy the endpoints that are on the dependency graph
-            groups = [group for group in toposort(endpoints_dep_map)]
+            groups = list(toposort(endpoints_dep_map))
             for group in groups:
                 for name in group:
                     if name not in processed:
@@ -4621,7 +4645,7 @@ async def folder_push(
 
             # First, we will deploy the datasources that need to be deployed.
             # We need to deploy the datasources from left to right as some datasources might have MV that depend on the column types of previous datasources. Ex: `test_change_column_type_landing_datasource` test
-            groups = [group for group in toposort(dependencies_graph_fork_downstream)]
+            groups = list(toposort(dependencies_graph_fork_downstream))
             groups.reverse()
             for group in groups:
                 for name in group:
@@ -4675,7 +4699,7 @@ async def folder_push(
             # If we have ENDPOINT_A ----> MV_PIPE_B -----> DATASOURCE_B ------> ENDPOINT_C
             # Where endpoint A is being used in the MV_PIPE_B, if we only modify the endpoint A
             # The dependencies graph will only contain the endpoint A and the MV_PIPE_B, but not the DATASOURCE_B and the ENDPOINT_C
-            groups = [group for group in toposort(dependencies_graph_fork_downstream)]
+            groups = list(toposort(dependencies_graph_fork_downstream))
             for group in groups:
                 for name in group:
                     if name in processed or not is_endpoint(resources_to_run_fork_downstream[name]):
@@ -4684,7 +4708,7 @@ async def folder_push(
                     endpoints_dep_map[name] = dependencies_graph_fork_downstream[name]
 
             # Now that we have the dependencies of the endpoints, we need to check that the resources has not been deployed yet and only care about the endpoints that depend on endpoints
-            groups = [group for group in toposort(endpoints_dep_map)]
+            groups = list(toposort(endpoints_dep_map))
 
             # As we have used the forkdownstream graph to get the dependencies of the endpoints, we have all the dependencies of the endpoints
             # But we need to deploy the endpoints and the dependencies of the endpoints from left to right
@@ -4709,7 +4733,7 @@ async def folder_push(
             # Now we should have the endpoints and datasources deployed, we can deploy the rest of the pipes (copy & sinks)
             # We need to rely on the forkdownstream graph as it contains all the modified pipes as well as the dependencies of the pipes
             # In this case, we don't need to generate a new graph as we did for the endpoints as the pipes are not going to be used as dependencies and the datasources are already deployed
-            groups = [group for group in toposort(dependencies_graph_fork_downstream)]
+            groups = list(toposort(dependencies_graph_fork_downstream))
             for group in groups:
                 for name in group:
                     if name in processed or is_materialized(resources_to_run_fork_downstream.get(name)):
@@ -4729,7 +4753,7 @@ async def folder_push(
             # Finally, we need to deploy the materialized views from right to left.
             # We need to rely on the forkdownstream graph as it contains all the modified materialized views as well as the dependencies of the materialized views
             # In this case, we don't need to generate a new graph as we did for the endpoints as the pipes are not going to be used as dependencies and the datasources are already deployed
-            groups = [group for group in toposort(dependencies_graph_fork_downstream)]
+            groups = list(toposort(dependencies_graph_fork_downstream))
             for group in groups:
                 for name in group:
                     if name in processed or not is_materialized(resources_to_run_fork_downstream.get(name)):
@@ -5732,36 +5756,3 @@ def is_file_a_datasource(filename: str) -> bool:
                 return True
 
     return False
-
-
-def update_connector_params(service: str, ds_params: Dict[str, Any], connector_required_params: List[str]) -> None:
-    """
-    Update connector parameters for a given service, ensuring required parameters exist.
-
-    :param service: The name of the service (e.g., 's3').
-    :param ds_params: The data source parameters to be checked.
-    :param connector_required_params: The list of required parameters for the connector.
-    :return: None
-    """
-
-    connector_at_least_one_required_param: List[str] = []
-
-    # Handle the "at least one param" requirement
-    if connector_at_least_one_required_param and not any(
-        key in ds_params for key in connector_at_least_one_required_param
-    ):
-        params = [
-            (ImportReplacements.get_datafile_param_for_linker_param(service, param) or param).upper()
-            for param in connector_at_least_one_required_param
-        ]
-        click.echo(FeedbackManager.error_updating_connector_missing_at_least_one_param(param=" or ".join(params)))
-        return
-
-    # Handle the mandatory params requirement
-    if not all(key in ds_params for key in connector_required_params):
-        params = [
-            (ImportReplacements.get_datafile_param_for_linker_param(service, param) or param).upper()
-            for param in connector_required_params
-        ]
-        click.echo(FeedbackManager.error_updating_connector_missing_params(param=", ".join(params)))
-        return

tinybird/datatypes.py

@@ -108,10 +108,6 @@ def is_type_date(type_to_check: str) -> bool:
     return date_type_pattern.match(type_to_check) is not None
 
 
-def string_test(x: str) -> bool:
-    return True
-
-
 def date_test(x: str) -> bool:
     return date_pattern.match(x) is not None
 

tinybird/feedback_manager.py

@@ -242,10 +242,11 @@ class FeedbackManager:
     error_organization_index = error_message(
         "Error selecting organization '{organization_index}'. Select a valid index or 0 to cancel"
     )
-    warning_none_organization = warning_message(
-        "Tinybird is now based on organizations. Please, go to the UI ({ui_host}) to follow the migration process. \nYour workspace will be created any way."
-    )
     error_while_fetching_orgs = error_message("Error while fetching organizations: {error}")
+    error_classic_workspace_creation_deprecated = error_message(
+        "The Classic experience is deprecated. New workspaces should be created in Forward. "
+        "Migration guide: https://www.tinybird.co/docs/forward/guides/migrate-from-classic"
+    )
     error_deleted_include = error_message(
         "Related include file {include_file} was deleted and it's used in {filename}. Delete or remove dependency from {filename}."
     )
@@ -864,6 +865,9 @@ class FeedbackManager:
     success_total_rows = success_message("** Total rows in {datasource}: {total_rows}")
     success_appended_datasource = success_message("** Data appended to Data Source '{datasource}' successfully!")
     success_replaced_datasource = success_message("** Data replaced in Data Source '{datasource}' successfully!")
+    success_import_job_queued = success_message("** {operation} import job queued: {job_id}")
+    success_import_job_completed = success_message("** {operation} import completed: {job_id}")
+    info_import_job_status = info_message("Check status: tb job details {job_id}")
     success_auth = success_message(
         "** Auth successful! \n** Configuration written to .tinyb file, consider adding it to .gitignore"
     )
@@ -879,6 +883,11 @@ class FeedbackManager:
     success_delete_rows_datasource = success_message(
         "** Data Source '{datasource}' rows deleted matching condition \"{delete_condition}\""
     )
+    success_lightweight_delete_rows_datasource = success_message(
+        "** Data Source '{datasource}' rows deleted matching condition \"{delete_condition}\""
+        "\n   Rows affected: {rows_affected}"
+        "\n   Partitions scanned: {partitions_scanned} (done: {partitions_done}, in progress: {partitions_in_progress})"
+    )
     success_dry_run_delete_rows_datasource = success_message(
         "** [DRY RUN] Data Source '{datasource}' rows '{rows}' matching condition \"{delete_condition}\" to be deleted"
     )

tinybird/sql.py

@@ -911,30 +911,6 @@ def engine_replicated_to_local(engine: str) -> str:
     return _RE_REPLICATED_MT.sub(_replace, engine.strip())
 
 
-def engine_patch_replicated_engine(engine: str, engine_full: Optional[str], new_table_name: str) -> Optional[str]:
-    """
-    >>> engine_patch_replicated_engine("ReplicatedMergeTree", "ReplicatedMergeTree('/clickhouse/tables/1-1/table_name', 'replica') PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192", 'table_name_staging')
-    "ReplicatedMergeTree('/clickhouse/tables/1-1/table_name_staging', 'replica') PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192"
-    >>> engine_patch_replicated_engine("ReplicatedMergeTree", "ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/sales_product_rank_rt_replicated_2', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY (purchase_location, sku_rank_lc, date)", 'sales_product_rank_rt_replicated_2_staging')
-    "ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/sales_product_rank_rt_replicated_2_staging', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY (purchase_location, sku_rank_lc, date)"
-    >>> engine_patch_replicated_engine("ReplicatedMergeTree", None, 't_000') is None
-    True
-    >>> engine_patch_replicated_engine("Log", "Log()", 't_000')
-    'Log()'
-    >>> engine_patch_replicated_engine("MergeTree", "MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024", 't_000')
-    'MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024'
-    """
-    if not engine_full:
-        return None
-    if engine.lower().startswith("Replicated".lower()):
-        parts = _RE_REPLICATED_ENGINE_SPLIT.split(engine_full)
-        paths = parts[2].split("/")
-        paths[-1] = new_table_name
-        zoo_path = "/".join(paths)
-        return "".join([*parts[:2], zoo_path, *parts[3:]])
-    return engine_full
-
-
 if __name__ == "__main__":
     print(  # noqa: T201
         _parse_table_structure(