tinybird-cli 6.5.7.dev0__py3-none-any.whl → 7.0.0.dev0__py3-none-any.whl

tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '6.5.7.dev0'
-__revision__ = '532ee82'
+__version__ = '7.0.0.dev0'
+__revision__ = '3fcfdc16'

tinybird/ch_utils/constants.py

@@ -263,4 +263,5 @@ VALID_QUERY_FORMATS = (
     "JSONCompact",
     "JSONStringsEachRowWithProgress",
     "ODBCDriver2",
+    "Markdown",
 )

tinybird/ch_utils/engine.py

@@ -1,3 +1,4 @@
+import functools
 import logging
 import re
 from collections import defaultdict
@@ -25,6 +26,9 @@ DEFAULT_EMPTY_PARAMETERS = ["ttl", "partition_key", "sorting_key"]
 # * `N` is a number
 SIMPLE_TTL_DEFINITION = re.compile(r"""^([a-zA-Z0-9_\-\.\(\)\ \/\*]*) \+ (toInterval[a-zA-Z]*\([0-9]+\))$""")
 
+# ClickHouse 25.8 quotes the engine name `Null`, as `null` is a keyword.
+_BACKQUOTED_ENGINE_NAME = re.compile(r"^`(\w+)`")
+
 
 class TableDetails:
     """
@@ -78,6 +82,16 @@ class TableDetails:
 
     @property
     def engine_full(self) -> Optional[str]:
+        """
+        >>> TableDetails({"engine_full": "`Null`", "engine": "Null"}).engine_full
+        'Null'
+        >>> TableDetails({"engine_full": "`Null`()", "engine": "Null"}).engine_full
+        'Null()'
+        >>> TableDetails({"engine_full": "Null", "engine": "Null"}).to_json()
+        {'engine_full': 'Null', 'engine': 'Null'}
+        >>> TableDetails({"engine_full": "`Null`", "engine": "Null"}).to_json()
+        {'engine_full': 'Null', 'engine': 'Null'}
+        """
         _engine_full: Optional[str] = self.details.get("engine_full", None)
         if not _engine_full:
             return None
@@ -86,7 +100,7 @@ class TableDetails:
         # We cannot remove index_granularity = 8192 blindly because it might be followed by other settings
         if settings and settings.strip().lower() == "index_granularity = 8192":
             _engine_full = _engine_full.replace(" SETTINGS index_granularity = 8192", "")
-        return engine_replicated_to_local(_engine_full)
+        return engine_replicated_to_local(_BACKQUOTED_ENGINE_NAME.sub(r"\1", _engine_full))
 
     @property
     def original_engine_full(self) -> Optional[str]:
@@ -111,8 +125,7 @@ class TableDetails:
 
     @property
     def version(self):
-        _version = self.details.get("version", None)
-        return _version
+        return self.details.get("version", None)
 
     def is_replicated(self) -> bool:
         engine: Optional[str] = self.details.get("engine", None)
@@ -183,6 +196,7 @@ class TableDetails:
         settings = self.details.get("settings", None)
         if settings and settings.strip().lower() != "index_granularity = 8192":
             return settings
+        return None
 
     @property
     def ttl(self):
@@ -190,23 +204,19 @@ class TableDetails:
 
     @property
     def ver(self):
-        _ver = self.details.get("ver", None)
-        return _ver
+        return self.details.get("ver", None)
 
     @property
     def is_deleted(self):
-        _is_deleted = self.details.get("is_deleted", None)
-        return _is_deleted
+        return self.details.get("is_deleted", None)
 
     @property
     def columns(self):
-        _columns = self.details.get("columns", None)
-        return _columns
+        return self.details.get("columns", None)
 
     @property
     def sign(self):
-        _sign = self.details.get("sign", None)
-        return _sign
+        return self.details.get("sign", None)
 
     @property
     def statistics(self) -> Dict[str, Any]:
@@ -444,38 +454,70 @@ def get_engine_config(engine: str):
     )
 
 
-def engine_params(columns, params: List[EngineParam], args: Dict):
-    params_values = []
+def validate_engine_config(
+    engine: str, args: dict, schema: Optional[str] = None, columns: Optional[List[Dict[str, Any]]] = None
+) -> None:
+    if schema is not None and columns is not None:
+        raise ValueError("You can not use 'schema' and 'columns' at the same time")
+    _, (params, options) = get_engine_config(engine)
+    schema_columns = parse_table_structure(schema) if schema is not None else (columns or [])
+    engine_settings = {key.replace("engine_", ""): value for key, value in args.items()}
+
+    for arg in engine_settings:
+        if not hasattr(TableDetails, arg):
+            raise ValueError(f"engine_{arg} is not a valid option")
+
     for p in params:
-        if p.required and p.name not in args:
+        if p.required and p.name not in engine_settings:
             raise ValueError(f"Missing required parameter '{p.name}'")
-        param_value = args.get(p.name, None) or p.default_value
+        param_value = engine_settings.get(p.name) or p.default_value
         if not param_value:
             continue
         if p.is_valid:
             check_is_valid(
-                valid_check=p.is_valid, check_type="parameter", columns=columns, tb_param=p.tb_param, value=param_value
+                valid_check=p.is_valid,
+                check_type="parameter",
+                columns=schema_columns,
+                tb_param=p.tb_param,
+                value=param_value,
             )
+
+    for o in options:
+        if o.required and o.name not in engine_settings:
+            raise ValueError(f"Missing required option '{o.name}'")
+        option_value = engine_settings.get(o.name) or o.default_value
+        if o.is_valid:
+            check_is_valid(
+                valid_check=o.is_valid,
+                check_type="option",
+                columns=schema_columns,
+                tb_param=o.tb_param,
+                value=option_value,
+            )
+
+
+def engine_params(params: List[EngineParam], args: Dict) -> List[str]:
+    params_values = []
+    for p in params:
+        param_value = args.get(p.name, None) or p.default_value
+        if not param_value:
+            continue
         params_values.append(param_value)
     return params_values
 
 
-def engine_options(columns, options: List[EngineOption], args: Dict):
+def engine_options(options: List[EngineOption], args: Dict) -> List[str]:
     options_values = []
     engine_settings = ""
 
     for o in options:
-        if o.required and o.name not in args:
-            raise ValueError(f"Missing required option '{o.name}'")
         option_value = args.get(o.name) or o.default_value
-        if o.is_valid:
-            check_is_valid(
-                valid_check=o.is_valid, check_type="option", columns=columns, tb_param=o.tb_param, value=option_value
-            )
-
         if option_value:
             if o.sql.lower() == "settings":
                 engine_settings = f"{o.sql} {option_value}"
+            # Conditional TTLs fail if we wrap the value in parentheses
+            elif o.sql.lower() == "ttl":
+                options_values.append(f"{o.sql} {option_value}")
             else:
                 options_values.append(f"{o.sql} ({option_value})")
 
@@ -491,7 +533,7 @@ def check_is_valid(
     columns: List[Dict[str, Any]],
     tb_param: str,
     value: str,
-):
+) -> None:
     """
     >>> check_is_valid(sorting_key_is_valid, 'option', ['column-name'], 'sorting_key', 'date')
 
@@ -508,15 +550,13 @@ def check_is_valid(
         raise ValueError(f"Invalid value '{value}' for {check_type} '{tb_param}', reason: {e}")
 
 
-def build_engine(
-    engine: str, columns: Optional[List], params: List[EngineParam], options: List[EngineOption], args: Dict
-):
-    return f"{engine}({', '.join(engine_params(columns, params, args))}) {' '.join(engine_options(columns, options, args))}".strip()
+def build_engine(engine: str, params: List[EngineParam], options: List[EngineOption], args: Dict) -> str:
+    return f"{engine}({', '.join(engine_params(params, args))}) {' '.join(engine_options(options, args))}".strip()
 
 
 def engine_full_from_dict(
     engine: str, args: dict, schema: Optional[str] = None, columns: Optional[List[Dict[str, Any]]] = None
-):
+) -> str:
     """
     >>> schema = ''
     >>> engine_full_from_dict('wadus', {}, schema=schema)
@@ -536,7 +576,9 @@ def engine_full_from_dict(
     >>> engine_full_from_dict('MergeTree', {'sorting_key': 'local_date, cod_store'}, schema=schema)
     'MergeTree() ORDER BY (local_date, cod_store)'
     >>> engine_full_from_dict('MergeTree', {'partition_key': 'toDate(timestamp)', 'sorting_key': 'local_date, cod_store', 'settings': 'index_granularity = 32, index_granularity_bytes = 2048', 'ttl': 'toDate(local_date) + INTERVAL 1 DAY'}, schema=schema)
-    'MergeTree() PARTITION BY (toDate(timestamp)) ORDER BY (local_date, cod_store) TTL (toDate(local_date) + INTERVAL 1 DAY) SETTINGS index_granularity = 32, index_granularity_bytes = 2048'
+    'MergeTree() PARTITION BY (toDate(timestamp)) ORDER BY (local_date, cod_store) TTL toDate(local_date) + INTERVAL 1 DAY SETTINGS index_granularity = 32, index_granularity_bytes = 2048'
+    >>> engine_full_from_dict('MergeTree', {'sorting_key': 'timestamp', 'ttl': 'timestamp + INTERVAL 1 DAY DELETE WHERE is_deleted = 1'}, schema=schema)
+    'MergeTree() ORDER BY (timestamp) TTL timestamp + INTERVAL 1 DAY DELETE WHERE is_deleted = 1'
 
     >>> schema = ''
     >>> engine_full_from_dict('CollapsingMergeTree', {'sign': 'sign_column'}, schema=schema)
@@ -605,20 +647,10 @@ def engine_full_from_dict(
     'ReplacingMergeTree(updated_at) PARTITION BY (tuple()) ORDER BY (project_id) SETTINGS index_granularity = 32'
     """
 
-    if schema is not None and columns is not None:
-        raise ValueError("You can not use 'schema' and 'columns' at the same time")
-    engine_config = get_engine_config(engine)
-    name, (params, options) = engine_config
-    if columns is None and schema is not None:
-        columns = parse_table_structure(schema)
-
+    validate_engine_config(engine, args, schema=schema, columns=columns)
+    name, (params, options) = get_engine_config(engine)
     engine_settings = {key.replace("engine_", ""): value for key, value in args.items()}
-
-    for arg in engine_settings:
-        if not hasattr(TableDetails, arg):
-            raise ValueError(f"engine_{arg} is not a valid option")
-
-    return build_engine(name, columns, params, options, engine_settings)
+    return build_engine(name, params, options, engine_settings)
 
 
 def engine_params_from_engine_full(engine_full: str) -> Dict[str, Any]:
@@ -694,8 +726,7 @@ def ttl_from_engine(engine: str) -> Optional[str]:
         return None
     settings_array = engine.split(" SETTINGS ")
     settings = " SETTINGS " + settings_array[1] if len(settings_array) > 1 else None
-    ttl = ttl_array[1][: -(len(settings))] if settings else ttl_array[1]
-    return ttl
+    return ttl_array[1][: -(len(settings))] if settings else ttl_array[1]
 
 
 def ttl_condition_from_engine_full(engine_full: Optional[str]) -> Optional[str]:
@@ -759,17 +790,22 @@ def ttl_condition_from_engine_full(engine_full: Optional[str]) -> Optional[str]:
         return None
 
 
+@functools.lru_cache(maxsize=1024)
+def _index_lines(create_table_query_expr: str) -> tuple[str, ...]:
+    from tinybird.sql_toolset import format_sql
+
+    return tuple(
+        line.strip() for line in format_sql(create_table_query_expr).splitlines() if line.strip().startswith("INDEX")
+    )
+
+
 def _parse_indexes(create_table_query_expr: str) -> List[TableIndex]:
-    if create_table_query_expr == "":
+    # format_sql is CPU-heavy and runs on the event loop. ClickHouse writes the
+    # INDEX keyword in uppercase in create_table_query.
+    if "INDEX" not in create_table_query_expr:
         return []
     try:
-        from tinybird.sql_toolset import format_sql
-
-        indexes = [
-            line.strip()
-            for line in format_sql(create_table_query_expr).splitlines()
-            if line.strip().startswith("INDEX")
-        ]
+        indexes = list(_index_lines(create_table_query_expr))
     except ModuleNotFoundError:
         # this is not needed from CLI
         return []

tinybird/client.py

@@ -47,14 +47,6 @@ class TimeoutException(Exception):
     pass
 
 
-class ReachRetryLimit(Exception):
-    pass
-
-
-class ConnectorNothingToLoad(Exception):
-    pass
-
-
 class JobException(Exception):
     pass
 
@@ -266,14 +258,13 @@ class TinyB:
 
     async def alter_tokens(self, name: str, scopes: List[str]):
         if not scopes:
-            return
+            return None
         scopes_url: str = "&".join([f"scope={scope}" for scope in scopes])
         url = f"/v0/tokens/{name}"
         if len(url + "?" + scopes_url) > TinyB.MAX_GET_LENGTH:
             return await self._req(url, method="PUT", data=scopes_url)
-        else:
-            url = url + "?" + scopes_url
-            return await self._req(url, method="PUT", data="")
+        url = url + "?" + scopes_url
+        return await self._req(url, method="PUT", data="")
 
     async def datasources(self, branch: Optional[str] = None, used_by: bool = False) -> List[Dict[str, Any]]:
         params = {}
@@ -294,16 +285,13 @@ class TinyB:
         return await self._req(f"/v0/variables/{name}")
 
     async def create_secret(self, name: str, value: str):
-        response = await self._req("/v0/variables", method="POST", data={"name": name, "value": value})
-        return response
+        return await self._req("/v0/variables", method="POST", data={"name": name, "value": value})
 
     async def update_secret(self, name: str, value: str):
-        response = await self._req(f"/v0/variables/{name}", method="PUT", data={"value": value})
-        return response
+        return await self._req(f"/v0/variables/{name}", method="PUT", data={"value": value})
 
     async def delete_secret(self, name: str):
-        response = await self._req(f"/v0/variables/{name}", method="DELETE")
-        return response
+        return await self._req(f"/v0/variables/{name}", method="DELETE")
 
     async def get_connections(self, service: Optional[str] = None):
         params = {}
@@ -412,6 +400,16 @@ class TinyB:
         format: str = "csv",
         replace_options: Optional[Set[str]] = None,
     ) -> Dict[str, Any]:
+        if mode in ("append", "replace"):
+            return await self.datasource_append_url(
+                table_name,
+                url,
+                mode=mode,
+                sql_condition=sql_condition,
+                format=format,
+                replace_options=replace_options,
+            )
+
         params = {"name": table_name, "url": url, "mode": mode, "debug": "blocks_block_log", "format": format}
 
         if sql_condition:
@@ -428,6 +426,33 @@ class TinyB:
 
         return await self.wait_for_job(res["id"], status_callback, backoff_multiplier=1.5, maximum_backoff_seconds=20)
 
+    async def datasource_append_url(
+        self,
+        datasource_name: str,
+        url: str,
+        mode: str = "append",
+        sql_condition: Optional[str] = None,
+        format: str = "csv",
+        replace_options: Optional[Set[str]] = None,
+    ) -> Dict[str, Any]:
+        params = {"format": format, "url": url}
+        if sql_condition:
+            params["replace_condition"] = sql_condition
+        if replace_options:
+            for option in replace_options:
+                params[option] = "true"
+
+        res = await self._req(
+            f"/v1/datasources/{quote(datasource_name, safe='')}/{mode}?{urlencode(params, safe='')}",
+            method="POST",
+            data=b"",
+        )
+        if not isinstance(res, dict):
+            raise RuntimeError("We couldn't confirm that your import started. Please try again.")
+        if "error" in res:
+            raise Exception(res["error"])
+        return res
+
     async def datasource_delete(self, datasource_name: str, force: bool = False, dry_run: bool = False):
         params = {"force": "true" if force else "false", "dry_run": "true" if dry_run else "false"}
         return await self._req(f"/v0/datasources/{datasource_name}?{urlencode(params)}", method="DELETE")
@@ -577,8 +602,7 @@ class TinyB:
         node_name = node["params"]["name"] if node.get("params", None) else node["name"]
         if datasource_name:
             params["datasource"] = datasource_name
-        response = await self._req(f"/v0/pipes/{pipe_name}/nodes/{node_name}/analysis?{urlencode(params)}")
-        return response
+        return await self._req(f"/v0/pipes/{pipe_name}/nodes/{node_name}/analysis?{urlencode(params)}")
 
     async def populate_node(
         self,
@@ -600,10 +624,7 @@ class TinyB:
             params.update({"populate_condition": populate_condition})
         if on_demand_compute:
             params.update({"on_demand_compute": "true"})
-        response = await self._req(
-            f"/v0/pipes/{pipe_name}/nodes/{node_name}/population?{urlencode(params)}", method="POST"
-        )
-        return response
+        return await self._req(f"/v0/pipes/{pipe_name}/nodes/{node_name}/population?{urlencode(params)}", method="POST")
 
     async def pipes(self, branch=None, dependencies: bool = False, node_attrs=None, attrs=None) -> List[Dict[str, Any]]:
         params = {
@@ -635,9 +656,8 @@ class TinyB:
         query_string = urlencode(params)
         if len(url + "?" + query_string) > TinyB.MAX_GET_LENGTH:
             return await self._req(f"/v0/pipes/{pipe_name_or_uid}.{format}", method="POST", data=params)
-        else:
-            url = url + "?" + query_string
-            return await self._req(url)
+        url = url + "?" + query_string
+        return await self._req(url)
 
     async def pipe_create(self, pipe_name: str, sql: str):
         return await self._req(
@@ -663,6 +683,7 @@ class TinyB:
         target_datasource: Optional[str] = None,
         schedule_cron: Optional[str] = None,
         mode: Optional[str] = None,
+        on_demand_compute: Optional[bool] = None,
     ):
         data = {"schedule_cron": schedule_cron}
 
@@ -672,6 +693,9 @@ class TinyB:
         if mode:
             data["mode"] = mode
 
+        if on_demand_compute is not None:
+            data["on_demand_compute"] = "true" if on_demand_compute else "false"
+
         return await self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/copy", method="PUT", data=data)
 
     async def pipe_remove_copy(self, pipe_name_or_id: str, node_id: str):
@@ -726,8 +750,7 @@ class TinyB:
 
         if len(sql) > TinyB.MAX_GET_LENGTH:
             return await self._req(f"/v0/sql?{urlencode(params)}", data=sql, method="POST")
-        else:
-            return await self._req(f"/v0/sql?q={quote(sql, safe='')}&{urlencode(params)}")
+        return await self._req(f"/v0/sql?q={quote(sql, safe='')}&{urlencode(params)}")
 
     async def jobs(self, status=None):
         jobs = (await self._req("/v0/jobs"))["jobs"]
@@ -828,7 +851,6 @@ class TinyB:
         assert_time_increase_percentage: Optional[float] = None,
         assert_bytes_read_increase_percentage: Optional[float] = None,
         assert_max_time: Optional[float] = None,
-        run_in_main: Optional[bool] = False,
     ):
         test: Dict[str, Any] = {
             test_type: {
@@ -858,20 +880,12 @@ class TinyB:
         ]
 
         data = json.dumps(regression_commands)
-        if run_in_main:
-            url = f"/v0/environments/{branch_id}/regression/main"
-        else:
-            url = f"/v0/environments/{branch_id}/regression"
+        url = f"/v0/environments/{branch_id}/regression"
         return await self._req(url, method="POST", data=data, headers={"Content-Type": "application/json"})
 
-    async def branch_regression_tests_file(
-        self, branch_id: str, regression_commands: List[Dict[str, Any]], run_in_main: Optional[bool] = False
-    ):
+    async def branch_regression_tests_file(self, branch_id: str, regression_commands: List[Dict[str, Any]]):
         data = json.dumps(regression_commands)
-        if run_in_main:
-            url = f"/v0/environments/{branch_id}/regression/main"
-        else:
-            url = f"/v0/environments/{branch_id}/regression"
+        url = f"/v0/environments/{branch_id}/regression"
         return await self._req(url, method="POST", data=data, headers={"Content-Type": "application/json"})
 
     async def delete_workspace(self, id: str, hard_delete_confirmation: Optional[str], version: str = "v0"):
@@ -922,7 +936,7 @@ class TinyB:
 
     async def add_workspaces_to_organization(self, organization_id: str, workspace_ids: List[str]):
         if not workspace_ids:
-            return
+            return None
         return await self._req(
             f"/v0/organizations/{organization_id}/workspaces",
             method="PUT",
@@ -1105,8 +1119,7 @@ class TinyB:
                 from tinybird.sql_toolset import format_sql
 
                 return format_sql(sql)
-            else:
-                return await self._sql_get_format_remote(sql, with_clickhouse_format)
+            return await self._sql_get_format_remote(sql, with_clickhouse_format)
         except ModuleNotFoundError:
             return await self._sql_get_format_remote(sql, with_clickhouse_format)
 
@@ -1168,8 +1181,7 @@ class TinyB:
         return next((connector for connector in result["connectors"] if connector_equals(connector, kwargs)), None)
 
     async def regions(self):
-        regions = await self._req("/v0/regions")
-        return regions
+        return await self._req("/v0/regions")
 
     async def datasource_query_copy(self, datasource_name: str, sql_query: str):
         params = {"copy_to": datasource_name}

tinybird/config.py

@@ -49,11 +49,7 @@ LEGACY_HOSTS = {
     "https://ui.europe-west2.gcp.tinybird.co": "https://cloud.tinybird.co/gcp/europe-west2",
     "https://ui.ap-east.aws.tinybird.co": "https://app.tinybird.co/aws/ap-east-1",
     "https://ui.ap-east-1.aws.tinybird.co": "https://app.tinybird.co/aws/ap-east-1",
-    "https://ui.split.tinybird.co": "https://app.tinybird.co/aws/split-us-east",
-    "https://ui.split.us-west-2.aws.tinybird.co": "https://app.tinybird.co/aws/split-us-west-2",
     "https://ui.northamerica-northeast2.gcp.tinybird.co": "https://app.tinybird.co/gcp/northamerica-northeast2",
-    "https://api.split.tinybird.co": "https://app.tinybird.co/aws/split-us-east",
-    "https://api.split.us-west-2.aws.tinybird.co": "https://app.tinybird.co/aws/split-us-west-2",
     "https://ui.wadus1.gcp.tinybird.co": "https://app.wadus.tinybird.co/gcp/wadus1",
     "https://ui.wadus2.gcp.tinybird.co": "https://app.wadus.tinybird.co/gcp/wadus2",
     "https://ui.wadus3.gcp.tinybird.co": "https://app.wadus.tinybird.co/gcp/wadus3",