tinybird-cli 6.5.6.dev0__py3-none-any.whl → 6.5.8.dev0__py3-none-any.whl

tinybird/sql_toolset.py

@@ -314,9 +314,9 @@ def _sql_get_used_tables_impl(
             msg = str(e)
             if "is restricted. Contact support@tinybird.co" in msg:
                 raise InvalidFunction(msg=msg) from e
-            elif "Unknown function tb_secret" in msg:
+            if "Unknown function tb_secret" in msg:
                 raise InvalidFunction(msg="Unknown function tb_secret. Usage: {{tb_secret('secret_name')}}") from e
-            elif "Unknown function tb_var" in msg:
+            if "Unknown function tb_var" in msg:
                 raise InvalidFunction(msg="Unknown function tb_var. Usage: {{tb_var('var_name')}}") from e
             raise
         # Do not cache this fallback result: the returned sql string can contain
@@ -487,13 +487,12 @@ def replacements_to_tuples(replacements: dict) -> dict:
     return parsed_replacements
 
 
-@lru_cache(maxsize=2**15)
+@lru_cache(maxsize=2**13)
 def replace_tables_chquery_cached(
     sql: str,
     sorted_replacements: Optional[tuple] = None,
     default_database: str = "",
     output_one_line: bool = False,
-    timestamp: Optional[datetime] = None,
     function_allow_list: Optional[FrozenSet[str]] = None,
     settings_allow_list: Optional[FrozenSet[str]] = None,
 ) -> str:
@@ -536,7 +535,6 @@ def replace_tables(
             sql,
             None,
             output_one_line=output_one_line,
-            timestamp=timestamp,
             function_allow_list=hashable_list,
             settings_allow_list=hashable_settings_list,
         )
@@ -610,7 +608,6 @@ def replace_tables(
             sql,
             None,
             output_one_line=output_one_line,
-            timestamp=timestamp,
             function_allow_list=hashable_list,
             settings_allow_list=hashable_settings_list,
         )
@@ -649,7 +646,6 @@ def replace_tables(
                 r,
                 default_database=default_database,
                 output_one_line=output_one_line,
-                timestamp=timestamp,
                 function_allow_list=hashable_list,
                 settings_allow_list=hashable_settings_list,
             )
@@ -658,16 +654,13 @@ def replace_tables(
                 sql,
                 None,
                 output_one_line=output_one_line,
-                timestamp=timestamp,
                 function_allow_list=hashable_list,
                 settings_allow_list=hashable_settings_list,
             )
 
     # Fix for empty database names in JOINs - remove empty backticks like ``.table_name
     # that are generated when chquery.replace_tables processes tuples with empty database names
-    sql = sql.replace("``.", "")
-
-    return sql
+    return sql.replace("``.", "")
 
 
 def is_invalid_resource(
@@ -706,3 +699,62 @@ def is_invalid_resource_from_current_workspace(
     valid_tables: Optional[Set[Tuple[str, str]]],
 ) -> bool:
     return bool(database == default_database and valid_tables and r not in valid_tables and r not in _replaced_with)
+
+
+def validate_ttl_expression(
+    columns: list[dict[str, str]],
+    ttl: Optional[str],
+    sorting_key: str,
+    primary_key: Optional[str] = None,
+    partition_key: Optional[str] = None,
+) -> Optional[str]:
+    if not ttl:
+        return None
+
+    kwargs = {"partition_key": partition_key} if partition_key else {}
+    error = chquery.validate_ttl(
+        ttl=ttl,
+        columns=columns,
+        sorting_key=sorting_key,
+        primary_key=primary_key or sorting_key,
+        **kwargs,
+    )
+    if error is None:
+        return None
+    return error.removeprefix("DB::Exception: ")
+
+
+TTL_ONLY_DROP_PARTS_SETTING = "ttl_only_drop_parts"
+
+
+def should_default_ttl_only_drop_parts(
+    columns: list[dict[str, str]],
+    ttl: Optional[str],
+    sorting_key: str,
+    partition_key: Optional[str],
+    primary_key: Optional[str] = None,
+) -> bool:
+    """Whether ``ttl_only_drop_parts = 1`` should be defaulted for this table."""
+    if not ttl or not partition_key:
+        return False
+    result = chquery.check_ttl_partition_compatibility(
+        columns=columns,
+        ttl=ttl,
+        sorting_key=sorting_key,
+        primary_key=primary_key or sorting_key,
+        partition_key=partition_key,
+    )
+    return result["checked"] and result["error"] is None
+
+
+def validate_partition_key_expression(
+    columns: list[dict[str, str]],
+    partition_key: Optional[str],
+) -> Optional[str]:
+    if not partition_key:
+        return None
+
+    error = chquery.validate_partition_key(columns=columns, partition_key=partition_key)
+    if error is None:
+        return None
+    return error.removeprefix("DB::Exception: ")

tinybird/syncasync.py

@@ -142,8 +142,7 @@ class Local:
             storage = self._get_storage()
             if key in storage:
                 return storage[key]
-            else:
-                raise AttributeError(f"{self!r} object has no attribute {key!r}")
+            raise AttributeError(f"{self!r} object has no attribute {key!r}")
 
     def __setattr__(self, key, value):
         if key in ("_context_refs", "_thread_critical", "_thread_lock", "_attr_name"):
@@ -151,6 +150,7 @@ class Local:
         with self._thread_lock:
             storage = self._get_storage()
             storage[key] = value
+        return None
 
     def __delattr__(self, key):
         with self._thread_lock:
@@ -252,56 +252,12 @@ def _iscoroutinefunction_or_partial(func: Any) -> bool:
     # this to exist. Code taken from CPython.
     if sys.version_info >= (3, 8):
         return asyncio.iscoroutinefunction(func)
-    else:
-        while inspect.ismethod(func):
-            func = func.__func__
-        while isinstance(func, functools.partial):
-            func = func.func
-
-        return asyncio.iscoroutinefunction(func)
+    while inspect.ismethod(func):
+        func = func.__func__
+    while isinstance(func, functools.partial):
+        func = func.func
 
-
-class ThreadSensitiveContext:
-    """Async context manager to manage context for thread sensitive mode
-    This context manager controls which thread pool executor is used when in
-    thread sensitive mode. By default, a single thread pool executor is shared
-    within a process.
-    In Python 3.7+, the ThreadSensitiveContext() context manager may be used to
-    specify a thread pool per context.
-    In Python 3.6, usage of this context manager has no effect.
-    This context manager is re-entrant, so only the outer-most call to
-    ThreadSensitiveContext will set the context.
-    """
-
-    def __init__(self):
-        self.token = None
-
-    if contextvars:
-
-        async def __aenter__(self):
-            try:
-                SyncToAsync.thread_sensitive_context.get()
-            except LookupError:
-                self.token = SyncToAsync.thread_sensitive_context.set(self)
-
-            return self
-
-        async def __aexit__(self, exc, value, tb):
-            if not self.token:
-                return
-
-            executor = SyncToAsync.context_to_thread_executor.pop(self, None)
-            if executor:
-                executor.shutdown()
-            SyncToAsync.thread_sensitive_context.reset(self.token)
-
-    else:
-
-        async def __aenter__(self):
-            return self
-
-        async def __aexit__(self, exc, value, tb):
-            pass
+    return asyncio.iscoroutinefunction(func)
 
 
 class AsyncToSync:
@@ -660,9 +616,8 @@ class SyncToAsync:
             if hasattr(asyncio, "current_task"):
                 # Python 3.7 and up
                 return asyncio.current_task()
-            else:
-                # Python 3.6
-                return asyncio.Task.current_task()
+            # Python 3.6
+            return asyncio.Task.current_task()
         except RuntimeError:
             return None
 

tinybird/tb_cli_modules/branch.py

@@ -264,8 +264,7 @@ async def release_rm(semver: str, oldest_rollback: bool, force: bool, yes: bool,
         if not oldest_rollback_semver:
             click.echo(FeedbackManager.info_release_no_rollback())
             return
-        else:
-            semver = oldest_rollback_semver
+        semver = oldest_rollback_semver
     await print_release_summary(config, semver, info=True, dry_run=dry_run)
     if dry_run or yes or click.confirm(FeedbackManager.warning_confirm_delete_release(semver=semver)):
         try:
@@ -555,17 +554,9 @@ async def data_branch(last_partition: bool, all: bool, wait: bool) -> None:
     default=False,
     help="Flag to skip execution of regression tests. This is handy for CI branches where regression might be flaky",
 )
-@click.option(
-    "--main",
-    is_flag=True,
-    default=False,
-    help="Run regression tests in the main Branch. For this flag to work all the resources in the Branch pipe endpoints need to exist in the main Branch.",
-)
 @click.pass_context
 @coro
-async def regression_tests(
-    ctx, filename: str, wait: bool, skip_regression_tests: Optional[bool] = False, main: Optional[bool] = False
-):
+async def regression_tests(ctx, filename: str, wait: bool, skip_regression_tests: Optional[bool] = False):
     """Regression test commands for Branches"""
     if skip_regression_tests:
         click.echo(FeedbackManager.warning_regression_skipped())
@@ -591,9 +582,7 @@ async def regression_tests(
         if current_main_workspace["id"] == config["id"]:
             raise CLIException(FeedbackManager.error_not_allowed_in_main_branch())
         try:
-            response = await client.branch_regression_tests_file(
-                config["id"], regression_tests_commands, run_in_main=main
-            )
+            response = await client.branch_regression_tests_file(config["id"], regression_tests_commands)
             if "job" not in response:
                 raise CLIBranchException(str(response))
             job_id = response["job"]["job_id"]
@@ -605,7 +594,7 @@ async def regression_tests(
         except Exception as e:
             raise CLIBranchException(FeedbackManager.error_exception(error=str(e)))
     elif not ctx.invoked_subcommand:
-        await _run_regression(type="coverage", wait=wait, run_in_main=main)
+        await _run_regression(type="coverage", wait=wait)
 
 
 async def _run_regression(
@@ -621,7 +610,6 @@ async def _run_regression(
     failfast: Optional[bool] = False,
     wait: Optional[bool] = False,
     skip: Optional[bool] = False,
-    run_in_main: Optional[bool] = False,
     **kwargs,
 ):
     if skip:
@@ -649,7 +637,6 @@ async def _run_regression(
             assert_time_increase_percentage=assert_time_increase_percentage,
             assert_bytes_read_increase_percentage=assert_bytes_read_increase_percentage,
             assert_max_time=assert_max_time,
-            run_in_main=run_in_main,
             **kwargs,
         )
         if "job" not in response:
@@ -729,12 +716,6 @@ async def _run_regression(
     default=False,
     help="Flag to skip execution of regression tests. This is handy for CI branches where regression might be flaky",
 )
-@click.option(
-    "--main",
-    is_flag=True,
-    default=False,
-    help="Run regression tests in the main Branch. For this flag to work all the resources in the Branch pipe endpoints need to exist in the main Branch.",
-)
 @coro
 async def coverage(
     pipe_name: str,
@@ -748,7 +729,6 @@ async def coverage(
     failfast: bool,
     wait: bool,
     skip_regression_tests: Optional[bool] = False,
-    main: Optional[bool] = False,
 ):
     await _run_regression(
         "coverage",
@@ -763,7 +743,6 @@ async def coverage(
         failfast,
         wait,
         skip_regression_tests,
-        run_in_main=main,
     )
 
 

tinybird/tb_cli_modules/cicd.py

@@ -327,6 +327,7 @@ class CICDGeneratorBase:
                 return FeedbackManager.warning_for_cicd_file(
                     file_name=cicd_file.file_name, warning_message=cicd_file.warning_message.format(**params)
                 )
+        return None
 
     def is_already_generated(self, path: str) -> bool:
         for cicd_file in self.cicd_files:
@@ -396,6 +397,7 @@ def ask_provider_interactively():
                 FeedbackManager.error_git_provider_index(host_index=provider_index, available_options=available_options)
             )
             provider_index = -1
+    return None
 
 
 async def init_cicd(

tinybird/tb_cli_modules/cli.py

@@ -261,8 +261,7 @@ async def init(
             raise CLIGitReleaseException(
                 FeedbackManager.error_dottinyb_not_ignored(git_working_dir=f"{cli_git_release.working_dir()}/")
             )
-        else:
-            click.echo(FeedbackManager.info_dottinyb_already_ignored())
+        click.echo(FeedbackManager.info_dottinyb_already_ignored())
 
         if (
             os.path.exists(f"{cli_git_release.working_dir()}/.diff_tmp")
@@ -271,14 +270,13 @@ async def init(
             raise CLIGitReleaseException(
                 FeedbackManager.error_dotdiff_not_ignored(git_working_dir=f"{cli_git_release.working_dir()}/")
             )
-        else:
-            click.echo(FeedbackManager.info_dotdifftemp_already_ignored())
+        click.echo(FeedbackManager.info_dotdifftemp_already_ignored())
 
         if "release" not in current_ws:
             raise CLIGitReleaseException(FeedbackManager.error_no_correct_token_for_init())
 
         # If we have a release and we are not overriding the commit, we check if we have a release already
-        elif current_ws.get("release") and not override_commit:
+        if current_ws.get("release") and not override_commit:
             final_response = FeedbackManager.error_release_already_set(
                 workspace=current_ws["name"], commit=current_ws["release"]["commit"]
             )
@@ -1034,7 +1032,7 @@ async def materialize(
                     pass
 
         filename = str(f.absolute())
-        to_run = await folder_push(
+        return await folder_push(
             cl,
             filenames=[filename],
             push_deps=push_deps,
@@ -1043,7 +1041,6 @@ async def materialize(
             no_versions=no_versions,
             verbose=verbose,
         )
-        return to_run
 
     def _save_local_pipe(pipe_file_name, pipe_datafile, pipe):
         base = Path("pipes")
@@ -1075,7 +1072,7 @@ async def materialize(
         check = option == 1
 
         filename = str(f_pipe.absolute())
-        to_run = await folder_push(
+        return await folder_push(
             cl,
             filenames=[filename],
             dry_run=False,
@@ -1090,7 +1087,6 @@ async def materialize(
             run_tests=False,
             verbose=verbose,
         )
-        return to_run
 
     async def _populate(pipe, node_name, f_pipe):
         if force_populate or click.confirm(FeedbackManager.prompt_populate(file=f_pipe)):

tinybird/tb_cli_modules/common.py

@@ -35,7 +35,6 @@ from packaging.version import Version
 from tinybird.client import (
     AuthException,
     AuthNoTokenException,
-    DoesNotExistException,
     JobException,
     OperationCanNotBePerformed,
     TinyB,
@@ -194,6 +193,7 @@ class AliasedGroup(click.Group):
         cm = click.Group.get_command(self, ctx, cmd_name)
         if cm is not None:
             return cm
+        return None
 
     def resolve_command(self, ctx, args):
         # always return the command's name, not the alias
@@ -454,6 +454,7 @@ def ask_for_region_interactively(regions):
             available_options = ", ".join(map(str, range(1, len(regions) + 1)))
             click.echo(FeedbackManager.error_region_index(host_index=region_index, available_options=available_options))
             region_index = -1
+    return None
 
 
 def get_region_info(ctx, region=None):
@@ -728,9 +729,8 @@ async def create_workspace_branch(
             job_response = await wait_job(config.get_client(), job_id, job_url, "Branch creation")
             if job_response is None:
                 raise CLIException(f"Empty job API response (job_id: {job_id}, job_url: {job_url})")
-            else:
-                response = job_response.get("result", {})
-                is_summary = "partitions" in response
+            response = job_response.get("result", {})
+            is_summary = "partitions" in response
 
         await switch_workspace(config, branch_name, only_environments=True)
         if is_summary and (bool(last_partition) or bool(all)):
@@ -757,10 +757,8 @@ async def print_branch_regression_tests_summary(client, job_id, host, response=N
         if isinstance(metric, float):
             if is_percentage:
                 return f"{round(metric, 3):+} %"
-            else:
-                return f"{round(metric, 3)} seconds"
-        else:
-            return metric
+            return f"{round(metric, 3)} seconds"
+        return metric
 
     failed = False
     response = await client.job(job_id) if job_id else response or {"progress": []}
@@ -970,7 +968,7 @@ async def push_data(
     sql_condition: Optional[str] = None,
     replace_options=None,
     concurrency: int = 1,
-    use_v1: bool = False,
+    wait: bool = True,
 ) -> Optional[list[str]]:
     if url and type(url) is tuple:
         url = url[0]
@@ -979,24 +977,6 @@ async def push_data(
     if not url:
         raise CLIException(FeedbackManager.error_missing_url(datasource=datasource_name))
 
-    def cb(res):
-        if cb.First:  # type: ignore[attr-defined]
-            blocks_to_process = len([x for x in res["block_log"] if x["status"] == "idle"])
-            if blocks_to_process:
-                cb.bar = click.progressbar(label=FeedbackManager.info_progress_blocks(), length=blocks_to_process)  # type: ignore[attr-defined]
-                cb.bar.update(0)  # type: ignore[attr-defined]
-                cb.First = False  # type: ignore[attr-defined]
-                cb.blocks_to_process = blocks_to_process  # type: ignore[attr-defined]
-        else:
-            done = len([x for x in res["block_log"] if x["status"] == "done"])
-            if done * 2 > cb.blocks_to_process:  # type: ignore[attr-defined]
-                cb.bar.label = FeedbackManager.info_progress_current_blocks()  # type: ignore[attr-defined]
-            cb.bar.update(done - cb.prev_done)  # type: ignore[attr-defined]
-            cb.prev_done = done  # type: ignore[attr-defined]
-
-    cb.First = True  # type: ignore[attr-defined]
-    cb.prev_done = 0  # type: ignore[attr-defined]
-
     click.echo(FeedbackManager.info_starting_import_process())
 
     if isinstance(url, list):
@@ -1006,18 +986,16 @@ async def push_data(
 
     async def process_url(
         datasource_name: str, url: str, mode: str, sql_condition: Optional[str], replace_options: Optional[Set[str]]
-    ):
+    ) -> str:
         parsed = urlparse(url)
-        if use_v1 and parsed.scheme in ("http", "https"):
-            raise CLIException("--experimental=use_v1 only supports local files.")
+        is_remote_url = parsed.scheme in ("http", "https")
         # poor man's format detection
         _format = get_format_from_filename_or_url(url)
-        if parsed.scheme in ("http", "https"):
-            res = await client.datasource_create_from_url(
+        if is_remote_url:
+            res = await client.datasource_append_url(
                 datasource_name,
                 url,
                 mode=mode,
-                status_callback=cb,
                 sql_condition=sql_condition,
                 format=_format,
                 replace_options=replace_options,
@@ -1030,66 +1008,35 @@ async def push_data(
                 sql_condition=sql_condition,
                 format=_format,
                 replace_options=replace_options,
-                use_v1=use_v1,
             )
 
-        if use_v1:
-            job_id = res.get("id") or res.get("import_id")
-            if not isinstance(job_id, str):
-                raise CLIException("The v1 import response did not include a job ID.")
-            return job_id
-
-        datasource_name = res["datasource"]["name"]
-        try:
-            datasource = await client.get_datasource(datasource_name)
-        except DoesNotExistException:
-            raise CLIException(FeedbackManager.error_datasource_does_not_exist(datasource=datasource_name))
-        except Exception as e:
-            raise CLIException(FeedbackManager.error_exception(error=str(e)))
-
-        total_rows = (datasource.get("statistics", {}) or {}).get("row_count", 0)
-        appended_rows = 0
-        parser = None
-
-        if res.get("error"):
-            raise CLIException(FeedbackManager.error_exception(error=res["error"]))
-        if res.get("errors"):
-            raise CLIException(FeedbackManager.error_exception(error=res["errors"]))
-        if res.get("blocks"):
-            for block in res["blocks"]:
-                if "process_return" in block and block["process_return"] is not None:
-                    process_return = block["process_return"][0]
-                    parser = process_return["parser"] if process_return.get("parser") else parser
-                    if parser and parser != "clickhouse":
-                        parser = process_return["parser"]
-                        appended_rows += process_return["lines"]
-
-        return parser, total_rows, appended_rows
+        job_id = res.get("id") or res.get("import_id")
+        if not isinstance(job_id, str):
+            raise CLIException("We couldn't confirm that your import started. Please try again.")
+        return job_id
 
     try:
         tasks = [process_url(datasource_name, url, mode, sql_condition, replace_options) for url in urls]
-        output = await gather_with_concurrency(concurrency, *tasks)
-        if use_v1:
-            return list(output)
-        parser, total_rows, appended_rows = list(output)[-1]
-    except AuthNoTokenException:
-        raise
-    except OperationCanNotBePerformed as e:
-        raise CLIException(FeedbackManager.error_operation_can_not_be_performed(error=e))
-    except Exception as e:
-        raise CLIException(FeedbackManager.error_exception(error=e))
-    else:
-        click.echo(FeedbackManager.success_progress_blocks())
-        if mode == "append" and parser and parser != "clickhouse":
-            click.echo(FeedbackManager.success_appended_rows(appended_rows=appended_rows))
-
+        job_ids = list(await gather_with_concurrency(concurrency, *tasks))
+        if not wait:
+            return job_ids
+        for job_id in job_ids:
+            await wait_job_no_ui(client, job_id)
+        datasource = await client.get_datasource(datasource_name)
+        total_rows = (datasource.get("statistics", {}) or {}).get("row_count", 0)
         click.echo(FeedbackManager.success_total_rows(datasource=datasource_name, total_rows=total_rows))
-
         if mode == "replace":
             click.echo(FeedbackManager.success_replaced_datasource(datasource=datasource_name))
         else:
             click.echo(FeedbackManager.success_appended_datasource(datasource=datasource_name))
+        click.echo(FeedbackManager.success_progress_blocks())
         click.echo(FeedbackManager.info_data_pushed(datasource=datasource_name))
+    except AuthNoTokenException:
+        raise
+    except OperationCanNotBePerformed as e:
+        raise CLIException(FeedbackManager.error_operation_can_not_be_performed(error=e))
+    except Exception as e:
+        raise CLIException(FeedbackManager.error_exception(error=e))
 
     return None
 
@@ -1230,8 +1177,7 @@ async def switch_workspace(config: CLIConfig, workspace_name_or_id: str, only_en
         if not workspace:
             if only_environments:
                 raise CLIException(FeedbackManager.error_branch(branch=workspace_name_or_id, cli="tb"))
-            else:
-                raise CLIException(FeedbackManager.error_workspace(workspace=workspace_name_or_id, cli="tb"))
+            raise CLIException(FeedbackManager.error_workspace(workspace=workspace_name_or_id, cli="tb"))
 
         config.set_token(workspace["token"])
         config.set_token_for_host(workspace["token"], config.get_host())
@@ -1358,11 +1304,11 @@ class ConnectionReplacements:
         """Maps prompt parameters to API parameters."""
 
         api_params = {}
-        for key in params.keys():
+        for key, value in params.items():
             try:
-                api_params[ConnectionReplacements._PARAMS_REPLACEMENTS[service][key]] = params[key]
+                api_params[ConnectionReplacements._PARAMS_REPLACEMENTS[service][key]] = value
             except KeyError:
-                api_params[key] = params[key]
+                api_params[key] = value
 
         api_params["service"] = service
         return api_params

tinybird/tb_cli_modules/config.py

@@ -121,8 +121,7 @@ class CLIConfig:
             return self._values[key].origin
         if self._parent:
             return self._parent.get_value_origin(key)
-        else:
-            return ConfigValueOrigin.NONE
+        return ConfigValueOrigin.NONE
 
     def persist_to_file(self, override_with_values: Optional["CLIConfig"] = None) -> None:
         if not self._path: