tinybird-cli 5.20.1.dev4__tar.gz → 5.21.1.dev1__tar.gz

{tinybird_cli-5.20.1.dev4 → tinybird_cli-5.21.1.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird_cli
-Version: 5.20.1.dev4
+Version: 5.21.1.dev1
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli
 Author: Tinybird
@@ -61,24 +61,15 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
-5.20.1.dev4
+5.21.1.dev1
 ***********
 
 - Internal changes.
 
-5.20.1.dev3
-***********
-
-- `Fixed` Mypy errors in base infrastructure and analytics modules.
-
-5.20.1.dev2
-***********
-
-- `Fixed` Templating when using `id` parameter and the value was not provided.
-
-5.20.1.dev1
+5.21.0
 ***********
 
+- `Added` `tb pipe populate` and `tb push <PIPE> --populate` has a new `--on-demand-compute` flag to enable Compute-Compute Separation in populates.
 - `Added` When trying to push a materialized view with an unoptimized join, the CLI will now warn the user.
 
 5.19.0

{tinybird_cli-5.20.1.dev4 → tinybird_cli-5.21.1.dev1}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '5.20.1.dev4'
-__revision__ = 'bb9222e'
+__version__ = '5.21.1.dev1'
+__revision__ = '39c9264'

{tinybird_cli-5.20.1.dev4 → tinybird_cli-5.21.1.dev1}/tinybird/client.py

@@ -569,6 +569,7 @@ class TinyB:
         populate_condition: Optional[str] = None,
         truncate: bool = True,
         unlink_on_populate_error: bool = False,
+        on_demand_compute: bool = False,
     ):
         params: Dict[str, Any] = {
             "truncate": "true" if truncate else "false",
@@ -578,6 +579,8 @@ class TinyB:
             params.update({"populate_subset": populate_subset})
         if populate_condition:
             params.update({"populate_condition": populate_condition})
+        if on_demand_compute:
+            params.update({"on_demand_compute": "true"})
         response = await self._req(
             f"/v0/pipes/{pipe_name}/nodes/{node_name}/population?{urlencode(params)}", method="POST"
         )

{tinybird_cli-5.20.1.dev4 → tinybird_cli-5.21.1.dev1}/tinybird/datafile_common.py

@@ -2894,6 +2894,7 @@ async def new_pipe(
     config: Any = None,
     fork_downstream: Optional[bool] = False,
     fork: Optional[bool] = False,
+    on_demand_compute: bool = False,
 ):
     # TODO use tb_client instead of calling the urls directly.
     host = tb_client.host
@@ -2981,6 +2982,8 @@ async def new_pipe(
         params["force"] = "true"
     if populate:
         params["populate"] = "true"
+        if on_demand_compute:
+            params["on_demand_compute"] = "true"
     if populate_condition:
         params["populate_condition"] = populate_condition
     if populate_subset:
@@ -3600,6 +3603,7 @@ async def exec_file(
     fork_downstream: Optional[bool] = False,
     fork: Optional[bool] = False,
     git_release: Optional[bool] = False,
+    on_demand_compute: bool = False,
 ):
     if debug:
         click.echo(FeedbackManager.debug_running_file(file=pp.pformat(r)))
@@ -3629,6 +3633,7 @@ async def exec_file(
             tests_check_requests_from_branch=tests_check_requests_from_branch,
             fork_downstream=fork_downstream,
             fork=fork,
+            on_demand_compute=on_demand_compute,
         )
         await update_tags_in_resource(r, "pipe", tb_client)
     elif r["resource"] == "datasources":
@@ -4239,6 +4244,7 @@ async def folder_push(
     use_main: bool = False,
     check_outdated: bool = True,
     hide_folders: bool = False,
+    on_demand_compute: bool = False,
 ):
     workspaces: List[Dict[str, Any]] = (await tb_client.user_workspaces_and_branches()).get("workspaces", [])
     current_ws: Dict[str, Any] = next(
@@ -4423,6 +4429,7 @@ async def folder_push(
                             fork_downstream,
                             fork,
                             git_release,
+                            on_demand_compute,
                         )
                         if not run_tests:
                             click.echo(

{tinybird_cli-5.20.1.dev4 → tinybird_cli-5.21.1.dev1}/tinybird/sql_template.py

@@ -74,7 +74,7 @@ class SQLTemplateException(ValueError):
 # replace_vars_smart(t)
 # print(generate(t, **{x: '' for x in names}))
 
-
+JOB_TIMESTAMP_PARAM = "job_timestamp"
 DEFAULT_PARAM_NAMES = ["format", "q"]
 RESERVED_PARAM_NAMES = [
     "__tb__semver",
@@ -93,6 +93,7 @@ RESERVED_PARAM_NAMES = [
     "tag",
     "template_parameters",
     "token",
+    JOB_TIMESTAMP_PARAM,
 ]
 
 parameter_types = [
@@ -2002,6 +2003,7 @@ def format_SQLTemplateException_message(e: SQLTemplateException, vars_and_types:
                     item.get("default") is None
                     and item.get("used_in", None) is None
                     and item.get("name") not in vars_with_default_none
+                    and item.get("name") is not JOB_TIMESTAMP_PARAM
                 ):
                     vars_with_default_none.append(item["name"])
 
@@ -2294,6 +2296,11 @@ def render_sql_template(
         processed_variables = preprocess_variables(variables, template_variables_with_types)
         variables.update(processed_variables)
 
+    # Handle job_timestamp special case providing the default value if not provided
+    if any(var["name"] == JOB_TIMESTAMP_PARAM for var in template_variables_with_types):
+        variables = variables or {}
+        variables.setdefault(JOB_TIMESTAMP_PARAM, datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
+
     if test_mode:
 
         def dummy(*args, **kwargs):

{tinybird_cli-5.20.1.dev4 → tinybird_cli-5.21.1.dev1}/tinybird/sql_toolset.py

@@ -456,6 +456,10 @@ def replace_tables(
                 sql, None, output_one_line=output_one_line, timestamp=timestamp, function_allow_list=hashable_list
             )
 
+    # Fix for empty database names in JOINs - remove empty backticks like ``.table_name
+    # that are generated when chquery.replace_tables processes tuples with empty database names
+    sql = sql.replace("``.", "")
+
     return sql
 
 

{tinybird_cli-5.20.1.dev4 → tinybird_cli-5.21.1.dev1}/tinybird/tb_cli_modules/cli.py

@@ -602,6 +602,12 @@ def check(filenames: List[str], debug: bool) -> None:
     help="The user token is required for sharing a datasource that contains the SHARED_WITH entry.",
     type=click.types.STRING,
 )
+@click.option(
+    "--on-demand-compute",
+    is_flag=True,
+    default=False,
+    help="Use on-demand compute instances for populate jobs. Only effective when used with --populate.",
+)
 @click.pass_context
 @coro
 async def push(
@@ -633,6 +639,7 @@ async def push(
     check_requests_from_main: bool,
     folder: str,
     user_token: Optional[str],
+    on_demand_compute: bool,
 ) -> None:
     """Push files to Tinybird."""
 
@@ -674,6 +681,7 @@ async def push(
         config=CLIConfig.get_project_config(),
         user_token=user_token,
         is_internal=is_internal,
+        on_demand_compute=on_demand_compute,
     )
 
 

{tinybird_cli-5.20.1.dev4 → tinybird_cli-5.21.1.dev1}/tinybird/tb_cli_modules/pipe.py

@@ -234,6 +234,12 @@ async def pipe_ls(ctx: Context, match: str, format_: str):
     default=False,
     help="Waits for populate jobs to finish, showing a progress bar. Disabled by default.",
 )
+@click.option(
+    "--on-demand-compute",
+    is_flag=True,
+    default=False,
+    help="Use on-demand compute instances for the populate job.",
+)
 @click.pass_context
 @coro
 async def pipe_populate(
@@ -244,6 +250,7 @@ async def pipe_populate(
     truncate: bool,
     unlink_on_populate_error: bool,
     wait: bool,
+    on_demand_compute: bool,
 ):
     """Populate the result of a Materialized Node into the target Materialized View"""
     cl = create_tb_client(ctx)
@@ -270,6 +277,7 @@ async def pipe_populate(
         populate_condition=sql_condition,
         truncate=truncate,
         unlink_on_populate_error=unlink_on_populate_error,
+        on_demand_compute=on_demand_compute,
     )
     if "job" not in response:
         raise CLIPipeException(response)

{tinybird_cli-5.20.1.dev4 → tinybird_cli-5.21.1.dev1}/tinybird/tb_cli_modules/token.py

@@ -167,7 +167,7 @@ def create(ctx: Context) -> None:
 
     You can create two types of tokens: JWT or Static.
 
-    * JWT tokens have a TTL and can only have the PIPES:READ scope.Their main use case is allow your users to call your endpoints without exposing your API key.
+    * JWT tokens have a TTL and can have PIPES:READ and DATASOURCES:READ scopes. Their main use case is allow your users to call your endpoints or read datasources without exposing your API key.
 
     * Static Tokens do not have a TTL and can have any valid scope (DATASOURCES:READ, DATASOURCES:APPEND, DATASOURCES:CREATE, DATASOURCES:DROP, PIPES:CREATE, PIPES:READ, PIPES:DROP).
 
@@ -175,6 +175,8 @@ def create(ctx: Context) -> None:
 
     tb token create jwt my_jwt_token --ttl 1h --scope PIPES:READ --resource my_pipe
 
+    tb token create jwt my_jwt_token --ttl 1h --scope DATASOURCES:READ --resource my_datasource --filter "column = 'value'"
+
     tb token create static my_static_token --scope PIPES:READ --resource my_pipe
 
     tb token create static my_static_token --scope DATASOURCES:READ --resource my_datasource
@@ -192,17 +194,22 @@ def create(ctx: Context) -> None:
 @click.option(
     "--scope",
     multiple=True,
-    type=click.Choice(["PIPES:READ"]),
+    type=click.Choice(["PIPES:READ", "DATASOURCES:READ"]),
     required=True,
-    help="Scope of the token (only PIPES:READ is allowed for JWT tokens)",
+    help="Scope of the token (only PIPES:READ and DATASOURCES:READ are allowed for JWT tokens)",
 )
 @click.option("--resource", multiple=True, required=True, help="Resource associated with the scope")
 @click.option(
     "--fixed-params", multiple=True, help="Fixed parameters in key=value format, multiple values separated by commas"
 )
+@click.option(
+    "--filter",
+    multiple=True,
+    help="SQL filter to apply when reading a datasource (only applicable for DATASOURCES:READ scope)",
+)
 @click.pass_context
 @coro
-async def create_jwt_token(ctx: Context, name: str, ttl: timedelta, scope, resource, fixed_params) -> None:
+async def create_jwt_token(ctx: Context, name: str, ttl: timedelta, scope, resource, fixed_params, filter) -> None:
     """Create a JWT token with a TTL specify."""
 
     obj: Dict[str, Any] = ctx.ensure_object(dict)
@@ -216,6 +223,10 @@ async def create_jwt_token(ctx: Context, name: str, ttl: timedelta, scope, resou
     if fixed_params and len(fixed_params) > len(scope):
         raise CLITokenException(FeedbackManager.error_number_of_fixed_params_and_resources_mismatch())
 
+    # Ensure the number of filters does not exceed the number of scope/resource pairs
+    if filter and len(filter) > len(scope):
+        raise CLITokenException("The number of SQL filters must match the number of scopes")
+
     # Parse fixed params
     parsed_fixed_params = parse_fixed_params(fixed_params) if fixed_params else []
 
@@ -224,15 +235,24 @@ async def create_jwt_token(ctx: Context, name: str, ttl: timedelta, scope, resou
     for i, params in enumerate(parsed_fixed_params):
         fixed_params_list[i] = params
 
+    # Create a list of filters for each scope/resource pair, defaulting to None if not provided
+    filters_list: List[Optional[str]] = [None] * len(scope)
+    for i, f in enumerate(filter):
+        filters_list[i] = f
+
     scopes = []
-    for sc, res, fparams in zip(scope, resource, fixed_params_list):
-        scopes.append(
-            {
-                "type": sc,
-                "resource": res,
-                "fixed_params": fparams,
-            }
-        )
+    for sc, res, fparams, filter in zip(scope, resource, fixed_params_list, filters_list):
+        scope_data = {
+            "type": sc,
+            "resource": res,
+        }
+
+        if sc == "PIPES:READ" and fparams:
+            scope_data["fixed_params"] = fparams
+        elif sc == "DATASOURCES:READ" and filter:
+            scope_data["filter"] = filter
+
+        scopes.append(scope_data)
 
     try:
         response = await client.create_jwt_token(name, expiration_time, scopes)

{tinybird_cli-5.20.1.dev4 → tinybird_cli-5.21.1.dev1}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird_cli
-Version: 5.20.1.dev4
+Version: 5.21.1.dev1
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli
 Author: Tinybird
@@ -61,24 +61,15 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
-5.20.1.dev4
+5.21.1.dev1
 ***********
 
 - Internal changes.
 
-5.20.1.dev3
-***********
-
-- `Fixed` Mypy errors in base infrastructure and analytics modules.
-
-5.20.1.dev2
-***********
-
-- `Fixed` Templating when using `id` parameter and the value was not provided.
-
-5.20.1.dev1
+5.21.0
 ***********
 
+- `Added` `tb pipe populate` and `tb push <PIPE> --populate` has a new `--on-demand-compute` flag to enable Compute-Compute Separation in populates.
 - `Added` When trying to push a materialized view with an unoptimized join, the CLI will now warn the user.
 
 5.19.0