tinybird-cli 3.8.1.dev1__tar.gz → 3.8.2.dev2__tar.gz

{tinybird-cli-3.8.1.dev1 → tinybird-cli-3.8.2.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 3.8.1.dev1
+Version: 3.8.2.dev2
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -16,9 +16,25 @@ Tinybird CLI
 The Tinybird command-line tool allows you to use all the Tinybird functionality directly from the command line. Additionally, it includes several functions to create and manage data projects easily.
 
 Changelog
-
 ---------
 
+3.8.2.dev2
+************
+
+- `Added` New `--policy` option for `create s3_iamrole` command that will generate different hints depending on the case
+
+3.8.2.dev1
+************
+
+- `Added` Support for connection names when doing `tb connection rm`
+
+3.8.1
+************
+
+- `Fixed` Avoid system vars evaluation when doing `tb fmt`
+- `Fixed` environment variables substitution for Data Source engine parameters.
+
+
 3.8.0
 ************
 

{tinybird-cli-3.8.1.dev1 → tinybird-cli-3.8.2.dev2}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '3.8.1.dev1'
-__revision__ = 'ee21c5f'
+__version__ = '3.8.2.dev2'
+__revision__ = 'b88d054'

{tinybird-cli-3.8.1.dev1 → tinybird-cli-3.8.2.dev2}/tinybird/client.py

@@ -997,8 +997,11 @@ class TinyB(object):
     async def get_s3_trust_policy(self) -> Dict[str, Any]:
         return await self._req("/v0/integrations/s3/policies/trust-policy")
 
-    async def get_s3_access_policy(self) -> Dict[str, Any]:
-        return await self._req("/v0/integrations/s3/policies/access-policy")
+    async def get_s3_access_write_policy(self) -> Dict[str, Any]:
+        return await self._req("/v0/integrations/s3/policies/write-access-policy")
+
+    async def get_s3_access_read_policy(self) -> Dict[str, Any]:
+        return await self._req("/v0/integrations/s3/policies/read-access-policy")
 
     async def sql_get_format(self, sql: str, with_clickhouse_format: bool = False) -> str:
         try:

{tinybird-cli-3.8.1.dev1 → tinybird-cli-3.8.2.dev2}/tinybird/datafile.py

@@ -727,7 +727,9 @@ class Datafile:
         return True
 
 
-def parse_datasource(filename: str, replace_includes: bool = True, content: Optional[str] = None) -> Datafile:
+def parse_datasource(
+    filename: str, replace_includes: bool = True, content: Optional[str] = None, skip_eval: bool = False
+) -> Datafile:
     basepath = ""
     if not content:
         with open(filename) as file:
@@ -737,7 +739,7 @@ def parse_datasource(filename: str, replace_includes: bool = True, content: Opti
         s = content
 
     try:
-        doc = parse(s, "default", basepath, replace_includes=replace_includes)
+        doc = parse(s, "default", basepath, replace_includes=replace_includes, skip_eval=skip_eval)
     except ParseException as e:
         raise click.ClickException(
             FeedbackManager.error_parsing_file(filename=filename, lineno=e.lineno, error=e)
@@ -749,7 +751,9 @@ def parse_datasource(filename: str, replace_includes: bool = True, content: Opti
     return doc
 
 
-def parse_pipe(filename: str, replace_includes: bool = True, content: Optional[str] = None) -> Datafile:
+def parse_pipe(
+    filename: str, replace_includes: bool = True, content: Optional[str] = None, skip_eval: bool = False
+) -> Datafile:
     basepath = ""
     if not content:
         with open(filename) as file:
@@ -760,7 +764,7 @@ def parse_pipe(filename: str, replace_includes: bool = True, content: Optional[s
 
     try:
         sql = ""
-        doc = parse(s, basepath=basepath, replace_includes=replace_includes)
+        doc = parse(s, basepath=basepath, replace_includes=replace_includes, skip_eval=skip_eval)
         for node in doc.nodes:
             sql = node.get("sql", "")
             if sql.strip()[0] == "%":
@@ -795,7 +799,9 @@ def parse_pipe(filename: str, replace_includes: bool = True, content: Optional[s
     return doc
 
 
-def parse_token(filename: str, replace_includes: bool = True, content: Optional[str] = None) -> Datafile:
+def parse_token(
+    filename: str, replace_includes: bool = True, content: Optional[str] = None, skip_eval: bool = False
+) -> Datafile:
     if not content:
         with open(filename) as file:
             s = file.read()
@@ -805,7 +811,7 @@ def parse_token(filename: str, replace_includes: bool = True, content: Optional[
 
     try:
         sql = ""
-        doc = parse(s, basepath=basepath, replace_includes=replace_includes)
+        doc = parse(s, basepath=basepath, replace_includes=replace_includes, skip_eval=skip_eval)
     except ParseException as e:
         raise click.ClickException(
             FeedbackManager.error_parsing_file(
@@ -835,14 +841,20 @@ def _unquote(x: str):
     return x
 
 
-def eval_var(s: str) -> str:
+def eval_var(s: str, skip: bool = False) -> str:
+    if skip:
+        return s
     # replace ENV variables
     # it's probably a bad idea to allow to get any env var
     return Template(s).safe_substitute(os.environ)
 
 
 def parse(
-    s: str, default_node: Optional[str] = None, basepath: str = ".", replace_includes: bool = True
+    s: str,
+    default_node: Optional[str] = None,
+    basepath: str = ".",
+    replace_includes: bool = True,
+    skip_eval: bool = False,
 ) -> Datafile:  # noqa: C901
     """
     Parses `s` string into a document
@@ -898,7 +910,7 @@ def parse(
     def assign_var(v: str) -> Callable[[VarArg(str), KwArg(Any)], None]:
         def _f(*args: str, **kwargs: Any):
             s = _unquote((" ".join(args)).strip())
-            parser_state.current_node[v.lower()] = eval_var(s)
+            parser_state.current_node[v.lower()] = eval_var(s, skip=skip_eval)
 
         return _f
 
@@ -1029,12 +1041,12 @@ def parse(
     def set_engine(*args: str, **kwargs: Any) -> None:
         __init_engine("ENGINE")
         engine_type = _unquote((" ".join(args)).strip())
-        parser_state.current_node["engine"]["type"] = engine_type
+        parser_state.current_node["engine"]["type"] = eval_var(engine_type, skip=skip_eval)
 
     def add_engine_var(v: str) -> Callable[[VarArg(str), KwArg(Any)], None]:
         def _f(*args: str, **kwargs: Any):
             __init_engine(f"ENGINE_{v}".upper())
-            engine_arg = _unquote((" ".join(args)).strip())
+            engine_arg = eval_var(_unquote((" ".join(args)).strip()), skip=skip_eval)
             parser_state.current_node["engine"]["args"].append((v, engine_arg))
 
         return _f
@@ -4470,11 +4482,12 @@ async def format_datasource(
     replace_includes: bool = False,
     datafile: Optional[Datafile] = None,
     for_deploy_diff: bool = False,
+    skip_eval: bool = False,
 ) -> str:
     if datafile:
         doc = datafile
     else:
-        doc = parse_datasource(filename, replace_includes=replace_includes)
+        doc = parse_datasource(filename, replace_includes=replace_includes, skip_eval=skip_eval)
 
     file_parts: List[str] = []
     if for_diff:
@@ -4707,11 +4720,12 @@ async def format_pipe(
     replace_includes: bool = False,
     datafile: Optional[Datafile] = None,
     for_deploy_diff: bool = False,
+    skip_eval: bool = False,
 ) -> str:
     if datafile:
         doc = datafile
     else:
-        doc = parse_pipe(filename, replace_includes=replace_includes)
+        doc = parse_pipe(filename, replace_includes=replace_includes, skip_eval=skip_eval)
 
     file_parts: List[str] = []
     format_sources(file_parts, doc)
@@ -4738,7 +4752,7 @@ async def format_pipe(
                 if "." in include_file
                 else eval_var(include_file)
             )
-            included_pipe = parse_pipe(include_file)
+            included_pipe = parse_pipe(include_file, skip_eval=skip_eval)
             pipe_nodes = doc.nodes.copy()
             for included_node in included_pipe.nodes.copy():
                 unrolled_included_node = next(

{tinybird-cli-3.8.1.dev1 → tinybird-cli-3.8.2.dev2}/tinybird/feedback_manager.py

@@ -205,7 +205,7 @@ class FeedbackManager:
     error_connection_already_exists = error_message(
         "Connection {name} already exists. Use 'tb connection ls' to list your connections"
     )
-    error_connection_does_not_exists = error_message("Connection {connection} does not exist")
+    error_connection_does_not_exists = error_message("Connection {connection_id} does not exist")
     error_connection_create = error_message("Connection {connection_name} could not be created: {error}")
     error_connection_integration_not_available = error_message("Connection could not be created: {error}")
     error_workspace = error_message("Workspace {workspace} not found. use 'tb workspace ls' to list your workspaces")

{tinybird-cli-3.8.1.dev1 → tinybird-cli-3.8.2.dev2}/tinybird/sql_template.py

@@ -1655,6 +1655,11 @@ def get_var_names_and_types(t, node_id=None):  # noqa: C901
         raise SQLTemplateException(e)
 
 
+@lru_cache(maxsize=256)
+def get_var_names_and_types_cached(t: Template):
+    return get_var_names_and_types(t)
+
+
 def wrap_vars(t):
     def _n(chunks, v):
         for x in chunks:
@@ -1746,8 +1751,55 @@ def get_template_and_variables(sql: str, name: Optional[str]):
         raise SQLTemplateException(e)
 
 
+def preprocess_variables(variables: dict, t: Template):
+    """
+    >>> preprocess_variables({"test": "1,2"}, Template("select * from table where f = {{Array(test, 'String')}}"))
+    {'test': ['1', '2']}
+    >>> preprocess_variables({"test": ['1', '2']}, Template("select * from table where f = {{Array(test, 'String')}}"))
+    {'test': ['1', '2']}
+    >>> preprocess_variables({"test": [1,2]}, Template("select * from table where f = {{Array(test, 'String')}}"))
+    {'test': ['1', '2']}
+    >>> preprocess_variables({"test": '24'}, Template("select * from table where f = {{Int32(test)}}"))
+    {}
+    >>> preprocess_variables({"test": "1,2,3"}, Template("select * from table where f = {{Array(test,'Int32')}}"))
+    {'test': [1, 2, 3]}
+    >>> preprocess_variables({"test": "1,2,msg"}, Template("select * from table where f = {{Array(test,'Int32')}}"))
+    {}
+    """
+    template_variables = get_var_names_and_types_cached(t)
+    processed_variables = {}
+    for variable, value in variables.items():
+        try:
+            template_vars = [t_var for t_var in template_variables if t_var["name"] == variable] or None
+            if template_vars is None or value is None:
+                continue
+
+            t_var = template_vars[0]
+            var_type = t_var.get("type")
+            if var_type is None:
+                continue
+
+            # For now, we only preprocess Array types
+            match = re.match(r"Array\((\w+)\)", var_type)
+            if match is None:
+                continue
+
+            array_type = match.group(1)
+            array_fn = type_fns.get("Array")
+            parsed_exp = array_fn(value, array_type)
+            processed_variables[variable] = ast.literal_eval(parsed_exp)
+        except Exception:
+            continue
+
+    return processed_variables
+
+
 def render_sql_template(
-    sql: str, variables: Optional[dict] = None, test_mode: bool = False, name: Optional[str] = None
+    sql: str,
+    variables: Optional[dict] = None,
+    test_mode: bool = False,
+    name: Optional[str] = None,
+    preprocess_variables_flag: bool = False,
 ) -> Tuple[str, dict]:
     """
     >>> render_sql_template("select * from table where f = {{Float32(foo)}}", { 'foo': -1 })
@@ -1934,10 +1986,28 @@ def render_sql_template(
     Traceback (most recent call last):
     ...
     tinybird.sql_template.SQLTemplateException: Template Syntax Error: Error parsing JSON: '' - Expecting value: line 1 column 1 (char 0)
+    >>> render_sql_template("% {% if defined(test) %}{% set _groupByCSV = ','.join(test) %} SELECT test as aa, {{Array(test, 'String')}} as test, {{_groupByCSV}} as a {% end %}", {"test": "1,2"}, preprocess_variables_flag=True)
+    ("%  SELECT test as aa, ['1','2'] as test, '1,2' as a ", {})
+    >>> render_sql_template("% {% if defined(test) %}{% set _groupByCSV = ','.join(test) %} SELECT test as aa, {{Array(test, 'String')}} as test, {{_groupByCSV}} as a {% end %}", {"test": ["1","2"]}, preprocess_variables_flag=True)
+    ("%  SELECT test as aa, ['1','2'] as test, '1,2' as a ", {})
+    >>> render_sql_template("% {% if defined(test) %}{% set _total = sum(test) %} SELECT test as aa, {{Array(test, 'Int32')}} as test, {{_total}} as a {% end %}", {"test": "1,2"}, preprocess_variables_flag=True)
+    ('%  SELECT test as aa, [1,2] as test, 3 as a ', {})
+    >>> render_sql_template("% {% if defined(test) %}{% set _groupByCSV = ','.join(test) %} SELECT test as aa, {{Array(test, 'String')}} as test, {{_groupByCSV}} as a {% end %}", {"test": "1,2"})
+    ("%  SELECT test as aa, ['1','2'] as test, '1,,,2' as a ", {})
+    >>> render_sql_template("% {% if defined(test) %}{% set _groupByCSV = ','.join(test) %} SELECT test as aa, {{Array(test, 'String')}} as test, {{_groupByCSV}} as a {% end %}", {"test": ["1","2"]})
+    ("%  SELECT test as aa, ['1','2'] as test, '1,2' as a ", {})
+    >>> render_sql_template("% {% if defined(test) %}{% set _total = sum(test) %} SELECT test as aa, {{Array(test, 'Int32')}} as test, {{_total}} as a {% end %}", {"test": "1,2"})
+    Traceback (most recent call last):
+    ...
+    ValueError: unsupported operand type(s) for +: 'int' and 'str'
     """
 
     t, template_variables = get_template_and_variables(sql, name)
 
+    if preprocess_variables_flag and variables is not None:
+        processed_variables = preprocess_variables(variables, t)
+        variables.update(processed_variables)
+
     if test_mode:
 
         def dummy(*args, **kwargs):

{tinybird-cli-3.8.1.dev1 → tinybird-cli-3.8.2.dev2}/tinybird/sql_toolset.py

@@ -164,6 +164,7 @@ def replace_tables(
     valid_tables: Optional[Set[Tuple[str, str]]] = None,
     output_one_line: bool = False,
     timestamp: Optional[datetime] = None,
+    preprocess_variables_flag: bool = False,
 ) -> str:
     """
     Given a query and a list of table replacements, returns the query after applying the table replacements.

{tinybird-cli-3.8.1.dev1 → tinybird-cli-3.8.2.dev2}/tinybird/tb_cli_modules/cli.py

@@ -797,9 +797,9 @@ async def fmt(
             click.echo(filename)
         extensions = Path(filename).suffixes
         if is_file_a_datasource(filename):
-            result = await format_datasource(filename)
+            result = await format_datasource(filename, skip_eval=True)
         elif (".pipe" in extensions) or (".incl" in extensions):
-            result = await format_pipe(filename, line_length)
+            result = await format_pipe(filename, line_length, skip_eval=True)
         else:
             click.echo("Unsupported file type. Supported files types are: .pipe, .incl and .datasource")
             return None

{tinybird-cli-3.8.1.dev1 → tinybird-cli-3.8.2.dev2}/tinybird/tb_cli_modules/connection.py

@@ -394,25 +394,39 @@ async def connection_create_bigquery(ctx: Context, no_validate: bool) -> None:
 
 
 @connection.command(name="rm")
-@click.argument("connection_id")
+@click.argument("connection_id_or_name")
 @click.option(
     "--force", default=False, help="Force connection removal even if there are datasources currently using it"
 )
 @click.pass_context
 @coro
-async def connection_rm(ctx: Context, connection_id: str, force: bool) -> None:
+async def connection_rm(ctx: Context, connection_id_or_name: str, force: bool) -> None:
     """Remove a connection."""
 
     obj: Dict[str, Any] = ctx.ensure_object(dict)
     client: TinyB = obj["client"]
 
     try:
-        await client.connector_delete(connection_id)
+        await client.connector_delete(connection_id_or_name)
     except DoesNotExistException:
-        raise CLIConnectionException(FeedbackManager.error_connection_does_not_exists(connection_id=connection_id))
+        connections = await client.connections()
+        connection = next(
+            (connection for connection in connections if connection["name"] == connection_id_or_name), None
+        )
+        if connection:
+            try:
+                await client.connector_delete(connection["id"])
+            except DoesNotExistException:
+                raise CLIConnectionException(
+                    FeedbackManager.error_connection_does_not_exists(connection_id=connection_id_or_name)
+                )
+        else:
+            raise CLIConnectionException(
+                FeedbackManager.error_connection_does_not_exists(connection_id=connection_id_or_name)
+            )
     except Exception as e:
         raise CLIConnectionException(FeedbackManager.error_exception(error=e))
-    click.echo(FeedbackManager.success_delete_connection(connection_id=connection_id))
+    click.echo(FeedbackManager.success_delete_connection(connection_id=connection_id_or_name))
 
 
 @connection.command(name="ls")
@@ -698,6 +712,7 @@ async def connection_create_gcs(
 @click.option("--connection-name", default=None, help="The name of the connection to identify it in Tinybird")
 @click.option("--role-arn", default=None, help="The ARN of the IAM role to use for the connection")
 @click.option("--region", default=None, help="The Amazon S3 region where the bucket is located")
+@click.option("--policy", default="write", help="The Amazon S3 access policy: write or read")
 @click.option(
     "--no-validate", is_flag=True, default=False, help="Do not validate S3 permissions during connection creation"
 )
@@ -708,6 +723,7 @@ async def connection_create_s3_iamrole(
     connection_name: Optional[str] = "",
     role_arn: Optional[str] = "",
     region: Optional[str] = "",
+    policy: str = "write",
     no_validate: Optional[bool] = False,
 ) -> None:
     """
@@ -723,7 +739,12 @@ async def connection_create_s3_iamrole(
     async def get_s3_policies():
         s3_access_policy: Dict[str, Any] = {}
         try:
-            s3_access_policy = await client.get_s3_access_policy()
+            if policy == "write":
+                s3_access_policy = await client.get_s3_access_write_policy()
+            elif policy == "read":
+                s3_access_policy = await client.get_s3_access_read_policy()
+            else:
+                raise Exception(f"Access policy {policy} not supported. Choose from 'read' or 'write'")
             if not len(s3_access_policy) > 0:
                 raise Exception("S3 Integration not supported in this region")
         except Exception as e:

{tinybird-cli-3.8.1.dev1 → tinybird-cli-3.8.2.dev2}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 3.8.1.dev1
+Version: 3.8.2.dev2
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -16,9 +16,25 @@ Tinybird CLI
 The Tinybird command-line tool allows you to use all the Tinybird functionality directly from the command line. Additionally, it includes several functions to create and manage data projects easily.
 
 Changelog
-
 ---------
 
+3.8.2.dev2
+************
+
+- `Added` New `--policy` option for `create s3_iamrole` command that will generate different hints depending on the case
+
+3.8.2.dev1
+************
+
+- `Added` Support for connection names when doing `tb connection rm`
+
+3.8.1
+************
+
+- `Fixed` Avoid system vars evaluation when doing `tb fmt`
+- `Fixed` environment variables substitution for Data Source engine parameters.
+
+
 3.8.0
 ************