tinybird-cli 3.8.2.dev1__tar.gz → 3.9.0__tar.gz

{tinybird-cli-3.8.2.dev1 → tinybird-cli-3.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 3.8.2.dev1
+Version: 3.9.0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -16,15 +16,17 @@ Tinybird CLI
 The Tinybird command-line tool allows you to use all the Tinybird functionality directly from the command line. Additionally, it includes several functions to create and manage data projects easily.
 
 Changelog
-
 ---------
 
-3.8.1
+3.9.0
 ************
 
+- `Added` Support for connection names when doing `tb connection rm`
+- `Added` New `--policy` option for `create s3_iamrole` command that will generate different hints depending on the case
 - `Fixed` Avoid system vars evaluation when doing `tb fmt`
 - `Fixed` environment variables substitution for Data Source engine parameters.
 
+
 3.8.0
 ************
 

{tinybird-cli-3.8.2.dev1 → tinybird-cli-3.9.0}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '3.8.2.dev1'
-__revision__ = 'f993ee3'
+__version__ = '3.9.0'
+__revision__ = 'e7e3bee'

{tinybird-cli-3.8.2.dev1 → tinybird-cli-3.9.0}/tinybird/client.py

@@ -997,8 +997,11 @@ class TinyB(object):
     async def get_s3_trust_policy(self) -> Dict[str, Any]:
         return await self._req("/v0/integrations/s3/policies/trust-policy")
 
-    async def get_s3_access_policy(self) -> Dict[str, Any]:
-        return await self._req("/v0/integrations/s3/policies/access-policy")
+    async def get_s3_access_write_policy(self) -> Dict[str, Any]:
+        return await self._req("/v0/integrations/s3/policies/write-access-policy")
+
+    async def get_s3_access_read_policy(self) -> Dict[str, Any]:
+        return await self._req("/v0/integrations/s3/policies/read-access-policy")
 
     async def sql_get_format(self, sql: str, with_clickhouse_format: bool = False) -> str:
         try:

{tinybird-cli-3.8.2.dev1 → tinybird-cli-3.9.0}/tinybird/feedback_manager.py

@@ -205,7 +205,7 @@ class FeedbackManager:
     error_connection_already_exists = error_message(
         "Connection {name} already exists. Use 'tb connection ls' to list your connections"
     )
-    error_connection_does_not_exists = error_message("Connection {connection} does not exist")
+    error_connection_does_not_exists = error_message("Connection {connection_id} does not exist")
     error_connection_create = error_message("Connection {connection_name} could not be created: {error}")
     error_connection_integration_not_available = error_message("Connection could not be created: {error}")
     error_workspace = error_message("Workspace {workspace} not found. use 'tb workspace ls' to list your workspaces")

{tinybird-cli-3.8.2.dev1 → tinybird-cli-3.9.0}/tinybird/sql_template.py

@@ -1562,7 +1562,20 @@ def get_var_data(content, node_id=None):  # noqa: C901
 
         return vars
 
-    parsed = ast.parse(content)
+    def parse_content(content, retries=0):
+        try:
+            parsed = ast.parse(content)
+            return parsed
+        except Exception as e:
+            if "AST constructor recursion depth mismatch" not in str(e):
+                raise e
+            retries += 1
+            if retries > 3:
+                raise e
+            return parse_content(content, retries)
+
+    parsed = parse_content(content)
+
     # calculate parents for each node for later checks
     for node in ast.walk(parsed):
         for child in ast.iter_child_nodes(node):
@@ -1655,6 +1668,11 @@ def get_var_names_and_types(t, node_id=None):  # noqa: C901
         raise SQLTemplateException(e)
 
 
+@lru_cache(maxsize=256)
+def get_var_names_and_types_cached(t: Template):
+    return get_var_names_and_types(t)
+
+
 def wrap_vars(t):
     def _n(chunks, v):
         for x in chunks:
@@ -1746,8 +1764,55 @@ def get_template_and_variables(sql: str, name: Optional[str]):
         raise SQLTemplateException(e)
 
 
+def preprocess_variables(variables: dict, t: Template):
+    """
+    >>> preprocess_variables({"test": "1,2"}, Template("select * from table where f = {{Array(test, 'String')}}"))
+    {'test': ['1', '2']}
+    >>> preprocess_variables({"test": ['1', '2']}, Template("select * from table where f = {{Array(test, 'String')}}"))
+    {'test': ['1', '2']}
+    >>> preprocess_variables({"test": [1,2]}, Template("select * from table where f = {{Array(test, 'String')}}"))
+    {'test': ['1', '2']}
+    >>> preprocess_variables({"test": '24'}, Template("select * from table where f = {{Int32(test)}}"))
+    {}
+    >>> preprocess_variables({"test": "1,2,3"}, Template("select * from table where f = {{Array(test,'Int32')}}"))
+    {'test': [1, 2, 3]}
+    >>> preprocess_variables({"test": "1,2,msg"}, Template("select * from table where f = {{Array(test,'Int32')}}"))
+    {}
+    """
+    template_variables = get_var_names_and_types_cached(t)
+    processed_variables = {}
+    for variable, value in variables.items():
+        try:
+            template_vars = [t_var for t_var in template_variables if t_var["name"] == variable] or None
+            if template_vars is None or value is None:
+                continue
+
+            t_var = template_vars[0]
+            var_type = t_var.get("type")
+            if var_type is None:
+                continue
+
+            # For now, we only preprocess Array types
+            match = re.match(r"Array\((\w+)\)", var_type)
+            if match is None:
+                continue
+
+            array_type = match.group(1)
+            array_fn = type_fns.get("Array")
+            parsed_exp = array_fn(value, array_type)
+            processed_variables[variable] = ast.literal_eval(parsed_exp)
+        except Exception:
+            continue
+
+    return processed_variables
+
+
 def render_sql_template(
-    sql: str, variables: Optional[dict] = None, test_mode: bool = False, name: Optional[str] = None
+    sql: str,
+    variables: Optional[dict] = None,
+    test_mode: bool = False,
+    name: Optional[str] = None,
+    preprocess_variables_flag: bool = False,
 ) -> Tuple[str, dict]:
     """
     >>> render_sql_template("select * from table where f = {{Float32(foo)}}", { 'foo': -1 })
@@ -1934,10 +1999,32 @@ def render_sql_template(
     Traceback (most recent call last):
     ...
     tinybird.sql_template.SQLTemplateException: Template Syntax Error: Error parsing JSON: '' - Expecting value: line 1 column 1 (char 0)
+    >>> render_sql_template("% {% if defined(test) %}{% set _groupByCSV = ','.join(test) %} SELECT test as aa, {{Array(test, 'String')}} as test, {{_groupByCSV}} as a {% end %}", {"test": "1,2"}, preprocess_variables_flag=True)
+    ("%  SELECT test as aa, ['1','2'] as test, '1,2' as a ", {})
+    >>> render_sql_template("% {% if defined(test) %}{% set _groupByCSV = ','.join(test) %} SELECT test as aa, {{Array(test, 'String')}} as test, {{_groupByCSV}} as a {% end %}", {"test": ["1","2"]}, preprocess_variables_flag=True)
+    ("%  SELECT test as aa, ['1','2'] as test, '1,2' as a ", {})
+    >>> render_sql_template("% {% if defined(test) %}{% set _total = sum(test) %} SELECT test as aa, {{Array(test, 'Int32')}} as test, {{_total}} as a {% end %}", {"test": "1,2"}, preprocess_variables_flag=True)
+    ('%  SELECT test as aa, [1,2] as test, 3 as a ', {})
+    >>> render_sql_template("% {% if defined(test) %}{% set _groupByCSV = ','.join(test) %} SELECT test as aa, {{Array(test, 'String')}} as test, {{_groupByCSV}} as a {% end %}", {"test": "1,2"})
+    ("%  SELECT test as aa, ['1','2'] as test, '1,,,2' as a ", {})
+    >>> render_sql_template("% {% if defined(test) %}{% set _groupByCSV = ','.join(test) %} SELECT test as aa, {{Array(test, 'String')}} as test, {{_groupByCSV}} as a {% end %}", {"test": ["1","2"]})
+    ("%  SELECT test as aa, ['1','2'] as test, '1,2' as a ", {})
+    >>> render_sql_template("% {% if defined(test) %}{% set _total = sum(test) %} SELECT test as aa, {{Array(test, 'Int32')}} as test, {{_total}} as a {% end %}", {"test": "1,2"})
+    Traceback (most recent call last):
+    ...
+    ValueError: unsupported operand type(s) for +: 'int' and 'str'
+    >>> render_sql_template("% SELECT {% if defined(x) %} x, 1", preprocess_variables_flag=True)
+    Traceback (most recent call last):
+    ...
+    tinybird.tornado_template.UnClosedIfError: Missing {% end %} block for if at line 1
     """
 
     t, template_variables = get_template_and_variables(sql, name)
 
+    if preprocess_variables_flag and variables is not None:
+        processed_variables = preprocess_variables(variables, t)
+        variables.update(processed_variables)
+
     if test_mode:
 
         def dummy(*args, **kwargs):

{tinybird-cli-3.8.2.dev1 → tinybird-cli-3.9.0}/tinybird/sql_toolset.py

@@ -164,6 +164,7 @@ def replace_tables(
     valid_tables: Optional[Set[Tuple[str, str]]] = None,
     output_one_line: bool = False,
     timestamp: Optional[datetime] = None,
+    preprocess_variables_flag: bool = False,
 ) -> str:
     """
     Given a query and a list of table replacements, returns the query after applying the table replacements.

{tinybird-cli-3.8.2.dev1 → tinybird-cli-3.9.0}/tinybird/tb_cli_modules/connection.py

@@ -394,25 +394,39 @@ async def connection_create_bigquery(ctx: Context, no_validate: bool) -> None:
 
 
 @connection.command(name="rm")
-@click.argument("connection_id")
+@click.argument("connection_id_or_name")
 @click.option(
     "--force", default=False, help="Force connection removal even if there are datasources currently using it"
 )
 @click.pass_context
 @coro
-async def connection_rm(ctx: Context, connection_id: str, force: bool) -> None:
+async def connection_rm(ctx: Context, connection_id_or_name: str, force: bool) -> None:
     """Remove a connection."""
 
     obj: Dict[str, Any] = ctx.ensure_object(dict)
     client: TinyB = obj["client"]
 
     try:
-        await client.connector_delete(connection_id)
+        await client.connector_delete(connection_id_or_name)
     except DoesNotExistException:
-        raise CLIConnectionException(FeedbackManager.error_connection_does_not_exists(connection_id=connection_id))
+        connections = await client.connections()
+        connection = next(
+            (connection for connection in connections if connection["name"] == connection_id_or_name), None
+        )
+        if connection:
+            try:
+                await client.connector_delete(connection["id"])
+            except DoesNotExistException:
+                raise CLIConnectionException(
+                    FeedbackManager.error_connection_does_not_exists(connection_id=connection_id_or_name)
+                )
+        else:
+            raise CLIConnectionException(
+                FeedbackManager.error_connection_does_not_exists(connection_id=connection_id_or_name)
+            )
     except Exception as e:
         raise CLIConnectionException(FeedbackManager.error_exception(error=e))
-    click.echo(FeedbackManager.success_delete_connection(connection_id=connection_id))
+    click.echo(FeedbackManager.success_delete_connection(connection_id=connection_id_or_name))
 
 
 @connection.command(name="ls")
@@ -698,6 +712,7 @@ async def connection_create_gcs(
 @click.option("--connection-name", default=None, help="The name of the connection to identify it in Tinybird")
 @click.option("--role-arn", default=None, help="The ARN of the IAM role to use for the connection")
 @click.option("--region", default=None, help="The Amazon S3 region where the bucket is located")
+@click.option("--policy", default="write", help="The Amazon S3 access policy: write or read")
 @click.option(
     "--no-validate", is_flag=True, default=False, help="Do not validate S3 permissions during connection creation"
 )
@@ -708,6 +723,7 @@ async def connection_create_s3_iamrole(
     connection_name: Optional[str] = "",
     role_arn: Optional[str] = "",
     region: Optional[str] = "",
+    policy: str = "write",
     no_validate: Optional[bool] = False,
 ) -> None:
     """
@@ -723,7 +739,12 @@ async def connection_create_s3_iamrole(
     async def get_s3_policies():
         s3_access_policy: Dict[str, Any] = {}
         try:
-            s3_access_policy = await client.get_s3_access_policy()
+            if policy == "write":
+                s3_access_policy = await client.get_s3_access_write_policy()
+            elif policy == "read":
+                s3_access_policy = await client.get_s3_access_read_policy()
+            else:
+                raise Exception(f"Access policy {policy} not supported. Choose from 'read' or 'write'")
             if not len(s3_access_policy) > 0:
                 raise Exception("S3 Integration not supported in this region")
         except Exception as e:

{tinybird-cli-3.8.2.dev1 → tinybird-cli-3.9.0}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 3.8.2.dev1
+Version: 3.9.0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -16,15 +16,17 @@ Tinybird CLI
 The Tinybird command-line tool allows you to use all the Tinybird functionality directly from the command line. Additionally, it includes several functions to create and manage data projects easily.
 
 Changelog
-
 ---------
 
-3.8.1
+3.9.0
 ************
 
+- `Added` Support for connection names when doing `tb connection rm`
+- `Added` New `--policy` option for `create s3_iamrole` command that will generate different hints depending on the case
 - `Fixed` Avoid system vars evaluation when doing `tb fmt`
 - `Fixed` environment variables substitution for Data Source engine parameters.
 
+
 3.8.0
 ************