tinybird-cli 3.2.1.dev5__tar.gz → 3.2.1.dev7__tar.gz

{tinybird-cli-3.2.1.dev5 → tinybird-cli-3.2.1.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 3.2.1.dev5
+Version: 3.2.1.dev7
 Summary: Tinybird Command Line Tool
 Home-page: https://docs.tinybird.co/cli.html
 Author: Tinybird
@@ -13,18 +13,27 @@ Provides-Extra: snowflake
 Tinybird CLI
 ============
 
-The Tinybird command-line tool allows you to use all the Tinybird functionality directly from the command line. Additionally, it includes several functions to create and manage data projects easily.
+The Tinybird command-line tool allows you to use all the Tinybird functionality directly from the command line. Additionally it includes several functions to create and manage data projects easily.
 
 Changelog
 
 ---------
 
+3.2.1.dev7
+************
+
+- `Added` `bi_stats`, `bi_stats_rt` and `endpoint_errors` to the list of service data sources that the CLI ignore when looking for dependencies of pipes being pushed.
+
+3.2.1.dev6
+************
+
+- `Added` S3 Iam Role connection creation
+
 3.2.1.dev5
 ************
 
 - `Fixed` Homepage URL in PyPI
 
-
 3.2.1.dev4
 ************
 

{tinybird-cli-3.2.1.dev5 → tinybird-cli-3.2.1.dev7}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://docs.tinybird.co/cli.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '3.2.1.dev5'
-__revision__ = '0395b82'
+__version__ = '3.2.1.dev7'
+__revision__ = '4224631'

{tinybird-cli-3.2.1.dev5 → tinybird-cli-3.2.1.dev7}/tinybird/client.py

@@ -991,6 +991,12 @@ class TinyB(object):
         response = await self._req(f"/v0/connectors/snowflake/warehouses?{urlencode(params)}", method="POST", data="")
         return response["warehouses"]
 
+    async def get_s3_trust_policy(self) -> Dict[str, Any]:
+        return await self._req("/v0/integrations/s3/policies/trust-policy")
+
+    async def get_s3_access_policy(self) -> Dict[str, Any]:
+        return await self._req("/v0/integrations/s3/policies/access-policy")
+
     async def sql_get_format(self, sql: str, with_clickhouse_format: bool = False) -> str:
         try:
             if with_clickhouse_format:

{tinybird-cli-3.2.1.dev5 → tinybird-cli-3.2.1.dev7}/tinybird/connector_settings.py

@@ -1,19 +1,13 @@
 from __future__ import annotations
 
-import logging
 from enum import Enum
 from typing import List, Optional, Union
 
 from pydantic import BaseModel
 
-from tinybird.data_connectors.credentials import (
-    GCSConnectorCredentials,
-    S3ConnectorCredentials,
-    S3IAMRoleConnectorCredentials,
-)
+from tinybird.data_connectors.credentials import GCSConnectorCredentials, IAMRoleAWSCredentials, S3ConnectorCredentials
 
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
+GCS_BASE_URL = "https://storage.googleapis.com"
 
 
 class DataConnectorType(str, Enum):
@@ -80,8 +74,8 @@ class S3IAMConnectorSetting(DataConnectorSetting):
     endpoint_url: Optional[str] = None
 
     @property
-    def credentials(self) -> S3IAMRoleConnectorCredentials:
-        return S3IAMRoleConnectorCredentials(
+    def credentials(self) -> IAMRoleAWSCredentials:
+        return IAMRoleAWSCredentials(
             role_arn=self.s3_iamrole_arn,
             external_id=self.s3_iamrole_external_id,
             region=self.s3_iamrole_region,
@@ -116,7 +110,7 @@ class BigQueryConnectorSetting(DataConnectorSetting):
 class GCSHmacConnectorSetting(DataConnectorSetting):
     gcs_hmac_access_id: str
     gcs_hmac_secret: str
-    endpoint_url: Optional[str] = None
+    endpoint_url: str = GCS_BASE_URL
 
     @property
     def credentials(self) -> GCSConnectorCredentials:

{tinybird-cli-3.2.1.dev5 → tinybird-cli-3.2.1.dev7}/tinybird/data_connectors/credentials.py

@@ -14,7 +14,7 @@ class S3ConnectorCredentials(ConnectorCredentials):
 
 
 @dataclasses.dataclass(frozen=True)
-class S3IAMRoleConnectorCredentials(ConnectorCredentials):
+class IAMRoleAWSCredentials(ConnectorCredentials):
     role_arn: str
     external_id: str
     region: str

{tinybird-cli-3.2.1.dev5 → tinybird-cli-3.2.1.dev7}/tinybird/datafile.py

@@ -81,6 +81,9 @@ INTERNAL_TABLES: Tuple[str, ...] = (
     "data_connectors_log",
     "kafka_ops_log",
     "datasources_storage",
+    "endpoint_errors",
+    "bi_stats_rt",
+    "bi_stats",
 )
 
 

{tinybird-cli-3.2.1.dev5 → tinybird-cli-3.2.1.dev7}/tinybird/feedback_manager.py

@@ -196,6 +196,7 @@ class FeedbackManager:
     )
     error_connection_does_not_exists = error_message("Connection {connection} does not exist")
     error_connection_create = error_message("Connection {connection_name} could not be created: {error}")
+    error_connection_integration_not_available = error_message("Connection could not be created: {error}")
     error_workspace = error_message("Workspace {workspace} not found. use 'tb workspace ls' to list your workspaces")
     error_branch = error_message(
         "Branch {branch} not found. use 'tb branch ls' to list your Branches, make sure you are authenticated using the right workspace token"
@@ -360,6 +361,15 @@ class FeedbackManager:
 ** Assign it the role "BigQuery Data Viewer"
 Ready? """
     )
+
+    prompt_s3_iamrole_connection_login_aws = prompt_message("""[1] Log into your AWS Console\n\n""")
+    prompt_s3_iamrole_connection_policy = prompt_message(
+        """\n[2] Go to IAM > Policies. Create a new policy with the following permissions. Please, replace <bucket> with your bucket name:\n\n{access_policy}\n\n(The policy has been copied to your clipboard)\n\n"""
+    )
+    prompt_s3_iamrole_connection_role = prompt_message(
+        """\n[3] Go to IAM > Roles. Create a new IAM Role using the following custom trust policy and attach the access policy you just created in the previous step:\n\n{trust_policy}\n\n(The policy has been copied to your clipboard)\n\n"""
+    )
+
     prompt_init_git_release_pull = prompt_message(
         "❓ Download the Data Project to continue, otherwise you can't initialize Workspace with Git. Execute '{pull_command}'?"
     )
@@ -390,6 +400,7 @@ Ready? """
         "Connection {name} already exists. Use 'tb connection ls' to list your connections"
     )
     info_creating_kafka_connection = info_message("** Creating new Kafka connection '{connection_name}'")
+    info_creating_s3_iamrole_connection = info_message("** Creating new S3 IAM Role connection '{connection_name}'")
 
     warning_remove_oldest_rollback = warning_message(
         "[WARNING] Will try to remove oldest rollback Release before promoting to live Release {semver}."

{tinybird-cli-3.2.1.dev5 → tinybird-cli-3.2.1.dev7}/tinybird/tb_cli_modules/common.py

@@ -1147,15 +1147,19 @@ async def print_branch_regression_tests_summary(client, job_id, host, response=N
                     step["run"]["metrics_summary"].get("passed", 0),
                     step["run"]["metrics_summary"].get("failed", 0),
                     format_metric(
-                        step["run"]["metrics_timing"]["mean response time"][2]
-                        if "mean response time" in step["run"]["metrics_timing"]
-                        else 0.0,
+                        (
+                            step["run"]["metrics_timing"]["mean response time"][2]
+                            if "mean response time" in step["run"]["metrics_timing"]
+                            else 0.0
+                        ),
                         is_percentage=True,
                     ),
                     format_metric(
-                        step["run"]["metrics_timing"]["mean read bytes"][2]
-                        if "mean read bytes" in step["run"]["metrics_timing"]
-                        else 0.0,
+                        (
+                            step["run"]["metrics_timing"]["mean read bytes"][2]
+                            if "mean read bytes" in step["run"]["metrics_timing"]
+                            else 0.0
+                        ),
                         is_percentage=True,
                     ),
                 ]
@@ -1622,6 +1626,12 @@ class ConnectionReplacements:
             "secret": "s3_secret_access_key",
             "region": "s3_region",
         },
+        "s3_iamrole": {
+            "service": "service",
+            "connection_name": "name",
+            "role_arn": "s3_iamrole_arn",
+            "region": "s3_iamrole_region",
+        },
         "gcs_hmac": {
             "service": "service",
             "connection_name": "name",

{tinybird-cli-3.2.1.dev5 → tinybird-cli-3.2.1.dev7}/tinybird/tb_cli_modules/connection.py

@@ -3,12 +3,15 @@
 # - If it makes sense and only when strictly necessary, you can create utility functions in this file.
 # - But please, **do not** interleave utility functions and command definitions.
 
+import json
 import os
+import sys
 from os import getcwd
 from pathlib import Path
 from typing import Any, Dict, Optional
 
 import click
+import pyperclip
 from click import Context
 
 from tinybird.client import DoesNotExistException, TinyB
@@ -620,3 +623,139 @@ async def connection_create_gcs(
 """
         )
     click.echo(FeedbackManager.success_connection_file_created(name=conn_file_name))
+
+
+@connection_create.command(
+    name="s3_iamrole", short_help="Creates a AWS S3 connection using IAM role authentication", hidden=True
+)
+@click.option("--connection-name", default=None, help="The name of the connection to identify it in Tinybird")
+@click.option("--role-arn", default=None, help="The ARN of the IAM role to use for the connection")
+@click.option("--region", default=None, help="The Amazon S3 region where the bucket is located")
+@click.option(
+    "--no-validate", is_flag=True, default=False, help="Do not validate S3 permissions during connection creation"
+)
+@click.pass_context
+@coro
+async def connection_create_s3_iamrole(
+    ctx: Context,
+    connection_name: Optional[str] = "",
+    role_arn: Optional[str] = "",
+    region: Optional[str] = "",
+    no_validate: Optional[bool] = False,
+) -> None:
+    """
+    Creates a S3 connection using IAM role authentication in the current workspace
+
+    \b
+    $ tb connection create s3_iamrole
+    """
+
+    obj: Dict[str, Any] = ctx.ensure_object(dict)
+    client: TinyB = obj["client"]
+
+    async def get_s3_policies():
+        s3_access_policy: Dict[str, Any] = {}
+        try:
+            s3_access_policy = await client.get_s3_access_policy()
+            if not len(s3_access_policy) > 0:
+                raise Exception("S3 Integration not supported in this region")
+        except Exception as e:
+            raise CLIConnectionException(FeedbackManager.error_connection_integration_not_available(error=str(e)))
+
+        s3_trust_policy: Dict[str, Any] = {}
+        try:
+            s3_trust_policy = await client.get_s3_trust_policy()
+            if not len(s3_trust_policy) > 0:
+                raise Exception("S3 Integration not supported in this region")
+        except Exception as e:
+            raise CLIConnectionException(FeedbackManager.error_connection_integration_not_available(error=str(e)))
+
+        return json.dumps(s3_access_policy, indent=4), json.dumps(s3_trust_policy, indent=4)
+
+    async def validate_s3_integration(role_arn: Optional[str], region: Optional[str]):
+        if no_validate is False:
+            access_policy, trust_policy = await get_s3_policies()
+
+            if not role_arn:
+                if not click.confirm(
+                    FeedbackManager.prompt_s3_iamrole_connection_login_aws(),
+                    show_default=False,
+                    prompt_suffix="Press y to continue:",
+                ):
+                    sys.exit(1)
+
+                pyperclip.copy(access_policy)
+                if not click.confirm(
+                    FeedbackManager.prompt_s3_iamrole_connection_policy(access_policy=access_policy),
+                    show_default=False,
+                    prompt_suffix="Press y to continue:",
+                ):
+                    sys.exit(1)
+
+                pyperclip.copy(trust_policy)
+                if not click.confirm(
+                    FeedbackManager.prompt_s3_iamrole_connection_role(trust_policy=trust_policy),
+                    show_default=False,
+                    prompt_suffix="Press y to continue:",
+                ):
+                    sys.exit(1)
+
+        if not role_arn:
+            role_arn = click.prompt("Enter the ARN of the role you just created")
+            validate_string_connector_param("Role ARN", role_arn)
+
+        if not region:
+            region = click.prompt("Enter the region where the bucket is located")
+            validate_string_connector_param("Region", region)
+
+        return role_arn, region
+
+    async def validate_connection_name(connection_name: Optional[str]) -> str:
+        if connection_name and no_validate is False:
+            if await client.get_connector(connection_name) is not None:
+                raise CLIConnectionException(FeedbackManager.info_connection_already_exists(name=connection_name))
+        else:
+            while not connection_name:
+                connection_name = click.prompt("Enter the name for this connection", default=None, show_default=False)
+                assert isinstance(connection_name, str)
+
+                if no_validate is False:
+                    if await client.get_connector(connection_name) is not None:
+                        click.echo(FeedbackManager.info_connection_already_exists(name=connection_name))
+                        connection_name = None
+        assert isinstance(connection_name, str)
+        return connection_name
+
+    async def create_connection(connection_name, role_arn, region) -> None:
+        conn_file_name = f"{connection_name}.connection"
+        conn_file_path = Path(getcwd(), conn_file_name)
+        service = DataConnectors.AMAZON_S3_IAMROLE
+
+        if os.path.isfile(conn_file_path):
+            raise CLIConnectionException(FeedbackManager.error_connection_file_already_exists(name=conn_file_name))
+
+        click.echo(FeedbackManager.info_creating_s3_iamrole_connection(connection_name=connection_name))
+
+        params = ConnectionReplacements.map_api_params_from_prompt_params(
+            service, connection_name=connection_name, role_arn=role_arn, region=region
+        )
+
+        click.echo("** Creating connection...")
+        try:
+            _ = await client.connection_create(params)
+        except Exception as e:
+            raise CLIConnectionException(
+                FeedbackManager.error_connection_create(connection_name=connection_name, error=str(e))
+            )
+
+        with open(conn_file_path, "w") as f:
+            f.write(
+                f"""TYPE {service}
+
+"""
+            )
+        click.echo(FeedbackManager.success_connection_file_created(name=conn_file_name))
+
+    role_arn, region = await validate_s3_integration(role_arn, region)
+    connection_name = await validate_connection_name(connection_name)
+    await create_connection(connection_name, role_arn, region)

{tinybird-cli-3.2.1.dev5 → tinybird-cli-3.2.1.dev7}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 3.2.1.dev5
+Version: 3.2.1.dev7
 Summary: Tinybird Command Line Tool
 Home-page: https://docs.tinybird.co/cli.html
 Author: Tinybird
@@ -13,18 +13,27 @@ Provides-Extra: snowflake
 Tinybird CLI
 ============
 
-The Tinybird command-line tool allows you to use all the Tinybird functionality directly from the command line. Additionally, it includes several functions to create and manage data projects easily.
+The Tinybird command-line tool allows you to use all the Tinybird functionality directly from the command line. Additionally it includes several functions to create and manage data projects easily.
 
 Changelog
 
 ---------
 
+3.2.1.dev7
+************
+
+- `Added` `bi_stats`, `bi_stats_rt` and `endpoint_errors` to the list of service data sources that the CLI ignore when looking for dependencies of pipes being pushed.
+
+3.2.1.dev6
+************
+
+- `Added` S3 Iam Role connection creation
+
 3.2.1.dev5
 ************
 
 - `Fixed` Homepage URL in PyPI
 
-
 3.2.1.dev4
 ************