tinybird 0.0.1.dev117__tar.gz → 0.0.1.dev118__tar.gz

{tinybird-0.0.1.dev117 → tinybird-0.0.1.dev118}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev117
+Version: 0.0.1.dev118
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev117 → tinybird-0.0.1.dev118}/tinybird/config.py

@@ -124,8 +124,12 @@ def get_display_host(ui_host: str):
     return LEGACY_HOSTS.get(ui_host, ui_host)
 
 
-def get_display_cloud_host(cloud_host: str):
-    return CLOUD_HOSTS.get(cloud_host, cloud_host)
+def get_display_cloud_host(api_host: str) -> str:
+    is_local = "localhost" in api_host
+    if is_local:
+        port = api_host.split(":")[-1]
+        return f"http://cloud.tinybird.co/local/{port}"
+    return CLOUD_HOSTS.get(api_host, api_host)
 
 
 class FeatureFlags:

{tinybird-0.0.1.dev117 → tinybird-0.0.1.dev118}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev117'
-__revision__ = 'f1a78c6'
+__version__ = '0.0.1.dev118'
+__revision__ = '5932fa5'

{tinybird-0.0.1.dev117 → tinybird-0.0.1.dev118}/tinybird/tb/modules/common.py

@@ -1754,27 +1754,26 @@ async def remove_release(
 async def run_aws_iamrole_connection_flow(
     client: TinyB,
     service: str,
-    policy: str = "read",
-):
+    environment: str,
+) -> Tuple[str, str, str]:
     if service == DataConnectorType.AMAZON_DYNAMODB:
-        raise NotImplementedError("DynamoDB is not supported yet")
+        raise NotImplementedError("DynamoDB is not supported")
 
-    resource_type = "table" if service == DataConnectorType.AMAZON_DYNAMODB else "bucket"
-    resource_name = click.prompt(f"Enter the name of the {resource_type}")
-    validate_string_connector_param(resource_type.capitalize(), resource_name)
+    bucket_name = click.prompt(
+        "📦 Bucket name (specific name recommended, use '*' for unrestricted access in IAM policy)",
+        prompt_suffix="\n> ",
+    )
+    validate_string_connector_param("Bucket", bucket_name)
 
-    resource_region = click.prompt(f"Enter the region where the {resource_type} is located")
-    validate_string_connector_param("Region", resource_region)
+    region = click.prompt("🌐 Region (the region where the bucket is located, e.g. 'us-east-1')", prompt_suffix="\n> ")
+    validate_string_connector_param("Region", region)
 
-    access_policy, trust_policy, external_id = await get_aws_iamrole_policies(client, service=service, policy=policy)
-    access_policy = access_policy.replace("<bucket>", resource_name)
+    access_policy, trust_policy, _ = await get_aws_iamrole_policies(client, service=service, policy="read")
+    access_policy = access_policy.replace("<bucket>", bucket_name)
 
-    if not click.confirm(
-        FeedbackManager.prompt_s3_iamrole_connection_login_aws(),
-        show_default=False,
-        prompt_suffix="Press y to continue:",
-    ):
-        sys.exit(1)
+    click.echo(FeedbackManager.prompt_s3_iamrole_connection_login_aws())
+    click.echo(FeedbackManager.click_enter_to_continue())
+    input()
 
     access_policy_copied = True
     try:
@@ -1782,16 +1781,58 @@ async def run_aws_iamrole_connection_flow(
     except Exception:
         access_policy_copied = False
 
-    if not click.confirm(
+    # Display message and wait for user to press Enter
+    click.echo(
+        FeedbackManager.prompt_s3_iamrole_connection_policy(
+            access_policy=access_policy, aws_region=region, bucket=bucket_name
+        )
+        if access_policy_copied
+        else FeedbackManager.prompt_s3_iamrole_connection_policy_not_copied(
+            access_policy=access_policy, aws_region=region, bucket=bucket_name
+        )
+    )
+    click.echo(FeedbackManager.click_enter_to_continue())
+    input()
+
+    trust_policy_copied = True
+    try:
+        pyperclip.copy(trust_policy)
+    except Exception:
+        trust_policy_copied = False
+
+    role_arn = click.prompt(
         (
-            FeedbackManager.prompt_s3_iamrole_connection_policy(access_policy=access_policy)
-            if access_policy_copied
-            else FeedbackManager.prompt_s3_iamrole_connection_policy_not_copied(access_policy=access_policy)
+            FeedbackManager.prompt_s3_iamrole_connection_role(
+                trust_policy=trust_policy,
+                aws_region=region,
+                bucket=bucket_name,
+                environment=environment.upper(),
+                step="3",
+            )
+            if trust_policy_copied
+            else FeedbackManager.prompt_s3_iamrole_connection_role_not_copied(
+                trust_policy=trust_policy,
+                aws_region=region,
+                bucket=bucket_name,
+                environment=environment.upper(),
+                step="3",
+            )
         ),
         show_default=False,
-        prompt_suffix="Press y to continue:",
-    ):
-        sys.exit(1)
+    )
+    validate_string_connector_param("Role ARN", role_arn)
+
+    return role_arn, region, bucket_name
+
+
+async def production_aws_iamrole_only(
+    prod_client: TinyB,
+    service: str,
+    region: str,
+    bucket_name: str,
+    environment: str,
+) -> Tuple[str, str, str]:
+    _, trust_policy, external_id = await get_aws_iamrole_policies(prod_client, service=service, policy="read")
 
     trust_policy_copied = True
     try:
@@ -1799,21 +1840,29 @@ async def run_aws_iamrole_connection_flow(
     except Exception:
         trust_policy_copied = False
 
-    if not click.confirm(
+    role_arn = click.prompt(
         (
-            FeedbackManager.prompt_s3_iamrole_connection_role(trust_policy=trust_policy)
+            FeedbackManager.prompt_s3_iamrole_connection_role(
+                trust_policy=trust_policy,
+                aws_region=region,
+                bucket=bucket_name,
+                environment=environment.upper(),
+                step="4",
+            )
             if trust_policy_copied
-            else FeedbackManager.prompt_s3_iamrole_connection_role_not_copied(trust_policy=trust_policy)
+            else FeedbackManager.prompt_s3_iamrole_connection_role_not_copied(
+                trust_policy=trust_policy,
+                aws_region=region,
+                bucket=bucket_name,
+                environment=environment.upper(),
+                step="4",
+            )
         ),
         show_default=False,
-        prompt_suffix="Press y to continue:",
-    ):
-        sys.exit(1)
-
-    role_arn = click.prompt("Enter the ARN of the role you just created")
+    )
     validate_string_connector_param("Role ARN", role_arn)
 
-    return role_arn, resource_region, external_id
+    return role_arn, region, external_id
 
 
 async def get_aws_iamrole_policies(client: TinyB, service: str, policy: str = "write"):
@@ -1846,16 +1895,35 @@ async def get_aws_iamrole_policies(client: TinyB, service: str, policy: str = "w
     return json.dumps(access_policy, indent=4), json.dumps(trust_policy, indent=4), external_id
 
 
-async def validate_aws_iamrole_connection_name(client: TinyB) -> str:
+def get_s3_connection_name(project_folder: str) -> str:
     connection_name = None
+    valid_pattern = r"^[a-zA-Z][a-zA-Z0-9_]*$"
+
     while not connection_name:
-        connection_name = click.prompt("Enter the name for this connection", default=None, show_default=False)
+        connection_name = click.prompt(
+            "🔗 Enter a name for your new Tinybird S3 connection (use alphanumeric characters, and underscores)",
+            prompt_suffix="\n> ",
+            show_default=True,
+        )
         assert isinstance(connection_name, str)
 
-        if await client.get_connector(connection_name) is not None:
-            click.echo(FeedbackManager.info_connection_already_exists(name=connection_name))
+        # Validate against invalid characters
+        if not re.match(valid_pattern, connection_name):
+            if not connection_name[0].isalpha():
+                click.echo("Error: Connection name must start with a letter.")
+            else:
+                click.echo("Error: Connection name can only contain letters, numbers, and underscores.")
             connection_name = None
-    assert isinstance(connection_name, str)
+            continue
+
+        # Check for existing connection with the same name
+        project_folder_path = Path(project_folder)
+        connection_files = list(project_folder_path.glob("*.connection"))
+        for conn_file in connection_files:
+            if conn_file.stem == connection_name:
+                click.echo(FeedbackManager.error_connection_file_already_exists(name=f"{connection_name}.connection"))
+                connection_name = None
+                break
     return connection_name
 
 

{tinybird-0.0.1.dev117 → tinybird-0.0.1.dev118}/tinybird/tb/modules/connection.py

@@ -3,6 +3,7 @@
 # - If it makes sense and only when strictly necessary, you can create utility functions in this file.
 # - But please, **do not** interleave utility functions and command definitions.
 
+import uuid
 from typing import Any, Dict, List, Optional
 
 import click
@@ -15,10 +16,11 @@ from tinybird.tb.modules.common import (
     _get_setting_value,
     coro,
     echo_safe_humanfriendly_tables_format_smart_table,
+    get_s3_connection_name,
+    production_aws_iamrole_only,
     run_aws_iamrole_connection_flow,
-    validate_aws_iamrole_connection_name,
 )
-from tinybird.tb.modules.create import generate_aws_iamrole_connection_file
+from tinybird.tb.modules.create import generate_aws_iamrole_connection_file_with_secret
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.project import Project
 
@@ -148,20 +150,50 @@ async def connection_create_s3(ctx: Context) -> None:
     project: Project = ctx.ensure_object(dict)["project"]
     obj: Dict[str, Any] = ctx.ensure_object(dict)
     client: TinyB = obj["client"]
+
     service = DataConnectorType.AMAZON_S3
-    connection_name = await validate_aws_iamrole_connection_name(client)
-    role_arn, region, external_id = await run_aws_iamrole_connection_flow(
+    click.echo(FeedbackManager.prompt_s3_connection_header())
+    connection_name = get_s3_connection_name(project.folder)
+    role_arn, region, bucket_name = await run_aws_iamrole_connection_flow(
         client,
         service=service,
-        policy="read",  # For now only read since we only support import from S3
+        environment=obj["env"],
+    )
+    unique_suffix = uuid.uuid4().hex[:8]  # Use first 8 chars of a UUID for brevity
+    secret_name = f"s3_role_arn_{connection_name}_{unique_suffix}"
+    await client.create_secret(name=secret_name, value=role_arn)
+
+    create_in_cloud = (
+        click.confirm(FeedbackManager.prompt_s3_iamrole_success_cloud(), default=True)
+        if obj["env"] == "local"
+        else False
     )
 
-    await generate_aws_iamrole_connection_file(
-        name=connection_name, service=service, role_arn=role_arn, region=region, folder=project.folder
+    if create_in_cloud:
+        prod_config = obj["config"]
+        host = prod_config["host"]
+        token = prod_config["token"]
+        prod_client = TinyB(
+            token=token,
+            host=host,
+            staging=False,
+        )
+        prod_role_arn, _, _ = await production_aws_iamrole_only(
+            prod_client, service=service, region=region, bucket_name=bucket_name, environment="cloud"
+        )
+        await prod_client.create_secret(name=secret_name, value=prod_role_arn)
+
+    connection_file_path = await generate_aws_iamrole_connection_file_with_secret(
+        name=connection_name,
+        service=service,
+        role_arn_secret_name=secret_name,
+        region=region,
+        folder=project.folder,
     )
-    if external_id:
-        click.echo(
-            FeedbackManager.success_s3_iam_connection_created(
-                connection_name=connection_name, external_id=external_id, role_arn=role_arn
-            )
+
+    click.echo(
+        FeedbackManager.prompt_s3_iamrole_success(
+            connection_name=connection_name,
+            connection_path=str(connection_file_path),
         )
+    )

{tinybird-0.0.1.dev117 → tinybird-0.0.1.dev118}/tinybird/tb/modules/create.py

@@ -362,26 +362,27 @@ def generate_pipe_file(name: str, content: str, folder: str) -> Path:
     return f.relative_to(folder)
 
 
-def generate_connection_file(name: str, content: str, folder: str) -> Path:
+def generate_connection_file(name: str, content: str, folder: str, skip_feedback: bool = False) -> Path:
     base = Path(folder) / "connections"
     if not base.exists():
         base.mkdir()
     f = base / (f"{name}.connection")
     with open(f"{f}", "w") as file:
         file.write(content)
-    click.echo(FeedbackManager.info_file_created(file=f.relative_to(folder)))
+    if not skip_feedback:
+        click.echo(FeedbackManager.info_file_created(file=f.relative_to(folder)))
     return f.relative_to(folder)
 
 
-async def generate_aws_iamrole_connection_file(
-    name: str, service: str, role_arn: str, region: str, folder: str
-) -> None:
+async def generate_aws_iamrole_connection_file_with_secret(
+    name: str, service: str, role_arn_secret_name: str, region: str, folder: str
+) -> Path:
     content = f"""TYPE {service}
-
-S3_ARN {role_arn}
+S3_ARN {{{{ tb_secret("{role_arn_secret_name}") }}}}
 S3_REGION {region}
 """
-    generate_connection_file(name, content, folder)
+    file_path = generate_connection_file(name, content, folder, skip_feedback=True)
+    return file_path
 
 
 def create_rules(folder: str, source: str, agent: str):

{tinybird-0.0.1.dev117 → tinybird-0.0.1.dev118}/tinybird/tb/modules/deployment.py

@@ -10,7 +10,7 @@ import click
 import requests
 
 from tinybird.tb.modules.cli import cli
-from tinybird.tb.modules.common import echo_safe_humanfriendly_tables_format_smart_table
+from tinybird.tb.modules.common import echo_safe_humanfriendly_tables_format_smart_table, get_display_cloud_host
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.project import Project
 
@@ -214,8 +214,14 @@ def deployment_create(
 
 
 @deployment_group.command(name="ls")
+@click.option(
+    "--include-deleted",
+    is_flag=True,
+    default=False,
+    help="Include deleted deployments. Disabled by default.",
+)
 @click.pass_context
-def deployment_ls(ctx: click.Context) -> None:
+def deployment_ls(ctx: click.Context, include_deleted: bool) -> None:
     """
     List all the deployments you have in the project.
     """
@@ -224,11 +230,12 @@ def deployment_ls(ctx: click.Context) -> None:
     TINYBIRD_API_KEY = client.token
     HEADERS = {"Authorization": f"Bearer {TINYBIRD_API_KEY}"}
     url = f"{client.host}/v1/deployments"
+    if include_deleted:
+        url += "?include_deleted=true"
 
     result = api_fetch(url, HEADERS)
-
-    status_map = {"data_ready": "Ready", "failed": "Failed"}
-    columns = ["ID", "Status", "Created at", "Live"]
+    status_map = {"data_ready": "Staging", "failed": "Failed", "deleted": "Deleted"}
+    columns = ["ID", "Status", "Created at"]
     table = []
     for deployment in result.get("deployments", []):
         if deployment.get("id") == "0":
@@ -237,12 +244,12 @@ def deployment_ls(ctx: click.Context) -> None:
         table.append(
             [
                 deployment.get("id"),
-                status_map.get(deployment.get("status"), "In progress"),
+                "Live" if deployment.get("live") else status_map.get(deployment.get("status", "In progress")),
                 datetime.fromisoformat(deployment.get("created_at")).strftime("%Y-%m-%d %H:%M:%S"),
-                deployment.get("live"),
             ]
         )
 
+    table.reverse()
     echo_safe_humanfriendly_tables_format_smart_table(table, column_names=columns)
 
 
@@ -401,14 +408,11 @@ def create_deployment(
 
         status = result.get("result")
         if status == "success":
-            # TODO: This is a hack to show the url in the case of region is public. The URL should be returned by the API
-            if client.host == "https://api.europe-west2.gcp.tinybird.co":
-                click.echo(
-                    FeedbackManager.gray(message="Deployment URL: ")
-                    + FeedbackManager.info(
-                        message=f"https://cloud.tinybird.co/gcp/europe-west2/{config.get('name')}/deployments/{deployment.get('id')}"
-                    )
-                )
+            host = get_display_cloud_host(client.host)
+            click.echo(
+                FeedbackManager.gray(message="Deployment URL: ")
+                + FeedbackManager.info(message=f"{host}/{config.get('name')}/deployments/{deployment.get('id')}")
+            )
 
             if wait:
                 click.echo(FeedbackManager.info(message="\n✓ Deployment submitted successfully"))

{tinybird-0.0.1.dev117 → tinybird-0.0.1.dev118}/tinybird/tb/modules/feedback_manager.py

@@ -496,19 +496,106 @@ class FeedbackManager:
 Ready? """
     )
 
-    prompt_s3_iamrole_connection_login_aws = prompt_message("""[1] Log into your AWS Console\n\n""")
+    # S3 IAM Role Connection Messages
+    prompt_s3_connection_header = info_highlight_message("""
+──────────────────────────────────────────────────────────────
+  S3 CONNECTION SETUP
+──────────────────────────────────────────────────────────────""")
+
+    prompt_s3_iamrole_connection_login_aws = prompt_message("""
+──────────────────────────────────────────────────────────────
+  STEP 1: AWS AUTHENTICATION
+──────────────────────────────────────────────────────────────
+
+Please log into your \033]8;;https://console.aws.amazon.com/\033\\AWS Console\033]8;;\033\\. We'll guide you through creating the necessary permissions.
+
+You'll be creating a single IAM Policy and Roles (you will need one per environment, Local and Cloud) to access your S3 data. Using IAM Roles improves security by providing temporary credentials and following least privilege principles.""")
+
     prompt_s3_iamrole_connection_policy = prompt_message(
-        """\n[2] Go to IAM > Policies. Create a new policy with the following permissions:\n\n{access_policy}\n\n(The policy has been copied to your clipboard)\n\n"""
+        """
+──────────────────────────────────────────────────────────────
+STEP 2: CREATE IAM POLICY
+──────────────────────────────────────────────────────────────
+
+1. Go to \033]8;;https://console.aws.amazon.com/iamv2/home?region={aws_region}#/policies/create\033\\AWS IAM > Create Policy\033]8;;\033\\
+2. Select the JSON tab
+3. Paste the following policy (already copied to clipboard):
+
+{access_policy}
+
+4. Name the policy something meaningful (e.g., TinybirdS3Access-{bucket})
+5. Click "Create policy"
+"""
     )
+    click_enter_to_continue = prompt_message("\nPress Enter to continue...")
+
     prompt_s3_iamrole_connection_policy_not_copied = prompt_message(
-        """\n[2] Go to IAM > Policies. Create a new policy with the following permissions. Please, copy this policy and replace <bucket> with your bucket name:\n\n{access_policy}\n\n"""
+        """
+──────────────────────────────────────────────────────────────
+STEP 2: CREATE IAM POLICY
+──────────────────────────────────────────────────────────────
+
+1. Go to \033]8;;https://console.aws.amazon.com/iamv2/home?region={aws_region}#/policies/create\033\\AWS IAM > Create Policy\033]8;;\033\\
+2. Select the JSON tab
+3. Copy and paste the following policy (replace <bucket> with your bucket name):
+
+{access_policy}
+
+4. Name the policy something meaningful (e.g., TinybirdS3Access-{bucket})
+5. Click "Create policy"
+"""
     )
+
     prompt_s3_iamrole_connection_role = prompt_message(
-        """\n[3] Go to IAM > Roles. Create a new IAM Role using the following custom trust policy and attach the access policy you just created in the previous step:\n\n{trust_policy}\n\n(The policy has been copied to your clipboard)\n\n"""
+        """
+──────────────────────────────────────────────────────────────
+STEP {step}: CREATE IAM ROLE FOR {environment} ENVIRONMENT
+──────────────────────────────────────────────────────────────
+
+1. Go to \033]8;;https://console.aws.amazon.com/iamv2/home?region={aws_region}#/roles/create\033\\AWS IAM > Create Role\033]8;;\033\\
+2. Choose "Custom trust policy" and paste the following (already copied to clipboard):
+
+{trust_policy}
+
+3. Click Next, search for and select the policy you just created
+4. Name the role something meaningful (e.g., TinybirdS3Role-{bucket}-{environment})
+5. Click "Create role"
+6. Copy the Role ARN from the role details page
+
+Please enter the ARN of the role you just created"""
     )
+
+    prompt_s3_iamrole_success_cloud = prompt_message(
+        "Would you like to create this connection in the cloud environment as well?"
+    )
+
     prompt_s3_iamrole_connection_role_not_copied = prompt_message(
-        """\n[3] Go to IAM > Roles. Create a new IAM Role using the following custom trust policy and attach the access policy you just created in the previous step:\n\n{trust_policy}\n\n"""
+        """
+──────────────────────────────────────────────────────────────
+STEP {step}: CREATE IAM ROLE FOR {environment} ENVIRONMENT
+──────────────────────────────────────────────────────────────
+
+1. Go to \033]8;;https://console.aws.amazon.com/iamv2/home?region={aws_region}#/roles/create\033\\AWS IAM > Create Role\033]8;;\033\\
+2. Choose "Custom trust policy" and paste the following:
+
+{trust_policy}
+
+3. Click Next, search for and select the policy you just created
+4. Name the role something meaningful (e.g., TinybirdS3Role-{bucket}-{environment})
+5. Click "Create role"
+6. Copy the Role ARN from the role details page
+
+Please enter the ARN of the role you just created"""
     )
+
+    prompt_s3_iamrole_success = success_message("""
+✅ S3 CONNECTION CONFIGURED SUCCESSFULLY
+
+  • File created at: {connection_path}
+  • You can now use this connection in your Data Sources with: IMPORT_CONNECTION_NAME '{connection_name}'
+  • \033]8;;https://www.tinybird.co/docs/forward/get-data-in/connectors/s3\033\\Learn more about our S3 Connector\033]8;;\033\\
+""")
+
     prompt_init_git_release_pull = prompt_message(
         "❓ Download the Data Project to continue, otherwise you can't initialize Workspace with Git. Execute '{pull_command}'?"
     )

{tinybird-0.0.1.dev117 → tinybird-0.0.1.dev118}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev117
+Version: 0.0.1.dev118
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird