tinybird 0.0.1.dev110__tar.gz → 0.0.1.dev112__tar.gz

{tinybird-0.0.1.dev110 → tinybird-0.0.1.dev112}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev110
+Version: 0.0.1.dev112
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev110 → tinybird-0.0.1.dev112}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev110'
-__revision__ = '034b6bf'
+__version__ = '0.0.1.dev112'
+__revision__ = 'eb9e376'

{tinybird-0.0.1.dev110 → tinybird-0.0.1.dev112}/tinybird/tb/modules/cli.py

@@ -6,6 +6,7 @@
 import json
 import logging
 import os
+import shutil
 import sys
 from os import getcwd
 from pathlib import Path
@@ -48,7 +49,13 @@ DEFAULT_PATTERNS: List[Tuple[str, Union[str, Callable[[str], str]]]] = [
 VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
 
 
-@click.group(cls=CatchAuthExceptions, context_settings={"help_option_names": ["-h", "--help"]})
+@click.group(
+    cls=CatchAuthExceptions,
+    context_settings={
+        "help_option_names": ["-h", "--help"],
+        "max_content_width": shutil.get_terminal_size().columns - 10,
+    },
+)
 @click.option(
     "--debug/--no-debug",
     default=False,

{tinybird-0.0.1.dev110 → tinybird-0.0.1.dev112}/tinybird/tb/modules/common.py

@@ -1751,90 +1751,69 @@ async def remove_release(
             click.echo(FeedbackManager.info_no_release_deleted())
 
 
-async def validate_aws_iamrole_integration(
+async def run_aws_iamrole_connection_flow(
     client: TinyB,
     service: str,
-    role_arn: Optional[str],
-    region: Optional[str],
-    policy: str = "write",
-    no_validate: Optional[bool] = False,
+    policy: str = "read",
 ):
-    if no_validate is False:
-        access_policy, trust_policy, external_id = await get_aws_iamrole_policies(
-            client, service=service, policy=policy
-        )
+    if service == DataConnectorType.AMAZON_DYNAMODB:
+        raise NotImplementedError("DynamoDB is not supported yet")
 
-        if not role_arn:
-            if not click.confirm(
-                FeedbackManager.prompt_s3_iamrole_connection_login_aws(),
-                show_default=False,
-                prompt_suffix="Press y to continue:",
-            ):
-                sys.exit(1)
+    resource_type = "table" if service == DataConnectorType.AMAZON_DYNAMODB else "bucket"
+    resource_name = click.prompt(f"Enter the name of the {resource_type}")
+    validate_string_connector_param(resource_type.capitalize(), resource_name)
 
-            access_policy_copied = True
-            try:
-                pyperclip.copy(access_policy)
-            except Exception:
-                access_policy_copied = False
+    resource_region = click.prompt(f"Enter the region where the {resource_type} is located")
+    validate_string_connector_param("Region", resource_region)
 
-            replacements_dict = {
-                "<bucket>": "<bucket> with your bucket name",
-                "<table_name>": "<table_name> with your DynamoDB table name",
-            }
+    access_policy, trust_policy, external_id = await get_aws_iamrole_policies(client, service=service, policy=policy)
+    access_policy = access_policy.replace("<bucket>", resource_name)
 
-            replacements = [
-                replacements_dict.get(replacement, "")
-                for replacement in replacements_dict.keys()
-                if replacement in access_policy
-            ]
+    if not click.confirm(
+        FeedbackManager.prompt_s3_iamrole_connection_login_aws(),
+        show_default=False,
+        prompt_suffix="Press y to continue:",
+    ):
+        sys.exit(1)
 
-            if not click.confirm(
-                (
-                    FeedbackManager.prompt_s3_iamrole_connection_policy(
-                        access_policy=access_policy, replacements=", ".join(replacements)
-                    )
-                    if access_policy_copied
-                    else FeedbackManager.prompt_s3_iamrole_connection_policy_not_copied(access_policy=access_policy)
-                ),
-                show_default=False,
-                prompt_suffix="Press y to continue:",
-            ):
-                sys.exit(1)
-
-            trust_policy_copied = True
-            try:
-                pyperclip.copy(trust_policy)
-            except Exception:
-                trust_policy_copied = False
-
-            if not click.confirm(
-                (
-                    FeedbackManager.prompt_s3_iamrole_connection_role(trust_policy=trust_policy)
-                    if trust_policy_copied
-                    else FeedbackManager.prompt_s3_iamrole_connection_role_not_copied(trust_policy=trust_policy)
-                ),
-                show_default=False,
-                prompt_suffix="Press y to continue:",
-            ):
-                sys.exit(1)
-    else:
-        try:
-            trust_policy = await client.get_trust_policy(service)
-            external_id = trust_policy["Statement"][0]["Condition"]["StringEquals"]["sts:ExternalId"]
-        except Exception:
-            external_id = ""
+    access_policy_copied = True
+    try:
+        pyperclip.copy(access_policy)
+    except Exception:
+        access_policy_copied = False
 
-    if not role_arn:
-        role_arn = click.prompt("Enter the ARN of the role you just created")
-        validate_string_connector_param("Role ARN", role_arn)
+    if not click.confirm(
+        (
+            FeedbackManager.prompt_s3_iamrole_connection_policy(access_policy=access_policy)
+            if access_policy_copied
+            else FeedbackManager.prompt_s3_iamrole_connection_policy_not_copied(access_policy=access_policy)
+        ),
+        show_default=False,
+        prompt_suffix="Press y to continue:",
+    ):
+        sys.exit(1)
 
-    if not region:
-        region_resource = "table" if service == DataConnectorType.AMAZON_DYNAMODB else "bucket"
-        region = click.prompt(f"Enter the region where the {region_resource} is located")
-        validate_string_connector_param("Region", region)
+    trust_policy_copied = True
+    try:
+        pyperclip.copy(trust_policy)
+    except Exception:
+        trust_policy_copied = False
 
-    return role_arn, region, external_id
+    if not click.confirm(
+        (
+            FeedbackManager.prompt_s3_iamrole_connection_role(trust_policy=trust_policy)
+            if trust_policy_copied
+            else FeedbackManager.prompt_s3_iamrole_connection_role_not_copied(trust_policy=trust_policy)
+        ),
+        show_default=False,
+        prompt_suffix="Press y to continue:",
+    ):
+        sys.exit(1)
+
+    role_arn = click.prompt("Enter the ARN of the role you just created")
+    validate_string_connector_param("Role ARN", role_arn)
+
+    return role_arn, resource_region, external_id
 
 
 async def get_aws_iamrole_policies(client: TinyB, service: str, policy: str = "write"):
@@ -1867,20 +1846,15 @@ async def get_aws_iamrole_policies(client: TinyB, service: str, policy: str = "w
     return json.dumps(access_policy, indent=4), json.dumps(trust_policy, indent=4), external_id
 
 
-async def validate_aws_iamrole_connection_name(
-    client: TinyB, connection_name: Optional[str], no_validate: Optional[bool] = False
-) -> str:
-    if connection_name and no_validate is False:
-        if await client.get_connector(connection_name, skip_bigquery=True) is not None:
-            raise CLIConnectionException(FeedbackManager.info_connection_already_exists(name=connection_name))
-    else:
-        while not connection_name:
-            connection_name = click.prompt("Enter the name for this connection", default=None, show_default=False)
-            assert isinstance(connection_name, str)
+async def validate_aws_iamrole_connection_name(client: TinyB) -> str:
+    connection_name = None
+    while not connection_name:
+        connection_name = click.prompt("Enter the name for this connection", default=None, show_default=False)
+        assert isinstance(connection_name, str)
 
-            if no_validate is False and await client.get_connector(connection_name) is not None:
-                click.echo(FeedbackManager.info_connection_already_exists(name=connection_name))
-                connection_name = None
+        if await client.get_connector(connection_name) is not None:
+            click.echo(FeedbackManager.info_connection_already_exists(name=connection_name))
+            connection_name = None
     assert isinstance(connection_name, str)
     return connection_name
 

{tinybird-0.0.1.dev110 → tinybird-0.0.1.dev112}/tinybird/tb/modules/connection.py

@@ -15,8 +15,12 @@ from tinybird.tb.modules.common import (
     _get_setting_value,
     coro,
     echo_safe_humanfriendly_tables_format_smart_table,
+    run_aws_iamrole_connection_flow,
+    validate_aws_iamrole_connection_name,
 )
+from tinybird.tb.modules.create import generate_aws_iamrole_connection_file
 from tinybird.tb.modules.feedback_manager import FeedbackManager
+from tinybird.tb.modules.project import Project
 
 DATA_CONNECTOR_SETTINGS: Dict[DataConnectorType, List[str]] = {
     DataConnectorType.KAFKA: [
@@ -123,3 +127,41 @@ async def connection_ls(ctx: Context, service: Optional[DataConnectorType] = Non
     column_names = [c.replace("kafka_", "") for c in columns]
     echo_safe_humanfriendly_tables_format_smart_table(table, column_names=column_names)
     click.echo("\n")
+
+
+@connection.group(name="create")
+@click.pass_context
+def connection_create(ctx: Context) -> None:
+    """Create a connection."""
+
+
+@connection_create.command(name="s3", short_help="Creates a AWS S3 connection.")
+@click.pass_context
+@coro
+async def connection_create_s3(ctx: Context) -> None:
+    """
+    Creates a AWS S3 connection.
+
+    \b
+    $ tb connection create s3
+    """
+    project: Project = ctx.ensure_object(dict)["project"]
+    obj: Dict[str, Any] = ctx.ensure_object(dict)
+    client: TinyB = obj["client"]
+    service = DataConnectorType.AMAZON_S3
+    connection_name = await validate_aws_iamrole_connection_name(client)
+    role_arn, region, external_id = await run_aws_iamrole_connection_flow(
+        client,
+        service=service,
+        policy="read",  # For now only read since we only support import from S3
+    )
+
+    await generate_aws_iamrole_connection_file(
+        name=connection_name, service=service, role_arn=role_arn, region=region, folder=project.folder
+    )
+    if external_id:
+        click.echo(
+            FeedbackManager.success_s3_iam_connection_created(
+                connection_name=connection_name, external_id=external_id, role_arn=role_arn
+            )
+        )

{tinybird-0.0.1.dev110 → tinybird-0.0.1.dev112}/tinybird/tb/modules/create.py

@@ -353,6 +353,17 @@ def generate_connection_file(name: str, content: str, folder: str) -> Path:
     return f.relative_to(folder)
 
 
+async def generate_aws_iamrole_connection_file(
+    name: str, service: str, role_arn: str, region: str, folder: str
+) -> None:
+    content = f"""TYPE {service}
+
+S3_ARN {role_arn}
+S3_REGION {region}
+"""
+    generate_connection_file(name, content, folder)
+
+
 def create_rules(folder: str, source: str, agent: str):
     if agent == "cursor":
         extension = ".cursorrules"

{tinybird-0.0.1.dev110 → tinybird-0.0.1.dev112}/tinybird/tb/modules/datafile/parse_datasource.py

@@ -1,8 +1,9 @@
 import os
-from typing import List, Optional
+from typing import Dict, Optional
 
 import click
 
+from tinybird.sql_template import render_template_with_secrets
 from tinybird.tb.modules.datafile.common import (
     Datafile,
     DatafileKind,
@@ -21,7 +22,7 @@ def parse_datasource(
     skip_eval: bool = False,
     hide_folders: bool = False,
     add_context_to_datafile_syntax_errors: bool = True,
-    secrets: Optional[List[str]] = None,
+    secrets: Optional[Dict[str, str]] = None,
 ) -> Datafile:
     basepath = ""
     if not content:
@@ -31,6 +32,7 @@ def parse_datasource(
     else:
         s = content
 
+    s = render_template_with_secrets(filename, s, secrets=secrets or {})
     filename = format_filename(filename, hide_folders)
     try:
         doc = parse(

{tinybird-0.0.1.dev110 → tinybird-0.0.1.dev112}/tinybird/tb/modules/datafile/parse_pipe.py

@@ -1,5 +1,5 @@
 import os
-from typing import List, Optional
+from typing import Dict, Optional
 
 import click
 
@@ -23,7 +23,7 @@ def parse_pipe(
     skip_eval: bool = False,
     hide_folders: bool = False,
     add_context_to_datafile_syntax_errors: bool = True,
-    secrets: Optional[List[str]] = None,
+    secrets: Optional[Dict[str, str]] = None,
 ) -> Datafile:
     basepath = ""
     if not content:

{tinybird-0.0.1.dev110 → tinybird-0.0.1.dev112}/tinybird/tb/modules/feedback_manager.py

@@ -498,7 +498,7 @@ Ready? """
 
     prompt_s3_iamrole_connection_login_aws = prompt_message("""[1] Log into your AWS Console\n\n""")
     prompt_s3_iamrole_connection_policy = prompt_message(
-        """\n[2] Go to IAM > Policies. Create a new policy with the following permissions. Please, replace {replacements}:\n\n{access_policy}\n\n(The policy has been copied to your clipboard)\n\n"""
+        """\n[2] Go to IAM > Policies. Create a new policy with the following permissions:\n\n{access_policy}\n\n(The policy has been copied to your clipboard)\n\n"""
     )
     prompt_s3_iamrole_connection_policy_not_copied = prompt_message(
         """\n[2] Go to IAM > Policies. Create a new policy with the following permissions. Please, copy this policy and replace <bucket> with your bucket name:\n\n{access_policy}\n\n"""

{tinybird-0.0.1.dev110 → tinybird-0.0.1.dev112}/tinybird/tb/modules/local.py

@@ -130,7 +130,7 @@ def get_docker_client() -> DockerClient:
         )
 
 
-def get_use_aws_creds():
+def get_use_aws_creds() -> dict[str, str]:
     credentials: dict[str, str] = {}
     try:
         # Get the boto3 session and credentials
@@ -149,14 +149,29 @@ def get_use_aws_creds():
             # Add region if available
             if session.region_name:
                 credentials["AWS_DEFAULT_REGION"] = session.region_name
-    except Exception:
-        # TODO (rbarbadillo): We should handle this better. If users don't have AWS credentials, most times it's fine
-        # but if they want to use S3, they'll need a warning.
-        pass
+
+            click.echo(
+                FeedbackManager.success(
+                    message=f"✓ AWS credentials found and will be passed to Tinybird Local (region: {session.region_name or 'not set'})"
+                )
+            )
+        else:
+            click.echo(
+                FeedbackManager.warning(
+                    message="△ No AWS credentials found. S3 operations will not work in Tinybird Local."
+                )
+            )
+    except Exception as e:
+        click.echo(
+            FeedbackManager.warning(
+                message=f"△ Error retrieving AWS credentials: {str(e)}. S3 operations will not work in Tinybird Local."
+            )
+        )
+
     return credentials
 
 
-def stop_tinybird_local(docker_client):
+def stop_tinybird_local(docker_client: DockerClient) -> None:
     """Stop the Tinybird container."""
     try:
         container = docker_client.containers.get(TB_CONTAINER_NAME)
@@ -165,7 +180,7 @@ def stop_tinybird_local(docker_client):
         pass
 
 
-def remove_tinybird_local(docker_client):
+def remove_tinybird_local(docker_client: DockerClient) -> None:
     """Remove the Tinybird container."""
     try:
         container = docker_client.containers.get(TB_CONTAINER_NAME)
@@ -181,7 +196,7 @@ def remove_tinybird_local(docker_client):
         pass
 
 
-def update_cli():
+def update_cli() -> None:
     click.echo(FeedbackManager.highlight(message="» Updating Tinybird CLI..."))
 
     try:
@@ -205,20 +220,20 @@ def update_cli():
 
 
 @cli.command()
-def update():
+def update() -> None:
     """Update Tinybird CLI to the latest version."""
     update_cli()
 
 
 @cli.command(name="upgrade", hidden=True)
-def upgrade():
+def upgrade() -> None:
     """Update Tinybird CLI to the latest version."""
     update_cli()
 
 
 @cli.group()
 @click.pass_context
-def local(ctx):
+def local(ctx: click.Context) -> None:
     """Manage the local Tinybird instance."""
 
 

{tinybird-0.0.1.dev110 → tinybird-0.0.1.dev112}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev110
+Version: 0.0.1.dev112
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird