tinybird 0.0.1.dev94__tar.gz → 0.0.1.dev95__tar.gz

{tinybird-0.0.1.dev94 → tinybird-0.0.1.dev95}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev94
+Version: 0.0.1.dev95
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev94 → tinybird-0.0.1.dev95}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev94'
-__revision__ = '97387e2'
+__version__ = '0.0.1.dev95'
+__revision__ = 'dc3767a'

{tinybird-0.0.1.dev94 → tinybird-0.0.1.dev95}/tinybird/tb/cli.py

@@ -15,6 +15,7 @@ import tinybird.tb.modules.datasource
 import tinybird.tb.modules.deployment
 import tinybird.tb.modules.endpoint
 import tinybird.tb.modules.fmt
+import tinybird.tb.modules.infra
 import tinybird.tb.modules.job
 import tinybird.tb.modules.local
 import tinybird.tb.modules.login

{tinybird-0.0.1.dev94 → tinybird-0.0.1.dev95}/tinybird/tb/modules/cli.py

@@ -398,7 +398,7 @@ async def create_ctx_client(ctx: Context, config: Dict[str, Any], cloud: bool, b
     if command in commands_without_ctx_client:
         return None
 
-    commands_always_cloud = ["pull", "playground"]
+    commands_always_cloud = ["pull", "playground", "infra"]
     commands_always_build = ["build", "test", "dev", "create"]
     commands_always_local: List[str] = []
     if (

{tinybird-0.0.1.dev94 → tinybird-0.0.1.dev95}/tinybird/tb/modules/common.py

@@ -81,6 +81,8 @@ SUPPORTED_FORMATS = ["csv", "ndjson", "json", "parquet"]
 OLDEST_ROLLBACK = "oldest_rollback"
 MAIN_BRANCH = "main"
 
+CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
+
 
 def obfuscate_token(value: Optional[str]) -> Optional[str]:
     if not value:

{tinybird-0.0.1.dev94 → tinybird-0.0.1.dev95}/tinybird/tb/modules/datafile/common.py

@@ -241,6 +241,11 @@ class Datafile:
                     raise DatafileValidationError(f"Materialized node {repr(node['name'])} missing target datasource")
                 if node.get("type", "").lower() == PipeNodeTypes.COPY:
                     self.validate_copy_node(node)
+            for token in self.tokens:
+                if token["permission"].upper() != "READ":
+                    raise DatafileValidationError(
+                        f"Invalid permission {token['permission']} for token {token['token_name']}. Only READ is allowed for pipes"
+                    )
         elif self.kind == DatafileKind.datasource:
             # TODO(eclbg):
             #  [x] Just one node
@@ -253,6 +258,11 @@ class Datafile:
             node = self.nodes[0]
             if "schema" not in node:
                 raise DatafileValidationError("SCHEMA is mandatory")
+            for token in self.tokens:
+                if token["permission"].upper() not in {"READ", "APPEND"}:
+                    raise DatafileValidationError(
+                        f"Invalid permission {token['permission']} for token {token['token_name']}. Only READ and APPEND are allowed for datasources"
+                    )
         else:
             # We cannot validate a datafile whose kind is unknown
             pass
@@ -1274,16 +1284,28 @@ def parse(
 
     @multiline_not_supported
     def add_token(*args: str, **kwargs: Any) -> None:  # token_name, permissions):
-        # lineno = kwargs["lineno"]
+        lineno = kwargs["lineno"]
         if len(args) < 2:
             raise DatafileSyntaxError(
-                message='TOKEN takes two params: token name and permissions e.g TOKEN "read api token" READ',
+                message='TOKEN takes two params: token name and permission e.g TOKEN "read api token" READ',
                 lineno=lineno,
                 pos=1,
             )
-        # TODO(eclbg): We should validate that the permissions are a valid string. We only support READ for pipes and
-        # APPEND for datasources
-        doc.tokens.append({"token_name": _unquote(args[0]), "permissions": args[1]})
+        if len(args) > 2:
+            raise DatafileSyntaxError(
+                f"Invalid number of arguments for TOKEN command: {len(args)}. Expected 2 arguments: token name and permission",
+                lineno=lineno,
+                pos=len("token") + len(args[0]) + 3,  # Naive handling of whitespace. Assuming there's 2
+            )
+        permission = args[1]
+        if permission.upper() not in ["READ", "APPEND"]:
+            raise DatafileSyntaxError(
+                f"Invalid permission: {permission}. Only READ and APPEND are supported",
+                lineno=lineno,
+                pos=len("token") + len(args[0]) + 3,  # Naive handling of whitespace. Assuming there's 2
+            )
+        token_name = _unquote(args[0])
+        doc.tokens.append({"token_name": token_name, "permission": permission.upper()})
 
     @not_supported_yet()
     def include(*args: str, **kwargs: Any) -> None:

{tinybird-0.0.1.dev94 → tinybird-0.0.1.dev95}/tinybird/tb/modules/deployment.py

@@ -4,7 +4,7 @@ import sys
 import time
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Dict, Optional, Union
+from typing import Any, Dict, Optional, Tuple, Union
 
 import click
 import requests
@@ -451,8 +451,10 @@ def create_deployment(
 
 def print_changes(result: dict, project: Project) -> None:
     deployment = result.get("deployment", {})
-    columns = ["status", "name", "path"]
+    resources_columns = ["status", "name", "path"]
     resources: list[list[Union[str, None]]] = []
+    tokens_columns = ["Change", "Token name", "Added permissions", "Removed permissions"]
+    tokens: list[Tuple[str, str, str, str]] = []
 
     for ds in deployment.get("new_datasource_names", []):
         resources.append(["new", ds, project.get_resource_path(ds, "datasource")])
@@ -481,8 +483,26 @@ def print_changes(result: dict, project: Project) -> None:
     for dc in deployment.get("deleted_data_connector_names", []):
         resources.append(["deleted", dc, project.get_resource_path(dc, "data_connector")])
 
+    for token_change in deployment.get("token_changes", []):
+        token_name = token_change.get("token_name")
+        change_type = token_change.get("change_type")
+        added_perms = []
+        removed_perms = []
+        permission_changes = token_change.get("permission_changes", {})
+        for perm in permission_changes.get("added_permissions", []):
+            added_perms.append(f"{perm['resource_name']}.{perm['resource_type']}:{perm['permission']}")
+        for perm in permission_changes.get("removed_permissions", []):
+            removed_perms.append(f"{perm['resource_name']}.{perm['resource_type']}:{perm['permission']}")
+
+        tokens.append((change_type, token_name, "\n".join(added_perms), "\n".join(removed_perms)))
+
     if resources:
         click.echo(FeedbackManager.highlight(message="\n» Changes to be deployed...\n"))
-        echo_safe_humanfriendly_tables_format_smart_table(resources, column_names=columns)
+        echo_safe_humanfriendly_tables_format_smart_table(resources, column_names=resources_columns)
     else:
         click.echo(FeedbackManager.highlight(message="\n» No changes to be deployed\n"))
+    if tokens:
+        click.echo(FeedbackManager.highlight(message="\n» Changes in tokens to be deployed...\n"))
+        echo_safe_humanfriendly_tables_format_smart_table(tokens, column_names=tokens_columns)
+    else:
+        click.echo(FeedbackManager.highlight(message="\n» No changes in tokens to be deployed\n"))

tinybird-0.0.1.dev95/tinybird/tb/modules/infra.py

@@ -0,0 +1,473 @@
+from pathlib import Path
+from typing import Optional
+
+import click
+from click import Context
+
+from tinybird.tb.modules.cli import cli
+
+from .common import CONTEXT_SETTINGS
+
+K8S_YML = """
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: %(namespace)s
+  labels:
+    name: tinybird
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: tinybird
+  namespace:  %(namespace)s
+  labels:
+    name: tinybird
+automountServiceAccountToken: true
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: tinybird
+  namespace:  %(namespace)s
+  labels:
+    name: tinybird
+  annotations:
+    service.beta.kubernetes.io/aws-load-balancer-type: 'external'
+    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: 'ip'
+    service.beta.kubernetes.io/aws-load-balancer-backend-protocol: 'tcp'
+    service.beta.kubernetes.io/aws-load-balancer-scheme: 'internet-facing'
+    service.beta.kubernetes.io/aws-load-balancer-ssl-negotiation-policy: 'ELBSecurityPolicy-TLS13-1-2-2021-06'
+    service.beta.kubernetes.io/aws-load-balancer-ssl-cert: '%(cert_arn)s'
+spec:
+  type: LoadBalancer
+  ports:
+    - port: 443
+      targetPort: http
+      protocol: TCP
+      name: https
+  selector:
+    name: tinybird
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tinybird
+  namespace:  %(namespace)s
+  labels:
+    name: tinybird
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: tinybird
+  template:
+    metadata:
+      labels:
+        name: tinybird
+    spec:
+      serviceAccountName: tinybird
+      containers:
+        - name: tinybird
+          image: "tinybirdco/tinybird-local:beta"
+          imagePullPolicy: Always
+          ports:
+            - name: http
+              containerPort: 7181
+              protocol: TCP
+"""
+
+TERRAFORM_FIRST_TEMPLATE = """
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.0"
+    }
+  }
+}
+
+provider "aws" {
+  region = "%(aws_region)s"
+}
+
+# Get the hosted zone data
+data "aws_route53_zone" "selected" {
+  name = "%(domain)s"
+}
+
+# Create ACM certificate
+resource "aws_acm_certificate" "cert" {
+  domain_name               = "*.${data.aws_route53_zone.selected.name}"
+  validation_method         = "DNS"
+  subject_alternative_names = [data.aws_route53_zone.selected.name]
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+# Create DNS records for certificate validation
+resource "aws_route53_record" "cert_validation" {
+  for_each = {
+    for dvo in aws_acm_certificate.cert.domain_validation_options : dvo.domain_name => {
+      name   = dvo.resource_record_name
+      record = dvo.resource_record_value
+      type   = dvo.resource_record_type
+    }
+  }
+
+  allow_overwrite = true
+  name            = each.value.name
+  records         = [each.value.record]
+  ttl             = 60
+  type            = each.value.type
+  zone_id         = data.aws_route53_zone.selected.zone_id
+}
+
+# Certificate validation
+resource "aws_acm_certificate_validation" "cert" {
+  certificate_arn         = aws_acm_certificate.cert.arn
+  validation_record_fqdns = [for record in aws_route53_record.cert_validation : record.fqdn]
+}
+
+output "certificate_arn" {
+  description = "The ARN of the ACM certificate"
+  value       = aws_acm_certificate.cert.arn
+}
+"""
+
+TERRAFORM_SECOND_TEMPLATE = """
+# Create Route 53 record for the load balancer
+resource "aws_route53_record" "tinybird" {
+  zone_id = data.aws_route53_zone.selected.zone_id
+  name    = "%(full_dns_name)s"
+  type    = "CNAME"
+  ttl     = 300
+  records = ["%(external_ip)s"]
+}
+
+output "tinybird_dns" {
+  description = "The DNS name for Tinybird"
+  value       = aws_route53_record.tinybird.fqdn
+}
+"""
+
+
+@cli.group(context_settings=CONTEXT_SETTINGS)
+@click.pass_context
+def infra(ctx: Context) -> None:
+    """Infra commands."""
+
+
+@infra.command(name="ls")
+@click.pass_context
+async def infra_ls(ctx: Context) -> None:
+    """List infra"""
+
+
+@infra.command(name="init")
+@click.option("--provider", default="aws", type=str, help="Infrastructure provider (aws, gcp, azure)")
+@click.option("--region", type=str, help="AWS region (for AWS provider)")
+@click.option("--domain", type=str, help="Route53 domain name (for AWS provider)")
+@click.option("--namespace", type=str, help="Kubernetes namespace for deployment")
+@click.option("--dns-record", type=str, help="DNS record name to create (without domain, e.g. 'tinybird')")
+@click.option(
+    "--auto-apply-terraform", is_flag=True, help="Automatically apply Terraform configuration without prompting"
+)
+@click.option("--auto-apply-dns", is_flag=True, help="Automatically apply DNS configuration without prompting")
+@click.option(
+    "--auto-apply-kubectl", is_flag=True, help="Automatically apply Kubernetes configuration without prompting"
+)
+@click.option("--skip-terraform", is_flag=True, help="Skip Terraform configuration and application")
+@click.option("--skip-kubectl", is_flag=True, help="Skip Kubernetes configuration and application")
+@click.option("--skip-dns", is_flag=True, help="Skip DNS configuration and application")
+@click.pass_context
+def infra_init(
+    ctx: Context,
+    provider: str,
+    region: Optional[str] = None,
+    domain: Optional[str] = None,
+    namespace: Optional[str] = None,
+    dns_record: Optional[str] = None,
+    auto_apply_terraform: bool = False,
+    auto_apply_dns: bool = False,
+    auto_apply_kubectl: bool = False,
+    skip_terraform: bool = False,
+    skip_kubectl: bool = False,
+    skip_dns: bool = False,
+) -> None:
+    """Init infra"""
+    # AWS-specific Terraform template creation
+    if provider.lower() != "aws":
+        click.echo("Provider not supported yet.")
+        return
+
+    # Create infra directory if it doesn't exist
+    infra_dir = Path("infra")
+    infra_dir.mkdir(exist_ok=True)
+    yaml_path = infra_dir / "k8s.yaml"
+    tf_path = infra_dir / "main.tf"
+
+    # Write the Terraform template
+    region = region or click.prompt("Enter aws region", default="us-east-1", type=str)
+    domain = domain or click.prompt("Enter route 53 domain name", type=str)
+
+    terraform_content = TERRAFORM_FIRST_TEMPLATE % {"aws_region": region, "domain": domain}
+
+    with open(tf_path, "w") as f:
+        f.write(terraform_content.lstrip())
+
+    click.echo(f"Creating Terraform configuration in {tf_path}")
+
+    # Apply Terraform configuration if user confirms
+    if not skip_terraform and (
+        auto_apply_terraform or click.confirm("Would you like to apply the Terraform configuration now?")
+    ):
+        import subprocess
+
+        # Initialize Terraform
+        click.echo("Initializing Terraform...")
+        init_result = subprocess.run(["terraform", "-chdir=infra", "init"], capture_output=True, text=True)
+
+        if init_result.returncode != 0:
+            click.echo("Terraform initialization failed:")
+            click.echo(init_result.stderr)
+            return
+
+        click.echo(init_result.stdout)
+
+        # Apply Terraform configuration
+        click.echo("\nApplying Terraform configuration...\n")
+        apply_result = subprocess.run(
+            ["terraform", "-chdir=infra", "apply", "-auto-approve"], capture_output=True, text=True
+        )
+
+        if apply_result.returncode != 0:
+            click.echo("Terraform apply failed:")
+            click.echo(apply_result.stderr)
+            return
+
+        click.echo(apply_result.stdout)
+
+        # Get the certificate ARN output
+        output_result = subprocess.run(
+            ["terraform", "-chdir=infra", "output", "certificate_arn"], capture_output=True, text=True
+        )
+
+        if output_result.returncode == 0:
+            cert_arn = output_result.stdout.strip().replace('"', "")
+
+            namespace = namespace or click.prompt("Enter namespace name", default="tinybird", type=str)
+            new_content = K8S_YML % {"namespace": namespace, "cert_arn": cert_arn}
+
+            with open(yaml_path, "w") as f:
+                f.write(new_content.lstrip())
+
+            click.echo(f"Created Kubernetes configuration with certificate ARN in {yaml_path}")
+
+            # Prompt to apply the k8s configuration
+            if not skip_kubectl and (
+                auto_apply_kubectl or click.confirm("Would you like to apply the Kubernetes configuration now?")
+            ):
+                import subprocess
+
+                # Get current kubectl context
+                current_context_result = subprocess.run(
+                    ["kubectl", "config", "current-context"], capture_output=True, text=True
+                )
+
+                current_context = (
+                    current_context_result.stdout.strip() if current_context_result.returncode == 0 else "unknown"
+                )
+
+                # Get available contexts
+                contexts_result = subprocess.run(
+                    ["kubectl", "config", "get-contexts", "-o", "name"], capture_output=True, text=True
+                )
+
+                if contexts_result.returncode != 0:
+                    click.echo("Failed to get kubectl contexts:")
+                    click.echo(contexts_result.stderr)
+                    return
+
+                available_contexts = [
+                    context.strip() for context in contexts_result.stdout.splitlines() if context.strip()
+                ]
+
+                if not available_contexts:
+                    click.echo("No kubectl contexts found. Please configure kubectl first.")
+                    return
+
+                # Prompt user to select a context
+                if len(available_contexts) == 1:
+                    selected_context = available_contexts[0]
+                    click.echo(f"Using the only available kubectl context: {selected_context}")
+                else:
+                    click.echo("\nAvailable kubectl contexts:")
+                    for i, context in enumerate(available_contexts):
+                        marker = " (current)" if context == current_context else ""
+                        click.echo(f"  {i + 1}. {context}{marker}")
+
+                    click.echo("")
+                    default_index = (
+                        available_contexts.index(current_context) + 1 if current_context in available_contexts else 1
+                    )
+
+                    selected_index = click.prompt(
+                        "Select kubectl context number to apply configuration",
+                        type=click.IntRange(1, len(available_contexts)),
+                        default=default_index,
+                    )
+
+                    selected_context = available_contexts[selected_index - 1]
+                    click.echo(f"Selected context: {selected_context}")
+
+                # Apply the configuration to the selected context
+                click.echo(f"Applying Kubernetes configuration to context '{selected_context}'...")
+                apply_result = subprocess.run(
+                    ["kubectl", "--context", selected_context, "apply", "-f", str(yaml_path)],
+                    capture_output=True,
+                    text=True,
+                )
+
+                if apply_result.returncode != 0:
+                    click.echo("Failed to apply Kubernetes configuration:")
+                    click.echo(apply_result.stderr)
+                else:
+                    click.echo("Kubernetes configuration applied successfully:")
+                    click.echo(apply_result.stdout)
+
+                    # Get the namespace from the applied configuration
+                    namespace = None
+                    with open(yaml_path, "r") as f:
+                        for line in f:
+                            if "namespace:" in line and not namespace:
+                                namespace = line.split("namespace:")[1].strip()
+                                break
+
+                    if not namespace:
+                        namespace = "tinybird"  # Default namespace
+
+                    click.echo(f"\nWaiting for load balancer to be provisioned in namespace '{namespace}'...")
+
+                    # Wait for the load balancer to get an external IP
+                    max_attempts = 30
+                    attempt = 0
+                    external_ip = None
+
+                    while attempt < max_attempts and not external_ip:
+                        attempt += 1
+
+                        # Get the service details
+                        get_service_result = subprocess.run(
+                            [
+                                "kubectl",
+                                "--context",
+                                selected_context,
+                                "-n",
+                                namespace,
+                                "get",
+                                "service",
+                                "tinybird",
+                                "-o",
+                                "jsonpath='{.status.loadBalancer.ingress[0].hostname}'",
+                            ],
+                            capture_output=True,
+                            text=True,
+                        )
+
+                        if get_service_result.returncode == 0:
+                            potential_ip = get_service_result.stdout.strip().replace("'", "")
+                            if potential_ip and potential_ip != "":
+                                external_ip = potential_ip
+                                break
+
+                        click.echo(
+                            f"Attempt {attempt}/{max_attempts}: Load balancer not ready yet, waiting 10 seconds..."
+                        )
+                        import time
+
+                        time.sleep(10)
+
+                    if external_ip:
+                        click.echo("\n✅ Load balancer provisioned successfully!")
+
+                        # Update the Terraform configuration with the load balancer DNS
+                        if not skip_dns and domain and tf_path.exists():
+                            click.echo("\nUpdating Terraform configuration with load balancer DNS...")
+
+                            with open(tf_path, "r") as f:
+                                tf_content = f.read()
+
+                            # Check if the Route 53 record already exists in the file
+                            if 'resource "aws_route53_record" "tinybird"' not in tf_content:
+                                # Get the DNS record name
+                                dns_record = dns_record or click.prompt(
+                                    "Enter DNS record name (without domain)", default="tinybird", type=str
+                                )
+
+                                # Create the full DNS name
+                                full_dns_name = f"{dns_record}.{domain}"
+
+                                # Use in the Terraform template
+                                route53_record = TERRAFORM_SECOND_TEMPLATE % {
+                                    "external_ip": external_ip,
+                                    "full_dns_name": full_dns_name,
+                                }
+
+                                # Append the Route 53 record to the Terraform file
+                                with open(tf_path, "a") as f:
+                                    f.write(route53_record.lstrip())
+
+                                click.echo("Added Route 53 record to Terraform configuration")
+                            else:
+                                # Update the existing Route 53 record
+                                updated_tf = tf_content.replace(
+                                    'records = ["LOAD_BALANCER_DNS_PLACEHOLDER"]', f'records = ["{external_ip}"]'
+                                )
+
+                                # Also handle case where there might be another placeholder or old value
+                                import re
+
+                                pattern = r'records\s*=\s*\[\s*"[^"]*"\s*\]'
+                                updated_tf = re.sub(pattern, f'records = ["{external_ip}"]', updated_tf)
+
+                                with open(tf_path, "w") as f:
+                                    f.write(updated_tf.lstrip())
+
+                                click.echo("Updated existing Route 53 record in Terraform configuration")
+
+                            # Apply the updated Terraform configuration
+                            if not skip_dns and (
+                                auto_apply_dns
+                                or click.confirm("Would you like to create the DNS record in Route 53 now?")
+                            ):
+                                click.echo("Applying updated Terraform configuration...")
+                                apply_result = subprocess.run(
+                                    ["terraform", "-chdir=infra", "apply", "-auto-approve"],
+                                    capture_output=True,
+                                    text=True,
+                                )
+
+                                if apply_result.returncode != 0:
+                                    click.echo("Failed to create DNS record:")
+                                    click.echo(apply_result.stderr)
+                                else:
+                                    click.echo("DNS record created successfully!")
+
+                                    # Get the DNS name from Terraform output
+                                    dns_output = subprocess.run(
+                                        ["terraform", "-chdir=infra", "output", "tinybird_dns"],
+                                        capture_output=True,
+                                        text=True,
+                                    )
+
+                                    if dns_output.returncode == 0:
+                                        dns_name = dns_output.stdout.strip().replace('"', "")
+                                        click.echo(f"\nYour Tinybird instance is now available at: https://{dns_name}")
+                                    else:
+                                        click.echo(
+                                            f"\nYour Tinybird instance should be available at: https://tinybird.{domain}"
+                                        )

{tinybird-0.0.1.dev94 → tinybird-0.0.1.dev95}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev94
+Version: 0.0.1.dev95
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev94 → tinybird-0.0.1.dev95}/tinybird.egg-info/SOURCES.txt

@@ -41,6 +41,7 @@ tinybird/tb/modules/endpoint.py
 tinybird/tb/modules/exceptions.py
 tinybird/tb/modules/feedback_manager.py
 tinybird/tb/modules/fmt.py
+tinybird/tb/modules/infra.py
 tinybird/tb/modules/job.py
 tinybird/tb/modules/llm.py
 tinybird/tb/modules/llm_utils.py