tinybird 0.0.1.dev99__py3-none-any.whl → 0.0.1.dev101__py3-none-any.whl

tinybird/client.py

@@ -915,6 +915,16 @@ class TinyB:
         }
         return await self._req(f"/v1/infra?{urlencode(params)}", method="POST")
 
+    async def infra_update(self, infra_id: str, organization_id: str, name: str, host: str) -> Dict[str, Any]:
+        params = {
+            "organization_id": organization_id,
+        }
+        if name:
+            params["name"] = name
+        if host:
+            params["host"] = host
+        return await self._req(f"/v1/infra/{infra_id}?{urlencode(params)}", method="PUT")
+
     async def infra_list(self, organization_id: str) -> List[Dict[str, Any]]:
         data = await self._req(f"/v1/infra?organization_id={organization_id}")
         return data.get("infras", [])

tinybird/sql.py

@@ -174,7 +174,7 @@ def try_to_fix_nullable_in_simple_aggregating_function(t: str) -> Optional[str]:
     return result
 
 
-def schema_to_sql_columns(schema: List[Dict[str, Any]]) -> List[str]:
+def schema_to_sql_columns(schema: List[Dict[str, Any]], skip_jsonpaths: bool = False) -> List[str]:
     """return an array with each column in SQL
     >>> schema_to_sql_columns([{'name': 'temperature', 'type': 'Float32', 'codec': None, 'default_value': None, 'nullable': False, 'normalized_name': 'temperature'}, {'name': 'temperature_delta', 'type': 'Float32', 'codec': 'CODEC(Delta(4), LZ4))', 'default_value': 'MATERIALIZED temperature', 'nullable': False, 'normalized_name': 'temperature_delta'}])
     ['`temperature` Float32', '`temperature_delta` Float32 MATERIALIZED temperature CODEC(Delta(4), LZ4))']
@@ -198,7 +198,7 @@ def schema_to_sql_columns(schema: List[Dict[str, Any]]) -> List[str]:
         else:
             _type = x["type"]
         parts = [col_name(name, backquotes=True), _type]
-        if x.get("jsonpath", None):
+        if x.get("jsonpath", None) and not skip_jsonpaths:
             parts.append(f"`json:{x['jsonpath']}`")
         if "default_value" in x and x["default_value"] not in ("", None):
             parts.append(x["default_value"])

tinybird/sql_toolset.py

@@ -10,6 +10,9 @@ from toposort import toposort
 
 from tinybird.ch_utils.constants import COPY_ENABLED_TABLE_FUNCTIONS, ENABLED_TABLE_FUNCTIONS
 
+# VALID_REMOTE is used to explicitly vet queries sent to sql_toolset. In this module, when a table used in a query has
+# VALID_REMOTE in the database portion (as in (VALID_REMOTE, "select * from cluster(tinybird, public.t_blabla)"), the
+# query is not blocked, even if the rhs (the "select * ..." bit) is not found in the various collections of allowed tables.
 VALID_REMOTE = "VALID_REMOTE"
 
 
@@ -351,14 +354,24 @@ def is_invalid_resource(
     valid_tables: Optional[Set[Tuple[str, str]]] = None,
 ) -> bool:
     return is_invalid_resource_from_other_workspace(
-        r, database, default_database, _replaced_with
+        r, database, default_database, _replaced_with, valid_tables
     ) or is_invalid_resource_from_current_workspace(r, database, default_database, _replaced_with, valid_tables)
 
 
 def is_invalid_resource_from_other_workspace(
-    r: Tuple[str, str], database: str, default_database: str, _replaced_with: Set[Tuple[str, str]]
+    r: Tuple[str, str],
+    database: str,
+    default_database: str,
+    _replaced_with: Set[Tuple[str, str]],
+    valid_tables: Optional[Set[Tuple[str, str]]],
 ) -> bool:
-    return database not in [default_database, "tinybird", VALID_REMOTE] and r not in _replaced_with
+    # return database not in [default_database, "tinybird", VALID_REMOTE] and r not in _replaced_with
+    return bool(
+        database not in [default_database, "tinybird", VALID_REMOTE]
+        and valid_tables
+        and r not in valid_tables
+        and r not in _replaced_with
+    )
 
 
 def is_invalid_resource_from_current_workspace(

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev99'
-__revision__ = '1e30c94'
+__version__ = '0.0.1.dev101'
+__revision__ = '5dc0d4b'

tinybird/tb/modules/datafile/common.py

@@ -183,6 +183,15 @@ class DatafileKind(Enum):
         return extension_map[extension]
 
 
+KAFKA_PARAMS = {
+    "kafka_connection_name",
+    "kafka_topic",
+    "kafka_group_id",
+}
+
+REQUIRED_KAFKA_PARAMS = KAFKA_PARAMS
+
+
 class Datafile:
     def __init__(self) -> None:
         self.maintainer: Optional[str] = None
@@ -248,9 +257,10 @@ class Datafile:
                         f"Invalid permission {token['permission']} for token {token['token_name']}. Only READ is allowed for pipes"
                     )
         elif self.kind == DatafileKind.datasource:
-            # TODO(eclbg):
             #  [x] Just one node
             #  [x] Engine is present
+            #  [x] Token permissions are valid
+            #  [x] If it's a kafka datasource, all required kafka params are present
             #  [ ] ...
             if len(self.nodes) > 1:
                 # Our users are not aware of data source data files being a single-node data file, hence this error
@@ -259,11 +269,20 @@ class Datafile:
             node = self.nodes[0]
             if "schema" not in node:
                 raise DatafileValidationError("SCHEMA is mandatory")
+            # Validate token permissions
             for token in self.tokens:
                 if token["permission"].upper() not in {"READ", "APPEND"}:
                     raise DatafileValidationError(
                         f"Invalid permission {token['permission']} for token {token['token_name']}. Only READ and APPEND are allowed for datasources"
                     )
+            # Validate Kafka params
+            if any(param in node for param in KAFKA_PARAMS) and not all(
+                param in node for param in REQUIRED_KAFKA_PARAMS
+            ):
+                missing = [param for param in REQUIRED_KAFKA_PARAMS if param not in node]
+                raise DatafileValidationError(
+                    f"Some Kafka params have been provided, but the following required ones are missing: {missing}"
+                )
         else:
             # We cannot validate a datafile whose kind is unknown
             pass
@@ -1427,6 +1446,7 @@ def parse(
         "include": include,
         "sql": sql("sql"),
         "version": version,
+        # TODO(eclbg): We should decide on a single place to define the kafka params. Definitely not here.
         "kafka_connection_name": assign_var("kafka_connection_name"),
         "kafka_topic": assign_var("kafka_topic"),
         "kafka_group_id": assign_var("kafka_group_id"),

tinybird/tb/modules/infra.py

@@ -1,9 +1,13 @@
 import json
+import subprocess
+import time
 import uuid
 from pathlib import Path
 from typing import Optional
 
 import click
+import pyperclip
+import requests
 from click import Context
 
 from tinybird.client import TinyB
@@ -40,28 +44,50 @@ metadata:
   namespace:  %(namespace)s
   labels:
     name: tinybird
-  annotations:
-    service.beta.kubernetes.io/aws-load-balancer-type: 'external'
-    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: 'ip'
-    service.beta.kubernetes.io/aws-load-balancer-backend-protocol: 'tcp'
-    service.beta.kubernetes.io/aws-load-balancer-scheme: 'internet-facing'
-    service.beta.kubernetes.io/aws-load-balancer-ssl-negotiation-policy: 'ELBSecurityPolicy-TLS13-1-2-2021-06'
-    service.beta.kubernetes.io/aws-load-balancer-ssl-cert: '%(cert_arn)s'
 spec:
-  type: LoadBalancer
+  type: ClusterIP
   ports:
-    - port: 443
+    - port: 80
       targetPort: http
       protocol: TCP
-      name: https
+      name: http
   selector:
     name: tinybird
 ---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  namespace: %(namespace)s
+  name: tinybird
+  annotations:
+    alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}]'
+    alb.ingress.kubernetes.io/scheme: internet-facing
+    alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS13-1-2-2021-06
+    alb.ingress.kubernetes.io/ssl-redirect: '443'
+    alb.ingress.kubernetes.io/target-type: 'ip'
+    alb.ingress.kubernetes.io/load-balancer-name: %(namespace)s
+    alb.ingress.kubernetes.io/success-codes: '200,301,302'
+spec:
+  ingressClassName: alb
+  tls:
+    - hosts:
+        - %(full_dns_name)s
+  rules:
+    - http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: tinybird
+                port:
+                  name: http
+---
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: tinybird
-  namespace:  %(namespace)s
+  namespace: %(namespace)s
 spec:
   serviceName: "tinybird"
   replicas: 1
@@ -85,6 +111,12 @@ spec:
           env:
             - name: TB_INFRA_TOKEN
               value: "%(infra_token)s"
+            - name: TB_INFRA_WORKSPACE
+              value: "%(infra_workspace)s"
+            - name: TB_INFRA_ORGANIZATION
+              value: "%(infra_organization)s"
+            - name: TB_INFRA_USER
+              value: "%(infra_user)s"
           volumeMounts:
           - name: clickhouse-data
             mountPath: /var/lib/clickhouse
@@ -115,12 +147,12 @@ provider "aws" {
 
 # Get the hosted zone data
 data "aws_route53_zone" "selected" {
-  name = "%(domain)s"
+  name = "%(dns_zone_name)s"
 }
 
 # Create ACM certificate
 resource "aws_acm_certificate" "cert" {
-  domain_name               = "*.${data.aws_route53_zone.selected.name}"
+  domain_name               = "%(dns_record)s.${data.aws_route53_zone.selected.name}"
   validation_method         = "DNS"
   subject_alternative_names = [data.aws_route53_zone.selected.name]
 
@@ -152,21 +184,23 @@ resource "aws_acm_certificate_validation" "cert" {
   certificate_arn         = aws_acm_certificate.cert.arn
   validation_record_fqdns = [for record in aws_route53_record.cert_validation : record.fqdn]
 }
-
-output "certificate_arn" {
-  description = "The ARN of the ACM certificate"
-  value       = aws_acm_certificate.cert.arn
-}
 """
 
 TERRAFORM_SECOND_TEMPLATE = """
 # Create Route 53 record for the load balancer
+data "aws_alb" "tinybird" {
+  name = "%(namespace)s"
+}
+
 resource "aws_route53_record" "tinybird" {
   zone_id = data.aws_route53_zone.selected.zone_id
   name    = "%(full_dns_name)s"
-  type    = "CNAME"
-  ttl     = 300
-  records = ["%(external_ip)s"]
+  type    = "A"
+  alias {
+    name                   = data.aws_alb.tinybird.dns_name
+    zone_id                = data.aws_alb.tinybird.zone_id
+    evaluate_target_health = true
+  }
 }
 
 output "tinybird_dns" {
@@ -184,46 +218,42 @@ def infra(ctx: Context) -> None:
 
 @infra.command(name="init")
 @click.option("--name", type=str, help="Name for identifying the self-managed infrastructure in Tinybird")
-@click.option("--provider", default="aws", type=str, help="Infrastructure provider (aws, gcp, azure)")
-@click.option("--region", type=str, help="AWS region (for AWS provider)")
-@click.option("--domain", type=str, help="Route53 domain name (for AWS provider)")
-@click.option("--namespace", type=str, help="Kubernetes namespace for deployment")
-@click.option("--dns-record", type=str, help="DNS record name to create (without domain, e.g. 'tinybird')")
-@click.option(
-    "--auto-apply-terraform", is_flag=True, help="Automatically apply Terraform configuration without prompting"
-)
-@click.option("--auto-apply-dns", is_flag=True, help="Automatically apply DNS configuration without prompting")
+@click.option("--provider", type=str, help="Infrastructure provider. Possible values are: aws, gcp, azure)")
+@click.option("--region", type=str, help="AWS region, when using aws as the provider")
+@click.option("--dns-zone-name", type=str, help="DNS zone name")
+@click.option("--namespace", type=str, help="Kubernetes namespace for the deployment")
+@click.option("--dns-record", type=str, help="DNS record name to create, without domain. For example, 'tinybird')")
+@click.option("--storage-class", type=str, help="Storage class for the k8s StatefulSet")
 @click.option(
-    "--auto-apply-kubectl", is_flag=True, help="Automatically apply Kubernetes configuration without prompting"
+    "--auto-apply", is_flag=True, help="Automatically apply Terraform and kubectl configuration without prompting"
 )
-@click.option("--skip-terraform", is_flag=True, help="Skip Terraform configuration and application")
-@click.option("--skip-kubectl", is_flag=True, help="Skip Kubernetes configuration and application")
-@click.option("--skip-dns", is_flag=True, help="Skip DNS configuration and application")
+@click.option("--skip-apply", is_flag=True, help="Skip Terraform and kubectl configuration and application")
 @click.pass_context
 def infra_init(
     ctx: Context,
     name: str,
     provider: str,
     region: Optional[str] = None,
-    domain: Optional[str] = None,
+    dns_zone_name: Optional[str] = None,
     namespace: Optional[str] = None,
     dns_record: Optional[str] = None,
     storage_class: Optional[str] = None,
-    auto_apply_terraform: bool = False,
-    auto_apply_dns: bool = False,
-    auto_apply_kubectl: bool = False,
-    skip_terraform: bool = False,
-    skip_kubectl: bool = False,
-    skip_dns: bool = False,
+    auto_apply: bool = False,
+    skip_apply: bool = False,
 ) -> None:
     """Init infra"""
+    # Check if provider is specified
+    if not provider:
+        click.echo("Error: --provider option is required. Specify a provider. Possible values are: aws, gcp, azure.")
+        return
+
     # AWS-specific Terraform template creation
     if provider.lower() != "aws":
         click.echo("Provider not supported yet.")
         return
 
     # Create infra directory if it doesn't exist
-    infra_dir = Path("infra")
+    infra_dir = Path(f"infra/{provider}")
     infra_dir.mkdir(exist_ok=True)
     yaml_path = infra_dir / "k8s.yaml"
     tf_path = infra_dir / "main.tf"
@@ -237,34 +267,34 @@ def infra_init(
                 config = json.load(f)
             click.echo("Loaded existing configuration from config.json")
         except json.JSONDecodeError:
-            click.echo("Warning: Could not parse existing config.json, will create a new one")
+            click.echo("Warning: Could not parse existing config.json. Creating a new file...")
 
     # Generate a random ID for default values
     random_id = str(uuid.uuid4())[:8]
 
     # Get or prompt for configuration values
-    name = name or click.prompt("Enter name", type=str)
-    region = region or config.get("region") or click.prompt("Enter aws region", default="us-east-1", type=str)
-    domain = domain or config.get("domain") or click.prompt("Enter route 53 domain name", type=str)
+    name = name or click.prompt("Enter the name for your self-managed region", type=str)
+    region = region or config.get("region") or click.prompt("Enter the AWS region", default="us-east-1", type=str)
+    dns_zone_name = dns_zone_name or config.get("dns_zone_name") or click.prompt("Enter the DNS zone name", type=str)
     namespace = (
         namespace
         or config.get("namespace")
-        or click.prompt("Enter namespace name", default=f"tinybird-{random_id}", type=str)
+        or click.prompt("Enter the Kubernetes namespace", default=f"tinybird-{random_id}", type=str)
     )
     dns_record = (
         dns_record
         or config.get("dns_record")
-        or click.prompt("Enter DNS record name (without domain)", default=f"tinybird-{random_id}", type=str)
+        or click.prompt("Enter the DNS record name, without domain", default=f"tinybird-{random_id}", type=str)
     )
     storage_class = config.get("storage_class") or click.prompt(
-        "Enter storage class", default="gp3-encrypted", type=str
+        "Enter the Kubernetes storage class", default="gp3-encrypted", type=str
     )
 
     # Save configuration
     config = {
         "provider": provider,
         "region": region,
-        "domain": domain,
+        "dns_zone_name": dns_zone_name,
         "namespace": namespace,
         "dns_record": dns_record,
         "storage_class": storage_class,
@@ -280,30 +310,48 @@ def infra_init(
     user_client = cli_config.get_client(token=cli_config.get_user_token() or "")
     user_workspaces = async_to_sync(user_client.user_workspaces_with_organization)()
     admin_org_id = user_workspaces.get("organization_id")
+    admin_org_name = user_workspaces.get("organization_name", "")
     infras = async_to_sync(client.infra_list)(organization_id=admin_org_id)
     infra = next((infra for infra in infras if infra["name"] == name), None)
     if not infra:
         click.echo(FeedbackManager.highlight(message=f"\n» Creating infrastructure '{name}' in Tinybird..."))
-        host = f"https://{dns_record}.{domain}"
+        host = f"https://{dns_record}.{dns_zone_name}"
         infra = async_to_sync(client.infra_create)(organization_id=admin_org_id, name=name, host=host)
 
     infra_token = infra["token"]
 
     # Write the Terraform template
-    terraform_content = TERRAFORM_FIRST_TEMPLATE % {"aws_region": region, "domain": domain}
+    terraform_content = TERRAFORM_FIRST_TEMPLATE % {
+        "aws_region": region,
+        "dns_zone_name": dns_zone_name,
+        "dns_record": dns_record,
+    }
 
     with open(tf_path, "w") as f:
         f.write(terraform_content.lstrip())
 
-    click.echo(f"Creating Terraform configuration in {tf_path}")
+    click.echo(f"Created Terraform configuration in {tf_path}")
 
-    # Apply Terraform configuration if user confirms
-    if not skip_terraform:
-        import subprocess
+    new_content = K8S_YML % {
+        "namespace": namespace,
+        "storage_class": storage_class,
+        "full_dns_name": f"{dns_record}.{dns_zone_name}",
+        "infra_token": infra_token,
+        "infra_workspace": cli_config.get("name", ""),
+        "infra_organization": admin_org_name,
+        "infra_user": cli_config.get_user_email() or "",
+    }
+
+    with open(yaml_path, "w") as f:
+        f.write(new_content.lstrip())
+
+    click.echo(f"Created Kubernetes configuration in {yaml_path}")
 
+    # Apply Terraform configuration if user confirms
+    if not skip_apply:
         # Initialize Terraform
         click.echo("Initializing Terraform...")
-        init_result = subprocess.run(["terraform", "-chdir=infra", "init"], capture_output=True, text=True)
+        init_result = subprocess.run(["terraform", f"-chdir={infra_dir}", "init"], capture_output=True, text=True)
 
         if init_result.returncode != 0:
             click.echo("Terraform initialization failed:")
@@ -312,7 +360,7 @@ def infra_init(
 
         # Run terraform plan first
         click.echo("\nRunning Terraform plan...\n")
-        plan_result = subprocess.run(["terraform", "-chdir=infra", "plan"], capture_output=True, text=True)
+        plan_result = subprocess.run(["terraform", f"-chdir={infra_dir}", "plan"], capture_output=True, text=True)
 
         if plan_result.returncode != 0:
             click.echo("Terraform plan failed:")
@@ -322,10 +370,10 @@ def infra_init(
         click.echo(plan_result.stdout)
 
         # Apply Terraform configuration if user confirms
-        if auto_apply_terraform or click.confirm("Would you like to apply the Terraform configuration now?"):
+        if auto_apply or click.confirm("Would you like to apply the Terraform configuration now?"):
             click.echo("\nApplying Terraform configuration...\n")
             apply_result = subprocess.run(
-                ["terraform", "-chdir=infra", "apply", "-auto-approve"], capture_output=True, text=True
+                ["terraform", f"-chdir={infra_dir}", "apply", "-auto-approve"], capture_output=True, text=True
             )
 
             if apply_result.returncode != 0:
@@ -335,309 +383,133 @@ def infra_init(
 
             click.echo(apply_result.stdout)
 
-            # Get the certificate ARN output
-            output_result = subprocess.run(
-                ["terraform", "-chdir=infra", "output", "certificate_arn"], capture_output=True, text=True
+        # Prompt to apply the k8s configuration
+        if not skip_apply and (
+            auto_apply or click.confirm("Would you like to apply the Kubernetes configuration now?")
+        ):
+            # Get current kubectl context
+            current_context_result = subprocess.run(
+                ["kubectl", "config", "current-context"], capture_output=True, text=True
             )
 
-            if output_result.returncode == 0:
-                cert_arn = output_result.stdout.strip().replace('"', "")
-
-                new_content = K8S_YML % {
-                    "namespace": namespace,
-                    "cert_arn": cert_arn,
-                    "storage_class": storage_class,
-                    "infra_token": infra_token,
-                }
-
-                with open(yaml_path, "w") as f:
-                    f.write(new_content.lstrip())
-
-                click.echo(f"Created Kubernetes configuration with certificate ARN in {yaml_path}")
-
-                # Prompt to apply the k8s configuration
-                if not skip_kubectl and (
-                    auto_apply_kubectl or click.confirm("Would you like to apply the Kubernetes configuration now?")
-                ):
-                    import subprocess
-
-                    # Get current kubectl context
-                    current_context_result = subprocess.run(
-                        ["kubectl", "config", "current-context"], capture_output=True, text=True
-                    )
-
-                    current_context = (
-                        current_context_result.stdout.strip() if current_context_result.returncode == 0 else "unknown"
-                    )
-
-                    # Get available contexts
-                    contexts_result = subprocess.run(
-                        ["kubectl", "config", "get-contexts", "-o", "name"], capture_output=True, text=True
-                    )
-
-                    if contexts_result.returncode != 0:
-                        click.echo("Failed to get kubectl contexts:")
-                        click.echo(contexts_result.stderr)
-                        return
-
-                    available_contexts = [
-                        context.strip() for context in contexts_result.stdout.splitlines() if context.strip()
-                    ]
-
-                    if not available_contexts:
-                        click.echo("No kubectl contexts found. Please configure kubectl first.")
-                        return
-
-                    # Prompt user to select a context
-                    if len(available_contexts) == 1:
-                        selected_context = available_contexts[0]
-                        click.echo(f"Using the only available kubectl context: {selected_context}")
-                    else:
-                        click.echo("\nAvailable kubectl contexts:")
-                        for i, context in enumerate(available_contexts):
-                            marker = " (current)" if context == current_context else ""
-                            click.echo(f"  {i + 1}. {context}{marker}")
-
-                        click.echo("")
-                        default_index = (
-                            available_contexts.index(current_context) + 1
-                            if current_context in available_contexts
-                            else 1
-                        )
-
-                        selected_index = click.prompt(
-                            "Select kubectl context number to apply configuration",
-                            type=click.IntRange(1, len(available_contexts)),
-                            default=default_index,
-                        )
-
-                        selected_context = available_contexts[selected_index - 1]
-                        click.echo(f"Selected context: {selected_context}")
-
-                    # Apply the configuration to the selected context
-                    click.echo(f"Applying Kubernetes configuration to context '{selected_context}'...")
-                    apply_result = subprocess.run(
-                        ["kubectl", "--context", selected_context, "apply", "-f", str(yaml_path)],
-                        capture_output=True,
-                        text=True,
-                    )
-
-                    if apply_result.returncode != 0:
-                        click.echo("Failed to apply Kubernetes configuration:")
-                        click.echo(apply_result.stderr)
-                    else:
-                        click.echo("Kubernetes configuration applied successfully:")
-                        click.echo(apply_result.stdout)
-
-                        # Get the namespace from the applied configuration
-                        namespace = None
-                        with open(yaml_path, "r") as f:
-                            for line in f:
-                                if "namespace:" in line and not namespace:
-                                    namespace = line.split("namespace:")[1].strip()
-                                    break
-
-                        if not namespace:
-                            namespace = "tinybird"  # Default namespace
-
-                        click.echo(f"\nWaiting for load balancer to be provisioned in namespace '{namespace}'...")
-
-                        # Wait for the load balancer to get an external IP
-                        max_attempts = 30
-                        attempt = 0
-                        external_ip = None
-
-                        while attempt < max_attempts and not external_ip:
-                            attempt += 1
-
-                            # Get the service details
-                            get_service_result = subprocess.run(
-                                [
-                                    "kubectl",
-                                    "--context",
-                                    selected_context,
-                                    "-n",
-                                    namespace,
-                                    "get",
-                                    "service",
-                                    "tinybird",
-                                    "-o",
-                                    "jsonpath='{.status.loadBalancer.ingress[0].hostname}'",
-                                ],
-                                capture_output=True,
-                                text=True,
-                            )
-
-                            if get_service_result.returncode == 0:
-                                potential_ip = get_service_result.stdout.strip().replace("'", "")
-                                if potential_ip and potential_ip != "":
-                                    external_ip = potential_ip
-                                    break
-
-                            click.echo(
-                                f"Attempt {attempt}/{max_attempts}: Load balancer not ready yet, waiting 10 seconds..."
-                            )
-                            import time
-
-                            time.sleep(10)
-
-                        if external_ip:
-                            click.echo("\nLoad balancer provisioned successfully.")
-
-                            # Update the Terraform configuration with the load balancer DNS
-                            if not skip_dns and domain and tf_path.exists():
-                                click.echo("\nUpdating Terraform configuration with load balancer DNS...")
-
-                                with open(tf_path, "r") as f:
-                                    tf_content = f.read()
-
-                                # Check if the Route 53 record already exists in the file
-                                if 'resource "aws_route53_record" "tinybird"' not in tf_content:
-                                    # Create the full DNS name
-                                    full_dns_name = f"{dns_record}.{domain}"
-
-                                    # Use in the Terraform template
-                                    route53_record = TERRAFORM_SECOND_TEMPLATE % {
-                                        "external_ip": external_ip,
-                                        "full_dns_name": full_dns_name,
-                                    }
+            current_context = (
+                current_context_result.stdout.strip() if current_context_result.returncode == 0 else "unknown"
+            )
+            # Get available contexts
+            contexts_result = subprocess.run(
+                ["kubectl", "config", "get-contexts", "-o", "name"], capture_output=True, text=True
+            )
 
-                                    # Append the Route 53 record to the Terraform file
-                                    with open(tf_path, "a") as f:
-                                        f.write(route53_record.lstrip())
+            if contexts_result.returncode != 0:
+                click.echo("Failed to get kubectl contexts:")
+                click.echo(contexts_result.stderr)
+                return
 
-                                    click.echo("Added Route 53 record to Terraform configuration")
-                                else:
-                                    # Update the existing Route 53 record
-                                    updated_tf = tf_content.replace(
-                                        'records = ["LOAD_BALANCER_DNS_PLACEHOLDER"]', f'records = ["{external_ip}"]'
-                                    )
+            available_contexts = [context.strip() for context in contexts_result.stdout.splitlines() if context.strip()]
 
-                                    # Also handle case where there might be another placeholder or old value
-                                    import re
+            if not available_contexts:
+                click.echo("No kubectl contexts found. Configure kubectl first.")
+                return
 
-                                    pattern = r'records\s*=\s*\[\s*"[^"]*"\s*\]'
-                                    updated_tf = re.sub(pattern, f'records = ["{external_ip}"]', updated_tf)
+            # Prompt user to select a context
+            if len(available_contexts) == 1:
+                selected_context = available_contexts[0]
+                click.echo(f"Using the only available kubectl context: {selected_context}")
+            else:
+                click.echo("\nAvailable kubectl contexts:")
+                for i, context in enumerate(available_contexts):
+                    marker = " (current)" if context == current_context else ""
+                    click.echo(f"  {i + 1}. {context}{marker}")
+
+                click.echo("")
+                default_index = (
+                    available_contexts.index(current_context) + 1 if current_context in available_contexts else 1
+                )
+
+                selected_index = click.prompt(
+                    "Select kubectl context number to apply configuration",
+                    type=click.IntRange(1, len(available_contexts)),
+                    default=default_index,
+                )
+
+                selected_context = available_contexts[selected_index - 1]
+                click.echo(f"Selected context: {selected_context}")
+
+            # Apply the configuration to the selected context
+            click.echo(f"Applying Kubernetes configuration to context '{selected_context}'...")
+            apply_result = subprocess.run(
+                ["kubectl", "--context", selected_context, "apply", "-f", str(yaml_path)],
+                capture_output=True,
+                text=True,
+            )
 
-                                    with open(tf_path, "w") as f:
-                                        f.write(updated_tf.lstrip())
+            if apply_result.returncode != 0:
+                click.echo("Failed to apply Kubernetes configuration:")
+                click.echo(apply_result.stderr)
+            else:
+                click.echo("Kubernetes configuration applied successfully:")
+                click.echo(apply_result.stdout)
+
+                # Get the namespace from the applied configuration
+                namespace = None
+                with open(yaml_path, "r") as f:
+                    for line in f:
+                        if "namespace:" in line and not namespace:
+                            namespace = line.split("namespace:")[1].strip()
+                            break
+
+                if not namespace:
+                    namespace = "tinybird"  # Default namespace
+
+                click.echo("\nWaiting for load balancer and DNS to be provisioned...")
+
+                max_attempts = 30  # 30 attempts * 10 seconds = 5 minutes
+                endpoint_url = f"https://{dns_record}.{dns_zone_name}"
+
+                with click.progressbar(
+                    range(max_attempts),
+                    label=f"Checking endpoint availability: {endpoint_url}",
+                    length=max_attempts,
+                    show_eta=True,
+                    show_percent=True,
+                    fill_char="█",
+                    empty_char="░",
+                ) as bar:
+                    for attempt in bar:
+                        try:
+                            response = requests.get(endpoint_url, allow_redirects=False, timeout=5)
+                            if response.status_code < 400:  # Consider any non-error response as success
+                                click.echo(click.style("\n✅ HTTPS endpoint is now accessible!", fg="green", bold=True))
+                                break
+                        except requests.RequestException:
+                            pass
+
+                        if attempt == max_attempts - 1:
+                            click.echo(
+                                click.style(
+                                    "\n⚠️  HTTPS endpoint not accessible after 5 minutes", fg="yellow", bold=True
+                                )
+                            )
+                            click.echo(
+                                "  This might be due to DNS propagation or the Load Balancer provisioning delays"
+                            )
+                            click.echo(f"  Please try accessing {endpoint_url} manually in a few minutes")
+                        else:
+                            time.sleep(10)
 
-                                    click.echo("Updated existing Route 53 record in Terraform configuration")
+    if not skip_apply:
+        # Print a summary with the endpoint URL
+        click.echo("\n" + "=" * 60)
+        click.echo("DEPLOYMENT SUMMARY".center(60))
+        click.echo("=" * 60)
+        click.echo("✅ Load balancer provisioned")
 
-                                # Run terraform plan for DNS changes
-                                click.echo("\nRunning Terraform plan for DNS changes...\n")
-                                plan_result = subprocess.run(
-                                    ["terraform", "-chdir=infra", "plan"], capture_output=True, text=True
-                                )
+        click.echo(f"\n🔗 Tinybird is available at: https://{dns_record}.{dns_zone_name}")
 
-                                if plan_result.returncode != 0:
-                                    click.echo("Terraform plan for DNS changes failed:")
-                                    click.echo(plan_result.stderr)
-                                else:
-                                    click.echo(plan_result.stdout)
-
-                                    # Apply the updated Terraform configuration
-                                    if not skip_dns and (
-                                        auto_apply_dns
-                                        or click.confirm("Would you like to create the DNS record in Route 53 now?")
-                                    ):
-                                        click.echo("Applying updated Terraform configuration...")
-                                        apply_result = subprocess.run(
-                                            ["terraform", "-chdir=infra", "apply", "-auto-approve"],
-                                            capture_output=True,
-                                            text=True,
-                                        )
-
-                                        if apply_result.returncode != 0:
-                                            click.echo("Failed to create DNS record:")
-                                            click.echo(apply_result.stderr)
-                                        else:
-                                            click.echo(apply_result.stdout)
-                                            click.echo("DNS record created successfully!")
-
-                                            # Get the DNS name from Terraform output
-                                            dns_output = subprocess.run(
-                                                ["terraform", "-chdir=infra", "output", "tinybird_dns"],
-                                                capture_output=True,
-                                                text=True,
-                                            )
-
-                                            if dns_output.returncode == 0:
-                                                dns_name = dns_output.stdout.strip().replace('"', "")
-                                                click.echo("\nDNS record created successfully!")
-                                                click.echo(
-                                                    "\nWaiting up to 5 minutes for HTTPS endpoint to become available..."
-                                                )
-
-                                                import time
-
-                                                import requests
-
-                                                max_attempts = 30  # 30 attempts * 10 seconds = 5 minutes
-                                                attempt = 0
-                                                while attempt < max_attempts:
-                                                    attempt += 1
-                                                    try:
-                                                        response = requests.get(
-                                                            f"https://{dns_name}", allow_redirects=False, timeout=5
-                                                        )
-                                                        response.raise_for_status()
-                                                        click.echo("\n✅ HTTPS endpoint is now accessible!")
-                                                        break
-                                                    except requests.RequestException:
-                                                        if attempt == max_attempts:
-                                                            click.echo(
-                                                                "\n⚠️  HTTPS endpoint not accessible after 5 minutes"
-                                                            )
-                                                            click.echo(
-                                                                "  This might be due to DNS propagation or the Load Balancer provisioning delays"
-                                                            )
-                                                            click.echo(
-                                                                "  Please try accessing the URL manually in a few minutes"
-                                                            )
-                                                        else:
-                                                            click.echo(
-                                                                f"Attempt {attempt}/{max_attempts}: Not ready yet, waiting 10 seconds..."
-                                                            )
-                                                            time.sleep(10)
-                                            else:
-                                                click.echo(
-                                                    f"\nYour Tinybird instance should be available at: https://tinybird.{domain}"
-                                                )
-
-    # Print a summary with the endpoint URL
-    click.echo("\n" + "=" * 60)
-    click.echo("DEPLOYMENT SUMMARY".center(60))
-    click.echo("=" * 60)
-
-    if not skip_kubectl and external_ip:
-        click.echo(f"✅ Load balancer provisioned: {external_ip}")
-
-    if not skip_dns and not skip_terraform and domain:
-        # Try to get the DNS name from Terraform output
-        dns_output = subprocess.run(
-            ["terraform", "-chdir=infra", "output", "tinybird_dns"], capture_output=True, text=True
+        click.echo(
+            "\n📌 Note: It may take a few minutes for DNS to propagate and the HTTPS certificate to be fully provisioned."
         )
 
-        if dns_output.returncode == 0:
-            dns_name = dns_output.stdout.strip().replace('"', "")
-            click.echo(f"✅ DNS record created: {dns_name}")
-            click.echo(f"\n🔗 Tinybird is available at: https://{dns_name}")
-        else:
-            # Fallback to constructed DNS name
-            full_dns_name = f"{dns_record}.{domain}"
-            click.echo(f"✅ DNS record created: {full_dns_name}")
-            click.echo(f"\n🔗 Tinybird is available at: https://{full_dns_name}")
-    elif not skip_kubectl and external_ip:
-        click.echo(f"\n🔗 Tinybird is available at: https://{external_ip}")
-        if domain:
-            click.echo(f"\n📝 Consider creating a DNS record: {dns_record}.{domain} → {external_ip}")
-
-    click.echo(
-        "\n📌 Note: It may take a few minutes for DNS to propagate and the HTTPS certificate to be fully provisioned."
-    )
-    click.echo("=" * 60)
-
 
 @infra.command(name="rm")
 @click.argument("name")
@@ -691,3 +563,62 @@ async def infra_ls(ctx: click.Context):
     click.echo(FeedbackManager.info(message="\n** Infras:"))
     echo_safe_humanfriendly_tables_format_smart_table(table_human_readable, column_names=columns)
     click.echo("\n")
+
+
+@infra.command(name="create")
+@click.option("--name", type=str, help="Name for identifying the self-managed infrastructure in Tinybird")
+@click.option("--host", type=str, help="Host for the infrastructure")
+@click.pass_context
+@coro
+async def infra_create(ctx: click.Context, name: str, host: str):
+    """Create a self-managed infrastructure"""
+    try:
+        client: TinyB = ctx.ensure_object(dict)["client"]
+        user_workspaces = await client.user_workspaces_with_organization()
+        admin_org_id = user_workspaces.get("organization_id")
+        if not admin_org_id:
+            raise CLIException("No organization associated to this workspace")
+        name = name or click.prompt("Enter name", type=str)
+        host = host or click.prompt("Enter host", type=str)
+        click.echo(FeedbackManager.highlight(message=f"\n» Creating infrastructure '{name}' in Tinybird..."))
+        infra = await client.infra_create(organization_id=admin_org_id, name=name, host=host)
+        click.echo(FeedbackManager.success(message=f"\n✓ Infrastructure '{name}' created"))
+        pyperclip.copy(infra["token"])
+        click.echo(FeedbackManager.info(message="Access token has been copied to your clipboard."))
+        click.echo(
+            FeedbackManager.info(message="Pass it as an environment variable in your deployment as TB_INFRA_TOKEN")
+        )
+    except Exception as e:
+        click.echo(FeedbackManager.error(message=f"✗ Error: {str(e)}"))
+
+
+@infra.command(name="update")
+@click.argument("infra_name")
+@click.option("--name", type=str, help="Name for identifying the self-managed infrastructure in Tinybird")
+@click.option("--host", type=str, help="Host for the infrastructure")
+@click.pass_context
+@coro
+async def infra_update(ctx: click.Context, infra_name: str, name: str, host: str):
+    """Update a self-managed infrastructure"""
+    try:
+        client: TinyB = ctx.ensure_object(dict)["client"]
+        user_workspaces = await client.user_workspaces_with_organization()
+        admin_org_id = user_workspaces.get("organization_id")
+        if not admin_org_id:
+            raise CLIException("No organization associated to this workspace")
+
+        if not name and not host:
+            click.echo(
+                FeedbackManager.warning(message="No name or host provided. Please provide either a name or a host.")
+            )
+            return
+
+        if name or host:
+            infras = await client.infra_list(organization_id=admin_org_id)
+            infra_id = next((infra["id"] for infra in infras if infra["name"] == infra_name), None)
+            if not infra_id:
+                raise CLIException(f"Infrastructure '{infra_name}' not found")
+            click.echo(FeedbackManager.highlight(message=f"\n» Updating infrastructure '{infra_name}' in Tinybird..."))
+            await client.infra_update(infra_id=infra_id, organization_id=admin_org_id, name=name, host=host)
+    except Exception as e:
+        click.echo(FeedbackManager.error(message=f"✗ Error: {str(e)}"))

{tinybird-0.0.1.dev99.dist-info → tinybird-0.0.1.dev101.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev99
+Version: 0.0.1.dev101
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev99.dist-info → tinybird-0.0.1.dev101.dist-info}/RECORD

@@ -1,5 +1,5 @@
 tinybird/__cli__.py,sha256=esPl5QDTzuQgHe5FuxWLm-fURFigGGwjnYLh9GuWUw4,232
-tinybird/client.py,sha256=aDDifEWkVxJcm4WYaw6uUgXhav-e7JUW1N-mgtEjV4A,55087
+tinybird/client.py,sha256=yc2BP2HiQhaUjU-t7nBuZEiJsb2nQMH3vBfOhKCciyU,55468
 tinybird/config.py,sha256=5UP_UZ2Qtlm5aOH5W7SbtN8r7X-8u3-r853joKqU5zs,6072
 tinybird/connectors.py,sha256=7Gjms7b5MAaBFGi3xytsJurCylprONpFcYrzp4Fw2Rc,15241
 tinybird/context.py,sha256=FfqYfrGX_I7PKGTQo93utaKPDNVYWelg4Hsp3evX5wM,1291
@@ -7,15 +7,15 @@ tinybird/datatypes.py,sha256=XNypumfqNjsvLJ5iNXnbVHRvAJe0aQwI3lS6Cxox-e0,10979
 tinybird/feedback_manager.py,sha256=YSjtFDJvc8y66j2J0iIkb3SVzDdYAJbzFL-JPQ26pak,68761
 tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
 tinybird/prompts.py,sha256=syxUvbdkzxIrIK8pkS-Y7nzBx7DMS2Z9HaiXx7AjvnQ,33912
-tinybird/sql.py,sha256=LBi74GxhNAYTb6m2-KNGpAkguSKh7rcvBbERbE7nalA,46195
+tinybird/sql.py,sha256=J35bhdpuu84HW2tiLp-cs_nzkRwPhiy1yPcFhcWMCR4,46248
 tinybird/sql_template.py,sha256=VH8io4n5eP2z6TEw111d8hcA9FKQerFjprKKCW2MODw,99127
 tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,10196
-tinybird/sql_toolset.py,sha256=32SNvxRFKQYWTvYPMJ_u3ukcd1hKZyEqx8T2cv2412w,14697
+tinybird/sql_toolset.py,sha256=KORVbNAUTfW1qo3U9oe7Z59xQ0QMsFhB0ji3HzY2JVo,15324
 tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
 tinybird/tornado_template.py,sha256=jjNVDMnkYFWXflmT8KU_Ssbo5vR8KQq3EJMk5vYgXRw,41959
 tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj0,3890
 tinybird/ch_utils/engine.py,sha256=BZuPM7MFS7vaEKK5tOMR2bwSAgJudPrJt27uVEwZmTY,40512
-tinybird/tb/__cli__.py,sha256=x_8CEyPeFA4ID-fNJaTMBbVu0OhEt_vS8s7qeQ6Q7rA,251
+tinybird/tb/__cli__.py,sha256=9fN3V2hgGADGGvafwXcIrRciQG80C2cmQs_OlsP8u-A,252
 tinybird/tb/cli.py,sha256=H_HaZhkimKgkryYXpBjHfY9Qtg-ZORiONU3psDNpzDk,1135
 tinybird/tb/modules/auth.py,sha256=L1IatO2arRSzys3t8px8xVt8uPWUL5EVD0sFzAV_uVU,9022
 tinybird/tb/modules/build.py,sha256=h5drdmDFX8NHts9dA2Zepao7KSgMAl3DZGyFufVZP78,11085
@@ -32,7 +32,7 @@ tinybird/tb/modules/endpoint.py,sha256=EhVoGAXsFz-83Fiwj1gI-I73iRRvL49d0W81un7hv
 tinybird/tb/modules/exceptions.py,sha256=4A2sSjCEqKUMqpP3WI00zouCWW4uLaghXXLZBSw04mY,3363
 tinybird/tb/modules/feedback_manager.py,sha256=7nNiOx7OMebiheLED1r0d75SbuXCNxyBmF4e20rCBNc,69511
 tinybird/tb/modules/fmt.py,sha256=qpf9APqKTKL2uphNgdbj4OMVyLkAxZn6dn4eHF99L5g,3553
-tinybird/tb/modules/infra.py,sha256=dJcPYN_hpcLpygl4IcAANeP7fUgziwMGqYadWd9IT5g,29105
+tinybird/tb/modules/infra.py,sha256=vb6WmVHFuU1ZhtQMh2wOzRd2x8g7CLt_2gR3EL_L2E4,22867
 tinybird/tb/modules/job.py,sha256=956Pj8BEEsiD2GZsV9RKKVM3I_CveOLgS82lykO5ukk,2963
 tinybird/tb/modules/llm.py,sha256=AC0VSphTOM2t-v1_3NLvNN_FIbgMo4dTyMqIv5nniPo,835
 tinybird/tb/modules/llm_utils.py,sha256=nS9r4FAElJw8yXtmdYrx-rtI2zXR8qXfi1QqUDCfxvg,3469
@@ -61,7 +61,7 @@ tinybird/tb/modules/datafile/build.py,sha256=jhfIJ2xt0N13XsLPe3iMQIyCPApHS13_Df2
 tinybird/tb/modules/datafile/build_common.py,sha256=rT7VJ5mnQ68R_8US91DAtkusfvjWuG_NObOzNgtN_ko,4562
 tinybird/tb/modules/datafile/build_datasource.py,sha256=CCU3eQ8Rax9RgHHfbAXDRL6rQ49N35h_GDQnGrUUUzA,17379
 tinybird/tb/modules/datafile/build_pipe.py,sha256=w-Wd08gZYAEcak9FdBijVfIU2_Wn_PPdgAZddPpoGTo,11382
-tinybird/tb/modules/datafile/common.py,sha256=i9Gvhz3JiR58MRBcYZDwqTqamQOj-46TnHU8Hm8bqmg,81399
+tinybird/tb/modules/datafile/common.py,sha256=ZRRx-9EXeHMOz6Lu9Y1TSWELF0ftfUHLLQAANjJZ334,82241
 tinybird/tb/modules/datafile/diff.py,sha256=-0J7PsBO64T7LOZSkZ4ZFHHCPvT7cKItnJkbz2PkndU,6754
 tinybird/tb/modules/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
 tinybird/tb/modules/datafile/fixture.py,sha256=si-9LB-LdKQSWDtVW82xDrHtFfko5bgBG1cvjqqrcPU,1064
@@ -81,8 +81,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
-tinybird-0.0.1.dev99.dist-info/METADATA,sha256=19bjkN1GWZCSNjE45Gk7lm_slBrHlbHxihZZT0PVeYQ,2585
-tinybird-0.0.1.dev99.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-tinybird-0.0.1.dev99.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-0.0.1.dev99.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
-tinybird-0.0.1.dev99.dist-info/RECORD,,
+tinybird-0.0.1.dev101.dist-info/METADATA,sha256=73sj_cIfixFVZVrtmkn_MwEYvtkMLO4GYeJCjJQ1KA0,2586
+tinybird-0.0.1.dev101.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+tinybird-0.0.1.dev101.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-0.0.1.dev101.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
+tinybird-0.0.1.dev101.dist-info/RECORD,,