tinybird 0.0.1.dev324__tar.gz → 0.0.1.dev326__tar.gz

{tinybird-0.0.1.dev324 → tinybird-0.0.1.dev326}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev324
+Version: 0.0.1.dev326
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev324 → tinybird-0.0.1.dev326}/tinybird/sql_template.py

@@ -1841,7 +1841,7 @@ def get_var_names_and_types(t, node_id=None):
         raise SQLTemplateException(e)
 
 
-@lru_cache(maxsize=256)
+@lru_cache(maxsize=512)
 def get_var_names_and_types_cached(t: Template):
     return get_var_names_and_types(t)
 

{tinybird-0.0.1.dev324 → tinybird-0.0.1.dev326}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev324'
-__revision__ = 'b4c5779'
+__version__ = '0.0.1.dev326'
+__revision__ = '4414501'

{tinybird-0.0.1.dev324 → tinybird-0.0.1.dev326}/tinybird/tb/client.py

@@ -996,10 +996,92 @@ class TinyB:
             data=json.dumps(connection_params),
         )
 
-    def kafka_list_topics(self, connection_id: str, timeout=5):
-        resp = self._req(f"/v0/connectors/{connection_id}/preview?preview_activity=false", timeout=timeout)
+    def kafka_list_topics(self, connection_id: str, timeout=10, retries=3):
+        resp = self._req(
+            f"/v0/connectors/{connection_id}/preview?preview_activity=false",
+            timeout=timeout,
+            retries=retries,
+        )
         return [x["topic"] for x in resp["preview"]]
 
+    def kafka_preview_group(self, connection_id: str, topic: str, group_id: str, timeout=30):
+        params = {
+            "log": "previewGroup",
+            "kafka_group_id": group_id,
+            "kafka_topic": topic,
+            "preview_group": "true",
+        }
+        return self._req(f"/v0/connectors/{connection_id}/preview?{urlencode(params)}", method="GET", timeout=timeout)
+
+    def kafka_preview_topic(self, connection_id: str, topic: str, group_id: str, timeout: int = 30) -> Dict[str, Any]:
+        """Preview a Kafka topic and return structured preview data.
+
+        Args:
+            connection_id: The ID of the Kafka connection
+            topic: The Kafka topic name to preview
+            group_id: The Kafka consumer group ID
+            timeout: Request timeout in seconds
+
+        Returns:
+            A dictionary containing:
+                - analysis: Dictionary with columns information
+                - preview: Dictionary with data and meta arrays
+                - earliestTimestamp: The earliest message timestamp (if available)
+        """
+        params = {
+            "max_records": "12",
+            "preview_activity": "true",
+            "preview_earliest_timestamp": "true",
+            "kafka_topic": topic,
+            "kafka_group_id": group_id,
+            "log": "previewTopic",
+        }
+        response = self._req(
+            f"/v0/connectors/{connection_id}/preview?{urlencode(params)}", method="GET", timeout=timeout
+        )
+
+        if not response:
+            return {
+                "analysis": {"columns": []},
+                "preview": {"data": [], "meta": []},
+                "earliestTimestamp": "",
+            }
+
+        # Extract preview data (similar to TypeScript previewKafkaTopic)
+        preview_data = response.get("preview", [])
+        if not preview_data:
+            return {
+                "analysis": {"columns": []},
+                "preview": {"data": [], "meta": []},
+                "earliestTimestamp": "",
+            }
+
+        topic_preview = preview_data[0]
+        analysis = topic_preview.get("analysis", {})
+        deserialized = topic_preview.get("deserialized", {})
+
+        # Extract columns from analysis
+        columns = analysis.get("columns", []) if analysis else []
+
+        # Extract data and meta from deserialized
+        base_data = deserialized.get("data", []) if deserialized else []
+        base_meta = deserialized.get("meta", []) if deserialized else []
+
+        # Extract earliest timestamp
+        earliest = response.get("earliest", [])
+        earliest_timestamp = earliest[0].get("timestamp", "") if earliest else ""
+
+        return {
+            "analysis": {
+                "columns": columns,
+            },
+            "preview": {
+                "data": base_data,
+                "meta": base_meta,
+            },
+            "earliestTimestamp": earliest_timestamp,
+        }
+
     def get_gcp_service_account_details(self) -> Dict[str, Any]:
         return self._req("/v0/datasources-bigquery-credentials")
 

{tinybird-0.0.1.dev324 → tinybird-0.0.1.dev326}/tinybird/tb/modules/connection.py

@@ -4,13 +4,13 @@
 # - But please, **do not** interleave utility functions and command definitions.
 
 import uuid
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional
 
 import click
 from click import Context
-from confluent_kafka.admin import AdminClient
 
 from tinybird.tb.client import TinyB
+from tinybird.tb.modules.build_common import process as build_project
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import (
     DataConnectorType,
@@ -18,21 +18,15 @@ from tinybird.tb.modules.common import (
     echo_safe_humanfriendly_tables_format_smart_table,
     get_gcs_connection_name,
     get_gcs_svc_account_creds,
-    get_kafka_connection_name,
     get_s3_connection_name,
     production_aws_iamrole_only,
     run_aws_iamrole_connection_flow,
     run_gcp_svc_account_connection_flow,
-    validate_kafka_auto_offset_reset,
-    validate_kafka_bootstrap_servers,
-    validate_kafka_key,
-    validate_kafka_schema_registry_url,
-    validate_kafka_secret,
 )
+from tinybird.tb.modules.connection_kafka import connection_create_kafka, echo_kafka_data, select_group_id, select_topic
 from tinybird.tb.modules.create import (
     generate_aws_iamrole_connection_file_with_secret,
     generate_gcs_connection_file_with_secrets,
-    generate_kafka_connection_with_secrets,
 )
 from tinybird.tb.modules.exceptions import CLIConnectionException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
@@ -315,198 +309,115 @@ def connection_create_gcs(ctx: Context) -> None:
     )
 
 
-@connection_create.command(name="kafka", short_help="Creates a Kafka connection.")
+@connection_create.command(name="kafka", help="Create a Kafka connection.")
+@click.option("--connection-name", default=None, help="The name of the connection to identify it in Tinybird")
+@click.option("--bootstrap-servers", default=None, help="Kafka Bootstrap Server in form mykafka.mycloud.com:9092")
+@click.option("--key", default=None, help="Key/User")
+@click.option("--secret", default=None, help="Secret/Password")
+@click.option(
+    "--auto-offset-reset", default=None, help="Offset reset, can be 'latest' or 'earliest'. Defaults to 'latest'."
+)
+@click.option("--schema-registry-url", default=None, help="Avro Confluent Schema Registry URL")
+@click.option(
+    "--sasl-mechanism",
+    default="PLAIN",
+    help="Authentication method for connection-based protocols. Defaults to 'PLAIN'",
+)
+@click.option(
+    "--security-protocol",
+    default="SASL_SSL",
+    help="Security protocol for connection-based protocols. Defaults to 'SASL_SSL'",
+)
+@click.option("--ssl-ca-pem", default=None, help="Path or content of the CA Certificate file in PEM format")
 @click.pass_context
-def connection_create_kafka_cmd(ctx: Context) -> None:
-    """
-    Creates a Kafka connection.
-
-    \b
-    $ tb connection create kafka
-    """
-    connection_create_kafka(ctx)
-
-
-def connection_create_kafka(ctx: Context) -> Tuple[str, str, str, str, str, str, str, str, List[str]]:
-    obj: Dict[str, Any] = ctx.ensure_object(dict)
-    click.echo(FeedbackManager.gray(message="\n» Creating Kafka connection..."))
+def connection_create_kafka_cmd(
+    ctx: Context,
+    connection_name: Optional[str],
+    bootstrap_servers: Optional[str],
+    key: Optional[str],
+    secret: Optional[str],
+    auto_offset_reset: Optional[str],
+    schema_registry_url: Optional[str],
+    sasl_mechanism: Optional[str],
+    security_protocol: Optional[str],
+    ssl_ca_pem: Optional[str],
+) -> None:
+    env: str = ctx.ensure_object(dict)["env"]
     project: Project = ctx.ensure_object(dict)["project"]
-    name = get_kafka_connection_name(project.folder, "")
-    default_bootstrap_servers = "localhost:9092"
-    bootstrap_servers = click.prompt(
-        FeedbackManager.highlight(
-            message=f"? Bootstrap servers (comma-separated list of host:port pairs) [{default_bootstrap_servers}]"
-        ),
-        default=default_bootstrap_servers,
-        show_default=False,
-    )
-    if not bootstrap_servers:
-        bootstrap_servers = click.prompt("Bootstrap Server")
-
-    validate_kafka_bootstrap_servers(bootstrap_servers)
-    secret_required = click.confirm(
-        FeedbackManager.highlight(message="  ? Do you want to store the bootstrap server in a .env.local file? [Y/n]"),
-        default=True,
-        show_default=False,
-    )
-    tb_secret_bootstrap_servers: Optional[str] = None
-    tb_secret_key: Optional[str] = None
-    tb_secret_secret: Optional[str] = None
-
-    if secret_required:
-        tb_secret_bootstrap_servers = str(click.prompt(FeedbackManager.highlight(message="    ? Secret name")))
-        try:
-            save_secret_to_env_file(project=project, name=tb_secret_bootstrap_servers, value=bootstrap_servers)
-        except Exception as e:
-            raise CLIConnectionException(FeedbackManager.error(message=str(e)))
-
-    key = click.prompt(FeedbackManager.highlight(message="? Kafka key"))
-
-    validate_kafka_key(key)
-
-    secret_required = click.confirm(
-        FeedbackManager.highlight(message="  ? Do you want to store the Kafka key in a .env.local file? [Y/n]"),
-        default=True,
-        show_default=False,
-    )
-
-    if secret_required:
-        tb_secret_key = str(click.prompt(FeedbackManager.highlight(message="    ? Secret name")))
-        try:
-            save_secret_to_env_file(project=project, name=tb_secret_key, value=key)
-        except Exception as e:
-            raise CLIConnectionException(FeedbackManager.error(message=str(e)))
-
-    secret = click.prompt(FeedbackManager.highlight(message="? Kafka secret"), hide_input=True)
-
-    validate_kafka_secret(secret)
-
-    secret_required = click.confirm(
-        FeedbackManager.highlight(message="  ? Do you want to store the Kafka secret in a .env.local file? [Y/n]"),
-        default=True,
-        show_default=False,
-    )
-
-    if secret_required:
-        tb_secret_secret = str(click.prompt(FeedbackManager.highlight(message="    ? Secret name")))
-        try:
-            save_secret_to_env_file(project=project, name=tb_secret_secret, value=secret)
-        except Exception as e:
-            raise CLIConnectionException(FeedbackManager.error(message=str(e)))
-
-    security_protocol_options = ["SASL_SSL", "PLAINTEXT"]
-    security_protocol = click.prompt(
-        FeedbackManager.highlight(message="? Security Protocol (SASL_SSL, PLAINTEXT) [SASL_SSL]"),
-        type=click.Choice(security_protocol_options),
-        show_default=False,
-        show_choices=False,
-        default="SASL_SSL",
-    )
-
-    sasl_mechanism_options = ["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512"]
-    sasl_mechanism = click.prompt(
-        FeedbackManager.highlight(message="? SASL Mechanism (PLAIN, SCRAM-SHA-256, SCRAM-SHA-512) [PLAIN]"),
-        type=click.Choice(sasl_mechanism_options),
-        show_default=False,
-        show_choices=False,
-        default="PLAIN",
+    client: TinyB = ctx.ensure_object(dict)["client"]
+    config: Dict[str, Any] = ctx.ensure_object(dict)["config"]
+    if env == "local":
+        click.echo(FeedbackManager.gray(message="» Building project..."))
+        build_project(project=project, tb_client=client, watch=False, config=config, silent=True)
+        click.echo(FeedbackManager.success(message="✓ Build completed"))
+
+    result = connection_create_kafka(
+        ctx,
+        connection_name=connection_name,
+        bootstrap_servers=bootstrap_servers,
+        key=key,
+        secret=secret,
+        auto_offset_reset=auto_offset_reset,
+        schema_registry_url=schema_registry_url,
+        sasl_mechanism=sasl_mechanism,
+        security_protocol=security_protocol,
+        ssl_ca_pem=ssl_ca_pem,
     )
 
-    create_in_cloud = (
-        click.confirm(
-            FeedbackManager.highlight(
-                message="? Would you like to create this connection in the cloud environment as well? [Y/n]"
-            ),
+    if not result["error"]:
+        yes = click.confirm(
+            FeedbackManager.highlight(message="? Would you like to preview data from the connection?"),
             default=True,
             show_default=False,
         )
-        if obj["env"] == "local" and (tb_secret_bootstrap_servers or tb_secret_key or tb_secret_secret)
-        else False
-    )
+        if yes:
+            connection_data(ctx, connection_name=result["name"])
+        else:
+            click.echo(FeedbackManager.gray(message="Skipping data preview."))
 
-    if create_in_cloud:
-        click.echo(FeedbackManager.gray(message="» Creating Secrets in cloud environment..."))
-        prod_config = obj["config"]
-        host = prod_config["host"]
-        token = prod_config["token"]
-        prod_client = TinyB(
-            token=token,
-            host=host,
-            staging=False,
-        )
-        if tb_secret_bootstrap_servers:
-            prod_client.create_secret(name=tb_secret_bootstrap_servers, value=bootstrap_servers)
-        if tb_secret_key:
-            prod_client.create_secret(name=tb_secret_key, value=key)
-        if tb_secret_secret:
-            prod_client.create_secret(name=tb_secret_secret, value=secret)
-        click.echo(FeedbackManager.success(message="✓ Secrets created!"))
-
-    schema_registry_url = click.prompt(
-        FeedbackManager.highlight(message="? Schema Registry URL (optional)"),
-        default="",
-        show_default=False,
-    )
-    if schema_registry_url:
-        validate_kafka_schema_registry_url(schema_registry_url)
-
-    auto_offset_reset_options = ["latest", "earliest"]
-    auto_offset_reset = click.prompt(
-        FeedbackManager.highlight(message="? Auto offset reset (latest, earliest) [latest]"),
-        type=click.Choice(auto_offset_reset_options),
-        default="latest",
-        show_default=False,
-        show_choices=False,
-    )
-    validate_kafka_auto_offset_reset(auto_offset_reset)
-    click.echo(FeedbackManager.gray(message="» Validating connection..."))
 
-    topics = list_kafka_topics(bootstrap_servers, key, secret, security_protocol, sasl_mechanism)
+@connection.command(name="data", help="Preview data from an existing connection.")
+@click.argument("connection_name", type=str)
+@click.option("--kafka-topic", type=str, help="The Kafka topic to preview")
+@click.option("--kafka-group-id", type=str, help="The Kafka group ID to use for preview")
+@click.pass_context
+def connection_data_cmd(
+    ctx: Context,
+    connection_name: str,
+    kafka_topic: Optional[str] = None,
+    kafka_group_id: Optional[str] = None,
+) -> None:
+    connection_data(ctx, connection_name, kafka_topic, kafka_group_id)
+
+
+def connection_data(
+    ctx: Context,
+    connection_name: str,
+    kafka_topic: Optional[str] = None,
+    kafka_group_id: Optional[str] = None,
+) -> None:
+    project: Project = ctx.ensure_object(dict)["project"]
+    client: TinyB = ctx.ensure_object(dict)["client"]
+    env: str = ctx.ensure_object(dict)["env"]
+    config: Dict[str, Any] = ctx.ensure_object(dict)["config"]
 
-    if topics is None:
-        raise CLIConnectionException(FeedbackManager.error(message="Invalid Kafka connection"))
+    if env == "local":
+        click.echo(FeedbackManager.gray(message="» Building project to access the latest connections..."))
+        build_project(project=project, tb_client=client, watch=False, config=config, silent=True)
+        click.echo(FeedbackManager.success(message="✓ Build completed"))
 
-    click.echo(FeedbackManager.success(message="✓ Connection is valid"))
-    generate_kafka_connection_with_secrets(
-        name=name,
-        bootstrap_servers=bootstrap_servers,
-        tb_secret_bootstrap_servers=tb_secret_bootstrap_servers,
-        key=key,
-        tb_secret_key=tb_secret_key,
-        secret=secret,
-        tb_secret_secret=tb_secret_secret,
-        security_protocol=security_protocol,
-        sasl_mechanism=sasl_mechanism,
-        folder=project.folder,
-    )
-    click.echo(FeedbackManager.info_file_created(file=f"connections/{name}.connection"))
-    click.echo(FeedbackManager.success(message="✓ Connection created!"))
-    return (
-        name,
-        bootstrap_servers,
-        key,
-        secret,
-        schema_registry_url,
-        auto_offset_reset,
-        sasl_mechanism,
-        security_protocol,
-        topics,
-    )
+    connection = next((c for c in client.connections() if c["name"] == connection_name), None)
+    if not connection:
+        raise CLIConnectionException(FeedbackManager.error(message=f"Connection {connection_name} not found."))
 
+    if connection["service"] != DataConnectorType.KAFKA:
+        raise CLIConnectionException(
+            FeedbackManager.error(
+                message=f"{connection['service']} connections are not supported yet for previewing data. Supported connections: kafka"
+            )
+        )
 
-def list_kafka_topics(bootstrap_servers, sasl_username, sasl_password, security_protocol, sasl_mechanism):
-    conf = {
-        "bootstrap.servers": bootstrap_servers,
-        "security.protocol": security_protocol,
-        "sasl.mechanism": sasl_mechanism,
-        "sasl.username": sasl_username,
-        "sasl.password": sasl_password,
-        "log_level": 0,
-    }
-
-    try:
-        client = AdminClient(conf)
-        metadata = client.list_topics(timeout=5)
-        return list(metadata.topics.keys())
-    except Exception:
-        return None
+    connection_id = connection["id"]
+    topic = select_topic(kafka_topic, connection_id, client)
+    group_id = select_group_id(kafka_group_id, topic, connection_id, client)
+    echo_kafka_data(connection_id, topic, group_id, client)