tinybird 0.0.1.dev291__py3-none-any.whl → 1.0.5__py3-none-any.whl

tinybird/tb/modules/datasource.py

@@ -8,9 +8,8 @@ import os
 import re
 import time
 import uuid
-from datetime import datetime
 from pathlib import Path
-from typing import List, Optional
+from typing import Optional
 from urllib.parse import urlparse
 
 import click
@@ -33,7 +32,13 @@ from tinybird.tb.modules.common import (
     push_data,
 )
 from tinybird.tb.modules.config import CLIConfig
-from tinybird.tb.modules.connection import connection_create_kafka
+from tinybird.tb.modules.connection_kafka import (
+    connection_create_kafka,
+    echo_kafka_data,
+    select_connection,
+    select_group_id,
+    select_topic,
+)
 from tinybird.tb.modules.create import (
     create_resources_from_prompt,
     generate_aws_iamrole_connection_file_with_secret,
@@ -651,11 +656,19 @@ def datasource_sync(ctx: Context, datasource_name: str, yes: bool):
 @click.option("--blank", is_flag=True, default=False, help="Create a blank data source")
 @click.option("--file", type=str, help="Create a data source from a local file")
 @click.option("--url", type=str, help="Create a data source from a remote URL")
-@click.option("--connection", type=str, help="Create a data source from a connection")
 @click.option("--prompt", type=str, help="Create a data source from a prompt")
+@click.option("--connection-name", type=str, help="Create a data source from a connection")
 @click.option("--s3", is_flag=True, default=False, help="Create a data source from a S3 connection")
 @click.option("--gcs", is_flag=True, default=False, help="Create a data source from a GCS connection")
 @click.option("--kafka", is_flag=True, default=False, help="Create a data source from a Kafka connection")
+@click.option("--kafka-topic", type=str, help="Kafka topic")
+@click.option("--kafka-group-id", type=str, help="Kafka group ID")
+@click.option(
+    "--kafka-auto-offset-reset",
+    type=click.Choice(["latest", "earliest"], case_sensitive=False),
+    help="Kafka auto offset reset",
+)
+@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
 @click.pass_context
 def datasource_create(
     ctx: Context,
@@ -663,359 +676,515 @@ def datasource_create(
     blank: bool,
     file: str,
     url: str,
-    connection: str,
+    connection_name: Optional[str],
     prompt: str,
     s3: bool,
     gcs: bool,
     kafka: bool,
+    kafka_topic: str,
+    kafka_group_id: str,
+    kafka_auto_offset_reset: str,
+    yes: bool,
 ):
-    project: Project = ctx.ensure_object(dict)["project"]
-    client: TinyB = ctx.ensure_object(dict)["client"]
-    config = ctx.ensure_object(dict)["config"]
-    env: str = ctx.ensure_object(dict)["env"]
+    wizard_data: dict[str, str | bool | float] = {
+        "wizard": "datasource_create",
+        "current_step": "start",
+    }
+    start_time = time.time()
 
-    if env == "cloud":
-        raise CLIDatasourceException(
-            FeedbackManager.error(message="`tb datasource create` is not available against Tinybird Cloud.")
-        )
+    if name:
+        wizard_data["datasource_name"] = name
 
-    datasource_types = {
-        "blank": ("Blank", "Create an empty one"),
-        "local_file": ("Local file", "A local file"),
-        "remote_url": ("Remote URL", "A remote file"),
-        "s3": ("S3", "Files on S3"),
-        "gcs": ("GCS", "Files on GCS"),
-        "kafka": ("Kafka", "Connect a Kafka topic"),
-        "prompt": ("Prompt", "Create a datasource from a prompt"),
-    }
-    datasource_type: Optional[str] = None
-    connection_file: Optional[str] = None
-    ds_content = """SCHEMA >
+    try:
+        project: Project = ctx.ensure_object(dict)["project"]
+        client: TinyB = ctx.ensure_object(dict)["client"]
+        config = ctx.ensure_object(dict)["config"]
+        env: str = ctx.ensure_object(dict)["env"]
+
+        if env == "cloud":
+            raise CLIDatasourceException(
+                FeedbackManager.error(message="`tb datasource create` is not available against Tinybird Cloud.")
+            )
+
+        datasource_types = {
+            "blank": ("Blank", "Create an empty one"),
+            "local_file": ("Local file", "A local file"),
+            "remote_url": ("Remote URL", "A remote file"),
+            "s3": ("S3", "Files on S3"),
+            "gcs": ("GCS", "Files on GCS"),
+            "kafka": ("Kafka", "Connect a Kafka topic"),
+            "prompt": ("Prompt", "Create a datasource from a prompt"),
+        }
+        datasource_type: Optional[str] = None
+        connection_file: Optional[str] = None
+        ds_content = """SCHEMA >
     `data` String `json:$`
+
+ENGINE "MergeTree"
+# ENGINE_SORTING_KEY "user_id, timestamp"
+# ENGINE_TTL "timestamp + toIntervalDay(60)"
+# Learn more at https://www.tinybird.co/docs/forward/dev-reference/datafiles/datasource-files
 """
-    wizard_mode = True
-    if file:
-        datasource_type = "local_file"
-        wizard_mode = False
-    elif url:
-        datasource_type = "remote_url"
-        wizard_mode = False
-    elif blank:
-        datasource_type = "blank"
-        wizard_mode = False
-    elif connection:
-        connection_files = project.get_connection_files()
-        connection_file = next((f for f in connection_files if f.endswith(f"{connection}.connection")), None)
-        if connection_file:
-            connection_content = Path(connection_file).read_text()
-            if project.is_kafka_connection(connection_content):
-                datasource_type = "kafka"
-            elif project.is_s3_connection(connection_content):
-                datasource_type = "s3"
-            elif project.is_gcs_connection(connection_content):
-                datasource_type = "gcs"
-    elif s3:
-        datasource_type = "s3"
-        wizard_mode = False
-    elif gcs:
-        datasource_type = "gcs"
-        wizard_mode = False
-    elif kafka:
-        datasource_type = "kafka"
-        wizard_mode = False
-    elif prompt:
-        datasource_type = "prompt"
-        wizard_mode = False
-
-    if datasource_type is None:
-        click.echo(
-            FeedbackManager.highlight(
-                message="? This command will create the schema (.datasource) for your data. Choose where from:"
-            )
-        )
+        wizard_mode = True
+        if file:
+            datasource_type = "local_file"
+            wizard_mode = False
+        elif url:
+            datasource_type = "remote_url"
+            wizard_mode = False
+        elif blank:
+            datasource_type = "blank"
+            wizard_mode = False
+        elif connection_name:
+            connection_files = project.get_connection_files()
+            connection_file = next((f for f in connection_files if f.endswith(f"{connection_name}.connection")), None)
+            if connection_file:
+                connection_content = Path(connection_file).read_text()
+                if project.is_kafka_connection(connection_content):
+                    datasource_type = "kafka"
+                elif project.is_s3_connection(connection_content):
+                    datasource_type = "s3"
+                elif project.is_gcs_connection(connection_content):
+                    datasource_type = "gcs"
+        elif s3:
+            datasource_type = "s3"
+            wizard_mode = False
+        elif gcs:
+            datasource_type = "gcs"
+            wizard_mode = False
+        elif kafka:
+            datasource_type = "kafka"
+            wizard_mode = False
+        elif prompt:
+            datasource_type = "prompt"
+            wizard_mode = False
         datasource_type_index = -1
 
-        dt_keys = list(datasource_types.keys())
-        while datasource_type_index == -1:
-            for index, key in enumerate(dt_keys):
+        if datasource_type is None:
+            wizard_data["current_step"] = "select_datasource_origin"
+            click.echo(
+                FeedbackManager.highlight(
+                    message="? This command will create the schema (.datasource) for your data. Choose where from:"
+                )
+            )
+
+            dt_keys = list(datasource_types.keys())
+            while datasource_type_index == -1:
+                for index, key in enumerate(dt_keys):
+                    click.echo(
+                        f"  [{index + 1}] {FeedbackManager.bold(message=datasource_types[key][0])}: {datasource_types[key][1]}"
+                    )
+                click.echo(FeedbackManager.gray(message="\nFiles can be either NDJSON, CSV or Parquet."))
                 click.echo(
-                    f"  [{index + 1}] {FeedbackManager.bold(message=datasource_types[key][0])}: {datasource_types[key][1]}"
+                    FeedbackManager.gray(
+                        message=("Tip: Run `tb datasource create --file | --url | --connection` to skip this step.")
+                    )
                 )
-            click.echo(FeedbackManager.gray(message="\nFiles can be either NDJSON, CSV or Parquet."))
+                datasource_type_index = click.prompt("\nSelect option", default=1)
+
+                if datasource_type_index == 0:
+                    click.echo(FeedbackManager.warning(message="Datasource type selection cancelled by user"))
+
+                    wizard_data["exit_reason"] = "user_cancelled_type_selection"
+                    wizard_data["duration_seconds"] = round(time.time() - start_time, 2)
+                    add_telemetry_event("system_info", **wizard_data)
+                    return None
+
+                try:
+                    datasource_type = dt_keys[int(datasource_type_index) - 1]
+                except Exception:
+                    datasource_type_index = -1
+
+        if datasource_type:
+            wizard_data["datasource_type"] = datasource_type
+
+        if not datasource_type:
             click.echo(
-                FeedbackManager.gray(
-                    message=("Tip: Run `tb datasource create --file | --url | --connection` to skip this step.")
+                FeedbackManager.error(
+                    message=f"Invalid option: {datasource_type_index}. Please select a valid option from the list above."
                 )
             )
-            datasource_type_index = click.prompt("\nSelect option", default=1)
 
-            if datasource_type_index == 0:
-                click.echo(FeedbackManager.warning(message="Datasource type selection cancelled by user"))
-                return None
+            wizard_data["exit_reason"] = "invalid_type_selection"
+            wizard_data["duration_seconds"] = round(time.time() - start_time, 2)
+            add_telemetry_event("system_info", **wizard_data)
+            return
 
-            try:
-                datasource_type = dt_keys[int(datasource_type_index) - 1]
-            except Exception:
-                datasource_type_index = -1
+        if datasource_type == "prompt":
+            click.echo(FeedbackManager.gray(message="\n» Creating .datasource file..."))
+            if not config.get("user_token"):
+                raise Exception("This action requires authentication. Run 'tb login' first.")
 
-    if not datasource_type:
-        click.echo(
-            FeedbackManager.error(
-                message=f"Invalid option: {datasource_type_index}. Please select a valid option from the list above."
+            instructions = (
+                "Create or update a Tinybird datasource (.datasource file) for this project. "
+                "Do not generate mock data or append data; those steps will run later programmatically."
             )
-        )
-        return
-
-    if datasource_type == "prompt":
-        click.echo(FeedbackManager.gray(message="\n» Creating .datasource file..."))
-        user_token = config.get("user_token")
-        if not user_token:
-            raise Exception("This action requires authentication. Run 'tb login' first.")
-        project_config = CLIConfig.get_project_config()
-        tb_client: TinyB = project_config.get_client(token=config.get("token"), host=config.get("host"))
-        create_resources_from_prompt(tb_client, user_token, prompt, project, feature="tb_datasource_create")
-        click.echo(FeedbackManager.success(message="✓ .datasource created!"))
-        return
-
-    connection_required = datasource_type in ("kafka", "s3", "gcs")
-
-    if connection_required:
-
-        def get_connection_files():
-            connection_files = []
-            if datasource_type == "kafka":
-                connection_files = project.get_kafka_connection_files()
-            elif datasource_type == "s3":
-                connection_files = project.get_s3_connection_files()
-            elif datasource_type == "gcs":
-                connection_files = project.get_gcs_connection_files()
-            return connection_files
-
-        connection_files = get_connection_files()
-
-        click.echo(FeedbackManager.gray(message="\n» Selecting connection..."))
-        connection_name = ""
-        topics: List[str] = []
-        if len(connection_files) == 0:
-            click.echo(FeedbackManager.error(message=f"✗ No {datasource_types[datasource_type][0]} connections found."))
-            if click.confirm(
-                FeedbackManager.highlight(
-                    message=f"\n? Do you want to create a {datasource_types[datasource_type][0]} connection? [Y/n]"
-                ),
-                show_default=False,
-                default=True,
-            ):
-                if datasource_type != "kafka":
-                    click.echo(FeedbackManager.gray(message="\n» Creating .connection file..."))
-                    default_connection_name = f"{datasource_type}_{generate_short_id()}"
-                    connection_name = click.prompt(
-                        FeedbackManager.highlight(message=f"? Connection name [{default_connection_name}]"),
-                        show_default=False,
-                        default=default_connection_name,
-                    )
-                if datasource_type == "kafka":
-                    (
-                        connection_name,
-                        bootstrap_servers,
-                        key,
-                        secret,
-                        schema_registry_url,
-                        auto_offset_reset,
-                        sasl_mechanism,
-                        security_protocol,
-                        topics,
-                    ) = connection_create_kafka(ctx)
-                elif datasource_type == "s3":
-                    generate_aws_iamrole_connection_file_with_secret(
-                        connection_name,
-                        service="s3",
-                        role_arn_secret_name="S3_ARN",
-                        region="eu-west-1",
-                        folder=project.folder,
-                        with_default_secret=True,
-                    )
-                elif datasource_type == "gcs":
-                    generate_gcs_connection_file_with_secrets(
-                        connection_name,
-                        service="gcs",
-                        svc_account_creds="GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON",
-                        folder=project.folder,
-                    )
-                if datasource_type != "kafka":
-                    click.echo(FeedbackManager.info_file_created(file=f"connections/{connection_name}.connection"))
-                    click.echo(FeedbackManager.success(message="✓ .connection created!"))
-                connection_files = get_connection_files()
+            if not prompt:
+                wizard_data["current_step"] = "enter_prompt"
+                prompt = click.prompt(FeedbackManager.highlight(message="? Enter your prompt"))
+            wizard_data["prompt"] = prompt
+
+            if name:
+                instructions += f" Name the datasource '{name}'."
+
+            created_resources = create_resources_from_prompt(
+                config,
+                project,
+                prompt,
+                feature="tb_datasource_create",
+                instructions=instructions,
+            )
+            if any(path.suffix == ".datasource" for path in created_resources):
+                click.echo(FeedbackManager.success(message="✓ .datasource created!"))
             else:
-                click.echo(FeedbackManager.info(message=f"→ Run `tb connection create {datasource_type}` to add one."))
-                return
-
-        if not connection_file:
-            if len(connection_files) > 1:
                 click.echo(
-                    FeedbackManager.highlight(
-                        message=f"? Multiple {datasource_types[datasource_type][0]} connections found. Please select one:"
+                    FeedbackManager.gray(
+                        message="△ No new datasource file detected. Existing resources may have been updated instead."
                     )
                 )
-                connection_index = -1
-                while connection_index == -1:
-                    for index, conn_file in enumerate(connection_files):
-                        conn_path = Path(conn_file)
-                        click.echo(f"  [{index + 1}] {conn_path.stem}")
-                    connection_index = click.prompt("\nSelect option", default=1)
-                    try:
-                        connection_file = connection_files[int(connection_index) - 1]
-                        connection_path = Path(connection_file)
-                        connection = connection_path.stem
-                    except Exception:
-                        connection_index = -1
-            else:
-                connection_file = connection_files[0]
-                connection_path = Path(connection_file)
-                connection = connection_path.stem
-                click.echo(FeedbackManager.info(message=f"Using connection: {connection}"))
-
-    click.echo(FeedbackManager.gray(message="\n» Creating .datasource file..."))
-
-    if datasource_type == "local_file":
-        if not file:
-            file = click.prompt(FeedbackManager.highlight(message="? Path"))
-            if file.startswith("~"):
-                file = os.path.expanduser(file)
-
-        folder_path = project.path
-        path = folder_path / file
-        if not path.exists():
-            path = Path(file)
-
-        data_format = path.suffix.lstrip(".")
-        ds_content = analyze_file(str(path), client, format=data_format)
-        default_name = normalize_datasource_name(path.stem)
-        name = name or click.prompt(
-            FeedbackManager.highlight(message=f"? Data source name [{default_name}]"),
-            default=default_name,
-            show_default=False,
-        )
 
-    if datasource_type == "remote_url":
-        if not url:
-            url = click.prompt(FeedbackManager.highlight(message="? URL"))
-        format = url.split(".")[-1]
-        ds_content = analyze_file(url, client, format)
-        default_name = normalize_datasource_name(Path(url).stem)
-        name = name or click.prompt(
-            FeedbackManager.highlight(message=f"? Data source name [{default_name}]"),
-            default=default_name,
-            show_default=False,
-        )
+            wizard_data["current_step"] = "completed"
+            wizard_data["duration_seconds"] = round(time.time() - start_time, 2)
+            add_telemetry_event("system_info", **wizard_data)
+            return
 
-    if datasource_type not in ("remote_url", "local_file"):
-        default_name = f"ds_{generate_short_id()}"
-        name = name or click.prompt(
-            FeedbackManager.highlight(message=f"? Data source name [{default_name}]"),
-            default=default_name,
-            show_default=False,
-        )
+        connection_required = datasource_type in ("kafka", "s3", "gcs")
+
+        if connection_required:
+            if env == "local":
+                should_build = click.confirm(
+                    FeedbackManager.highlight(message="\n? Do you want to build the project before continue? [Y/n]"),
+                    show_default=False,
+                    default=True,
+                )
+                if should_build:
+                    click.echo(FeedbackManager.gray(message="» Building project before continue..."))
+                    build_project(project=project, tb_client=client, watch=False, config=config, silent=True)
+                    click.echo(FeedbackManager.success(message="✓ Build completed!\n"))
+                else:
+                    click.echo(FeedbackManager.gray(message="Skipping build...\n"))
+
+            wizard_data["current_step"] = "select_connection"
+
+            connections = client.connections(datasource_type)
+            connection_type = datasource_types[datasource_type][0]
+            new_connection_created = False
+            if len(connections) == 0:
+                click.echo(FeedbackManager.info(message=f"No {connection_type} connections found."))
+                if click.confirm(
+                    FeedbackManager.highlight(
+                        message=f"\n? Do you want to create a {connection_type} connection? [Y/n]"
+                    ),
+                    show_default=False,
+                    default=True,
+                ):
+                    wizard_data["created_new_connection"] = True
+                    if datasource_type == "kafka":
+                        result = connection_create_kafka(ctx)
+                        connection_name = result["name"]
+                    elif datasource_type == "s3":
+                        click.echo(FeedbackManager.gray(message="\n» Creating .connection file..."))
+                        default_connection_name = f"{datasource_type}_{generate_short_id()}"
+                        s3_connection_name: str = click.prompt(
+                            FeedbackManager.highlight(message=f"? Connection name [{default_connection_name}]"),
+                            show_default=False,
+                            default=default_connection_name,
+                        )
+                        connection_name = s3_connection_name
+                        wizard_data["connection_name"] = s3_connection_name
+                        generate_aws_iamrole_connection_file_with_secret(
+                            s3_connection_name,
+                            service="s3",
+                            role_arn_secret_name="S3_ARN",
+                            region="eu-west-1",
+                            folder=project.folder,
+                            with_default_secret=True,
+                        )
+                    elif datasource_type == "gcs":
+                        click.echo(FeedbackManager.gray(message="\n» Creating .connection file..."))
+                        default_connection_name = f"{datasource_type}_{generate_short_id()}"
+                        gcs_connection_name: str = click.prompt(
+                            FeedbackManager.highlight(message=f"? Connection name [{default_connection_name}]"),
+                            show_default=False,
+                            default=default_connection_name,
+                        )
+                        connection_name = gcs_connection_name
+                        wizard_data["connection_name"] = gcs_connection_name
+                        generate_gcs_connection_file_with_secrets(
+                            gcs_connection_name,
+                            service="gcs",
+                            svc_account_creds="GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON",
+                            folder=project.folder,
+                        )
+                    new_connection_created = True
+                    if env == "local" and new_connection_created:
+                        click.echo(FeedbackManager.gray(message="\n» Building project to access the new connection..."))
+                        build_project(project=project, tb_client=client, watch=False, config=config, silent=True)
+                        click.echo(FeedbackManager.success(message="✓ Build completed!"))
+                else:
+                    click.echo(
+                        FeedbackManager.info(message=f"→ Run `tb connection create {datasource_type}` to add one.")
+                    )
+                    wizard_data["exit_reason"] = "user_declined_connection_creation"
+                    wizard_data["duration_seconds"] = round(time.time() - start_time, 2)
+                    add_telemetry_event("system_info", **wizard_data)
+                    return
+
+            # Only prompt for connection selection if connection_name wasn't provided via CLI
+            if not connection_name:
+                wizard_data["selected_connection_from_multiple"] = True
+                connection = select_connection(None, datasource_type, connections, client)
+                connection_id = connection["id"]
+                connection_name = connection["name"]
 
-    if datasource_type == "kafka":
-        connections = client.connections("kafka")
-        kafka_connection_files = project.get_kafka_connection_files()
+        click.echo(FeedbackManager.gray(message="\n» Creating .datasource file..."))
 
-        # if we have no topics from before and no connections, we need to build the project
-        if len(topics) == 0 and len(kafka_connection_files) != len(connections):
-            click.echo(
-                FeedbackManager.error(message=f"✗ Some {datasource_types[datasource_type][0]} connections are missing.")
+        if datasource_type == "local_file":
+            wizard_data["current_step"] = "file_input"
+            if not file:
+                file = click.prompt(FeedbackManager.highlight(message="? Path"))
+                if file.startswith("~"):
+                    file = os.path.expanduser(file)
+
+            folder_path = project.path
+            path = folder_path / file
+            if not path.exists():
+                path = Path(file)
+
+            data_format = path.suffix.lstrip(".")
+            ds_content = analyze_file(str(path), client, format=data_format)
+            default_name = normalize_datasource_name(path.stem)
+            wizard_data["current_step"] = "enter_name"
+            name = name or click.prompt(
+                FeedbackManager.highlight(message=f"? Data source name [{default_name}]"),
+                default=default_name,
+                show_default=False,
             )
-            if click.confirm(
-                FeedbackManager.highlight(message="? Do you want to build your project before continue? [Y/n]"),
+            wizard_data["datasource_name"] = name
+
+            if name == default_name:
+                wizard_data["used_default_name"] = True
+
+        if datasource_type == "remote_url":
+            wizard_data["current_step"] = "file_input"
+            if not url:
+                url = click.prompt(FeedbackManager.highlight(message="? URL"))
+            format = url.split(".")[-1]
+            ds_content = analyze_file(url, client, format)
+            default_name = normalize_datasource_name(Path(url).stem)
+            wizard_data["current_step"] = "enter_name"
+            name = name or click.prompt(
+                FeedbackManager.highlight(message=f"? Data source name [{default_name}]"),
+                default=default_name,
                 show_default=False,
-                default=True,
-            ):
-                click.echo(FeedbackManager.gray(message="» Building project..."))
-                build_project(project=project, tb_client=client, watch=False, silent=True)
-                click.echo(FeedbackManager.success(message="✓ Build completed!"))
-                connections = client.connections("kafka")
+            )
+            wizard_data["datasource_name"] = name
 
-        connection_id = next((c["id"] for c in connections if c["name"] == connection), connection)
+            if name == default_name:
+                wizard_data["used_default_name"] = True
 
-        if not topics:
-            try:
-                topics = client.kafka_list_topics(connection_id) if connection_id else []
-            except Exception:
-                topics = []
-
-        if len(topics) > 1:
-            click.echo(FeedbackManager.highlight(message="? Multiple topics found. Please select one:"))
-            topic_index = -1
-            while topic_index == -1:
-                for index, topic in enumerate(topics):
-                    click.echo(f"  [{index + 1}] {topic}")
-                topic_index = click.prompt("\nSelect option", default=1)
-                try:
-                    topic = topics[int(topic_index) - 1]
-                except Exception:
-                    topic_index = -1
-        else:
-            topic = topics[0] if len(topics) > 0 else "topic_0"
+        if datasource_type not in ("remote_url", "local_file"):
+            wizard_data["current_step"] = "enter_name"
+            default_name = f"ds_{generate_short_id()}"
+            name = name or click.prompt(
+                FeedbackManager.highlight(message=f"? Data source name [{default_name}]"),
+                default=default_name,
+                show_default=False,
+            )
+            wizard_data["datasource_name"] = name
+
+            if name == default_name:
+                wizard_data["used_default_name"] = True
 
-        group_id = generate_kafka_group_id(topic)
+        if datasource_type == "kafka":
+            if not connection_name:
+                raise CLIDatasourceException(FeedbackManager.error(message="A Kafka connection name is required."))
 
-        ds_content += f"""
-KAFKA_CONNECTION_NAME {connection}
-KAFKA_TOPIC {topic}
-KAFKA_GROUP_ID {group_id}
+            wizard_data["current_step"] = "kafka_configuration"
+            connections = client.connections("kafka")
+
+            # Kafka configuration values - preserve CLI values if provided
+            kafka_topic_value: Optional[str] = kafka_topic or None
+            kafka_group_id_value: Optional[str] = kafka_group_id or None
+            kafka_auto_offset_reset_value: Optional[str] = kafka_auto_offset_reset or None
+            kafka_connection_id: Optional[str] = next(
+                (c["id"] for c in connections if c["name"] == connection_name), None
+            )
+
+            if connection_name and kafka_connection_id is None:
+                raise CLIDatasourceException(
+                    FeedbackManager.error(message=f"No Kafka connection found with name '{connection_name}'.")
+                )
+
+            confirmed = yes
+            change_topic = False
+            change_group_id = False
+            change_connection = False
+            change_auto_offset_reset = False
+
+            while not confirmed:
+                # Select connection if not set or if user wants to change it
+                if kafka_connection_id is None or change_connection:
+                    selected_connection = select_connection(kafka_connection_id, datasource_type, connections, client)
+                    kafka_connection_id = selected_connection["id"]
+                    connection_name = selected_connection["name"]
+                    change_connection = False
+
+                # Select topic if not set
+                if (kafka_topic_value is None or change_topic) and kafka_connection_id is not None:
+                    kafka_topic_value = select_topic(kafka_topic_value, kafka_connection_id, client)
+                    change_topic = False
+
+                # Select group ID if not set or if user wants to change it
+                if (
+                    (kafka_group_id_value is None or change_group_id)
+                    and kafka_connection_id is not None
+                    and kafka_topic_value is not None
+                ):
+                    kafka_group_id_value = select_group_id(
+                        kafka_group_id_value, kafka_topic_value, kafka_connection_id, client
+                    )
+                    change_group_id = False  # Reset flag
+
+                # Select auto offset reset if not set or if user wants to change it
+                if kafka_auto_offset_reset_value is None or change_auto_offset_reset:
+                    default_auto_offset = kafka_auto_offset_reset_value if kafka_auto_offset_reset_value else "latest"
+                    kafka_auto_offset_reset_value = click.prompt(
+                        FeedbackManager.highlight(message="? Auto offset reset"),
+                        type=click.Choice(["latest", "earliest"], case_sensitive=False),
+                        default=default_auto_offset,
+                        show_default=True,
+                    )
+                    change_auto_offset_reset = False  # Reset flag
+
+                # Show preview - at this point kafka_connection_id is guaranteed to be set
+                assert kafka_connection_id is not None
+                assert connection_name is not None
+                assert kafka_topic_value is not None
+                assert kafka_group_id_value is not None
+                echo_kafka_data(kafka_connection_id, connection_name, kafka_topic_value, kafka_group_id_value, client)
+
+                # Confirmation step
+                wizard_data["current_step"] = "kafka_confirmation"
+                click.echo(FeedbackManager.highlight(message="\n? What would you like to do?"))
+                click.echo("  [1] Create .datasource file with this configuration")
+                click.echo("  [2] Connection")
+                click.echo("  [3] Topic")
+                click.echo("  [4] Group ID")
+                click.echo("  [5] Auto offset reset")
+                click.echo("  [6] Cancel")
+
+                choice = click.prompt("\nSelect option", default=1, type=int)
+
+                if choice == 1:
+                    confirmed = True
+                elif choice == 2:
+                    change_connection = True
+                    # Reset topic and group ID since they are connection-specific
+                    kafka_topic_value = None
+                    kafka_group_id_value = None
+                elif choice == 3:
+                    kafka_topic_value = None  # Reset to prompt again
+                    kafka_group_id_value = None  # Reset group ID since it's topic-specific
+                elif choice == 4:
+                    change_group_id = True  # Set flag to re-prompt with current value as default
+                elif choice == 5:
+                    change_auto_offset_reset = True  # Set flag to re-prompt with current value as default
+                elif choice == 6:
+                    wizard_data["exit_reason"] = "user_cancelled_kafka_configuration"
+                    wizard_data["duration_seconds"] = round(time.time() - start_time, 2)
+                    add_telemetry_event("system_info", **wizard_data)
+                    return None
+                else:
+                    click.echo(FeedbackManager.error(message="Invalid option. Please select 1-6."))
+
+            ds_content += f"""
+KAFKA_CONNECTION_NAME {connection_name}
+KAFKA_TOPIC {kafka_topic_value}
+KAFKA_GROUP_ID {kafka_group_id_value}
+KAFKA_AUTO_OFFSET_RESET {kafka_auto_offset_reset_value}
+# Learn more at https://www.tinybird.co/docs/forward/get-data-in/connectors/kafka#kafka-datasource-settings
 """
 
-    if datasource_type == "s3":
-        if not connection:
-            connections = client.connections("s3")
-            connection = next((c["name"] for c in connections if c["name"] == connection), connection)
-        ds_content += f"""
-IMPORT_CONNECTION_NAME "{connection}"
+        if datasource_type == "s3":
+            # Use connection_name from CLI if provided, otherwise look it up from selected connection_id
+            s3_conn_name: Optional[str] = connection_name
+            if not s3_conn_name:
+                s3_connections = client.connections("s3")
+                s3_conn_name = next((c["name"] for c in s3_connections if c["id"] == connection_id), None)
+            ds_content += f"""
+IMPORT_CONNECTION_NAME "{s3_conn_name}"
 IMPORT_BUCKET_URI "s3://my-bucket/*.csv"
 IMPORT_SCHEDULE "@auto"
 """
 
-    if datasource_type == "gcs":
-        if not connection:
-            connections = client.connections("gcs")
-            connection = next((c["name"] for c in connections if c["name"] == connection), connection)
-        ds_content += f"""
-IMPORT_CONNECTION_NAME "{connection}"
+        if datasource_type == "gcs":
+            # Use connection_name from CLI if provided, otherwise look it up from selected connection_id
+            gcs_conn_name: Optional[str] = connection_name
+            if not gcs_conn_name:
+                gcs_connections = client.connections("gcs")
+                gcs_conn_name = next((c["name"] for c in gcs_connections if c["id"] == connection_id), None)
+            ds_content += f"""
+IMPORT_CONNECTION_NAME "{gcs_conn_name}"
 IMPORT_BUCKET_URI "gs://my-bucket/*.csv"
 IMPORT_SCHEDULE "@auto"
 """
 
-    click.echo(FeedbackManager.info(message=f"/datasources/{name}.datasource"))
-    datasources_path = project.path / "datasources"
-    if not datasources_path.exists():
-        datasources_path.mkdir()
-    ds_file = datasources_path / f"{name}.datasource"
-    if not ds_file.exists():
-        ds_file.touch()
-    ds_file.write_text(ds_content)
-    click.echo(FeedbackManager.success(message="✓ .datasource created!"))
-
-    if wizard_mode:
-        last_tip_message = "\nTip: To skip the interactive prompts, pass flags to this command, e.g."
-        last_tip_command = ""
-        if datasource_type == "local_file":
-            last_tip_command = f"`tb datasource create --file {file} --name {name}`."
-        elif datasource_type == "remote_url":
-            last_tip_command = f"`tb datasource create --url {url} --name {name}`."
-        elif datasource_type == "blank":
-            last_tip_command = f"`tb datasource create --blank --name {name}`."
-        elif datasource_type in ("s3", "gcs", "kafka"):
-            last_tip_command = f"`tb datasource create --{datasource_type} --name {name} --connection {connection}`."
+        wizard_data["current_step"] = "create_datasource_file"
+        click.echo(FeedbackManager.info(message=f"/datasources/{name}.datasource"))
+        datasources_path = project.path / "datasources"
+        if not datasources_path.exists():
+            datasources_path.mkdir()
+        ds_file = datasources_path / f"{name}.datasource"
+        if not ds_file.exists():
+            ds_file.touch()
+        ds_file.write_text(ds_content)
+        click.echo(FeedbackManager.success(message="✓ .datasource created!"))
 
-        click.echo(FeedbackManager.gray(message=(f"{last_tip_message} {last_tip_command}")))
+        if wizard_mode:
+            last_tip_message = "\nTip: To skip the interactive prompts, pass flags to this command, e.g."
+            last_tip_command = ""
+            if datasource_type == "local_file":
+                last_tip_command = f"`tb datasource create --file {file} --name {name}`."
+            elif datasource_type == "remote_url":
+                last_tip_command = f"`tb datasource create --url {url} --name {name}`."
+            elif datasource_type == "blank":
+                last_tip_command = f"`tb datasource create --blank --name {name}`."
+            elif datasource_type in ("s3", "gcs", "kafka"):
+                if datasource_type == "kafka":
+                    conn_name = connection_name
+                elif datasource_type == "s3":
+                    conn_name = s3_conn_name
+                else:
+                    conn_name = gcs_conn_name
+                last_tip_command = (
+                    f"`tb datasource create --{datasource_type} --name {name} --connection-name {conn_name}`."
+                )
 
+            click.echo(FeedbackManager.gray(message=(f"{last_tip_message} {last_tip_command}")))
 
-def generate_short_id():
-    return str(uuid.uuid4())[:4]
+        wizard_data["current_step"] = "completed"
+        wizard_data["duration_seconds"] = round(time.time() - start_time, 2)
+        add_telemetry_event("system_info", **wizard_data)
+
+    except Exception as e:
+        wizard_data["duration_seconds"] = round(time.time() - start_time, 2)
+
+        current_exception: Optional[BaseException] = e
+        while current_exception:
+            if isinstance(current_exception, KeyboardInterrupt):
+                wizard_data["exit_reason"] = "user_interrupted"
+                add_telemetry_event("system_info", **wizard_data)
+                raise
+            current_exception = current_exception.__cause__ or current_exception.__context__
+
+        wizard_data["error_message"] = str(e)
+        add_telemetry_event("wizard_error", **wizard_data)
+        raise
 
 
-def generate_kafka_group_id(topic: str):
-    return f"{topic}_{int(datetime.timestamp(datetime.now()))}"
+def generate_short_id():
+    return str(uuid.uuid4())[:4]
 
 
 def analyze_quarantine(datasource_name: str, project: Project, client: TinyB):