tinybird 0.0.1.dev228__py3-none-any.whl → 0.0.1.dev229__py3-none-any.whl

tinybird/datafile/common.py

@@ -122,7 +122,12 @@ VALID_PIPE_NODE_TYPES = {
     PipeNodeTypes.STREAM,
     PipeNodeTypes.DATA_SINK,
 }
-VISIBLE_PIPE_NODE_TYPES = {PipeNodeTypes.MATERIALIZED, PipeNodeTypes.COPY, PipeNodeTypes.ENDPOINT}
+VISIBLE_PIPE_NODE_TYPES = {
+    PipeNodeTypes.MATERIALIZED,
+    PipeNodeTypes.COPY,
+    PipeNodeTypes.ENDPOINT,
+    PipeNodeTypes.DATA_SINK,
+}
 
 
 class DataFileExtensions:
@@ -182,6 +187,22 @@ class MaterializedParameters(Parameters):
     ACCEPTED_ATTRIBUTES = PipeParameters.ACCEPTED_ATTRIBUTES.union(MANDATORY_ATTRIBUTES)
 
 
+class SinkParameters(Parameters):
+    # For Kafka sinks
+    KAFKA_MANDATORY_ATTRIBUTES = PipeParameters.MANDATORY_ATTRIBUTES.union(
+        {"export_connection_name", "export_kafka_topic", "export_schedule"}
+    )
+    KAFKA_ACCEPTED_ATTRIBUTES = PipeParameters.ACCEPTED_ATTRIBUTES.union(KAFKA_MANDATORY_ATTRIBUTES)
+
+    # For S3/GCS sinks
+    BLOB_MANDATORY_ATTRIBUTES = PipeParameters.MANDATORY_ATTRIBUTES.union(
+        {"export_connection_name", "export_schedule", "export_bucket_uri", "export_file_template"}
+    )
+    BLOB_ACCEPTED_ATTRIBUTES = PipeParameters.ACCEPTED_ATTRIBUTES.union(BLOB_MANDATORY_ATTRIBUTES).union(
+        {"export_format", "export_compression", "export_write_strategy", "export_strategy"}
+    )
+
+
 DATAFILE_NEW_LINE = "\n"
 DATAFILE_INDENT = " " * 4
 
@@ -305,6 +326,90 @@ class Datafile:
                     f"Materialized node {repr(node['name'])} has an invalid attribute ({MaterializedParameters.canonical_name(key)})"
                 )
 
+    def validate_sink_node(self, node: Dict[str, Any]):
+        export_connection_name = node.get("export_connection_name")
+
+        if not export_connection_name:
+            raise DatafileValidationError(
+                f"Sink node {repr(node['name'])} is missing required parameter 'export_connection_name'"
+            )
+
+        # Determine connection type to validate appropriate parameters
+        # First, try to determine from presence of Kafka-specific parameters
+        has_kafka_topic = "export_kafka_topic" in node
+        has_s3_gcs_params = any(param in node for param in ["export_bucket_uri", "export_file_template"])
+
+        # If both types of parameters are present, that's an error
+        if has_kafka_topic and has_s3_gcs_params:
+            raise DatafileValidationError(
+                f"Sink node {repr(node['name'])} has mixed Kafka and S3/GCS parameters. Use either Kafka parameters (export_kafka_topic) or S3/GCS parameters (export_bucket_uri, export_file_template), not both."
+            )
+
+        # If we have Kafka-specific parameters, treat as Kafka sink
+        if has_kafka_topic:
+            # Kafka sink validation
+            if missing := [param for param in SinkParameters.KAFKA_MANDATORY_ATTRIBUTES if param not in node]:
+                raise DatafileValidationError(
+                    f"Kafka sink node {repr(node['name'])} is missing required parameters: {missing}"
+                )
+
+            # For Kafka sinks, only specific parameters should be present
+            kafka_specific_params = (
+                {"export_kafka_topic", "export_schedule", "export_connection_name"}
+                | PipeParameters.MANDATORY_ATTRIBUTES
+                | PipeParameters.ACCEPTED_ATTRIBUTES
+            )
+            for key in node.keys():
+                if key not in kafka_specific_params:
+                    raise DatafileValidationError(
+                        f"Kafka sink node {repr(node['name'])} has invalid parameter '{key}'. Only export_kafka_topic, export_schedule, and export_connection_name are allowed for Kafka sinks."
+                    )
+
+        # If we have S3/GCS-specific parameters, treat as S3/GCS sink
+        elif has_s3_gcs_params:
+            # S3/GCS sink validation
+            if missing := [param for param in SinkParameters.BLOB_MANDATORY_ATTRIBUTES if param not in node]:
+                raise DatafileValidationError(
+                    f"S3/GCS sink node {repr(node['name'])} is missing required parameters: {missing}"
+                )
+
+            # Check that only valid parameters are present
+            for key in node.keys():
+                if key not in SinkParameters.BLOB_ACCEPTED_ATTRIBUTES:
+                    raise DatafileValidationError(
+                        f"S3/GCS sink node {repr(node['name'])} has invalid parameter '{key}'"
+                    )
+
+        # If no type-specific parameters are present, we can't determine the type
+        # This means the sink is missing required parameters for any sink type
+        else:
+            # Check if we have any export parameters at all besides connection_name and schedule
+            export_params = {
+                k
+                for k in node.keys()
+                if k.startswith("export_") and k not in {"export_connection_name", "export_schedule"}
+            }
+            if not export_params:
+                raise DatafileValidationError(
+                    f"Sink node {repr(node['name'])} is missing required parameters. "
+                    f"For Kafka sinks, provide 'export_kafka_topic'. "
+                    f"For S3/GCS sinks, provide 'export_bucket_uri' and 'export_file_template'."
+                )
+            else:
+                # There are some export parameters, but they don't match known patterns
+                raise DatafileValidationError(
+                    f"Sink node {repr(node['name'])} has unrecognized export parameters: {export_params}. "
+                    f"For Kafka sinks, use 'export_kafka_topic'. "
+                    f"For S3/GCS sinks, use 'export_bucket_uri' and 'export_file_template'."
+                )
+
+        # Validate schedule format (common for both Kafka and S3/GCS)
+        export_schedule = node.get("export_schedule")
+        if export_schedule and export_schedule != ON_DEMAND and not croniter.is_valid(export_schedule):
+            raise DatafileValidationError(
+                f"Sink node {repr(node['name'])} has invalid export_schedule '{export_schedule}'. Must be @on-demand or a valid cron expression."
+            )
+
     def validate(self):
         if self.kind == DatafileKind.pipe:
             # TODO(eclbg):
@@ -325,9 +430,11 @@ class Datafile:
                     self.validate_materialized_node(node)
                 if node_type == PipeNodeTypes.COPY:
                     self.validate_copy_node(node)
+                if node_type == PipeNodeTypes.DATA_SINK:
+                    self.validate_sink_node(node)
                 if node_type not in VALID_PIPE_NODE_TYPES:
                     raise DatafileValidationError(
-                        f"Invalid node type ({node_type}) in pipe {repr(node['name'])}. Allowed node types: {VISIBLE_PIPE_NODE_TYPES}"
+                        f"Invalid node '{repr(node['name'])}' of type ({node_type}). Allowed node types: {VISIBLE_PIPE_NODE_TYPES}"
                     )
             for token in self.tokens:
                 if token["permission"].upper() != "READ":

tinybird/prompts.py

@@ -676,7 +676,7 @@ EXPORT_FILE_TEMPLATE "daily_prices"
 EXPORT_SCHEDULE "*/5 * * * *"
 EXPORT_FORMAT "csv"
 EXPORT_COMPRESSION "gz"
-EXPORT_WRITE_STRATEGY "truncate"
+EXPORT_STRATEGY "truncate"
 </sink_pipe_instructions>
 """
 

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev228'
-__revision__ = '1a9e75c'
+__version__ = '0.0.1.dev229'
+__revision__ = '814718e'

tinybird/tb/modules/connection.py

@@ -319,79 +319,23 @@ async def connection_create_gcs(ctx: Context) -> None:
 
 
 @connection_create.command(name="kafka", short_help="Creates a Kafka connection.")
-@click.option(
-    "--name",
-    default=None,
-    help="The name of your Kafka connection. If not provided, it's set as the bootstrap server",
-)
-@click.option("--bootstrap-servers", help="Kafka Bootstrap Server in form mykafka.mycloud.com:9092")
-@click.option("--key", help="Key")
-@click.option("--secret", help="Secret")
-@click.option(
-    "--auto-offset-reset", default=None, help="Offset reset, can be 'latest' or 'earliest'. Defaults to 'latest'."
-)
-@click.option("--schema-registry-url", default=None, help="Avro Confluent Schema Registry URL")
-@click.option(
-    "--sasl-mechanism",
-    default=None,
-    help="Authentication method for connection-based protocols. Defaults to 'PLAIN'",
-)
-@click.option(
-    "--security-protocol",
-    default="SASL_SSL",
-    help="Security protocol for connection-based protocols. Defaults to 'SASL_SSL'",
-)
-@click.option("--skip-secrets", is_flag=True, help="Skip Tinybird Secret creation")
 @click.pass_context
 @coro
-async def connection_create_kafka_cmd(
-    ctx: Context,
-    name: str,
-    bootstrap_servers: str,
-    key: str,
-    secret: str,
-    schema_registry_url: str,
-    auto_offset_reset: str,
-    sasl_mechanism: str,
-    security_protocol: str,
-    skip_secrets: bool,
-) -> None:
+async def connection_create_kafka_cmd(ctx: Context) -> None:
     """
     Creates a Kafka connection.
 
     \b
     $ tb connection create kafka
     """
-    await connection_create_kafka(
-        ctx,
-        name,
-        bootstrap_servers,
-        key,
-        secret,
-        schema_registry_url,
-        auto_offset_reset,
-        sasl_mechanism,
-        security_protocol,
-        skip_secrets,
-    )
+    await connection_create_kafka(ctx)
 
 
-async def connection_create_kafka(
-    ctx: Context,
-    name="",
-    bootstrap_servers="",
-    key="",
-    secret="",
-    schema_registry_url="",
-    auto_offset_reset="",
-    sasl_mechanism="",
-    security_protocol="SASL_SSL",
-    skip_secrets=False,
-) -> Tuple[str, str, str, str, str, str, str, str, List[str]]:
+async def connection_create_kafka(ctx: Context) -> Tuple[str, str, str, str, str, str, str, str, List[str]]:
     obj: Dict[str, Any] = ctx.ensure_object(dict)
     click.echo(FeedbackManager.gray(message="\n» Creating Kafka connection..."))
     project: Project = ctx.ensure_object(dict)["project"]
-    name = get_kafka_connection_name(project.folder, name)
+    name = get_kafka_connection_name(project.folder, "")
     default_bootstrap_servers = "localhost:9092"
     bootstrap_servers = click.prompt(
         FeedbackManager.highlight(
@@ -404,7 +348,7 @@ async def connection_create_kafka(
         bootstrap_servers = click.prompt("Bootstrap Server")
 
     validate_kafka_bootstrap_servers(bootstrap_servers)
-    secret_required = not skip_secrets and click.confirm(
+    secret_required = click.confirm(
         FeedbackManager.highlight(message="  ? Do you want to store the bootstrap server in a .env.local file? [Y/n]"),
         default=True,
         show_default=False,
@@ -420,12 +364,11 @@ async def connection_create_kafka(
         except Exception as e:
             raise CLIConnectionException(FeedbackManager.error(message=str(e)))
 
-    if not key:
-        key = click.prompt(FeedbackManager.highlight(message="? Kafka key"))
+    key = click.prompt(FeedbackManager.highlight(message="? Kafka key"))
 
     validate_kafka_key(key)
 
-    secret_required = not skip_secrets and click.confirm(
+    secret_required = click.confirm(
         FeedbackManager.highlight(message="  ? Do you want to store the Kafka key in a .env.local file? [Y/n]"),
         default=True,
         show_default=False,
@@ -438,12 +381,11 @@ async def connection_create_kafka(
         except Exception as e:
             raise CLIConnectionException(FeedbackManager.error(message=str(e)))
 
-    if not secret:
-        secret = click.prompt(FeedbackManager.highlight(message="? Kafka secret"), hide_input=True)
+    secret = click.prompt(FeedbackManager.highlight(message="? Kafka secret"), hide_input=True)
 
     validate_kafka_secret(secret)
 
-    secret_required = not skip_secrets and click.confirm(
+    secret_required = click.confirm(
         FeedbackManager.highlight(message="  ? Do you want to store the Kafka secret in a .env.local file? [Y/n]"),
         default=True,
         show_default=False,
@@ -456,15 +398,14 @@ async def connection_create_kafka(
         except Exception as e:
             raise CLIConnectionException(FeedbackManager.error(message=str(e)))
 
-    if not sasl_mechanism:
-        sasl_mechanism_options = ["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512"]
-        sasl_mechanism = click.prompt(
-            FeedbackManager.highlight(message="? SASL Mechanism (PLAIN, SCRAM-SHA-256, SCRAM-SHA-512) [PLAIN]"),
-            type=click.Choice(sasl_mechanism_options),
-            show_default=False,
-            show_choices=False,
-            default="PLAIN",
-        )
+    sasl_mechanism_options = ["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512"]
+    sasl_mechanism = click.prompt(
+        FeedbackManager.highlight(message="? SASL Mechanism (PLAIN, SCRAM-SHA-256, SCRAM-SHA-512) [PLAIN]"),
+        type=click.Choice(sasl_mechanism_options),
+        show_default=False,
+        show_choices=False,
+        default="PLAIN",
+    )
 
     create_in_cloud = (
         click.confirm(
@@ -474,9 +415,7 @@ async def connection_create_kafka(
             default=True,
             show_default=False,
         )
-        if obj["env"] == "local"
-        and not skip_secrets
-        and (tb_secret_bootstrap_servers or tb_secret_key or tb_secret_secret)
+        if obj["env"] == "local" and (tb_secret_bootstrap_servers or tb_secret_key or tb_secret_secret)
         else False
     )
 
@@ -498,9 +437,25 @@ async def connection_create_kafka(
             await prod_client.create_secret(name=tb_secret_secret, value=secret)
         click.echo(FeedbackManager.success(message="✓ Secrets created!"))
 
-    schema_registry_url and validate_kafka_schema_registry_url(schema_registry_url)
-    auto_offset_reset and validate_kafka_auto_offset_reset(auto_offset_reset)
+    schema_registry_url = click.prompt(
+        FeedbackManager.highlight(message="? Schema Registry URL (optional)"),
+        default="",
+        show_default=False,
+    )
+    if schema_registry_url:
+        validate_kafka_schema_registry_url(schema_registry_url)
+
+    auto_offset_reset_options = ["latest", "earliest"]
+    auto_offset_reset = click.prompt(
+        FeedbackManager.highlight(message="? Auto offset reset (latest, earliest) [latest]"),
+        type=click.Choice(auto_offset_reset_options),
+        default="latest",
+        show_default=False,
+        show_choices=False,
+    )
+    validate_kafka_auto_offset_reset(auto_offset_reset)
     click.echo(FeedbackManager.gray(message="» Validating connection..."))
+    security_protocol = "SASL_SSL"
     topics = list_kafka_topics(bootstrap_servers, key, secret, security_protocol, sasl_mechanism)
 
     if topics is None:
@@ -515,7 +470,7 @@ async def connection_create_kafka(
         tb_secret_key=tb_secret_key,
         secret=secret,
         tb_secret_secret=tb_secret_secret,
-        security_protocol=security_protocol,
+        security_protocol="SASL_SSL",
         sasl_mechanism=sasl_mechanism,
         folder=project.folder,
     )

{tinybird-0.0.1.dev228.dist-info → tinybird-0.0.1.dev229.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev228
+Version: 0.0.1.dev229
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev228.dist-info → tinybird-0.0.1.dev229.dist-info}/RECORD

@@ -3,7 +3,7 @@ tinybird/context.py,sha256=FfqYfrGX_I7PKGTQo93utaKPDNVYWelg4Hsp3evX5wM,1291
 tinybird/datatypes.py,sha256=r4WCvspmrXTJHiPjjyOTiZyZl31FO3Ynkwq4LQsYm6E,11059
 tinybird/feedback_manager.py,sha256=1INQFfRfuMCb9lfB8KNf4r6qC2khW568hoHjtk-wshI,69305
 tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
-tinybird/prompts.py,sha256=BRCHfFKsVU9XBckQHIkluG5TCtgKo-MYMs3fFuR7zmQ,45322
+tinybird/prompts.py,sha256=dg8Q8Q29X69j-PPP8KGooMz7-M3FNXEYqC0Fj0LcUbw,45316
 tinybird/sql.py,sha256=BufnOgclQokDyihtuXesOwHBsebN6wRXIxO5wKRkOwE,48299
 tinybird/sql_template.py,sha256=WjsTBjpQLVBHGZbY2dZuhZUurFR-rbJ_KRRy5vx4Y5E,99967
 tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,10196
@@ -12,12 +12,12 @@ tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
 tinybird/tornado_template.py,sha256=jjNVDMnkYFWXflmT8KU_Ssbo5vR8KQq3EJMk5vYgXRw,41959
 tinybird/ch_utils/constants.py,sha256=yEKR11gLCL-irEXXF9QwShaR0JLXiBTlaxfolcCIoqY,4097
 tinybird/ch_utils/engine.py,sha256=X4tE9OrfaUy6kO9cqVEzyI9cDcmOF3IAssRRzsTsfEQ,40781
-tinybird/datafile/common.py,sha256=W8HtbKVLe8Fr8Q1x3xw2DA-vroi8EZfvDJptAxiJVzQ,91784
+tinybird/datafile/common.py,sha256=p--Zn09A4CRaSUQKLDoN3ITAX6E_mOsfFjK8aY9fVRk,97217
 tinybird/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
 tinybird/datafile/parse_connection.py,sha256=tRyn2Rpr1TeWet5BXmMoQgaotbGdYep1qiTak_OqC5E,1825
 tinybird/datafile/parse_datasource.py,sha256=ssW8QeFSgglVFi3sDZj_HgkJiTJ2069v2JgqnH3CkDE,1825
 tinybird/datafile/parse_pipe.py,sha256=xf4m0Tw44QWJzHzAm7Z7FwUoUUtr7noMYjU1NiWnX0k,3880
-tinybird/tb/__cli__.py,sha256=6UmdzNjS-aX3yoDOzf3ApvaWpj5UPI69igaZUUi7P1A,247
+tinybird/tb/__cli__.py,sha256=abf6p2ld9Q1KITecyuYWTXBq4NqGO5evhTrj1jhr90M,247
 tinybird/tb/check_pypi.py,sha256=i3l2L8IajeB7sreikR7oPlYJki9MtS3c_M4crnmbByc,760
 tinybird/tb/cli.py,sha256=0xYk2Ip4vb3nNFbxfTdG3VoIgdRvUKVbUVU_mviErPA,1107
 tinybird/tb/client.py,sha256=FKj61vY9STPW03kfVcxYuY1_csI-kP-mc1ERQfqJtg8,56505
@@ -27,7 +27,7 @@ tinybird/tb/modules/cicd.py,sha256=H22MaMsqAeDv-OmeHFOlUOAmfNzEzTFaEV9TQ0g4ehs,7
 tinybird/tb/modules/cli.py,sha256=zTUob6oSZszCx-lk6MJbQ_VuNOXBo8b0DOHPWezzMOg,15997
 tinybird/tb/modules/common.py,sha256=F6oaoFZ3aBxEMjiDKYhthsEIUqSFPkcdlMJ7h7A49Ac,83114
 tinybird/tb/modules/config.py,sha256=VnzYVUo4q1RBEEMMce4_OCrKp4erhgkRPHElydVlKj0,11488
-tinybird/tb/modules/connection.py,sha256=dAOv8z3ym9Tt62j7AI8R9PgFwgiCIFdgIMpUiMdtxaQ,18906
+tinybird/tb/modules/connection.py,sha256=KHZgu7sK1C4UBgwkKAMeNdKB0RrTls0LG3YoaUfeAfw,17648
 tinybird/tb/modules/copy.py,sha256=zHN1d5NA-MFsgbk2kKJq2P9qA8dNOnIsIa60QpVnSwc,4458
 tinybird/tb/modules/create.py,sha256=56_x6nwj_oAr72X7AvdxXAuXCDKGJp-w1dP6E2XdxqU,23376
 tinybird/tb/modules/datasource.py,sha256=UiQXDkSEmQIPj4NcUD_I4bAJe9IGVmlVW07yGgSu6kY,40941
@@ -82,8 +82,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
-tinybird-0.0.1.dev228.dist-info/METADATA,sha256=-XnISmLBDTR6JLoON6Q6W3v4uFAflsOiBq2yHlQOfFI,1682
-tinybird-0.0.1.dev228.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-tinybird-0.0.1.dev228.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-0.0.1.dev228.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
-tinybird-0.0.1.dev228.dist-info/RECORD,,
+tinybird-0.0.1.dev229.dist-info/METADATA,sha256=L3f2CdrGIrB29xoxfCof0f5s38emd119j3lnUUCZ3P8,1682
+tinybird-0.0.1.dev229.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+tinybird-0.0.1.dev229.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-0.0.1.dev229.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
+tinybird-0.0.1.dev229.dist-info/RECORD,,