tinybird 0.0.1.dev140__py3-none-any.whl → 0.0.1.dev141__py3-none-any.whl

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev140'
-__revision__ = 'aa2a66a'
+__version__ = '0.0.1.dev141'
+__revision__ = '2f7ef0f'

tinybird/tb/modules/datafile/common.py

@@ -197,8 +197,12 @@ REQUIRED_BLOB_STORAGE_PARAMS = {
     "import_schedule",
     "import_bucket_uri",
 }
-
 BLOB_STORAGE_PARAMS = REQUIRED_BLOB_STORAGE_PARAMS.union({"import_from_timestamp"})
+VALID_BLOB_STORAGE_CRON_VALUES = {
+    "@once",
+    "@on-demand",
+    "@auto",
+}
 
 
 class Datafile:
@@ -294,13 +298,17 @@ class Datafile:
                     f"Some Kafka params have been provided, but the following required ones are missing: {missing}"
                 )
             # Validate S3 params
-            if any(param in node for param in BLOB_STORAGE_PARAMS) and not all(
-                param in node for param in REQUIRED_BLOB_STORAGE_PARAMS
-            ):
-                missing = [param for param in REQUIRED_BLOB_STORAGE_PARAMS if param not in node]
-                raise DatafileValidationError(
-                    f"Some connection params have been provided, but the following required ones are missing: {missing}"
-                )
+            if any(param in node for param in BLOB_STORAGE_PARAMS):
+                if not all(param in node for param in REQUIRED_BLOB_STORAGE_PARAMS):
+                    missing = [param for param in REQUIRED_BLOB_STORAGE_PARAMS if param not in node]
+                    raise DatafileValidationError(
+                        f"Some connection params have been provided, but the following required ones are missing: {missing}"
+                    )
+                if node["import_schedule"] not in VALID_BLOB_STORAGE_CRON_VALUES:
+                    raise DatafileValidationError(
+                        f"Invalid import schedule '{node['import_schedule']}'. Only {sorted(VALID_BLOB_STORAGE_CRON_VALUES)} values are allowed"
+                    )
+
         else:
             # We cannot validate a datafile whose kind is unknown
             pass
@@ -1325,11 +1333,52 @@ def parse(
 
     def kafka_key_avro_deserialization(*args: str, **kwargs: Any):
         raise DatafileSyntaxError(
-            f'{kwargs["cmd"].upper()} has been deprecated. Use "KAFKA_KEY_FORMAT avro" instead',
+            f'{kwargs["cmd"].upper()} has been deprecated. Use "KAFKA_KEY_FORMAT avro" in the corresponding .datasource file instead',
+            lineno=kwargs["lineno"],
+            pos=1,
+        )
+
+    def kafka_target_partitions(*args: str, **kwargs: Any):
+        warnings.append(
+            DatafileParseWarning(
+                message=f"{kwargs['cmd'].upper()} has been deprecated and will be ignored.",
+            )
+        )
+
+    def import_service(*args: str, **kwargs: Any):
+        warnings.append(
+            DatafileParseWarning(
+                message=(
+                    f"{kwargs['cmd'].upper()} has been deprecated and will be ignored. If you're using an S3 or GCS "
+                    "connection, you don't need this setting anymore."
+                )
+            )
+        )
+
+    def import_strategy(*args: str, **kwargs: Any):
+        raise DatafileSyntaxError(
+            f"{kwargs['cmd'].upper()} has been deprecated. It is now fixed to 'append'",
+            lineno=kwargs["lineno"],
+            pos=1,
+        )
+
+    def kafka_store_binary_headers(*args: str, **kwargs: Any):
+        raise DatafileSyntaxError(
+            f"{kwargs['cmd'].upper()} has been deprecated. When KAFKA_STORE_HEADERS is True, __headers is always of type Map(String, String)",
             lineno=kwargs["lineno"],
             pos=1,
         )
 
+    def export_service(*args: str, **kwargs: Any):
+        warnings.append(
+            DatafileParseWarning(
+                message=(
+                    f"{kwargs['cmd'].upper()} has been deprecated and will be ignored. If you're using an S3 or GCS "
+                    "connection, you don't need this setting anymore."
+                )
+            )
+        )
+
     @deprecated(severity="error")
     def sources(x: str, **kwargs: Any) -> None:
         pass  # Deprecated
@@ -1469,6 +1518,7 @@ def parse(
     def version(*args: str, **kwargs: Any) -> None:
         pass  # whatever, it's deprecated
 
+    @not_supported_yet()
     def shared_with(*args: str, **kwargs: Any) -> None:
         for entries in args:
             # In case they specify multiple workspaces
@@ -1508,79 +1558,90 @@ def parse(
             else:
                 doc.filtering_tags += filtering_tags
 
-    cmds = {
-        "source": sources,
-        "schema": schema,
-        "indexes": indexes,
-        "engine": set_engine,
-        "partition_key": assign_var("partition_key"),
-        "sorting_key": assign_var("sorting_key"),
-        "primary_key": assign_var("primary_key"),
-        "sampling_key": assign_var("sampling_key"),
-        "ttl": assign_var("ttl"),
-        "settings": assign_var("settings"),
-        "node": node,
-        "scope": scope,
-        "description": description,
-        "type": assign_node_var("type"),
-        "datasource": assign_node_var("datasource"),
-        "tags": tags,
-        "target_datasource": assign_node_var("target_datasource"),
-        "copy_schedule": assign_node_var(CopyParameters.COPY_SCHEDULE),
-        "copy_mode": assign_node_var("mode"),
-        "mode": assign_node_var("mode"),
-        "resource": assign_node_var("resource"),
-        "filter": assign_node_var("filter"),
-        "token": add_token,
-        "include": include,
-        "sql": sql("sql"),
-        "version": version,
-        # TODO(eclbg): We should decide on a single place to define the kafka params. Definitely not here.
-        "kafka_connection_name": assign_var("kafka_connection_name"),
-        "kafka_topic": assign_var("kafka_topic"),
-        "kafka_group_id": assign_var("kafka_group_id"),
-        "kafka_bootstrap_servers": assign_var("kafka_bootstrap_servers"),
-        "kafka_key": assign_var("kafka_key"),
-        "kafka_secret": assign_var("kafka_secret"),
-        "kafka_schema_registry_url": assign_var("kafka_schema_registry_url"),
-        "kafka_target_partitions": assign_var("kafka_target_partitions"),
-        "kafka_auto_offset_reset": assign_var("kafka_auto_offset_reset"),
-        "kafka_store_raw_value": assign_var("kafka_store_raw_value"),
-        "kafka_store_headers": assign_var("kafka_store_headers"),
-        "kafka_store_binary_headers": assign_var("kafka_store_binary_headers"),
-        "kafka_key_format": assign_var("kafka_key_format"),
-        "kafka_value_format": assign_var("kafka_value_format"),
-        "kafka_key_avro_deserialization": kafka_key_avro_deserialization,
-        "kafka_ssl_ca_pem": assign_var("kafka_ssl_ca_pem"),
-        "kafka_security_protocol": assign_var("kafka_security_protocol"),
-        "kafka_sasl_mechanism": assign_var("kafka_sasl_mechanism"),
-        "import_service": assign_var("import_service"),
-        "s3_region": assign_var("s3_region"),
-        "s3_arn": assign_var("s3_arn"),
-        "s3_access_key": assign_var("s3_access_key"),
-        "s3_secret": assign_var("s3_secret"),
-        "gcs_service_account_credentials_json": assign_var_json("gcs_service_account_credentials_json"),
-        "import_connection_name": assign_var("import_connection_name"),
-        "import_schedule": assign_var("import_schedule"),
-        "import_strategy": assign_var("import_strategy"),
-        "import_external_datasource": assign_var("import_external_datasource"),
-        "import_bucket_uri": assign_var("import_bucket_uri"),
-        "import_from_timestamp": assign_var("import_from_timestamp"),
-        "import_query": assign_var("import_query"),
-        "import_table_arn": assign_var("import_table_arn"),
-        "import_export_bucket": assign_var("import_export_bucket"),
-        "shared_with": shared_with,
-        "export_service": assign_var("export_service"),
-        "export_connection_name": assign_var("export_connection_name"),
-        "export_schedule": assign_var("export_schedule"),
-        "export_bucket_uri": assign_var("export_bucket_uri"),
-        "export_file_template": assign_var("export_file_template"),
-        "export_format": assign_var("export_format"),
-        "export_strategy": assign_var("export_strategy"),
-        "export_compression": assign_var("export_compression"),
-        "export_write_strategy": assign_var("export_write_strategy"),
-        "export_kafka_topic": assign_var("export_kafka_topic"),
-        "forward_query": sql("forward_query"),
+    cmds_per_datafile_kind: dict[DatafileKind, dict[str, Callable]] = {
+        DatafileKind.datasource: {
+            "description": description,
+            "token": add_token,
+            "source": sources,
+            "schema": schema,
+            "indexes": indexes,
+            "engine": set_engine,
+            "partition_key": assign_var("partition_key"),
+            "sorting_key": assign_var("sorting_key"),
+            "primary_key": assign_var("primary_key"),
+            "sampling_key": assign_var("sampling_key"),
+            "ttl": assign_var("ttl"),
+            "tags": tags,
+            "include": include,
+            "version": version,
+            "kafka_connection_name": assign_var("kafka_connection_name"),
+            "kafka_topic": assign_var("kafka_topic"),
+            "kafka_group_id": assign_var("kafka_group_id"),
+            "kafka_auto_offset_reset": assign_var("kafka_auto_offset_reset"),
+            "kafka_store_raw_value": assign_var("kafka_store_raw_value"),
+            "kafka_store_headers": assign_var("kafka_store_headers"),
+            "kafka_store_binary_headers": kafka_store_binary_headers,
+            "kafka_key_format": assign_var("kafka_key_format"),
+            "kafka_value_format": assign_var("kafka_value_format"),
+            "kafka_target_partitions": kafka_target_partitions,  # Deprecated
+            "import_connection_name": assign_var("import_connection_name"),
+            "import_schedule": assign_var("import_schedule"),
+            "import_strategy": import_strategy,  # Deprecated, always append
+            "import_bucket_uri": assign_var("import_bucket_uri"),
+            "import_from_timestamp": assign_var("import_from_timestamp"),
+            "import_service": import_service,  # Deprecated
+            "import_external_datasource": assign_var("import_external_datasource"),  # Deprecated, BQ and SFK
+            "import_query": assign_var("import_query"),  # Deprecated, BQ and SFK
+            "import_table_arn": assign_var("import_table_arn"),  # Only for DynamoDB
+            "import_export_bucket": assign_var("import_export_bucket"),  # For DynamoDB
+            "shared_with": shared_with,  # Not supported yet
+            "export_service": export_service,  # Deprecated
+            "forward_query": sql("forward_query"),
+            "export_connection_name": assign_var("export_connection_name"),
+            "export_schedule": assign_var("export_schedule"),
+            "export_bucket_uri": assign_var("export_bucket_uri"),
+            "export_file_template": assign_var("export_file_template"),
+            "export_format": assign_var("export_format"),
+            "export_strategy": assign_var("export_strategy"),
+            "export_compression": assign_var("export_compression"),
+            "export_write_strategy": assign_var("export_write_strategy"),
+            "export_kafka_topic": assign_var("export_kafka_topic"),
+        },
+        DatafileKind.pipe: {
+            "node": node,
+            "scope": scope,
+            "description": description,
+            "type": assign_node_var("type"),
+            "datasource": assign_node_var("datasource"),
+            "tags": tags,
+            "target_datasource": assign_node_var("target_datasource"),
+            "copy_schedule": assign_node_var(CopyParameters.COPY_SCHEDULE),
+            "copy_mode": assign_node_var("mode"),
+            "mode": assign_node_var("mode"),
+            "filter": assign_node_var("filter"),
+            "token": add_token,
+            "include": include,
+            "sql": sql("sql"),
+            "version": version,
+        },
+        DatafileKind.connection: {
+            "description": description,
+            "type": assign_node_var("type"),
+            "kafka_bootstrap_servers": assign_var("kafka_bootstrap_servers"),
+            "kafka_key": assign_var("kafka_key"),
+            "kafka_secret": assign_var("kafka_secret"),
+            "kafka_schema_registry_url": assign_var("kafka_schema_registry_url"),
+            "kafka_ssl_ca_pem": assign_var("kafka_ssl_ca_pem"),
+            "kafka_security_protocol": assign_var("kafka_security_protocol"),
+            "kafka_sasl_mechanism": assign_var("kafka_sasl_mechanism"),
+            "kafka_key_avro_deserialization": kafka_key_avro_deserialization,
+            "s3_region": assign_var("s3_region"),
+            "s3_arn": assign_var("s3_arn"),
+            "s3_access_key": assign_var("s3_access_key"),
+            "s3_secret": assign_var("s3_secret"),
+            "gcs_service_account_credentials_json": assign_var_json("gcs_service_account_credentials_json"),
+            "include": include,
+        },
     }
 
     engine_vars = set()
@@ -1591,11 +1652,19 @@ def parse(
         for o in options:
             engine_vars.add(o.name)
     for v in engine_vars:
-        cmds[f"engine_{v}"] = add_engine_var(v)
+        cmds_per_datafile_kind[DatafileKind.datasource][f"engine_{v}"] = add_engine_var(v)
 
     if default_node:
         node(default_node)
 
+    def get_all_cmds() -> dict[str, Callable]:
+        all_cmds: dict[str, Callable] = {}
+        for _, cmds in cmds_per_datafile_kind.items():
+            all_cmds.update(cmds)
+        return all_cmds
+
+    cmds = cmds_per_datafile_kind[kind] if kind is not None else get_all_cmds()
+
     lineno = 1
     try:
         while lineno <= len(lines):
@@ -1646,7 +1715,14 @@ def parse(
                         if cmd.lower() in cmds:
                             cmds[cmd.lower()](*args, lineno=lineno, replace_includes=replace_includes, cmd=cmd)
                         else:
-                            raise click.ClickException(FeedbackManager.error_option(option=cmd.upper()))
+                            error_msg = f"{cmd.upper()} is not a valid option"
+                            if kind:
+                                error_msg += f" in {kind.value} files."
+                            raise DatafileSyntaxError(
+                                message=error_msg,
+                                lineno=lineno,
+                                pos=0,
+                            )
                 else:
                     parser_state.multiline_string += line
             lineno += 1
@@ -1686,7 +1762,6 @@ def parse(
 
 class ImportReplacements:
     _REPLACEMENTS: Tuple[Tuple[str, str, Optional[str]], ...] = (
-        ("import_service", "service", None),
         ("import_strategy", "mode", "replace"),
         ("import_connection_name", "connection", None),
         ("import_schedule", "cron", ON_DEMAND),

{tinybird-0.0.1.dev140.dist-info → tinybird-0.0.1.dev141.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev140
+Version: 0.0.1.dev141
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev140.dist-info → tinybird-0.0.1.dev141.dist-info}/RECORD

@@ -12,7 +12,7 @@ tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
 tinybird/tornado_template.py,sha256=jjNVDMnkYFWXflmT8KU_Ssbo5vR8KQq3EJMk5vYgXRw,41959
 tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj0,3890
 tinybird/ch_utils/engine.py,sha256=BZuPM7MFS7vaEKK5tOMR2bwSAgJudPrJt27uVEwZmTY,40512
-tinybird/tb/__cli__.py,sha256=ca_E2br4_GaxuhT-XfoJYkuzUd-v6Z8yRK7watCF2hE,252
+tinybird/tb/__cli__.py,sha256=HatbYmPEFnba3j4qc8NOBm4CILHsE8TZVBx4rD__eLc,252
 tinybird/tb/cli.py,sha256=6X7pMjscB1yDsnzBaZBnF4pCBJ7tZgCC500CtPEP-qQ,1106
 tinybird/tb/client.py,sha256=aaPKq5C77e72kR7IMv9WrvnvNki8mKMOTi9EsCp0eUc,55962
 tinybird/tb/config.py,sha256=HLMHbJg6BjeGZ2KiJA-BCCVnk8w959xsSaDEEePakZg,3981
@@ -59,7 +59,7 @@ tinybird/tb/modules/datafile/build.py,sha256=d_h3pRFDPFrDKGhpFx2iejY25GuB2k8yfNo
 tinybird/tb/modules/datafile/build_common.py,sha256=LU24kAQmxDJIyoIapDaYG-SU3P4FrMG9UBf8m9PgVSI,4565
 tinybird/tb/modules/datafile/build_datasource.py,sha256=nXEQ0qHdq2ai7jJTv8H2d7eeDPBYzLn8VY7zMtOYb8M,17382
 tinybird/tb/modules/datafile/build_pipe.py,sha256=6Cwjf3BKEF3-oQ9PipsQfK-Z43nSwtA4qJAUoysI7Uc,11385
-tinybird/tb/modules/datafile/common.py,sha256=p3EyhmO-zp5ZzdJcG6H5S60TTahhQn1fjK_ovXCk2J8,86185
+tinybird/tb/modules/datafile/common.py,sha256=NIqXpsj7u8uaeSKcUZM-85Kxg35pd08DVAdbVuHmvTU,89400
 tinybird/tb/modules/datafile/diff.py,sha256=MTmj53RYjER4neLgWVjabn-FKVFgh8h8uYiBo55lFQg,6757
 tinybird/tb/modules/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
 tinybird/tb/modules/datafile/fixture.py,sha256=DrRWivcvo_1rn7LlVUnHcXccdgx9yVj63mzBkUwCzk8,1420
@@ -79,8 +79,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
-tinybird-0.0.1.dev140.dist-info/METADATA,sha256=uI3BQRF3MVPNyMo39wVfQl5jVgZd5s1bIRpqoy3dScw,1612
-tinybird-0.0.1.dev140.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-tinybird-0.0.1.dev140.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-0.0.1.dev140.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
-tinybird-0.0.1.dev140.dist-info/RECORD,,
+tinybird-0.0.1.dev141.dist-info/METADATA,sha256=fKlvi9USAvvAJMIZlfVRRWGxJrAXIPtR3Bv0hSWEivU,1612
+tinybird-0.0.1.dev141.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+tinybird-0.0.1.dev141.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-0.0.1.dev141.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
+tinybird-0.0.1.dev141.dist-info/RECORD,,