tinybird 0.0.1.dev140__tar.gz → 0.0.1.dev142__tar.gz

{tinybird-0.0.1.dev140 → tinybird-0.0.1.dev142}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev140
+Version: 0.0.1.dev142
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev140 → tinybird-0.0.1.dev142}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev140'
-__revision__ = 'aa2a66a'
+__version__ = '0.0.1.dev142'
+__revision__ = 'f5afbf8'

{tinybird-0.0.1.dev140 → tinybird-0.0.1.dev142}/tinybird/tb/modules/cli.py

@@ -8,7 +8,7 @@ import logging
 import os
 import shutil
 import sys
-from os import getcwd
+from os import environ, getcwd
 from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
@@ -31,7 +31,7 @@ from tinybird.tb.modules.common import (
     getenv_bool,
     try_update_config_with_remote,
 )
-from tinybird.tb.modules.config import CLIConfig
+from tinybird.tb.modules.config import CURRENT_VERSION, CLIConfig
 from tinybird.tb.modules.datafile.build import build_graph
 from tinybird.tb.modules.datafile.pull import folder_pull
 from tinybird.tb.modules.feedback_manager import FeedbackManager
@@ -61,6 +61,12 @@ VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
 @click.option("--token", help="Use auth token, defaults to TB_TOKEN envvar, then to the .tinyb file.")
 @click.option("--user-token", help="Use user token, defaults to TB_USER_TOKEN envvar, then to the .tinyb file.")
 @click.option("--host", help="Use custom host, defaults to TB_HOST envvar, then to https://api.tinybird.co")
+@click.option(
+    "--version-warning/--no-version-warning",
+    envvar="TB_VERSION_WARNING",
+    default=True,
+    help="Don't print version warning message if there's a new available version. You can use TB_VERSION_WARNING envar",
+)
 @click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens.")
 @click.option("--cloud/--local", is_flag=True, default=False, help="Run against cloud or local.")
 @click.option("--staging", is_flag=True, default=False, help="Run against a staging deployment.")
@@ -73,6 +79,7 @@ async def cli(
     token: str,
     user_token: str,
     host: str,
+    version_warning: bool,
     show_tokens: bool,
     cloud: bool,
     staging: bool,
@@ -89,6 +96,23 @@ async def cli(
     if getenv_bool("TB_DISABLE_SSL_CHECKS", False):
         click.echo(FeedbackManager.warning_disabled_ssl_checks())
 
+    if not environ.get("PYTEST", None) and version_warning and not token:
+        from tinybird.check_pypi import CheckPypi
+
+        latest_version = await CheckPypi().get_latest_version()
+
+        if "x.y.z" in CURRENT_VERSION:
+            click.echo(FeedbackManager.warning_development_cli())
+
+        if "x.y.z" not in CURRENT_VERSION and latest_version != CURRENT_VERSION:
+            click.echo(
+                FeedbackManager.warning(message=f"** New version available. {CURRENT_VERSION} -> {latest_version}")
+            )
+            click.echo(
+                FeedbackManager.warning(
+                    message="** Run `tb update` to update or `export TB_VERSION_WARNING=0` to skip the check.\n"
+                )
+            )
     if debug:
         logging.basicConfig(level=logging.DEBUG)
 

{tinybird-0.0.1.dev140 → tinybird-0.0.1.dev142}/tinybird/tb/modules/datafile/common.py

@@ -197,8 +197,12 @@ REQUIRED_BLOB_STORAGE_PARAMS = {
     "import_schedule",
     "import_bucket_uri",
 }
-
 BLOB_STORAGE_PARAMS = REQUIRED_BLOB_STORAGE_PARAMS.union({"import_from_timestamp"})
+VALID_BLOB_STORAGE_CRON_VALUES = {
+    "@once",
+    "@on-demand",
+    "@auto",
+}
 
 
 class Datafile:
@@ -294,13 +298,17 @@ class Datafile:
                     f"Some Kafka params have been provided, but the following required ones are missing: {missing}"
                 )
             # Validate S3 params
-            if any(param in node for param in BLOB_STORAGE_PARAMS) and not all(
-                param in node for param in REQUIRED_BLOB_STORAGE_PARAMS
-            ):
-                missing = [param for param in REQUIRED_BLOB_STORAGE_PARAMS if param not in node]
-                raise DatafileValidationError(
-                    f"Some connection params have been provided, but the following required ones are missing: {missing}"
-                )
+            if any(param in node for param in BLOB_STORAGE_PARAMS):
+                if not all(param in node for param in REQUIRED_BLOB_STORAGE_PARAMS):
+                    missing = [param for param in REQUIRED_BLOB_STORAGE_PARAMS if param not in node]
+                    raise DatafileValidationError(
+                        f"Some connection params have been provided, but the following required ones are missing: {missing}"
+                    )
+                if node["import_schedule"] not in VALID_BLOB_STORAGE_CRON_VALUES:
+                    raise DatafileValidationError(
+                        f"Invalid import schedule '{node['import_schedule']}'. Only {sorted(VALID_BLOB_STORAGE_CRON_VALUES)} values are allowed"
+                    )
+
         else:
             # We cannot validate a datafile whose kind is unknown
             pass
@@ -1325,11 +1333,52 @@ def parse(
 
     def kafka_key_avro_deserialization(*args: str, **kwargs: Any):
         raise DatafileSyntaxError(
-            f'{kwargs["cmd"].upper()} has been deprecated. Use "KAFKA_KEY_FORMAT avro" instead',
+            f'{kwargs["cmd"].upper()} has been deprecated. Use "KAFKA_KEY_FORMAT avro" in the corresponding .datasource file instead',
+            lineno=kwargs["lineno"],
+            pos=1,
+        )
+
+    def kafka_target_partitions(*args: str, **kwargs: Any):
+        warnings.append(
+            DatafileParseWarning(
+                message=f"{kwargs['cmd'].upper()} has been deprecated and will be ignored.",
+            )
+        )
+
+    def import_service(*args: str, **kwargs: Any):
+        warnings.append(
+            DatafileParseWarning(
+                message=(
+                    f"{kwargs['cmd'].upper()} has been deprecated and will be ignored. If you're using an S3 or GCS "
+                    "connection, you don't need this setting anymore."
+                )
+            )
+        )
+
+    def import_strategy(*args: str, **kwargs: Any):
+        raise DatafileSyntaxError(
+            f"{kwargs['cmd'].upper()} has been deprecated. It is now fixed to 'append'",
+            lineno=kwargs["lineno"],
+            pos=1,
+        )
+
+    def kafka_store_binary_headers(*args: str, **kwargs: Any):
+        raise DatafileSyntaxError(
+            f"{kwargs['cmd'].upper()} has been deprecated. When KAFKA_STORE_HEADERS is True, __headers is always of type Map(String, String)",
             lineno=kwargs["lineno"],
             pos=1,
         )
 
+    def export_service(*args: str, **kwargs: Any):
+        warnings.append(
+            DatafileParseWarning(
+                message=(
+                    f"{kwargs['cmd'].upper()} has been deprecated and will be ignored. If you're using an S3 or GCS "
+                    "connection, you don't need this setting anymore."
+                )
+            )
+        )
+
     @deprecated(severity="error")
     def sources(x: str, **kwargs: Any) -> None:
         pass  # Deprecated
@@ -1469,6 +1518,7 @@ def parse(
     def version(*args: str, **kwargs: Any) -> None:
         pass  # whatever, it's deprecated
 
+    @not_supported_yet()
     def shared_with(*args: str, **kwargs: Any) -> None:
         for entries in args:
             # In case they specify multiple workspaces
@@ -1508,79 +1558,90 @@ def parse(
             else:
                 doc.filtering_tags += filtering_tags
 
-    cmds = {
-        "source": sources,
-        "schema": schema,
-        "indexes": indexes,
-        "engine": set_engine,
-        "partition_key": assign_var("partition_key"),
-        "sorting_key": assign_var("sorting_key"),
-        "primary_key": assign_var("primary_key"),
-        "sampling_key": assign_var("sampling_key"),
-        "ttl": assign_var("ttl"),
-        "settings": assign_var("settings"),
-        "node": node,
-        "scope": scope,
-        "description": description,
-        "type": assign_node_var("type"),
-        "datasource": assign_node_var("datasource"),
-        "tags": tags,
-        "target_datasource": assign_node_var("target_datasource"),
-        "copy_schedule": assign_node_var(CopyParameters.COPY_SCHEDULE),
-        "copy_mode": assign_node_var("mode"),
-        "mode": assign_node_var("mode"),
-        "resource": assign_node_var("resource"),
-        "filter": assign_node_var("filter"),
-        "token": add_token,
-        "include": include,
-        "sql": sql("sql"),
-        "version": version,
-        # TODO(eclbg): We should decide on a single place to define the kafka params. Definitely not here.
-        "kafka_connection_name": assign_var("kafka_connection_name"),
-        "kafka_topic": assign_var("kafka_topic"),
-        "kafka_group_id": assign_var("kafka_group_id"),
-        "kafka_bootstrap_servers": assign_var("kafka_bootstrap_servers"),
-        "kafka_key": assign_var("kafka_key"),
-        "kafka_secret": assign_var("kafka_secret"),
-        "kafka_schema_registry_url": assign_var("kafka_schema_registry_url"),
-        "kafka_target_partitions": assign_var("kafka_target_partitions"),
-        "kafka_auto_offset_reset": assign_var("kafka_auto_offset_reset"),
-        "kafka_store_raw_value": assign_var("kafka_store_raw_value"),
-        "kafka_store_headers": assign_var("kafka_store_headers"),
-        "kafka_store_binary_headers": assign_var("kafka_store_binary_headers"),
-        "kafka_key_format": assign_var("kafka_key_format"),
-        "kafka_value_format": assign_var("kafka_value_format"),
-        "kafka_key_avro_deserialization": kafka_key_avro_deserialization,
-        "kafka_ssl_ca_pem": assign_var("kafka_ssl_ca_pem"),
-        "kafka_security_protocol": assign_var("kafka_security_protocol"),
-        "kafka_sasl_mechanism": assign_var("kafka_sasl_mechanism"),
-        "import_service": assign_var("import_service"),
-        "s3_region": assign_var("s3_region"),
-        "s3_arn": assign_var("s3_arn"),
-        "s3_access_key": assign_var("s3_access_key"),
-        "s3_secret": assign_var("s3_secret"),
-        "gcs_service_account_credentials_json": assign_var_json("gcs_service_account_credentials_json"),
-        "import_connection_name": assign_var("import_connection_name"),
-        "import_schedule": assign_var("import_schedule"),
-        "import_strategy": assign_var("import_strategy"),
-        "import_external_datasource": assign_var("import_external_datasource"),
-        "import_bucket_uri": assign_var("import_bucket_uri"),
-        "import_from_timestamp": assign_var("import_from_timestamp"),
-        "import_query": assign_var("import_query"),
-        "import_table_arn": assign_var("import_table_arn"),
-        "import_export_bucket": assign_var("import_export_bucket"),
-        "shared_with": shared_with,
-        "export_service": assign_var("export_service"),
-        "export_connection_name": assign_var("export_connection_name"),
-        "export_schedule": assign_var("export_schedule"),
-        "export_bucket_uri": assign_var("export_bucket_uri"),
-        "export_file_template": assign_var("export_file_template"),
-        "export_format": assign_var("export_format"),
-        "export_strategy": assign_var("export_strategy"),
-        "export_compression": assign_var("export_compression"),
-        "export_write_strategy": assign_var("export_write_strategy"),
-        "export_kafka_topic": assign_var("export_kafka_topic"),
-        "forward_query": sql("forward_query"),
+    cmds_per_datafile_kind: dict[DatafileKind, dict[str, Callable]] = {
+        DatafileKind.datasource: {
+            "description": description,
+            "token": add_token,
+            "source": sources,
+            "schema": schema,
+            "indexes": indexes,
+            "engine": set_engine,
+            "partition_key": assign_var("partition_key"),
+            "sorting_key": assign_var("sorting_key"),
+            "primary_key": assign_var("primary_key"),
+            "sampling_key": assign_var("sampling_key"),
+            "ttl": assign_var("ttl"),
+            "tags": tags,
+            "include": include,
+            "version": version,
+            "kafka_connection_name": assign_var("kafka_connection_name"),
+            "kafka_topic": assign_var("kafka_topic"),
+            "kafka_group_id": assign_var("kafka_group_id"),
+            "kafka_auto_offset_reset": assign_var("kafka_auto_offset_reset"),
+            "kafka_store_raw_value": assign_var("kafka_store_raw_value"),
+            "kafka_store_headers": assign_var("kafka_store_headers"),
+            "kafka_store_binary_headers": kafka_store_binary_headers,
+            "kafka_key_format": assign_var("kafka_key_format"),
+            "kafka_value_format": assign_var("kafka_value_format"),
+            "kafka_target_partitions": kafka_target_partitions,  # Deprecated
+            "import_connection_name": assign_var("import_connection_name"),
+            "import_schedule": assign_var("import_schedule"),
+            "import_strategy": import_strategy,  # Deprecated, always append
+            "import_bucket_uri": assign_var("import_bucket_uri"),
+            "import_from_timestamp": assign_var("import_from_timestamp"),
+            "import_service": import_service,  # Deprecated
+            "import_external_datasource": assign_var("import_external_datasource"),  # Deprecated, BQ and SFK
+            "import_query": assign_var("import_query"),  # Deprecated, BQ and SFK
+            "import_table_arn": assign_var("import_table_arn"),  # Only for DynamoDB
+            "import_export_bucket": assign_var("import_export_bucket"),  # For DynamoDB
+            "shared_with": shared_with,  # Not supported yet
+            "export_service": export_service,  # Deprecated
+            "forward_query": sql("forward_query"),
+            "export_connection_name": assign_var("export_connection_name"),
+            "export_schedule": assign_var("export_schedule"),
+            "export_bucket_uri": assign_var("export_bucket_uri"),
+            "export_file_template": assign_var("export_file_template"),
+            "export_format": assign_var("export_format"),
+            "export_strategy": assign_var("export_strategy"),
+            "export_compression": assign_var("export_compression"),
+            "export_write_strategy": assign_var("export_write_strategy"),
+            "export_kafka_topic": assign_var("export_kafka_topic"),
+        },
+        DatafileKind.pipe: {
+            "node": node,
+            "scope": scope,
+            "description": description,
+            "type": assign_node_var("type"),
+            "datasource": assign_node_var("datasource"),
+            "tags": tags,
+            "target_datasource": assign_node_var("target_datasource"),
+            "copy_schedule": assign_node_var(CopyParameters.COPY_SCHEDULE),
+            "copy_mode": assign_node_var("mode"),
+            "mode": assign_node_var("mode"),
+            "filter": assign_node_var("filter"),
+            "token": add_token,
+            "include": include,
+            "sql": sql("sql"),
+            "version": version,
+        },
+        DatafileKind.connection: {
+            "description": description,
+            "type": assign_node_var("type"),
+            "kafka_bootstrap_servers": assign_var("kafka_bootstrap_servers"),
+            "kafka_key": assign_var("kafka_key"),
+            "kafka_secret": assign_var("kafka_secret"),
+            "kafka_schema_registry_url": assign_var("kafka_schema_registry_url"),
+            "kafka_ssl_ca_pem": assign_var("kafka_ssl_ca_pem"),
+            "kafka_security_protocol": assign_var("kafka_security_protocol"),
+            "kafka_sasl_mechanism": assign_var("kafka_sasl_mechanism"),
+            "kafka_key_avro_deserialization": kafka_key_avro_deserialization,
+            "s3_region": assign_var("s3_region"),
+            "s3_arn": assign_var("s3_arn"),
+            "s3_access_key": assign_var("s3_access_key"),
+            "s3_secret": assign_var("s3_secret"),
+            "gcs_service_account_credentials_json": assign_var_json("gcs_service_account_credentials_json"),
+            "include": include,
+        },
     }
 
     engine_vars = set()
@@ -1591,11 +1652,19 @@ def parse(
         for o in options:
             engine_vars.add(o.name)
     for v in engine_vars:
-        cmds[f"engine_{v}"] = add_engine_var(v)
+        cmds_per_datafile_kind[DatafileKind.datasource][f"engine_{v}"] = add_engine_var(v)
 
     if default_node:
         node(default_node)
 
+    def get_all_cmds() -> dict[str, Callable]:
+        all_cmds: dict[str, Callable] = {}
+        for _, cmds in cmds_per_datafile_kind.items():
+            all_cmds.update(cmds)
+        return all_cmds
+
+    cmds = cmds_per_datafile_kind[kind] if kind is not None else get_all_cmds()
+
     lineno = 1
     try:
         while lineno <= len(lines):
@@ -1646,7 +1715,14 @@ def parse(
                         if cmd.lower() in cmds:
                             cmds[cmd.lower()](*args, lineno=lineno, replace_includes=replace_includes, cmd=cmd)
                         else:
-                            raise click.ClickException(FeedbackManager.error_option(option=cmd.upper()))
+                            error_msg = f"{cmd.upper()} is not a valid option"
+                            if kind:
+                                error_msg += f" in {kind.value} files."
+                            raise DatafileSyntaxError(
+                                message=error_msg,
+                                lineno=lineno,
+                                pos=0,
+                            )
                 else:
                     parser_state.multiline_string += line
             lineno += 1
@@ -1686,7 +1762,6 @@ def parse(
 
 class ImportReplacements:
     _REPLACEMENTS: Tuple[Tuple[str, str, Optional[str]], ...] = (
-        ("import_service", "service", None),
         ("import_strategy", "mode", "replace"),
         ("import_connection_name", "connection", None),
         ("import_schedule", "cron", ON_DEMAND),

{tinybird-0.0.1.dev140 → tinybird-0.0.1.dev142}/tinybird/tb/modules/feedback_manager.py

@@ -726,9 +726,9 @@ STEP 3: ADD KEY TO SERVICE ACCOUNT
     )
     warning_fixture_not_found = warning_message("** Warning: No fixture found for the datasource {datasource_name}")
     warning_update_version = warning_message(
-        '** UPDATE AVAILABLE: please run "pip install tinybird=={latest_version}" to update or `export TB_VERSION_WARNING=0` to skip the check.'
+        "** VERSION {latest_version} AVAILABLE: please run `tb update` to update or `export TB_VERSION_WARNING=0` to skip the check."
     )
-    warning_current_version = warning_message("** current: tinybird {current_version}\n")
+    warning_current_version = warning_message("** Current version: {current_version}\n")
     warning_confirm_truncate_datasource = prompt_message(
         "Do you want to truncate {datasource}? Once truncated, your data can't be recovered"
     )

{tinybird-0.0.1.dev140 → tinybird-0.0.1.dev142}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev140
+Version: 0.0.1.dev142
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird