tinybird 0.0.1.dev174__tar.gz → 0.0.1.dev175__tar.gz

{tinybird-0.0.1.dev174 → tinybird-0.0.1.dev175}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev174
+Version: 0.0.1.dev175
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev174 → tinybird-0.0.1.dev175}/tinybird/datafile.py

@@ -4848,7 +4848,7 @@ def format_include(file_parts: List[str], doc: Datafile, unroll_includes: bool =
     assert doc.raw is not None
 
     include = [line for line in doc.raw if "INCLUDE" in line and ".incl" in line]
-    if len(include):
+    if include:
         file_parts.append(include[0])
         file_parts.append(DATAFILE_NEW_LINE)
     return file_parts

{tinybird-0.0.1.dev174 → tinybird-0.0.1.dev175}/tinybird/sql_template.py

@@ -1657,7 +1657,7 @@ def get_var_data(content, node_id=None):
                         # It will be overriden in later definitions or left as is otherwise.
                         # args[0] check is used to avoid adding unnamed parameters found in
                         # templates like: `split_to_array('')`
-                        if len(args) and isinstance(args[0], list):
+                        if args and isinstance(args[0], list):
                             raise ValueError(f'"{args[0]}" can not be used as a variable name')
                         if len(args) > 0 and args[0] not in vars and args[0]:
                             vars[args[0]] = {
@@ -1669,7 +1669,7 @@ def get_var_data(content, node_id=None):
                         if "default" not in kwargs:
                             default = kwargs.get("default", args[2] if len(args) > 2 and args[2] else None)
                             kwargs["default"] = check_default_value(default)
-                        if len(args):
+                        if args:
                             if isinstance(args[0], list):
                                 raise ValueError(f'"{args[0]}" can not be used as a variable name')
                             vars[args[0]] = {
@@ -1678,7 +1678,7 @@ def get_var_data(content, node_id=None):
                             }
                     elif func in parameter_types:
                         # avoid variable names to be None
-                        if len(args) and args[0] is not None:
+                        if args and args[0] is not None:
                             # if this is a cast use the function name to get the type
                             if "default" not in kwargs:
                                 default = kwargs.get("default", args[1] if len(args) > 1 else None)

{tinybird-0.0.1.dev174 → tinybird-0.0.1.dev175}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev174'
-__revision__ = '39f0b89'
+__version__ = '0.0.1.dev175'
+__revision__ = '943d7b4'

{tinybird-0.0.1.dev174 → tinybird-0.0.1.dev175}/tinybird/tb/modules/datafile/common.py

@@ -110,6 +110,19 @@ class PipeNodeTypes:
     DATA_SINK = "sink"
     COPY = "copy"
     STREAM = "stream"
+    EMPTY = ""
+
+
+VALID_PIPE_NODE_TYPES = {
+    PipeNodeTypes.MATERIALIZED,
+    PipeNodeTypes.STANDARD,
+    PipeNodeTypes.COPY,
+    PipeNodeTypes.EMPTY,
+    PipeNodeTypes.ENDPOINT,
+    PipeNodeTypes.STREAM,
+    PipeNodeTypes.DATA_SINK,
+}
+VISIBLE_PIPE_NODE_TYPES = {PipeNodeTypes.MATERIALIZED, PipeNodeTypes.COPY, PipeNodeTypes.ENDPOINT}
 
 
 class DataFileExtensions:
@@ -130,10 +143,43 @@ class CopyModes:
         return node_mode.lower() in CopyModes.valid_modes
 
 
-class CopyParameters:
+class Parameters:
+    ACCEPTED_ATTRIBUTES: set[str] = set()
+    MANDATORY_ATTRIBUTES: set[str] = set()
+    _PARAMS_ALIASES: dict[str, str] = {"name": "node", "mode": "copy_mode"}
+
+    @classmethod
+    def valid_params(cls) -> set[str]:
+        return cls.ACCEPTED_ATTRIBUTES
+
+    @classmethod
+    def required_params(cls) -> set[str]:
+        return cls.MANDATORY_ATTRIBUTES
+
+    @staticmethod
+    def canonical_name(name: str):
+        return Parameters._PARAMS_ALIASES.get(name, name)
+
+
+class PipeParameters(Parameters):
+    MANDATORY_ATTRIBUTES = {"name", "sql", "type"}
+    ACCEPTED_ATTRIBUTES = {"description"}.union(MANDATORY_ATTRIBUTES)
+
+
+class CopyParameters(Parameters):
     TARGET_DATASOURCE = "target_datasource"
     COPY_SCHEDULE = "copy_schedule"
     COPY_MODE = "copy_mode"
+    COPY_MODE_ALIAS = "mode"  # we need this because bot MODE and COPY_MODE go to `mode` variable inside the node
+    MANDATORY_ATTRIBUTES = PipeParameters.MANDATORY_ATTRIBUTES.union({TARGET_DATASOURCE})
+    ACCEPTED_ATTRIBUTES = PipeParameters.ACCEPTED_ATTRIBUTES.union(MANDATORY_ATTRIBUTES).union(
+        {COPY_SCHEDULE, COPY_MODE_ALIAS}
+    )
+
+
+class MaterializedParameters(Parameters):
+    MANDATORY_ATTRIBUTES = PipeParameters.MANDATORY_ATTRIBUTES.union({"datasource"})
+    ACCEPTED_ATTRIBUTES = PipeParameters.ACCEPTED_ATTRIBUTES.union(MANDATORY_ATTRIBUTES)
 
 
 DATAFILE_NEW_LINE = "\n"
@@ -228,8 +274,10 @@ class Datafile:
         self.kind = kind
 
     def validate_copy_node(self, node: Dict[str, Any]):
-        if "target_datasource" not in node:
-            raise DatafileValidationError("COPY node missing target datasource")
+        if missing := [param for param in CopyParameters.required_params() if param not in node]:
+            raise DatafileValidationError(
+                f"Some copy node params have been provided, but the following required ones are missing: {missing}"
+            )
         # copy mode must be append or replace
         if node.get("mode") and node["mode"] not in ["append", "replace"]:
             raise DatafileValidationError("COPY node mode must be append or replace")
@@ -240,6 +288,22 @@ class Datafile:
             and not croniter.is_valid(node["copy_schedule"])
         ):
             raise DatafileValidationError("COPY node schedule must be @on-demand or a valid cron expression.")
+        for key in node.keys():
+            if key not in CopyParameters.valid_params():
+                raise DatafileValidationError(
+                    f"Copy node {repr(node['name'])} has an invalid attribute ({CopyParameters.canonical_name(key)})"
+                )
+
+    def validate_materialized_node(self, node: Dict[str, Any]):
+        if missing := [param for param in MaterializedParameters.required_params() if param not in node]:
+            raise DatafileValidationError(
+                f"Some materialized node params have been provided, but the following required ones are missing: {missing}"
+            )
+        for key in node.keys():
+            if key not in MaterializedParameters.valid_params():
+                raise DatafileValidationError(
+                    f"Materialized node {repr(node['name'])} has an invalid attribute ({MaterializedParameters.canonical_name(key)})"
+                )
 
     def validate(self):
         if self.kind == DatafileKind.pipe:
@@ -250,19 +314,21 @@ class Datafile:
             #  [x] Only one materialized node
             #  [x] Only one node of any specific type
             #  (rbarbadillo): there's a HUGE amount of validations in api_pipes.py, we should somehow merge them
-            for node in self.nodes:
-                if "sql" not in node:
-                    raise DatafileValidationError(f"SQL missing for node {repr(node['name'])}")
             non_standard_nodes_count = 0
             for node in self.nodes:
-                if node.get("type", "").lower() not in {PipeNodeTypes.STANDARD, ""}:
+                node_type = node.get("type", "").lower()
+                if node_type not in {PipeNodeTypes.STANDARD, ""}:
                     non_standard_nodes_count += 1
                     if non_standard_nodes_count > 1:
                         raise DatafileValidationError("Multiple non-standard nodes in pipe. There can only be one")
-                if node.get("type", "").lower() == PipeNodeTypes.MATERIALIZED and "datasource" not in node:
-                    raise DatafileValidationError(f"Materialized node {repr(node['name'])} missing target datasource")
-                if node.get("type", "").lower() == PipeNodeTypes.COPY:
+                if node_type == PipeNodeTypes.MATERIALIZED:
+                    self.validate_materialized_node(node)
+                if node_type == PipeNodeTypes.COPY:
                     self.validate_copy_node(node)
+                if node_type not in VALID_PIPE_NODE_TYPES:
+                    raise DatafileValidationError(
+                        f"Invalid node type ({node_type}) in pipe {repr(node['name'])}. Allowed node types: {VISIBLE_PIPE_NODE_TYPES}"
+                    )
             for token in self.tokens:
                 if token["permission"].upper() != "READ":
                     raise DatafileValidationError(
@@ -289,17 +355,15 @@ class Datafile:
                         f"Invalid permission {token['permission']} for token {token['token_name']}. Only READ and APPEND are allowed for datasources"
                     )
             # Validate Kafka params
-            if any(param in node for param in KAFKA_PARAMS) and not all(
-                param in node for param in REQUIRED_KAFKA_PARAMS
+            if any(param in node for param in KAFKA_PARAMS) and (
+                missing := [param for param in REQUIRED_KAFKA_PARAMS if param not in node]
             ):
-                missing = [param for param in REQUIRED_KAFKA_PARAMS if param not in node]
                 raise DatafileValidationError(
                     f"Some Kafka params have been provided, but the following required ones are missing: {missing}"
                 )
             # Validate S3 params
             if any(param in node for param in BLOB_STORAGE_PARAMS):
-                if not all(param in node for param in REQUIRED_BLOB_STORAGE_PARAMS):
-                    missing = [param for param in REQUIRED_BLOB_STORAGE_PARAMS if param not in node]
+                if missing := [param for param in REQUIRED_BLOB_STORAGE_PARAMS if param not in node]:
                     raise DatafileValidationError(
                         f"Some connection params have been provided, but the following required ones are missing: {missing}"
                     )

{tinybird-0.0.1.dev174 → tinybird-0.0.1.dev175}/tinybird/tb/modules/datafile/format_common.py

@@ -53,7 +53,7 @@ def format_include(file_parts: List[str], doc: Datafile, unroll_includes: bool =
     assert doc.raw is not None
 
     include = [line for line in doc.raw if "INCLUDE" in line and ".incl" in line]
-    if len(include):
+    if include:
         file_parts.append(include[0])
         file_parts.append(DATAFILE_NEW_LINE)
     return file_parts

{tinybird-0.0.1.dev174 → tinybird-0.0.1.dev175}/tinybird/tb/modules/datasource.py

@@ -107,8 +107,8 @@ async def datasource_ls(ctx: Context, match: Optional[str], format_: str):
 
 
 @datasource.command(name="append")
-@click.argument("datasource_name")
-@click.argument("url", nargs=-1)
+@click.argument("datasource_name", required=True)
+@click.argument("url", nargs=-1, required=True)
 @click.option("--concurrency", help="How many files to submit concurrently", default=1, hidden=True)
 @click.pass_context
 @coro
@@ -137,8 +137,8 @@ async def datasource_append(
 
 
 @datasource.command(name="replace")
-@click.argument("datasource_name")
-@click.argument("url", nargs=-1)
+@click.argument("datasource_name", required=True)
+@click.argument("url", nargs=-1, required=True)
 @click.option("--sql-condition", default=None, help="SQL WHERE condition to replace data", hidden=True)
 @click.option("--skip-incompatible-partition-key", is_flag=True, default=False, hidden=True)
 @click.pass_context

{tinybird-0.0.1.dev174 → tinybird-0.0.1.dev175}/tinybird/tb/modules/tinyunit/tinyunit.py

@@ -317,7 +317,7 @@ def test_run_summary(results: List[TestSummaryResults], only_fail: bool = False,
             if test.error:
                 click.secho(test.error, fg=test.status.color, bold=True, nl=True, err=True)
 
-    if len(total_counts):
+    if total_counts:
         click.echo("\nTotals:")
         for key_status, value_total in total_counts.items():
             code_summary = f"Total {key_status.description}: {value_total}"

{tinybird-0.0.1.dev174 → tinybird-0.0.1.dev175}/tinybird/tb_cli_modules/connection.py

@@ -293,7 +293,7 @@ async def connection_create_snowflake(
     if not warehouse:
         warehouses = await client.get_snowflake_warehouses(account, username, password, role) or []
         warehouses_names = [w["name"] for w in warehouses]
-        default_warehouse = warehouses_names[0] if len(warehouses_names) else ""
+        default_warehouse = warehouses_names[0] if warehouses_names else ""
         warehouse = click.prompt(
             "Warehouse (optional)",
             type=click.types.Choice(warehouses_names, case_sensitive=False),

{tinybird-0.0.1.dev174 → tinybird-0.0.1.dev175}/tinybird/tb_cli_modules/fmt.py

@@ -83,7 +83,7 @@ async def fmt(
 
                 click.echo(FeedbackManager.success_generated_local_file(file=filename))
 
-    if len(failed):
+    if failed:
         click.echo(FeedbackManager.error_failed_to_format_files(number=len(failed)))
         for f in failed:
             click.echo(f"tb fmt {f} --yes")

{tinybird-0.0.1.dev174 → tinybird-0.0.1.dev175}/tinybird/tb_cli_modules/tinyunit/tinyunit.py

@@ -317,7 +317,7 @@ def test_run_summary(results: List[TestSummaryResults], only_fail: bool = False,
             if test.error:
                 click.secho(test.error, fg=test.status.color, bold=True, nl=True, err=True)
 
-    if len(total_counts):
+    if total_counts:
         click.echo("\nTotals:")
         for key_status, value_total in total_counts.items():
             code_summary = f"Total {key_status.description}: {value_total}"

{tinybird-0.0.1.dev174 → tinybird-0.0.1.dev175}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev174
+Version: 0.0.1.dev175
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird