tinybird 0.0.1.dev174__py3-none-any.whl → 0.0.1.dev175__py3-none-any.whl

tinybird/sql_template.py

@@ -1657,7 +1657,7 @@ def get_var_data(content, node_id=None):
                         # It will be overriden in later definitions or left as is otherwise.
                         # args[0] check is used to avoid adding unnamed parameters found in
                         # templates like: `split_to_array('')`
-                        if len(args) and isinstance(args[0], list):
+                        if args and isinstance(args[0], list):
                             raise ValueError(f'"{args[0]}" can not be used as a variable name')
                         if len(args) > 0 and args[0] not in vars and args[0]:
                             vars[args[0]] = {
@@ -1669,7 +1669,7 @@ def get_var_data(content, node_id=None):
                         if "default" not in kwargs:
                             default = kwargs.get("default", args[2] if len(args) > 2 and args[2] else None)
                             kwargs["default"] = check_default_value(default)
-                        if len(args):
+                        if args:
                             if isinstance(args[0], list):
                                 raise ValueError(f'"{args[0]}" can not be used as a variable name')
                             vars[args[0]] = {
@@ -1678,7 +1678,7 @@ def get_var_data(content, node_id=None):
                             }
                     elif func in parameter_types:
                         # avoid variable names to be None
-                        if len(args) and args[0] is not None:
+                        if args and args[0] is not None:
                             # if this is a cast use the function name to get the type
                             if "default" not in kwargs:
                                 default = kwargs.get("default", args[1] if len(args) > 1 else None)

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev174'
-__revision__ = '39f0b89'
+__version__ = '0.0.1.dev175'
+__revision__ = '943d7b4'

tinybird/tb/modules/datafile/common.py

@@ -110,6 +110,19 @@ class PipeNodeTypes:
     DATA_SINK = "sink"
     COPY = "copy"
     STREAM = "stream"
+    EMPTY = ""
+
+
+VALID_PIPE_NODE_TYPES = {
+    PipeNodeTypes.MATERIALIZED,
+    PipeNodeTypes.STANDARD,
+    PipeNodeTypes.COPY,
+    PipeNodeTypes.EMPTY,
+    PipeNodeTypes.ENDPOINT,
+    PipeNodeTypes.STREAM,
+    PipeNodeTypes.DATA_SINK,
+}
+VISIBLE_PIPE_NODE_TYPES = {PipeNodeTypes.MATERIALIZED, PipeNodeTypes.COPY, PipeNodeTypes.ENDPOINT}
 
 
 class DataFileExtensions:
@@ -130,10 +143,43 @@ class CopyModes:
         return node_mode.lower() in CopyModes.valid_modes
 
 
-class CopyParameters:
+class Parameters:
+    ACCEPTED_ATTRIBUTES: set[str] = set()
+    MANDATORY_ATTRIBUTES: set[str] = set()
+    _PARAMS_ALIASES: dict[str, str] = {"name": "node", "mode": "copy_mode"}
+
+    @classmethod
+    def valid_params(cls) -> set[str]:
+        return cls.ACCEPTED_ATTRIBUTES
+
+    @classmethod
+    def required_params(cls) -> set[str]:
+        return cls.MANDATORY_ATTRIBUTES
+
+    @staticmethod
+    def canonical_name(name: str):
+        return Parameters._PARAMS_ALIASES.get(name, name)
+
+
+class PipeParameters(Parameters):
+    MANDATORY_ATTRIBUTES = {"name", "sql", "type"}
+    ACCEPTED_ATTRIBUTES = {"description"}.union(MANDATORY_ATTRIBUTES)
+
+
+class CopyParameters(Parameters):
     TARGET_DATASOURCE = "target_datasource"
     COPY_SCHEDULE = "copy_schedule"
     COPY_MODE = "copy_mode"
+    COPY_MODE_ALIAS = "mode"  # we need this because bot MODE and COPY_MODE go to `mode` variable inside the node
+    MANDATORY_ATTRIBUTES = PipeParameters.MANDATORY_ATTRIBUTES.union({TARGET_DATASOURCE})
+    ACCEPTED_ATTRIBUTES = PipeParameters.ACCEPTED_ATTRIBUTES.union(MANDATORY_ATTRIBUTES).union(
+        {COPY_SCHEDULE, COPY_MODE_ALIAS}
+    )
+
+
+class MaterializedParameters(Parameters):
+    MANDATORY_ATTRIBUTES = PipeParameters.MANDATORY_ATTRIBUTES.union({"datasource"})
+    ACCEPTED_ATTRIBUTES = PipeParameters.ACCEPTED_ATTRIBUTES.union(MANDATORY_ATTRIBUTES)
 
 
 DATAFILE_NEW_LINE = "\n"
@@ -228,8 +274,10 @@ class Datafile:
         self.kind = kind
 
     def validate_copy_node(self, node: Dict[str, Any]):
-        if "target_datasource" not in node:
-            raise DatafileValidationError("COPY node missing target datasource")
+        if missing := [param for param in CopyParameters.required_params() if param not in node]:
+            raise DatafileValidationError(
+                f"Some copy node params have been provided, but the following required ones are missing: {missing}"
+            )
         # copy mode must be append or replace
         if node.get("mode") and node["mode"] not in ["append", "replace"]:
             raise DatafileValidationError("COPY node mode must be append or replace")
@@ -240,6 +288,22 @@ class Datafile:
             and not croniter.is_valid(node["copy_schedule"])
         ):
             raise DatafileValidationError("COPY node schedule must be @on-demand or a valid cron expression.")
+        for key in node.keys():
+            if key not in CopyParameters.valid_params():
+                raise DatafileValidationError(
+                    f"Copy node {repr(node['name'])} has an invalid attribute ({CopyParameters.canonical_name(key)})"
+                )
+
+    def validate_materialized_node(self, node: Dict[str, Any]):
+        if missing := [param for param in MaterializedParameters.required_params() if param not in node]:
+            raise DatafileValidationError(
+                f"Some materialized node params have been provided, but the following required ones are missing: {missing}"
+            )
+        for key in node.keys():
+            if key not in MaterializedParameters.valid_params():
+                raise DatafileValidationError(
+                    f"Materialized node {repr(node['name'])} has an invalid attribute ({MaterializedParameters.canonical_name(key)})"
+                )
 
     def validate(self):
         if self.kind == DatafileKind.pipe:
@@ -250,19 +314,21 @@ class Datafile:
             #  [x] Only one materialized node
             #  [x] Only one node of any specific type
             #  (rbarbadillo): there's a HUGE amount of validations in api_pipes.py, we should somehow merge them
-            for node in self.nodes:
-                if "sql" not in node:
-                    raise DatafileValidationError(f"SQL missing for node {repr(node['name'])}")
             non_standard_nodes_count = 0
             for node in self.nodes:
-                if node.get("type", "").lower() not in {PipeNodeTypes.STANDARD, ""}:
+                node_type = node.get("type", "").lower()
+                if node_type not in {PipeNodeTypes.STANDARD, ""}:
                     non_standard_nodes_count += 1
                     if non_standard_nodes_count > 1:
                         raise DatafileValidationError("Multiple non-standard nodes in pipe. There can only be one")
-                if node.get("type", "").lower() == PipeNodeTypes.MATERIALIZED and "datasource" not in node:
-                    raise DatafileValidationError(f"Materialized node {repr(node['name'])} missing target datasource")
-                if node.get("type", "").lower() == PipeNodeTypes.COPY:
+                if node_type == PipeNodeTypes.MATERIALIZED:
+                    self.validate_materialized_node(node)
+                if node_type == PipeNodeTypes.COPY:
                     self.validate_copy_node(node)
+                if node_type not in VALID_PIPE_NODE_TYPES:
+                    raise DatafileValidationError(
+                        f"Invalid node type ({node_type}) in pipe {repr(node['name'])}. Allowed node types: {VISIBLE_PIPE_NODE_TYPES}"
+                    )
             for token in self.tokens:
                 if token["permission"].upper() != "READ":
                     raise DatafileValidationError(
@@ -289,17 +355,15 @@ class Datafile:
                         f"Invalid permission {token['permission']} for token {token['token_name']}. Only READ and APPEND are allowed for datasources"
                     )
             # Validate Kafka params
-            if any(param in node for param in KAFKA_PARAMS) and not all(
-                param in node for param in REQUIRED_KAFKA_PARAMS
+            if any(param in node for param in KAFKA_PARAMS) and (
+                missing := [param for param in REQUIRED_KAFKA_PARAMS if param not in node]
             ):
-                missing = [param for param in REQUIRED_KAFKA_PARAMS if param not in node]
                 raise DatafileValidationError(
                     f"Some Kafka params have been provided, but the following required ones are missing: {missing}"
                 )
             # Validate S3 params
             if any(param in node for param in BLOB_STORAGE_PARAMS):
-                if not all(param in node for param in REQUIRED_BLOB_STORAGE_PARAMS):
-                    missing = [param for param in REQUIRED_BLOB_STORAGE_PARAMS if param not in node]
+                if missing := [param for param in REQUIRED_BLOB_STORAGE_PARAMS if param not in node]:
                     raise DatafileValidationError(
                         f"Some connection params have been provided, but the following required ones are missing: {missing}"
                     )

tinybird/tb/modules/datafile/format_common.py

@@ -53,7 +53,7 @@ def format_include(file_parts: List[str], doc: Datafile, unroll_includes: bool =
     assert doc.raw is not None
 
     include = [line for line in doc.raw if "INCLUDE" in line and ".incl" in line]
-    if len(include):
+    if include:
         file_parts.append(include[0])
         file_parts.append(DATAFILE_NEW_LINE)
     return file_parts

tinybird/tb/modules/datasource.py

@@ -107,8 +107,8 @@ async def datasource_ls(ctx: Context, match: Optional[str], format_: str):
 
 
 @datasource.command(name="append")
-@click.argument("datasource_name")
-@click.argument("url", nargs=-1)
+@click.argument("datasource_name", required=True)
+@click.argument("url", nargs=-1, required=True)
 @click.option("--concurrency", help="How many files to submit concurrently", default=1, hidden=True)
 @click.pass_context
 @coro
@@ -137,8 +137,8 @@ async def datasource_append(
 
 
 @datasource.command(name="replace")
-@click.argument("datasource_name")
-@click.argument("url", nargs=-1)
+@click.argument("datasource_name", required=True)
+@click.argument("url", nargs=-1, required=True)
 @click.option("--sql-condition", default=None, help="SQL WHERE condition to replace data", hidden=True)
 @click.option("--skip-incompatible-partition-key", is_flag=True, default=False, hidden=True)
 @click.pass_context

tinybird/tb/modules/tinyunit/tinyunit.py

@@ -317,7 +317,7 @@ def test_run_summary(results: List[TestSummaryResults], only_fail: bool = False,
             if test.error:
                 click.secho(test.error, fg=test.status.color, bold=True, nl=True, err=True)
 
-    if len(total_counts):
+    if total_counts:
         click.echo("\nTotals:")
         for key_status, value_total in total_counts.items():
             code_summary = f"Total {key_status.description}: {value_total}"

{tinybird-0.0.1.dev174.dist-info → tinybird-0.0.1.dev175.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev174
+Version: 0.0.1.dev175
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev174.dist-info → tinybird-0.0.1.dev175.dist-info}/RECORD

@@ -5,14 +5,14 @@ tinybird/feedback_manager.py,sha256=1INQFfRfuMCb9lfB8KNf4r6qC2khW568hoHjtk-wshI,
 tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
 tinybird/prompts.py,sha256=yLXzHB7_P0zjriIvI9UsfnIRwmcwLSEA1tcJMzYJqjw,36456
 tinybird/sql.py,sha256=C_B81wwv3BsqyXGhF5oTk9DcTUkrp7NwIFqSzd3Dmjc,47854
-tinybird/sql_template.py,sha256=mK0yeRFctbXTAu0VNMjIzoFBJoh9PoniAVgEatA5SG4,99832
+tinybird/sql_template.py,sha256=hWW8JawSWLl9GeWPYkC_Yrxj7P0MHEVMJ0Px9bedEgM,99817
 tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,10196
 tinybird/sql_toolset.py,sha256=KORVbNAUTfW1qo3U9oe7Z59xQ0QMsFhB0ji3HzY2JVo,15324
 tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
 tinybird/tornado_template.py,sha256=jjNVDMnkYFWXflmT8KU_Ssbo5vR8KQq3EJMk5vYgXRw,41959
 tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj0,3890
 tinybird/ch_utils/engine.py,sha256=X4tE9OrfaUy6kO9cqVEzyI9cDcmOF3IAssRRzsTsfEQ,40781
-tinybird/tb/__cli__.py,sha256=qwBoknIQp-vIaHCpdb58D9NUc-GpfFDJ91MCeKskmaI,247
+tinybird/tb/__cli__.py,sha256=mNNVJC46uQmnvvav2NlnK3gYzkaOlbEhJB0WGPr1imA,247
 tinybird/tb/check_pypi.py,sha256=rW4QmDRbtgKdUUwJCnBkVjmTjZSZGN-XgZhx7vMkC0w,1009
 tinybird/tb/cli.py,sha256=u3eGOhX0MHkuT6tiwaZ0_3twqLmqKXDAOxF7yV_Nn9Q,1075
 tinybird/tb/client.py,sha256=CSBl_JRuioPyY0H8Ac96dJ9wQXDXfrvK2lwqlOxKGoY,55715
@@ -25,7 +25,7 @@ tinybird/tb/modules/config.py,sha256=ziqW_t_mRVvWOd85VoB4vKyvgMkEfpXDf9H4v38p2xc
 tinybird/tb/modules/connection.py,sha256=7oOR7x4PhBcm1ETFFCH2YJ_3oeGXjAbmx1cnZX9_L70,9014
 tinybird/tb/modules/copy.py,sha256=2Mm4FWKehOG7CoOhiF1m9UZJgJn0W1_cMolqju8ONYg,5805
 tinybird/tb/modules/create.py,sha256=_s_vdb9pjq1ORajE_gFMxSGVuns1imNgiD5H8KuI2gQ,17231
-tinybird/tb/modules/datasource.py,sha256=V314rkpdVxVMjsp5qcSCTqDlmp4Vu--qM07BoWh-aqs,17783
+tinybird/tb/modules/datasource.py,sha256=mZ9N9LxbNA9QoKSMbxcMiQkdaQkfFqLeFB9_88e3_68,17843
 tinybird/tb/modules/deployment.py,sha256=j-6zGvlAlr__ERRJg98-jEtwpR3ZbUJQeRJIj5YS94I,27070
 tinybird/tb/modules/deprecations.py,sha256=rrszC1f_JJeJ8mUxGoCxckQTJFBCR8wREf4XXXN-PRc,4507
 tinybird/tb/modules/dev_server.py,sha256=57FCKuWpErwYUYgHspYDkLWEm9F4pbvVOtMrFXX1fVU,10129
@@ -60,11 +60,11 @@ tinybird/tb/modules/datafile/build.py,sha256=d_h3pRFDPFrDKGhpFx2iejY25GuB2k8yfNo
 tinybird/tb/modules/datafile/build_common.py,sha256=LU24kAQmxDJIyoIapDaYG-SU3P4FrMG9UBf8m9PgVSI,4565
 tinybird/tb/modules/datafile/build_datasource.py,sha256=nXEQ0qHdq2ai7jJTv8H2d7eeDPBYzLn8VY7zMtOYb8M,17382
 tinybird/tb/modules/datafile/build_pipe.py,sha256=6Cwjf3BKEF3-oQ9PipsQfK-Z43nSwtA4qJAUoysI7Uc,11385
-tinybird/tb/modules/datafile/common.py,sha256=WOHlXXPLs1RkdPLXA7qLHOGGey7d1Kn8PPdOe-3zVtE,89720
+tinybird/tb/modules/datafile/common.py,sha256=AYmU8IMKPEzdXNOtWcIrIn_voNncPADR_HzsxERi50g,92197
 tinybird/tb/modules/datafile/diff.py,sha256=MTmj53RYjER4neLgWVjabn-FKVFgh8h8uYiBo55lFQg,6757
 tinybird/tb/modules/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
 tinybird/tb/modules/datafile/fixture.py,sha256=DrRWivcvo_1rn7LlVUnHcXccdgx9yVj63mzBkUwCzk8,1420
-tinybird/tb/modules/datafile/format_common.py,sha256=WaNV4tXrQU5gjV6MJP-5TGqg_Bre6ilNS8emvFl-X3c,1967
+tinybird/tb/modules/datafile/format_common.py,sha256=1V1ZQuphp8EH2hT-IfZQEgz_0b9QqY177nwX8aMUte4,1962
 tinybird/tb/modules/datafile/format_datasource.py,sha256=iWbeXruxC7OBmjNgurWt6ymcJlYzxfKwkGnhpcoSKEo,6190
 tinybird/tb/modules/datafile/format_pipe.py,sha256=DUGdmlzI146YDqTwW-7kSIOXocz4AH-md_LFGUm9hrc,7436
 tinybird/tb/modules/datafile/parse_datasource.py,sha256=p-qSdmDF7xbXt4PrQQc8q8aFa93kVbLpU4hIi6d3QH4,1764
@@ -72,7 +72,7 @@ tinybird/tb/modules/datafile/parse_pipe.py,sha256=WuQlYW51tflbcp2iFapJ7bxa9IkegD
 tinybird/tb/modules/datafile/pipe_checker.py,sha256=xv7vyjN5dPc2hcw9RnLBq2VkR4nte-8bhYDT10qceQY,24620
 tinybird/tb/modules/datafile/playground.py,sha256=94tOydeg5iQ3TQAdEWQWxLhx5Emz6xh0bEwLSao44-Y,56568
 tinybird/tb/modules/datafile/pull.py,sha256=l6bIglY8b-tTIWgEYezf4kXjS0QHAVz4iOWLuNwe7Ps,5970
-tinybird/tb/modules/tinyunit/tinyunit.py,sha256=tVynlV16gt1ex0fDmCPmcABQYyzjVnnycHq1Ou3kU0c,11718
+tinybird/tb/modules/tinyunit/tinyunit.py,sha256=qc3OHiXYSOv3TQCFRYrMnR47xJbUH0LnV4FBcBtWN1s,11713
 tinybird/tb/modules/tinyunit/tinyunit_lib.py,sha256=hGh1ZaXC1af7rKnX7222urkj0QJMhMWclqMy59dOqwE,1922
 tinybird/tb_cli_modules/cicd.py,sha256=0lMkb6CVOFZl5HOwgY8mK4T4mgI7O8335UngLXtCc-c,13851
 tinybird/tb_cli_modules/common.py,sha256=G08f1_x5YSB6n-ncRj4tB2jhtjAjlWN-QD4xMWbTNQU,83033
@@ -80,8 +80,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
-tinybird-0.0.1.dev174.dist-info/METADATA,sha256=ByV0Ug-OYQaawT9rhCxOq05EaNrW5dKguHZLqws1rw0,1607
-tinybird-0.0.1.dev174.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-tinybird-0.0.1.dev174.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-0.0.1.dev174.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
-tinybird-0.0.1.dev174.dist-info/RECORD,,
+tinybird-0.0.1.dev175.dist-info/METADATA,sha256=eEO3ZslBJ4cRAUYq881Y5_TU1QyHEbAOC_QAdwYnpPo,1607
+tinybird-0.0.1.dev175.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+tinybird-0.0.1.dev175.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-0.0.1.dev175.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
+tinybird-0.0.1.dev175.dist-info/RECORD,,