tinybird 0.0.1.dev228__tar.gz → 0.0.1.dev313__tar.gz

{tinybird-0.0.1.dev228 → tinybird-0.0.1.dev313}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev228
+Version: 0.0.1.dev313
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -8,20 +8,24 @@ Author-email: support@tinybird.co
 Requires-Python: >=3.9, <3.14
 Description-Content-Type: text/x-rst
 Requires-Dist: aiofiles==24.1.0
-Requires-Dist: anthropic==0.42.0
+Requires-Dist: anthropic==0.55.0
 Requires-Dist: boto3
 Requires-Dist: click<8.2,>=8.1.6
 Requires-Dist: clickhouse-toolset==0.34.dev0
 Requires-Dist: colorama==0.4.6
-Requires-Dist: confluent-kafka==2.8.0
+Requires-Dist: confluent-kafka==2.8.2
 Requires-Dist: cryptography~=41.0.0
 Requires-Dist: croniter==1.3.15
 Requires-Dist: docker==7.1.0
 Requires-Dist: GitPython~=3.1.32
 Requires-Dist: humanfriendly~=8.2
+Requires-Dist: plotext==5.3.2
 Requires-Dist: prompt_toolkit==3.0.48
-Requires-Dist: pydantic~=2.8.0
-Requires-Dist: pyperclip==1.8.2
+Requires-Dist: logfire-api==4.2.0
+Requires-Dist: pydantic~=2.11.7
+Requires-Dist: pydantic-ai-slim[anthropic]~=0.5.0
+Requires-Dist: pydantic-ai-slim[retries]~=0.5.0
+Requires-Dist: pyperclip==1.9.0
 Requires-Dist: pyyaml<6.1,>=6.0
 Requires-Dist: requests<3,>=2.28.1
 Requires-Dist: shandy-sqlfmt==0.11.1
@@ -35,6 +39,7 @@ Requires-Dist: packaging<24,>=23.1
 Requires-Dist: llm>=0.19
 Requires-Dist: thefuzz==0.22.1
 Requires-Dist: python-dotenv==1.1.0
+Requires-Dist: pyjwt[crypto]==2.9.0
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{tinybird-0.0.1.dev228 → tinybird-0.0.1.dev313}/tinybird/ch_utils/constants.py

@@ -255,4 +255,12 @@ VALID_QUERY_FORMATS = (
     "JSONStrings",
     "Prometheus",
     "Native",
+    "RowBinaryWithNamesAndTypes",
+    "TabSeparated",
+    "JSONCompactEachRowWithNamesAndTypes",
+    "TabSeparatedWithNamesAndTypes",
+    "JSONCompactEachRow",
+    "JSONCompact",
+    "JSONStringsEachRowWithProgress",
+    "ODBCDriver2",
 )

{tinybird-0.0.1.dev228 → tinybird-0.0.1.dev313}/tinybird/ch_utils/engine.py

@@ -134,8 +134,9 @@ class TableDetails:
         _version = self.details.get("version", None)
         return _version
 
-    def is_replicated(self):
-        return "Replicated" in self.details.get("engine", None)
+    def is_replicated(self) -> bool:
+        engine: Optional[str] = self.details.get("engine", None)
+        return engine is not None and "Replicated" in engine
 
     def is_mergetree_family(self) -> bool:
         return self.engine is not None and "mergetree" in self.engine.lower()

{tinybird-0.0.1.dev228 → tinybird-0.0.1.dev313}/tinybird/client.py

@@ -569,6 +569,7 @@ class TinyB:
         populate_condition: Optional[str] = None,
         truncate: bool = True,
         unlink_on_populate_error: bool = False,
+        on_demand_compute: bool = False,
     ):
         params: Dict[str, Any] = {
             "truncate": "true" if truncate else "false",
@@ -578,6 +579,8 @@ class TinyB:
             params.update({"populate_subset": populate_subset})
         if populate_condition:
             params.update({"populate_condition": populate_condition})
+        if on_demand_compute:
+            params.update({"on_demand_compute": "true"})
         response = await self._req(
             f"/v0/pipes/{pipe_name}/nodes/{node_name}/population?{urlencode(params)}", method="POST"
         )

{tinybird-0.0.1.dev228 → tinybird-0.0.1.dev313}/tinybird/config.py

@@ -38,13 +38,10 @@ LEGACY_HOSTS = {
     "https://api.wadus3.gcp.tinybird.co": "https://app.wadus.tinybird.co/gcp/wadus3",
     "https://api.wadus4.gcp.tinybird.co": "https://app.wadus.tinybird.co/gcp/wadus4",
     "https://api.wadus5.gcp.tinybird.co": "https://app.wadus.tinybird.co/gcp/wadus5",
-    "https://api.wadus6.gcp.tinybird.co": "https://app.wadus.tinybird.co/gcp/wadus6",
     "https://api.wadus1.aws.tinybird.co": "https://app.wadus.tinybird.co/aws/wadus1",
     "https://api.wadus2.aws.tinybird.co": "https://app.wadus.tinybird.co/aws/wadus2",
     "https://api.wadus3.aws.tinybird.co": "https://app.wadus.tinybird.co/aws/wadus3",
     "https://api.wadus4.aws.tinybird.co": "https://app.wadus.tinybird.co/aws/wadus4",
-    "https://api.wadus5.aws.tinybird.co": "https://app.wadus.tinybird.co/aws/wadus5",
-    "https://api.wadus6.aws.tinybird.co": "https://app.wadus.tinybird.co/aws/wadus6",
     "https://ui.tinybird.co": "https://app.tinybird.co/gcp/europe-west3",
     "https://ui.us-east.tinybird.co": "https://app.tinybird.co/gcp/us-east4",
     "https://ui.us-east.aws.tinybird.co": "https://app.tinybird.co/aws/us-east-1",
@@ -62,13 +59,10 @@ LEGACY_HOSTS = {
     "https://ui.wadus3.gcp.tinybird.co": "https://app.wadus.tinybird.co/gcp/wadus3",
     "https://ui.wadus4.gcp.tinybird.co": "https://app.wadus.tinybird.co/gcp/wadus4",
     "https://ui.wadus5.gcp.tinybird.co": "https://app.wadus.tinybird.co/gcp/wadus5",
-    "https://ui.wadus6.gcp.tinybird.co": "https://app.wadus.tinybird.co/gcp/wadus6",
     "https://ui.wadus1.aws.tinybird.co": "https://app.wadus.tinybird.co/aws/wadus1",
     "https://ui.wadus2.aws.tinybird.co": "https://app.wadus.tinybird.co/aws/wadus2",
     "https://ui.wadus3.aws.tinybird.co": "https://app.wadus.tinybird.co/aws/wadus3",
     "https://ui.wadus4.aws.tinybird.co": "https://app.wadus.tinybird.co/aws/wadus4",
-    "https://ui.wadus5.aws.tinybird.co": "https://app.wadus.tinybird.co/aws/wadus5",
-    "https://ui.wadus6.aws.tinybird.co": "https://app.wadus.tinybird.co/aws/wadus6",
 }
 
 

{tinybird-0.0.1.dev228 → tinybird-0.0.1.dev313}/tinybird/connectors.py

@@ -369,13 +369,7 @@ class Snowflake(Connector):
             the_type = "String"
             if t.startswith("NUMBER"):
                 the_type = "Int32"
-            if (
-                t.startswith("FLOAT")
-                or t.startswith("DOUBLE")
-                or t.startswith("REAL")
-                or t.startswith("NUMERIC")
-                or t.startswith("DECIMAL")
-            ):
+            if t.startswith(("FLOAT", "DOUBLE", "REAL", "NUMERIC", "DECIMAL")):
                 the_type = "Float32"
             if t == "DATE":
                 the_type = "Date"

{tinybird-0.0.1.dev228 → tinybird-0.0.1.dev313}/tinybird/context.py

@@ -3,15 +3,15 @@ from typing import TYPE_CHECKING
 
 # Avoid circular import error
 if TYPE_CHECKING:
-    from tinybird.user import User
+    from hfi.hfi_workspace_data import HfiWorkspaceData
+
 
 workspace_id: ContextVar[str] = ContextVar("workspace_id")
-workspace: ContextVar["User"] = ContextVar("workspace")
+hfi_workspace_data: ContextVar["HfiWorkspaceData"] = ContextVar("hfi_workspace_data")
 table_id: ContextVar[str] = ContextVar("table_id")
 hfi_frequency: ContextVar[float] = ContextVar("hfi_frequency")
 hfi_frequency_gatherer: ContextVar[float] = ContextVar("hfi_frequency_gatherer")
 use_gatherer: ContextVar[bool] = ContextVar("use_gatherer")
-allow_gatherer_fallback: ContextVar[bool] = ContextVar("allow_gatherer_fallback")
 gatherer_allow_s3_backup_on_user_errors: ContextVar[bool] = ContextVar("gatherer_allow_s3_backup_on_user_errors")
 disable_template_security_validation: ContextVar[bool] = ContextVar("disable_template_security_validation")
 origin: ContextVar[str] = ContextVar("origin")

{tinybird-0.0.1.dev228 → tinybird-0.0.1.dev313}/tinybird/datafile/common.py

@@ -122,7 +122,12 @@ VALID_PIPE_NODE_TYPES = {
     PipeNodeTypes.STREAM,
     PipeNodeTypes.DATA_SINK,
 }
-VISIBLE_PIPE_NODE_TYPES = {PipeNodeTypes.MATERIALIZED, PipeNodeTypes.COPY, PipeNodeTypes.ENDPOINT}
+VISIBLE_PIPE_NODE_TYPES = {
+    PipeNodeTypes.MATERIALIZED,
+    PipeNodeTypes.COPY,
+    PipeNodeTypes.ENDPOINT,
+    PipeNodeTypes.DATA_SINK,
+}
 
 
 class DataFileExtensions:
@@ -179,7 +184,23 @@ class CopyParameters(Parameters):
 
 class MaterializedParameters(Parameters):
     MANDATORY_ATTRIBUTES = PipeParameters.MANDATORY_ATTRIBUTES.union({"datasource"})
-    ACCEPTED_ATTRIBUTES = PipeParameters.ACCEPTED_ATTRIBUTES.union(MANDATORY_ATTRIBUTES)
+    ACCEPTED_ATTRIBUTES = PipeParameters.ACCEPTED_ATTRIBUTES.union(MANDATORY_ATTRIBUTES).union({"deployment_method"})
+
+
+class SinkParameters(Parameters):
+    # For Kafka sinks
+    KAFKA_MANDATORY_ATTRIBUTES = PipeParameters.MANDATORY_ATTRIBUTES.union(
+        {"export_connection_name", "export_kafka_topic", "export_schedule"}
+    )
+    KAFKA_ACCEPTED_ATTRIBUTES = PipeParameters.ACCEPTED_ATTRIBUTES.union(KAFKA_MANDATORY_ATTRIBUTES)
+
+    # For S3/GCS sinks
+    BLOB_MANDATORY_ATTRIBUTES = PipeParameters.MANDATORY_ATTRIBUTES.union(
+        {"export_connection_name", "export_schedule", "export_bucket_uri", "export_file_template"}
+    )
+    BLOB_ACCEPTED_ATTRIBUTES = PipeParameters.ACCEPTED_ATTRIBUTES.union(BLOB_MANDATORY_ATTRIBUTES).union(
+        {"export_format", "export_compression", "export_write_strategy", "export_strategy"}
+    )
 
 
 DATAFILE_NEW_LINE = "\n"
@@ -273,6 +294,13 @@ class Datafile:
     def set_kind(self, kind: DatafileKind):
         self.kind = kind
 
+    def validate_standard_node(self, node: Dict[str, Any]):
+        for key in node.keys():
+            if key not in PipeParameters.valid_params():
+                raise DatafileValidationError(
+                    f"Standard node {repr(node['name'])} has an invalid attribute ({PipeParameters.canonical_name(key)})"
+                )
+
     def validate_copy_node(self, node: Dict[str, Any]):
         if missing := [param for param in CopyParameters.required_params() if param not in node]:
             raise DatafileValidationError(
@@ -305,15 +333,95 @@ class Datafile:
                     f"Materialized node {repr(node['name'])} has an invalid attribute ({MaterializedParameters.canonical_name(key)})"
                 )
 
+    def validate_sink_node(self, node: Dict[str, Any]):
+        export_connection_name = node.get("export_connection_name")
+
+        if not export_connection_name:
+            raise DatafileValidationError(
+                f"Sink node {repr(node['name'])} is missing required parameter 'export_connection_name'"
+            )
+
+        # Determine connection type to validate appropriate parameters
+        # First, try to determine from presence of Kafka-specific parameters
+        has_kafka_topic = "export_kafka_topic" in node
+        has_s3_gcs_params = any(param in node for param in ["export_bucket_uri", "export_file_template"])
+
+        # If both types of parameters are present, that's an error
+        if has_kafka_topic and has_s3_gcs_params:
+            raise DatafileValidationError(
+                f"Sink node {repr(node['name'])} has mixed Kafka and S3/GCS parameters. Use either Kafka parameters (export_kafka_topic) or S3/GCS parameters (export_bucket_uri, export_file_template), not both."
+            )
+
+        # If we have Kafka-specific parameters, treat as Kafka sink
+        if has_kafka_topic:
+            # Kafka sink validation
+            if missing := [param for param in SinkParameters.KAFKA_MANDATORY_ATTRIBUTES if param not in node]:
+                raise DatafileValidationError(
+                    f"Kafka sink node {repr(node['name'])} is missing required parameters: {missing}"
+                )
+
+            # For Kafka sinks, only specific parameters should be present
+            kafka_specific_params = (
+                {"export_kafka_topic", "export_schedule", "export_connection_name"}
+                | PipeParameters.MANDATORY_ATTRIBUTES
+                | PipeParameters.ACCEPTED_ATTRIBUTES
+            )
+            for key in node.keys():
+                if key not in kafka_specific_params:
+                    raise DatafileValidationError(
+                        f"Kafka sink node {repr(node['name'])} has invalid parameter '{key}'. Only export_kafka_topic, export_schedule, and export_connection_name are allowed for Kafka sinks."
+                    )
+
+        # If we have S3/GCS-specific parameters, treat as S3/GCS sink
+        elif has_s3_gcs_params:
+            # S3/GCS sink validation
+            if missing := [param for param in SinkParameters.BLOB_MANDATORY_ATTRIBUTES if param not in node]:
+                raise DatafileValidationError(
+                    f"S3/GCS sink node {repr(node['name'])} is missing required parameters: {missing}"
+                )
+
+            # Check that only valid parameters are present
+            for key in node.keys():
+                if key not in SinkParameters.BLOB_ACCEPTED_ATTRIBUTES:
+                    raise DatafileValidationError(
+                        f"S3/GCS sink node {repr(node['name'])} has invalid parameter '{key}'"
+                    )
+
+        # If no type-specific parameters are present, we can't determine the type
+        # This means the sink is missing required parameters for any sink type
+        else:
+            # Check if we have any export parameters at all besides connection_name and schedule
+            export_params = {
+                k
+                for k in node.keys()
+                if k.startswith("export_") and k not in {"export_connection_name", "export_schedule"}
+            }
+            if not export_params:
+                raise DatafileValidationError(
+                    f"Sink node {repr(node['name'])} is missing required parameters. "
+                    f"For Kafka sinks, provide 'export_kafka_topic'. "
+                    f"For S3/GCS sinks, provide 'export_bucket_uri' and 'export_file_template'."
+                )
+            else:
+                # There are some export parameters, but they don't match known patterns
+                raise DatafileValidationError(
+                    f"Sink node {repr(node['name'])} has unrecognized export parameters: {export_params}. "
+                    f"For Kafka sinks, use 'export_kafka_topic'. "
+                    f"For S3/GCS sinks, use 'export_bucket_uri' and 'export_file_template'."
+                )
+
+        # Validate schedule format (common for both Kafka and S3/GCS)
+        export_schedule = node.get("export_schedule")
+        if export_schedule and export_schedule != ON_DEMAND and not croniter.is_valid(export_schedule):
+            raise DatafileValidationError(
+                f"Sink node {repr(node['name'])} has invalid export_schedule '{export_schedule}'. Must be @on-demand or a valid cron expression."
+            )
+
     def validate(self):
         if self.kind == DatafileKind.pipe:
-            # TODO(eclbg):
-            #  [x] node names are unique
-            #  [x] SQL in all nodes
-            #  [x] Materialized nodes have target datasource
-            #  [x] Only one materialized node
-            #  [x] Only one node of any specific type
-            #  (rbarbadillo): there's a HUGE amount of validations in api_pipes.py, we should somehow merge them
+            if len(self.nodes) == 0:
+                raise DatafileValidationError("Pipe data file must have at least one node")
+
             non_standard_nodes_count = 0
             for node in self.nodes:
                 node_type = node.get("type", "").lower()
@@ -325,10 +433,15 @@ class Datafile:
                     self.validate_materialized_node(node)
                 if node_type == PipeNodeTypes.COPY:
                     self.validate_copy_node(node)
+                if node_type == PipeNodeTypes.DATA_SINK:
+                    self.validate_sink_node(node)
+                if node_type in {PipeNodeTypes.STANDARD, ""}:
+                    self.validate_standard_node(node)
                 if node_type not in VALID_PIPE_NODE_TYPES:
                     raise DatafileValidationError(
-                        f"Invalid node type ({node_type}) in pipe {repr(node['name'])}. Allowed node types: {VISIBLE_PIPE_NODE_TYPES}"
+                        f"Invalid node '{repr(node['name'])}' of type ({node_type}). Allowed node types: {VISIBLE_PIPE_NODE_TYPES}"
                     )
+
             for token in self.tokens:
                 if token["permission"].upper() != "READ":
                     raise DatafileValidationError(
@@ -354,6 +467,15 @@ class Datafile:
                     raise DatafileValidationError(
                         f"Invalid permission {token['permission']} for token {token['token_name']}. Only READ and APPEND are allowed for datasources"
                     )
+
+            # Validate sorting key if present
+            if "engine" in node and isinstance(node["engine"], dict) and "args" in node["engine"]:
+                for arg_name, arg_value in node["engine"]["args"]:
+                    if arg_name.lower() == "sorting_key":
+                        # Check for sorting key constraints
+                        self._validate_sorting_key(arg_value, node)
+                        break
+
             # Validate Kafka params
             if any(param in node for param in KAFKA_PARAMS) and (
                 missing := [param for param in REQUIRED_KAFKA_PARAMS if param not in node]
@@ -376,6 +498,151 @@ class Datafile:
             # We cannot validate a datafile whose kind is unknown
             pass
 
+    def _validate_sorting_key(self, sorting_key: str, node: Dict[str, Any]) -> None:
+        """
+        Validates that a sorting key doesn't reference:
+        - Nullable columns
+        - AggregateFunction types
+        - Engine version columns for ReplacingMergeTree
+        """
+        if sorting_key == "tuple()" or not sorting_key:
+            return  # Empty sorting key is valid
+
+        engine_ver_column = self._extract_engine_ver_column(node)
+        schema_columns = {col["name"]: col for col in node["columns"]}
+        sorting_key_columns = self._parse_sorting_key_columns(sorting_key, engine_ver_column)
+
+        self._validate_columns_against_schema(sorting_key_columns, schema_columns)
+
+    def _extract_engine_ver_column(self, node: Dict[str, Any]) -> Optional[str]:
+        engine_info = node.get("engine", {})
+
+        if not isinstance(engine_info, dict):
+            return None
+
+        engine_type = engine_info.get("type", "")
+        if engine_type != "ReplacingMergeTree":
+            return None
+
+        engine_args = engine_info.get("args", [])
+        for arg_name, arg_value in engine_args:
+            if arg_name == "ver":
+                return arg_value
+
+        return None
+
+    def _parse_sorting_key_columns(self, sorting_key: str, engine_ver_column: Optional[str]) -> List[str]:
+        """Parse sorting key to extract column names and validate constraints."""
+        # Validate ENGINE_VER column constraint early
+        if engine_ver_column and engine_ver_column in sorting_key:
+            raise DatafileValidationError(
+                f"ENGINE_VER column '{engine_ver_column}' cannot be included in the sorting key for ReplacingMergeTree. "
+                f"Including the version column in the sorting key prevents deduplication because rows with different "
+                f"versions will have different sorting keys and won't be considered duplicates. The sorting key should "
+                f"define the record identity (what makes it unique), while ENGINE_VER tracks which version to keep."
+            )
+
+        # Remove tuple() wrapper if present
+        column_str = sorting_key
+        if column_str.startswith("tuple(") and column_str.endswith(")"):
+            column_str = column_str[6:-1]
+
+        sorting_key_columns = []
+
+        for part in column_str.split(","):
+            part = part.strip()
+
+            if self._is_aggregate_function_expression(part):
+                raise DatafileValidationError(
+                    f"Sorting key contains aggregate function expression '{part}'. Aggregate function expressions cannot be used in sorting keys."
+                )
+
+            # Extract column names from the part
+            extracted_columns = self._extract_column_names_from_part(part)
+            sorting_key_columns.extend(extracted_columns)
+
+        return sorting_key_columns
+
+    def _is_aggregate_function_expression(self, part: str) -> bool:
+        """Check if a sorting key part is an aggregate function expression."""
+        if not ("(" in part and part.endswith(")")):
+            return False
+
+        func_start = part.find("(")
+        func_name = part[:func_start].strip().lower()
+
+        aggregate_function_names = {
+            "sum",
+            "count",
+            "avg",
+            "min",
+            "max",
+            "any",
+            "grouparray",
+            "groupuniqarray",
+            "uniq",
+            "summerge",
+            "countmerge",
+            "avgmerge",
+            "minmerge",
+            "maxmerge",
+            "anymerge",
+            "grouparraymerge",
+            "groupuniqarraymerge",
+            "uniqmerge",
+        }
+
+        return func_name in aggregate_function_names
+
+    def _extract_column_names_from_part(self, part: str) -> List[str]:
+        """Extract column names from a sorting key part."""
+        columns = []
+
+        if "(" in part and part.endswith(")"):
+            # Function expression - extract column names from inside parentheses
+            func_start = part.find("(")
+            inner_content = part[func_start + 1 : -1].strip()
+            for inner_part in inner_content.split(","):
+                inner_part = inner_part.strip().strip("`")
+                if inner_part and inner_part.isidentifier():
+                    columns.append(inner_part)
+        elif part:
+            # Simple column name
+            column_name = part.strip("`")
+            if column_name:
+                columns.append(column_name)
+
+        return columns
+
+    def _validate_columns_against_schema(
+        self, sorting_key_columns: List[str], schema_columns: Dict[str, Dict[str, Any]]
+    ) -> None:
+        """Validate each column in the sorting key against the schema."""
+        if not schema_columns:
+            return  # No schema information available, can't validate
+
+        for col_name in sorting_key_columns:
+            if col_name not in schema_columns:
+                continue
+
+            self._validate_single_column(col_name, schema_columns[col_name])
+
+    def _validate_single_column(self, col_name: str, column_info: Dict[str, Any]) -> None:
+        """Validate a single column for use in sorting keys."""
+        col_type = column_info.get("type", "").lower()
+
+        # we need to check any presence of Nullable in the column type
+        is_nullable = column_info.get("nullable", False) or "Nullable(" in column_info.get("type", "")
+
+        if is_nullable:
+            raise DatafileValidationError(
+                f"Sorting key contains nullable column '{col_name}'. Nullable columns cannot be used in sorting keys."
+            )
+        if "aggregatefunction" in col_type:
+            raise DatafileValidationError(
+                f"Sorting key contains column '{col_name}' with AggregateFunction type. AggregateFunction columns cannot be used in sorting keys."
+            )
+
 
 def format_filename(filename: str, hide_folders: bool = False):
     return os.path.basename(filename) if hide_folders else filename
@@ -1265,6 +1532,8 @@ def parse(
     default_node: Optional[str] = None,
     basepath: str = ".",
     replace_includes: bool = True,
+    # TODO(eclbg): I think we could remove `skip_eval` in Forward, and pin it to False. This would let us remove some
+    # other functions like `eval_var` that obscure things a bit.
     skip_eval: bool = False,
 ) -> ParseResult:
     lines = list(StringIO(s, newline=None))
@@ -1375,11 +1644,18 @@ def parse(
 
         parser_state.current_node["indexes"] = indexes
 
-    def assign_var(v: str) -> Callable[[VarArg(str), KwArg(Any)], None]:
+    def assign_var(v: str, allowed_values: Optional[set[str]] = None) -> Callable[[VarArg(str), KwArg(Any)], None]:
         @multiline_not_supported
         def _f(*args: str, **kwargs: Any):
             s = _unquote((" ".join(args)).strip())
-            parser_state.current_node[v.lower()] = eval_var(s, skip=skip_eval)
+            val = eval_var(s, skip=skip_eval)
+            if allowed_values and val.lower() not in {v.lower() for v in allowed_values}:
+                raise DatafileSyntaxError(
+                    f"{val} is not an allowed value for {kwargs['cmd'].upper()}. Use one of: {allowed_values}",
+                    lineno=kwargs["lineno"],
+                    pos=1,
+                )
+            parser_state.current_node[v.lower()] = val
 
         return _f
 
@@ -1474,6 +1750,10 @@ def parse(
         else:
             doc.description = description
 
+    def kafka_ssl_ca_pem(*args: str, **kwargs: Any) -> None:
+        kafka_ssl_ca_pem = ("\n".join(args)).strip()
+        parser_state.current_node["kafka_ssl_ca_pem"] = kafka_ssl_ca_pem
+
     def sql(var_name: str, **kwargs: Any) -> Callable[[str, KwArg(Any)], None]:
         # TODO(eclbg): We shouldn't allow SQL in datasource files
         def _f(sql: str, *args: Any, **kwargs: Any) -> None:
@@ -1587,11 +1867,26 @@ def parse(
     def version(*args: str, **kwargs: Any) -> None:
         pass  # whatever, it's deprecated
 
-    @not_supported_yet()
     def shared_with(*args: str, **kwargs: Any) -> None:
+        # Count total workspaces collected
+        total_workspaces = 0
+
         for entries in args:
-            # In case they specify multiple workspaces
-            doc.shared_with += [workspace.strip() for workspace in entries.splitlines()]
+            # In case they specify multiple workspaces, handle both line-separated and comma-separated values
+            lines = _unquote(entries).splitlines()
+            for line in lines:
+                # Split by comma and strip whitespace from each workspace name
+                workspaces = [workspace.strip().rstrip(",") for workspace in line.split(",") if workspace.strip()]
+                doc.shared_with += workspaces
+                total_workspaces += len(workspaces)
+
+        # Validate that at least one workspace was provided
+        if total_workspaces == 0:
+            raise DatafileSyntaxError(
+                "SHARED_WITH requires at least one workspace name",
+                lineno=kwargs["lineno"],
+                pos=1,
+            )
 
     def __init_engine(v: str):
         if not parser_state.current_node:
@@ -1608,6 +1903,8 @@ def parse(
         def _f(*args: str, **kwargs: Any):
             __init_engine(f"ENGINE_{v}".upper())
             engine_arg = eval_var(_unquote((" ".join(args)).strip()), skip=skip_eval)
+            if v.lower() == "ttl" and not engine_arg:
+                return
             parser_state.current_node["engine"]["args"].append((v, engine_arg))
 
         return _f
@@ -1663,9 +1960,10 @@ def parse(
             "import_query": assign_var("import_query"),  # Deprecated, BQ and SFK
             "import_table_arn": assign_var("import_table_arn"),  # Only for DynamoDB
             "import_export_bucket": assign_var("import_export_bucket"),  # For DynamoDB
-            "shared_with": shared_with,  # Not supported yet
+            "shared_with": shared_with,
             "export_service": export_service,  # Deprecated
             "forward_query": sql("forward_query"),
+            "backfill": assign_var("backfill", allowed_values={"skip"}),
             # ENGINE_* commands are added dynamically after this dict's definition
         },
         DatafileKind.pipe: {
@@ -1684,6 +1982,7 @@ def parse(
             "include": include,
             "sql": sql("sql"),
             "version": version,
+            "deployment_method": assign_var("deployment_method", allowed_values={"alter"}),
             "export_connection_name": assign_var("export_connection_name"),
             "export_schedule": assign_var("export_schedule"),
             "export_bucket_uri": assign_var("export_bucket_uri"),
@@ -1701,9 +2000,13 @@ def parse(
             "kafka_key": assign_var("kafka_key"),
             "kafka_secret": assign_var("kafka_secret"),
             "kafka_schema_registry_url": assign_var("kafka_schema_registry_url"),
-            "kafka_ssl_ca_pem": assign_var("kafka_ssl_ca_pem"),
+            "kafka_ssl_ca_pem": kafka_ssl_ca_pem,
             "kafka_security_protocol": assign_var("kafka_security_protocol"),
             "kafka_sasl_mechanism": assign_var("kafka_sasl_mechanism"),
+            "kafka_sasl_oauthbearer_method": assign_var("kafka_sasl_oauthbearer_method"),
+            "kafka_sasl_oauthbearer_aws_region": assign_var("kafka_sasl_oauthbearer_aws_region"),
+            "kafka_sasl_oauthbearer_aws_role_arn": assign_var("kafka_sasl_oauthbearer_aws_role_arn"),
+            "kafka_sasl_oauthbearer_aws_external_id": assign_var("kafka_sasl_oauthbearer_aws_external_id"),
             "kafka_key_avro_deserialization": kafka_key_avro_deserialization_deprecated,
             "s3_region": assign_var("s3_region"),
             "s3_arn": assign_var("s3_arn"),
@@ -1748,11 +2051,7 @@ def parse(
                 lexer = list(sa)
             if lexer:
                 cmd, args = lexer[0], lexer[1:]
-                if (
-                    parser_state.multiline
-                    and cmd.lower() in cmds
-                    and not (line.startswith(" ") or line.startswith("\t"))
-                ):
+                if parser_state.multiline and cmd.lower() in cmds and not line.startswith((" ", "\t")):
                     cmds[parser_state.command](
                         parser_state.multiline_string,
                         lineno=lineno,
@@ -2205,7 +2504,7 @@ def is_file_a_datasource(filename: str) -> bool:
 
         for line in lines:
             trimmed_line = line.strip().lower()
-            if trimmed_line.startswith("schema") or trimmed_line.startswith("engine"):
+            if trimmed_line.startswith(("schema", "engine")):
                 return True
 
     return False

{tinybird-0.0.1.dev228 → tinybird-0.0.1.dev313}/tinybird/datafile/parse_pipe.py

@@ -11,7 +11,7 @@ from tinybird.datafile.common import (
     parse,
 )
 from tinybird.datafile.exceptions import IncludeFileNotFoundException, ParseException
-from tinybird.sql_template import get_template_and_variables, render_sql_template, secret_template_key
+from tinybird.sql_template import get_template_and_variables, render_sql_template
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tornado_template import UnClosedIfError
 
@@ -54,7 +54,7 @@ def parse_pipe(
             if sql.strip()[0] == "%":
                 secrets_list: Optional[List[str]] = None
                 if secrets:
-                    secrets_list = [f"{secret_template_key(secret)}" for secret in secrets.keys()]
+                    secrets_list = list(secrets.keys())
                 # Setting test_mode=True to ignore errors on required parameters and
                 # secrets_in_test_mode=False to raise errors on missing secrets
                 sql, _, variable_warnings = render_sql_template(

{tinybird-0.0.1.dev228 → tinybird-0.0.1.dev313}/tinybird/feedback_manager.py

@@ -598,6 +598,12 @@ Ready? """
     warning_confirm_delete_branch = prompt_message("Do you want to remove '{branch}' Branch?")
     warning_confirm_delete_release = prompt_message("Do you want to remove Release {semver}?")
     warning_confirm_rollback_release = prompt_message("Do you want to rollback current Release {semver} to {rollback}?")
+    warning_confirm_on_demand_compute = warning_message(
+        "On-demand compute will incur additional costs beyond your regular usage.\n"
+        "This feature uses dedicated compute resources that are billed separately.\n"
+        "You can read more about the pricing at https://www.tinybird.co/docs/classic/work-with-data/process-and-copy/materialized-views#compute-compute-separation-for-populates\n\n"
+        "Do you want to proceed with on-demand compute?"
+    )
 
     warning_confirm_delete_token = prompt_message("Do you want to delete Token {token}?")
     warning_confirm_refresh_token = prompt_message("Do you want to refresh Token {token}?")