tinybird 2.0.1.dev0__tar.gz → 3.0.1__tar.gz

{tinybird-2.0.1.dev0 → tinybird-3.0.1}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 2.0.1.dev0
+Version: 3.0.1
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
 Author-email: support@tinybird.co
-Requires-Python: >=3.9, <3.14
+Requires-Python: >=3.10, <3.14
 Description-Content-Type: text/x-rst
 Requires-Dist: aiofiles==24.1.0
 Requires-Dist: anthropic==0.55.0
@@ -57,9 +57,15 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
-2.0.0
+3.0.1
 *******
 
+- `Experimental` Added hidden `--with-connections` flag to `tb build` and `tb dev` commands for enabling connection datasources (S3, Kafka, GCS) in branches. This is an experimental feature and is not yet publicly documented.
+
+3.0.0
+*******
+
+- `Removed` Python 3.9 support. The minimum supported Python version is now 3.10.
 - `Removed` BigQuery and Snowflake connector support from Forward CLI build and deployment commands
 
 1.1.14

{tinybird-2.0.1.dev0 → tinybird-3.0.1}/tinybird/ch_utils/engine.py

@@ -14,7 +14,6 @@ from ..sql import (
 )
 
 DEFAULT_EMPTY_PARAMETERS = ["ttl", "partition_key", "sorting_key"]
-DEFAULT_JOIN_EMPTY_PARAMETERS = ["join_strictness", "join_type", "key_columns"]
 
 # Currently we only support the simplest TTLs
 # f(X) + toIntervalZ(N)
@@ -62,25 +61,6 @@ class TableDetails:
     >>> ed.to_datafile()
     'ENGINE "MergeTree"\\nENGINE_PARTITION_KEY "toYear(timestamp)"\\nENGINE_SORTING_KEY "timestamp, cityHash64(location)"\\nENGINE_SAMPLING_KEY "cityHash64(location)"\\nENGINE_SETTINGS "index_granularity = 32, index_granularity_bytes = 2048"\\nENGINE_TTL "toDate(timestamp) + INTERVAL 1 DAY"'
 
-    >>> ed = TableDetails({"engine_full": "Join(ANY, LEFT, id)", "engine": "Join", "partition_key": "", "sorting_key": "", "primary_key": "", "sampling_key": ""})
-    >>> ed.engine_full
-    'Join(ANY, LEFT, id)'
-    >>> ed.engine
-    'Join'
-    >>> ed.to_json()
-    {'engine_full': 'Join(ANY, LEFT, id)', 'engine': 'Join', 'join_strictness': 'ANY', 'join_type': 'LEFT', 'key_columns': 'id'}
-    >>> ed.to_datafile()
-    'ENGINE "Join"\\nENGINE_JOIN_STRICTNESS "ANY"\\nENGINE_JOIN_TYPE "LEFT"\\nENGINE_KEY_COLUMNS "id"'
-
-    >>> ed = TableDetails({"database": "d_01", "name": "t_01", "engine": "Join", "join_strictness": "ANY", "join_type": "LEFT", "key_columns": "id"})
-    >>> ed.engine_full == None
-    True
-    >>> ed.engine
-    'Join'
-    >>> ed.to_json()
-    {'engine_full': None, 'engine': 'Join', 'join_strictness': 'ANY', 'join_type': 'LEFT', 'key_columns': 'id'}
-    >>> ed.to_datafile()
-    'ENGINE "Join"\\nENGINE_JOIN_STRICTNESS "ANY"\\nENGINE_JOIN_TYPE "LEFT"\\nENGINE_KEY_COLUMNS "id"'
     >>> ed = TableDetails({ "engine_full": "MergeTree() PARTITION BY toYear(timestamp) ORDER BY (timestamp, cityHash64(location)) SAMPLE BY cityHash64(location) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1, merge_with_ttl_timeout = 1800 TTL toDate(timestamp) + INTERVAL 1 DAY"})
     >>> ed.engine_full
     'MergeTree() PARTITION BY toYear(timestamp) ORDER BY (timestamp, cityHash64(location)) SAMPLE BY cityHash64(location) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1, merge_with_ttl_timeout = 1800 TTL toDate(timestamp) + INTERVAL 1 DAY'
@@ -222,21 +202,6 @@ class TableDetails:
         _sign = self.details.get("sign", None)
         return _sign
 
-    @property
-    def join_strictness(self):
-        _join_strictness = self.details.get("join_strictness", None)
-        return _join_strictness
-
-    @property
-    def join_type(self):
-        _join_type = self.details.get("join_type", None)
-        return _join_type
-
-    @property
-    def key_columns(self):
-        _key_columns = self.details.get("key_columns", None)
-        return _key_columns
-
     @property
     def statistics(self) -> Dict[str, Any]:
         return {
@@ -268,12 +233,6 @@ class TableDetails:
             d["sampling_key"] = self.sampling_key
         if self.settings:
             d["settings"] = self.settings
-        if self.join_strictness:
-            d["join_strictness"] = self.join_strictness
-        if self.join_type:
-            d["join_type"] = self.join_type
-        if self.key_columns:
-            d["key_columns"] = self.key_columns
         if self.ver:
             d["ver"] = self.ver
         if self.is_deleted:
@@ -292,10 +251,7 @@ class TableDetails:
             d = {**d, **engine_params}
 
         if include_empty_details:
-            if self.engine and self.engine.lower() == "join":
-                d = set_empty_details(d, DEFAULT_JOIN_EMPTY_PARAMETERS)
-            else:
-                d = set_empty_details(d, DEFAULT_EMPTY_PARAMETERS)
+            d = set_empty_details(d, DEFAULT_EMPTY_PARAMETERS)
 
         if exclude:
             for attr in exclude:
@@ -480,20 +436,6 @@ ENABLED_ENGINES = [
         ],
         MERGETREE_OPTIONS,
     ),
-    # Join(join_strictness, join_type, k1[, k2, ...])
-    engine_config(
-        "Join",
-        [
-            # https://github.com/ClickHouse/ClickHouse/blob/fa8e4e4735b932f08b6beffcb2d069b72de34401/src/Storages/StorageJoin.cpp
-            EngineParam(
-                name="join_strictness", required=True, is_valid=case_insensitive_check(["ANY", "ALL", "SEMI", "ANTI"])
-            ),
-            EngineParam(
-                name="join_type", required=True, is_valid=case_insensitive_check(["LEFT", "INNER", "RIGHT", "FULL"])
-            ),
-            EngineParam(name="key_columns", required=True, is_valid=columns_are_valid),
-        ],
-    ),
     # Null()
     engine_config("Null"),
 ]
@@ -599,7 +541,7 @@ def engine_full_from_dict(
     >>> engine_full_from_dict('wadus', {}, schema=schema)
     Traceback (most recent call last):
     ...
-    ValueError: Engine wadus is not supported, supported engines include: MergeTree, ReplacingMergeTree, SummingMergeTree, AggregatingMergeTree, CollapsingMergeTree, VersionedCollapsingMergeTree, Join, Null
+    ValueError: Engine wadus is not supported, supported engines include: MergeTree, ReplacingMergeTree, SummingMergeTree, AggregatingMergeTree, CollapsingMergeTree, VersionedCollapsingMergeTree, Null
     >>> schema = ''
     >>> engine_full_from_dict('null', {}, schema=schema)
     'Null()'
@@ -607,19 +549,6 @@ def engine_full_from_dict(
     >>> engine_full_from_dict('null', {}, columns=[])
     'Null()'
 
-    >>> schema = 'cid Int32'
-    >>> engine_full_from_dict('Join', {'join_strictness': 'ANY', 'join_type': 'LEFT', 'key_columns': 'cid'}, schema=schema)
-    'Join(ANY, LEFT, cid)'
-    >>> engine_full_from_dict('Join', {'join_strictness': 'ANY', 'join_type': 'LEFT', 'key_columns': 'cid'}, columns=[{'name': 'cid', 'type': 'Int32', 'codec': None, 'default_value': None, 'nullable': False, 'normalized_name': 'cid'}])
-    'Join(ANY, LEFT, cid)'
-    >>> schema = 'cid1 Int32, cid2 Int8'
-    >>> engine_full_from_dict('Join', {'join_strictness': 'ANY', 'join_type': 'LEFT', 'key_columns': 'cid1, cid2'}, schema=schema)
-    'Join(ANY, LEFT, cid1, cid2)'
-    >>> engine_full_from_dict('Join', {'join_strictness': 'ANY', 'join_type': 'OUTER', 'key_columns': 'cid'}, schema=schema)
-    Traceback (most recent call last):
-    ...
-    ValueError: Invalid value 'OUTER' for parameter 'engine_join_type', reason: valid values are LEFT, INNER, RIGHT, FULL
-
     >>> schema = ''
     >>> engine_full_from_dict('MergeTree', {}, schema=schema)
     'MergeTree() ORDER BY (tuple())'
@@ -715,10 +644,6 @@ def engine_params_from_engine_full(engine_full: str) -> Dict[str, Any]:
     """
     >>> engine_params_from_engine_full("Null()")
     {}
-    >>> engine_params_from_engine_full("Join(ANY, LEFT, id)")
-    {'join_strictness': 'ANY', 'join_type': 'LEFT', 'key_columns': 'id'}
-    >>> engine_params_from_engine_full("Join(ANY, LEFT, k1, k2)")
-    {'join_strictness': 'ANY', 'join_type': 'LEFT', 'key_columns': 'k1, k2'}
     >>> engine_params_from_engine_full("AggregatingMergeTree('/clickhouse/tables/{layer}-{shard}/d_f837aa.sales_by_country_rt__v0_staging_t_00c3091e7530472caebda05e97288a1d', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY (purchase_location, cod_device, date) SETTINGS index_granularity = 8192")
     {}
     >>> engine_params_from_engine_full("ReplicatedSummingMergeTree('/clickhouse/tables/{layer}-{shard}/d_abcf3e.t_69f9da31f4524995b8911e1b24c80ab4', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY (date, purchase_location, sku_rank_lc) SETTINGS index_granularity = 8192")

{tinybird-2.0.1.dev0 → tinybird-3.0.1}/tinybird/config.py

@@ -137,7 +137,3 @@ class FeatureFlags:
     @classmethod
     def is_localhost(cls) -> bool:
         return "SET_LOCALHOST" in environ
-
-    @classmethod
-    def enable_snowflake_connector_command(cls) -> bool:
-        return "ENABLE_SNOWFLAKE_CONNECTOR_COMMAND" in environ

{tinybird-2.0.1.dev0 → tinybird-3.0.1}/tinybird/datafile/common.py

@@ -33,6 +33,13 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
 # I'll try not to make logic changes, just error reporting changes
 # from tinybird.sql import parse_indexes_structure, parse_table_structure, schema_to_sql_columns
 
+# Pre-compiled regex patterns
+_PATTERN_INDEX_ENTRY = re.compile(
+    r"(\w+)\s+([\w\s*\[\]\*\(\),\'\"-><.]+)\s+TYPE\s+(\w+)(?:\(([\w\s*.,]+)\))?(?:\s+GRANULARITY\s+(\d+))?"
+)
+_PATTERN_SIMPLE_AGG_FUNC = re.compile(r"SimpleAggregateFunction\((\w+),\s*(?!(?:Nullable))([\w,.()]+)\)")
+_PATTERN_VERSION_NUMBER = re.compile(r"[0-9]+$")
+
 
 class DatafileValidationError(Exception): ...
 
@@ -271,6 +278,74 @@ VALID_BLOB_STORAGE_CRON_VALUES = {
 }
 
 
+def extract_column_names_from_sorting_key_part(part: str) -> List[str]:
+    """
+    Extract actual column names from a sorting key part (which might be an expression).
+
+    Examples:
+    - "shop" -> ["shop"]
+    - "`column_name`" -> ["column_name"]
+    - "ifNull(ad_id, '')" -> ["ad_id"]
+    - "toDate(timestamp)" -> ["timestamp"]
+    - "concat(first_name, last_name)" -> ["first_name", "last_name"]
+    """
+    columns = []
+
+    if "(" in part and part.endswith(")"):
+        # Function expression - extract column names from inside parentheses
+        func_start = part.find("(")
+        inner_content = part[func_start + 1 : -1].strip()
+        for inner_part in inner_content.split(","):
+            inner_part = inner_part.strip().strip("`")
+            if inner_part and inner_part.isidentifier():
+                columns.append(inner_part)
+    elif part:
+        # Simple column name
+        column_name = part.strip("`")
+        if column_name:
+            columns.append(column_name)
+
+    return columns
+
+
+def parse_sorting_key_column_names(sorting_key: str) -> List[str]:
+    """
+    Extract all column names from a sorting key expression.
+
+    Examples:
+    - "shop, event_date, channel" -> ["shop", "event_date", "channel"]
+    - "shop, event_date, ifNull(ad_id, ''), event_id" -> ["shop", "event_date", "ad_id", "event_id"]
+    - "tuple(shop, toDate(timestamp))" -> ["shop", "timestamp"]
+    """
+    # Remove tuple() wrapper if present
+    column_str = sorting_key
+    if column_str.startswith("tuple(") and column_str.endswith(")"):
+        column_str = column_str[6:-1]
+
+    sorting_key_columns = []
+
+    # Use regex to find all sorting key parts, respecting parentheses and quotes
+    # Pattern matches: function calls with args, or simple identifiers
+    # This handles cases like: ifNull(col, ''), toDate(timestamp), simple_col
+    pattern = r"""
+        (?:                      # Non-capturing group for the whole part
+            \w+\s*\([^)]*\)      # Function call: word + ( + anything except ) + )
+            |                     # OR
+            `[^`]+`              # Backtick-quoted identifier
+            |                     # OR
+            \w+                  # Simple word/identifier
+        )
+    """
+
+    for match in re.finditer(pattern, column_str, re.VERBOSE):
+        part = match.group(0).strip()
+        # Extract column names from the part (handles both simple columns and function expressions)
+        extracted_columns = extract_column_names_from_sorting_key_part(part)
+        sorting_key_columns.extend(extracted_columns)
+
+    return sorting_key_columns
+
+
 class Datafile:
     def __init__(self) -> None:
         self.maintainer: Optional[str] = None
@@ -559,7 +634,7 @@ class Datafile:
                 )
 
             # Extract column names from the part
-            extracted_columns = self._extract_column_names_from_part(part)
+            extracted_columns = extract_column_names_from_sorting_key_part(part)
             sorting_key_columns.extend(extracted_columns)
 
         return sorting_key_columns
@@ -595,26 +670,6 @@ class Datafile:
 
         return func_name in aggregate_function_names
 
-    def _extract_column_names_from_part(self, part: str) -> List[str]:
-        """Extract column names from a sorting key part."""
-        columns = []
-
-        if "(" in part and part.endswith(")"):
-            # Function expression - extract column names from inside parentheses
-            func_start = part.find("(")
-            inner_content = part[func_start + 1 : -1].strip()
-            for inner_part in inner_content.split(","):
-                inner_part = inner_part.strip().strip("`")
-                if inner_part and inner_part.isidentifier():
-                    columns.append(inner_part)
-        elif part:
-            # Simple column name
-            column_name = part.strip("`")
-            if column_name:
-                columns.append(column_name)
-
-        return columns
-
     def _validate_columns_against_schema(
         self, sorting_key_columns: List[str], schema_columns: Dict[str, Dict[str, Any]]
     ) -> None:
@@ -783,10 +838,7 @@ def parse_indexes_structure(indexes: Optional[List[str]]) -> List[TableIndex]:
                 pos=leading_whitespaces + 1,
             )
 
-        match = re.match(
-            r"(\w+)\s+([\w\s*\[\]\*\(\),\'\"-><.]+)\s+TYPE\s+(\w+)(?:\(([\w\s*.,]+)\))?(?:\s+GRANULARITY\s+(\d+))?",
-            index,
-        )
+        match = _PATTERN_INDEX_ENTRY.match(index)
         if match:
             index_name, a, index_type, value, granularity = match.groups()
             index_expr = f"{index_type}({value})" if value else index_type
@@ -1323,7 +1375,7 @@ def try_to_fix_nullable_in_simple_aggregating_function(t: str) -> Optional[str]:
     # as it is done with other aggregate functions.
     # If not, the aggregation could return incorrect results.
     result = None
-    if match := re.search(r"SimpleAggregateFunction\((\w+),\s*(?!(?:Nullable))([\w,.()]+)\)", t):
+    if match := _PATTERN_SIMPLE_AGG_FUNC.search(t):
         fn = match.group(1)
         inner_type = match.group(2)
         result = f"SimpleAggregateFunction({fn}, Nullable({inner_type}))"
@@ -2468,7 +2520,7 @@ def get_name_version(ds: str) -> Dict[str, Any]:
         return {"name": tk[0], "version": None}
     elif len(tk) == 2:
         if len(tk[1]):
-            if tk[1][0] == "v" and re.match("[0-9]+$", tk[1][1:]):
+            if tk[1][0] == "v" and _PATTERN_VERSION_NUMBER.match(tk[1][1:]):
                 return {"name": tk[0], "version": int(tk[1][1:])}
             else:
                 return {"name": tk[0] + "__" + tk[1], "version": None}

{tinybird-2.0.1.dev0 → tinybird-3.0.1}/tinybird/feedback_manager.py

@@ -305,12 +305,6 @@ class FeedbackManager:
     error_unknown_kafka_connection = error_message(
         "Unknown Kafka connection in Data Source '{datasource}'. Hint: you can create it with the 'tb connection create kafka' command.\n** See https://www.tinybird.co/docs/ingest/kafka.html?highlight=kafka#using-include-to-store-connection-settings to learn more."
     )
-    error_unknown_bq_connection = error_message(
-        "Unknown BigQuery connection in Data Source '{datasource}'. Hint: you can create it with the 'tb connection create bigquery' command."
-    )
-    error_unknown_snowflake_connection = error_message(
-        "Unknown Snowflake connection in Data Source '{datasource}'. Hint: you can create it with the 'tb connection create snowflake' command."
-    )
     error_unknown_connection_service = error_message("Unknown connection service '{service}'")
     error_missing_connection_name = error_message("Missing IMPORT_CONNECTION_NAME in '{datasource}'.")
     error_missing_external_datasource = error_message(
@@ -417,9 +411,6 @@ class FeedbackManager:
     error_updating_tag = error_message("Error updating tag: {error}")
     error_tag_generic = error_message("There was an issue updating tags. {error}")
     error_tag_not_found = error_message("Tag {tag_name} not found.")
-    error_snowflake_connector_not_enabled = error_message(
-        "🚨🚨🚨 [DEPRECATED] Snowflake connector is deprecated. Can't create the connector. **"
-    )
 
     info_incl_relative_path = info_message("** Relative path {path} does not exist, skipping.")
     info_ignoring_incl_file = info_message(
@@ -468,13 +459,6 @@ class FeedbackManager:
     prompt_ws_name = prompt_message("❓ Workspace name:")
     prompt_ws_template = prompt_message("❓ Starter template:")
 
-    prompt_bigquery_account = prompt_message(
-        """** Log into your Google Cloud Platform Console as a project editor and go to https://console.cloud.google.com/iam-admin/iam
-** Grant access to this principal: {service_account}
-** Assign it the role "BigQuery Data Viewer"
-Ready? """
-    )
-
     prompt_s3_iamrole_connection_login_aws = prompt_message("""[1] Log into your AWS Console\n\n""")
     prompt_s3_iamrole_connection_policy = prompt_message(
         """\n[2] Go to IAM > Policies. Create a new policy with the following permissions. Please, replace {replacements}:\n\n{access_policy}\n\n(The policy has been copied to your clipboard)\n\n"""
@@ -509,12 +493,6 @@ Ready? """
     error_bump_release = error_message(
         "Error: Bump the --semver flag, example 'tb --semver 0.0.2 {command}'. If you are in CI, bump the VERSION envvar in .tinyenv\n"
     )
-    error_bigquery_improper_permissions = error_message(
-        "Error: no access detected. It might take a minute to detect the new permissions.\n"
-    )
-    error_snowflake_improper_permissions = error_message(
-        "Snowflake connection is not valid. Please check your credentials and try again.\n"
-    )
     error_connection_improper_permissions = error_message(
         "Connection is not valid. Please check your credentials and try again.\n"
     )
@@ -646,9 +624,6 @@ Ready? """
     warning_tag_remove = prompt_message(
         "Tag {tag_name} is used by {resources_len} resources. Do you want to remove it?"
     )
-    warning_bigquery_connector_deprecated = warning_message(
-        "🚨🚨🚨 [DEPRECATED] BigQuery connector is deprecated. Use ingestion from GCS instead. https://www.tinybird.co/docs/get-data-in/guides/ingest-from-bigquery-using-google-cloud-storage **"
-    )
     warning_s3_access_key_secret_deprecated = warning_message(
         "🚨🚨🚨 [DEPRECATED] S3 (Access Key + Secret) connector is deprecated. Use S3 IAM role instead. https://www.tinybird.co/docs/get-data-in/connectors/s3 **"
     )

{tinybird-2.0.1.dev0 → tinybird-3.0.1}/tinybird/sql.py

@@ -39,6 +39,7 @@ _RE_INDEX_ENTRY = re.compile(
     r"(\w+)\s+([\w\s*\[\]\*\(\),\'\"-><.]+)\s+TYPE\s+(\w+)(?:\(([\w\s*.,]+)\))?(?:\s+GRANULARITY\s+(\d+))?"
 )
 _RE_REPLICATED_MT = re.compile(r"Replicated(.*)MergeTree\(([^\)]*)\)(.*)")
+_RE_REPLICATED_ENGINE_SPLIT = re.compile(r"(Replicated.*MergeTree\(')([^']*)('.*)")
 
 
 @dataclass
@@ -836,7 +837,7 @@ def engine_can_be_replicated(engine: Optional[str]) -> bool:
     """
     >>> engine_can_be_replicated('MergeTree() order by tuple()')
     True
-    >>> engine_can_be_replicated('JOIN(ANY, LEFT, foo)')
+    >>> engine_can_be_replicated('Null')
     False
     >>> engine_can_be_replicated('ReplicatingMergeTree() order by tuple()')
     True
@@ -855,7 +856,7 @@ def engine_supports_delete(engine: Optional[str]) -> bool:
     """
     >>> engine_supports_delete('MergeTree() order by tuple()')
     True
-    >>> engine_supports_delete('JOIN(ANY, LEFT, foo)')
+    >>> engine_supports_delete('Null')
     False
     >>> engine_supports_delete('ReplicatingMergeTree() order by tuple()')
     True
@@ -873,8 +874,8 @@ def engine_replicated_to_local(engine: str) -> str:
     'MergeTree() order by (test)'
     >>> engine_replicated_to_local("ReplicatedReplacingMergeTree('/clickhouse/tables/{layer}-{shard}/test.foo', '{replica}', timestamp) order by (test)")
     'ReplacingMergeTree(timestamp) order by (test)'
-    >>> engine_replicated_to_local("Join(ANY, LEFT, test)")
-    'Join(ANY, LEFT, test)'
+    >>> engine_replicated_to_local("Null")
+    'Null'
     >>> engine_replicated_to_local("ReplicatedVersionedCollapsingMergeTree('/clickhouse/tables/{layer}-{shard}/test.foo', '{replica}', sign,version) ORDER BY pk TTL toDate(local_timeplaced) + toIntervalDay(3) SETTINGS index_granularity = 8192")
     'VersionedCollapsingMergeTree(sign, version) ORDER BY pk TTL toDate(local_timeplaced) + toIntervalDay(3) SETTINGS index_granularity = 8192'
     """
@@ -911,7 +912,7 @@ def engine_patch_replicated_engine(engine: str, engine_full: Optional[str], new_
     if not engine_full:
         return None
     if engine.lower().startswith("Replicated".lower()):
-        parts = re.split(r"(Replicated.*MergeTree\(')([^']*)('.*)", engine_full)
+        parts = _RE_REPLICATED_ENGINE_SPLIT.split(engine_full)
         paths = parts[2].split("/")
         paths[-1] = new_table_name
         zoo_path = "/".join(paths)