tinybird-toolset 2.2.2__tar.gz → 2.4.0__tar.gz

{tinybird_toolset-2.2.2/src/tinybird_toolset.egg-info → tinybird_toolset-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tinybird-toolset
-Version: 2.2.2
+Version: 2.4.0
 Home-page: https://gitlab.com/tinybird/clickhouse-toolset
 Author: Tinybird.co
 Author-email: support@tinybird.co

{tinybird_toolset-2.2.2 → tinybird_toolset-2.4.0}/README.md

@@ -60,6 +60,15 @@ The build is a multi-stage process orchestrated by `conf.py`:
 2. **Toolset Build** - Compiles `functions/*.cpp` into `libCHToolset.a`
 3. **Extension Build** - Compiles `src/query.cpp`, links everything into `_query.so`
 
+On MacOS, you'll need the gnu version of multiple tools. You can get them with:
+```
+brew install grep findutils gpatch
+```
+And you'll have to add their locations to your PATH. Homebrew will output which command to run for each.
+
+You'll also need `objcopy` or `llvm-objcopy` in your path. The latter is more practical and will be
+in the bin path of your homebrew llvm installation, so you'll have to do something like: `export PATH="/opt/homebrew/opt/llvm@19/bin:$PATH"`
+
 ```bash
 make build-3.11    # Build for Python 3.11
 make test-3.11     # Build and test
@@ -148,9 +157,10 @@ Then, you will compile the dependencies and the module itself. You need a modern
 If your system Clang is not the correct version, set the CC and CXX environment variables before building:
 
 ```bash
-# Example for macOS with Homebrew llvm@19
-export CC=/opt/homebrew/opt/llvm@19/bin/clang
-export CXX=/opt/homebrew/opt/llvm@19/bin/clang++
+# Example for macOS with Homebrew llvm@21
+export PATH=/opt/homebrew/opt/llvm@21/bin:$PATH
+export CC=/opt/homebrew/opt/llvm@21/bin/clang
+export CXX=/opt/homebrew/opt/llvm@21/bin/clang++
 ```
 
 The best option is to use the Makefile targets which will use virtualenv to install dependencies, build the packages, install them too and run tests:
@@ -171,13 +181,30 @@ You need to be able to compile ClickHouse for MacOS so we follow [their guide](h
 * Install Xcode and Command Line Tools
 * Install the necessary tools (cmake ninja libtool gettext llvm gcc ccache findutils grep). For the LLVM version, check the [ClickHouse build documentation](https://clickhouse.com/docs/en/development/build) for the minimum required Clang version.
 * Make sure your local clang is pointing to the llvm installation and not the default from OSX / Xcode. You can check it by running `clang --version` - it should show "Homebrew clang" and the correct version.
-* Set the CC and CXX environment variables before building:
+* Add Homebrew LLVM and the GNU userland tools to your `PATH` before building. ClickHouse's build on macOS expects tools such as `llvm-objcopy`, `llvm-strip`, `find`, and `grep` to resolve to the Homebrew/GNU versions instead of the default BSD/macOS tools.
 ```bash
-# Example for llvm@19
-export CC=/opt/homebrew/opt/llvm@19/bin/clang
-export CXX=/opt/homebrew/opt/llvm@19/bin/clang++
+export PATH="/opt/homebrew/opt/llvm@21/bin:/opt/homebrew/opt/findutils/libexec/gnubin:/opt/homebrew/opt/grep/libexec/gnubin:$PATH"
+export CC=/opt/homebrew/opt/llvm@21/bin/clang
+export CXX=/opt/homebrew/opt/llvm@21/bin/clang++
+```
+* You can persist those settings in `~/.zshrc`:
+```bash
+echo '' >> ~/.zshrc
+echo '# tinybird/clickhouse-toolset build environment' >> ~/.zshrc
+echo 'export PATH="/opt/homebrew/opt/llvm@21/bin:/opt/homebrew/opt/findutils/libexec/gnubin:/opt/homebrew/opt/grep/libexec/gnubin:$PATH"' >> ~/.zshrc
+echo 'export CC="/opt/homebrew/opt/llvm@21/bin/clang"' >> ~/.zshrc
+echo 'export CXX="/opt/homebrew/opt/llvm@21/bin/clang++"' >> ~/.zshrc
+source ~/.zshrc
+```
+
+* Follow the normal build. When validating a new machine or toolchain, prefer building a single Python version first:
+```bash
+make build-3.11
+```
+Then run the full build only after that succeeds:
+```bash
+make build
 ```
-* Follow the normal build (`make build`)
 
 #### Clean environment if you already did some compilation before
 To clean your environment you need to make sure that the repository is clean and updated with the expected values. To do that you can use:
@@ -187,6 +214,8 @@ make distclean
 git clean -fdx
 ```
 
+If you manually configured `ch_build` or `ts_build` with a different CMake generator earlier, remove them before rebuilding. The project configures the toolset with `Ninja`, and stale caches created with `Unix Makefiles` will fail with a generator mismatch.
+
 #### Make sure you have the expeted version from the submodules
 ```bash
 git submodule sync && git submodule update --init --recursive

{tinybird_toolset-2.2.2 → tinybird_toolset-2.4.0}/setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, Extension
 
 NAME = 'tinybird-toolset'
-VERSION = '2.2.2'
+VERSION = '2.4.0'
 
 try:
     from conf import *

{tinybird_toolset-2.2.2 → tinybird_toolset-2.4.0}/src/chtoolset/query.py

@@ -19,6 +19,7 @@ from chtoolset._query import replace_tables, \
     set_row_binary_encoder_buffer_config, \
     set_row_binary_encoder_thread_buffer_size, \
     get_columns_from_create_query, \
+    normalize_table_column, \
     parse_create_materialized_view_target_table as _parse_create_materialized_view_target_table
 
 from typing import Optional, TypedDict

{tinybird_toolset-2.2.2 → tinybird_toolset-2.4.0/src/tinybird_toolset.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tinybird-toolset
-Version: 2.2.2
+Version: 2.4.0
 Home-page: https://gitlab.com/tinybird/clickhouse-toolset
 Author: Tinybird.co
 Author-email: support@tinybird.co

{tinybird_toolset-2.2.2 → tinybird_toolset-2.4.0}/src/tinybird_toolset.egg-info/SOURCES.txt

@@ -19,6 +19,7 @@ tests/test_get_columns_from_create_query.py
 tests/test_get_left_table.py
 tests/test_internal_cache.py
 tests/test_normalize_query_keep_names.py
+tests/test_normalize_table_column.py
 tests/test_parse_create_materialized_view_target_table.py
 tests/test_replace_tables.py
 tests/test_replace_tables_backward_compat.py

tinybird_toolset-2.4.0/tests/test_normalize_table_column.py

@@ -0,0 +1,431 @@
+import unittest
+
+from chtoolset import query as chquery
+
+
+class TestNormalizeTableColumn(unittest.TestCase):
+    def test_normalizes_low_cardinality_with_nullable_flag(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "status",
+                "type": "LowCardinality(String)",
+                "nullable": True,
+            },
+        )
+
+        self.assertEqual(column["name"], "status")
+        self.assertEqual(column["type"], "LowCardinality(String)")
+        self.assertTrue(column["nullable"])
+
+    def test_uses_normalized_name_alias(self):
+        column = chquery.normalize_table_column(
+            column={"name": "Name", "normalized_name": "name", "type": "String"},
+        )
+
+        self.assertEqual(column["name"], "name")
+        self.assertEqual(column["type"], "String")
+        self.assertFalse(column["nullable"])
+
+    def test_detects_nullable_simple_aggregate_function(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "metric",
+                "type": "SimpleAggregateFunction(sum, Nullable(Int64))",
+            },
+        )
+
+        self.assertEqual(
+            column["type"], "SimpleAggregateFunction(sum, Nullable(Int64))"
+        )
+        self.assertTrue(column["nullable"])
+
+    def test_plain_nullable_wrapper_does_not_force_nullable_flag(self):
+        column = chquery.normalize_table_column(
+            column={"name": "discount", "type": "Nullable(Decimal64(4))"},
+        )
+
+        self.assertEqual(column["type"], "Nullable(Decimal64(4))")
+        self.assertTrue(column["nullable"])
+
+    def test_normalizes_decimal_spacing(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "amount",
+                "normalized_name": "amount",
+                "type": "Decimal(18,4)",
+                "nullable": False,
+            },
+        )
+
+        self.assertEqual(column["type"], "Decimal(18, 4)")
+        self.assertFalse(column["nullable"])
+
+    def test_normalizes_decimal_spacing_with_nullable_flag(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "tax",
+                "normalized_name": "tax",
+                "type": "Decimal(10,2)",
+                "nullable": True,
+            },
+        )
+
+        self.assertEqual(column["type"], "Decimal(10, 2)")
+        self.assertTrue(column["nullable"])
+
+    def test_preserves_decimal64_alias(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "discount",
+                "normalized_name": "discount",
+                "type": "Nullable(Decimal64(4))",
+                "nullable": False,
+            },
+        )
+
+        self.assertEqual(column["type"], "Nullable(Decimal64(4))")
+        self.assertTrue(column["nullable"])
+
+    def test_preserves_datetime64_precision(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "created_at",
+                "normalized_name": "created_at",
+                "type": "DateTime64(3)",
+                "nullable": False,
+            },
+        )
+
+        self.assertEqual(column["type"], "DateTime64(3)")
+        self.assertFalse(column["nullable"])
+
+    def test_normalizes_json_dynamic_param_spacing(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "json",
+                "normalized_name": "json",
+                "type": "JSON(max_dynamic_types=2, max_dynamic_paths=16)",
+                "nullable": False,
+            },
+        )
+
+        self.assertEqual(
+            column["type"], "JSON(max_dynamic_types = 2, max_dynamic_paths = 16)"
+        )
+        self.assertFalse(column["nullable"])
+
+    def test_normalizes_nested_decimal_spacing(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "agg",
+                "normalized_name": "agg",
+                "type": "AggregateFunction(sum, Decimal(10,2))",
+                "nullable": False,
+            },
+        )
+
+        self.assertEqual(column["type"], "AggregateFunction(sum, Decimal(10, 2))")
+        self.assertFalse(column["nullable"])
+
+    def test_normalizes_multiline_complex_type_spacing(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "agg",
+                "normalized_name": "agg",
+                "type": """AggregateFunction(
+                    argMax, Nullable(UUID), Tuple(UInt8, DateTime64(3), DateTime64(3), DateTime64(3))
+                )""",
+                "nullable": False,
+            },
+        )
+
+        self.assertEqual(
+            column["type"],
+            "AggregateFunction(argMax, Nullable(UUID), Tuple(UInt8, DateTime64(3), DateTime64(3), DateTime64(3)))",
+        )
+        self.assertFalse(column["nullable"])
+
+    def test_preserves_space_before_timezone_string(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "ts",
+                "normalized_name": "ts",
+                "type": "DateTime64(3, 'Europe/Vienna')",
+                "nullable": False,
+            },
+        )
+
+        self.assertEqual(column["type"], "DateTime64(3, 'Europe/Vienna')")
+        self.assertFalse(column["nullable"])
+
+    def test_nullable_flag_preserves_raw_type(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "reason",
+                "normalized_name": "reason",
+                "type": "String",
+                "nullable": True,
+            },
+        )
+
+        self.assertEqual(column["type"], "String")
+        self.assertTrue(column["nullable"])
+
+    def test_nullable_flag_preserves_low_cardinality_type(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "status",
+                "normalized_name": "status",
+                "type": "LowCardinality(String)",
+                "nullable": True,
+            },
+        )
+
+        self.assertEqual(column["type"], "LowCardinality(String)")
+        self.assertTrue(column["nullable"])
+
+    def test_preserves_existing_nullable_low_cardinality_wrapper(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "status",
+                "normalized_name": "status",
+                "type": "LowCardinality(Nullable(String))",
+                "nullable": False,
+            },
+        )
+
+        self.assertEqual(column["type"], "LowCardinality(Nullable(String))")
+        self.assertTrue(column["nullable"])
+
+    def test_normalizes_default_value_into_specifier_and_expression(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "created_at",
+                "normalized_name": "created_at",
+                "type": "DateTime64(3)",
+                "nullable": False,
+                "default_value": "DEFAULT now()",
+            },
+        )
+
+        self.assertEqual(column["type"], "DateTime64(3)")
+        self.assertEqual(column["default_specifier"], "DEFAULT")
+        self.assertEqual(column["default_expression"], "now()")
+
+    def test_preserves_split_default_specifier_and_expression(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "country",
+                "normalized_name": "country",
+                "type": "String",
+                "nullable": False,
+                "default_value": "DEFAULT 'Unknown'",
+            },
+        )
+
+        self.assertEqual(column["default_specifier"], "DEFAULT")
+        self.assertEqual(column["default_expression"], "'Unknown'")
+
+    def test_normalizes_comment_codec_ttl_and_primary_key(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "ts",
+                "normalized_name": "ts",
+                "type": "DateTime64(3)",
+                "nullable": False,
+                "comment": "'event time'",
+                "codec": "CODEC(ZSTD(1))",
+                "ttl": "ts + toIntervalDay(30)",
+                "is_primary_key": True,
+            },
+        )
+
+        self.assertEqual(column["type"], "DateTime64(3)")
+        self.assertEqual(column["comment"], "'event time'")
+        self.assertEqual(column["codec"], "CODEC(ZSTD(1))")
+        self.assertEqual(column["ttl"], "ts + toIntervalDay(30)")
+        self.assertTrue(column["is_primary_key"])
+
+    def test_preserves_normalized_name_for_output_name(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "CamelCase",
+                "normalized_name": "camel_case",
+                "type": "String",
+            },
+        )
+
+        self.assertEqual(column["name"], "camel_case")
+        self.assertEqual(column["type"], "String")
+
+    def test_normalizes_string_scalar(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "id",
+                "normalized_name": "id",
+                "type": "String",
+                "nullable": False,
+            },
+        )
+
+        self.assertEqual(column["type"], "String")
+
+    def test_normalizes_bool_scalar(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "is_active",
+                "normalized_name": "is_active",
+                "type": "Bool",
+                "nullable": False,
+            },
+        )
+
+        self.assertEqual(column["type"], "Bool")
+
+    def test_normalizes_uint64_scalar(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "count",
+                "normalized_name": "count",
+                "type": "UInt64",
+                "nullable": False,
+            },
+        )
+
+        self.assertEqual(column["type"], "UInt64")
+
+    def test_normalizes_array_payload(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "tags",
+                "normalized_name": "tags",
+                "type": "Array(String)",
+                "nullable": False,
+            },
+        )
+
+        self.assertEqual(column["type"], "Array(String)")
+
+    def test_normalizes_map_payload(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "metadata",
+                "normalized_name": "metadata",
+                "type": "Map(String, String)",
+                "nullable": False,
+            },
+        )
+
+        self.assertEqual(column["type"], "Map(String, String)")
+
+    def test_normalizes_low_cardinality_payload(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "category",
+                "normalized_name": "category",
+                "type": "LowCardinality(String)",
+                "nullable": False,
+            },
+        )
+
+        self.assertEqual(column["type"], "LowCardinality(String)")
+
+    def test_normalizes_nullable_string_field(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "phone",
+                "normalized_name": "phone",
+                "type": "Nullable(String)",
+                "nullable": False,
+            },
+        )
+
+        self.assertEqual(column["type"], "Nullable(String)")
+
+    def test_normalizes_nullable_datetime64_field(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "deleted_at",
+                "normalized_name": "deleted_at",
+                "type": "Nullable(DateTime64(3))",
+                "nullable": False,
+            },
+        )
+
+        self.assertEqual(column["type"], "Nullable(DateTime64(3))")
+
+    def test_normalizes_decimal_business_field(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "price",
+                "normalized_name": "price",
+                "type": "Decimal(12,2)",
+                "nullable": False,
+            },
+        )
+
+        self.assertEqual(column["type"], "Decimal(12, 2)")
+
+    def test_preserves_decimal64_business_field(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "amount",
+                "normalized_name": "amount",
+                "type": "Decimal64(4)",
+                "nullable": False,
+            },
+        )
+
+        self.assertEqual(column["type"], "Decimal64(4)")
+
+    def test_normalizes_float64_business_field(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "score",
+                "normalized_name": "score",
+                "type": "Float64",
+                "nullable": False,
+            },
+        )
+
+        self.assertEqual(column["type"], "Float64")
+
+    def test_default_now_for_created_at(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "created_at",
+                "normalized_name": "created_at",
+                "type": "DateTime64(3)",
+                "nullable": False,
+                "default_value": "DEFAULT now()",
+            },
+        )
+
+        self.assertEqual(column["default_specifier"], "DEFAULT")
+        self.assertEqual(column["default_expression"], "now()")
+
+    def test_default_true_for_bool(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "is_active",
+                "normalized_name": "is_active",
+                "type": "Bool",
+                "nullable": False,
+                "default_value": "DEFAULT true",
+            },
+        )
+
+        self.assertEqual(column["default_expression"], "true")
+
+    def test_default_zero_for_retry_count(self):
+        column = chquery.normalize_table_column(
+            column={
+                "name": "retry_count",
+                "normalized_name": "retry_count",
+                "type": "UInt32",
+                "nullable": False,
+                "default_value": "DEFAULT 0",
+            },
+        )
+
+        self.assertEqual(column["default_expression"], "0")

{tinybird_toolset-2.2.2 → tinybird_toolset-2.4.0}/tests/test_replace_tables.py

@@ -1144,6 +1144,25 @@ class TestDisabledFunctions(unittest.TestCase):
         expected = "SELECT * FROM (SELECT count() from system.numbers LIMIT 2) AS a, (SELECT count() from system.numbers LIMIT 2) AS a SETTINGS join_use_nulls = 1"
         self.assertEqual(chquery.format(expected), chquery.format(replaced))
 
+    def test_allow_analytics_query_settings(self):
+        sql = """SELECT * FROM a SETTINGS
+            max_threads = 1,
+            max_memory_usage = 1000000,
+            max_execution_time = 60,
+            use_query_condition_cache = 0
+        """
+        replacement = {("default", "a"): ("", "(SELECT count() FROM system.numbers LIMIT 1)")}
+        replaced = chquery.replace_tables(sql, replacement, default_database="default")
+        out = chquery.format(replaced)
+        self.assertIn("system.numbers", out)
+        for key in (
+            "max_threads",
+            "max_memory_usage",
+            "max_execution_time",
+            "use_query_condition_cache",
+        ):
+            self.assertIn(key, out)
+
     def test_disallow_enabled_settings(self):
         sql = "SELECT * FROM a, a SETTINGS join_use_nulls=1"
         replacement = {

{tinybird_toolset-2.2.2 → tinybird_toolset-2.4.0}/tests/test_tables.py

@@ -2025,6 +2025,23 @@ SELECT *
         self.assertEqual(chquery.tables("SELECT count() from system.numbers SETTINGS aggregate_functions_null_for_empty=1"),
                          [('system', 'numbers', '')])
 
+    def test_allow_analytics_query_settings(self):
+        sql = """SELECT count() FROM system.numbers LIMIT 1 SETTINGS
+            max_threads = 1,
+            max_memory_usage = 1000000,
+            max_execution_time = 60,
+            optimize_aggregation_in_order = 1,
+            use_skip_indexes = 1,
+            use_skip_indexes_if_final = 0,
+            optimize_move_to_prewhere = 1,
+            query_plan_optimize_lazy_materialization = 0,
+            min_bytes_to_use_direct_io = 0,
+            enable_filesystem_cache = 1,
+            use_query_cache = 0,
+            use_query_condition_cache = 0
+        """
+        self.assertEqual(chquery.tables(sql), [('system', 'numbers', '')])
+
     def test_disallow_blocked_settings(self):
         with self.assertRaisesRegex(ValueError, """DB::Exception: Usage of setting 'aggregate_functions_null_for_empty' is restricted. Contact support@tinybird.co if you require access to this feature"""):
             chquery.tables("SELECT count() from system.numbers SETTINGS aggregate_functions_null_for_empty=1",