tinybird-toolset 2.4.13__tar.gz → 2.5.0__tar.gz

{tinybird_toolset-2.4.13/src/tinybird_toolset.egg-info → tinybird_toolset-2.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tinybird-toolset
-Version: 2.4.13
+Version: 2.5.0
 Home-page: https://gitlab.com/tinybird/clickhouse-toolset
 Author: Tinybird.co
 Author-email: support@tinybird.co

{tinybird_toolset-2.4.13 → tinybird_toolset-2.5.0}/conf.py

@@ -94,9 +94,11 @@ class ClickHouseBuildExt(build_ext):
         if not os.path.exists(os.path.join(CLICKHOUSE_PATH, 'CMakeLists.txt')):
             raise RuntimeError('Git submodules are not initialized. Run: `git submodule update --init --recursive`.')
 
-        # One of the libraries, chosen randomly
+        # One of the libraries, chosen randomly (+ the TTL functions one, added later)
         if os.path.isfile(os.path.join(CLICKHOUSE_BUILD_PATH,
-                                       'src/AggregateFunctions/libclickhouse_aggregate_functions.a')):
+                                       'src/AggregateFunctions/libclickhouse_aggregate_functions.a')) \
+                and os.path.isfile(os.path.join(CLICKHOUSE_BUILD_PATH,
+                                                'src/Functions/libclickhouse_ttl_functions.a')):
             return
 
         # Avoid segfault caused by missing global context
@@ -138,9 +140,18 @@ class ClickHouseBuildExt(build_ext):
         # Removes TSA warnings (we don't compile with a libc++ with annotations)
         apply_patch("patches/tsa.patch")
 
-        # Temporary for debugging: some functions are missing from libdbms.a, this is in the private repo pmaster already so we should include it from the repo
+        # Builds the functions needed by validate_ttl into libclickhouse_ttl_functions.a,
+        # which is linked with --whole-archive/-force_load (see extra_libs).
+        # NOTE: after editing this patch (or any other), the early-return above skips the
+        # ClickHouse build; delete ch_build/src/Functions/libclickhouse_ttl_functions.a
+        # (or the whole ch_build) to force it to run again.
         apply_patch("patches/dbmsfunctions.patch")
 
+        # Range-check narrow-integer Field inserts so JSON/Dynamic paths widen instead of
+        # silently wrapping (1944 -> -104). Backport of the ColumnVector::tryInsert fix;
+        # remove once the fix lands in the pinned ClickHouse submodule.
+        apply_patch("patches/ColumnVectorTryInsert.patch")
+
         if sys.platform == 'darwin':
             # We disable whole_archive options destined for AppleClang, which is unsupported but keeps being a PITA
             apply_patch("patches/darwin.patch")
@@ -243,6 +254,14 @@ class ClickHouseBuildExt(build_ext):
                               ['--config', 'Release',
                                '--target', 'src/AggregateFunctions/libclickhouse_aggregate_functions.a'])
 
+        # Functions for TTL validation, also with MinSize as performance is not a concern.
+        # This library is linked with --whole-archive/-force_load (see extra_libs) so that
+        # its REGISTER_FUNCTION static registrars run and DB::registerFunctions() can
+        # register all of them without listing them one by one.
+        subprocess.check_call([cmake_cmd, '--build', CLICKHOUSE_BUILD_PATH] +
+                              ['--config', 'Release',
+                               '--target', 'src/Functions/libclickhouse_ttl_functions.a'])
+
         # Now build with Release just the parser, which is where we want speed
         subprocess.check_call([cmake_cmd, CLICKHOUSE_PATH] + cmake_args + ['-DCMAKE_BUILD_TYPE=Release'],
                               cwd=CLICKHOUSE_BUILD_PATH)
@@ -375,6 +394,15 @@ class CustomBuildWithFromCH(build_ext):
         # - Remove .o. These files only contain function declaration and we don't need them
         ch_libs = list(filter(lambda lib: lib and lib.find('whole-archive') == -1 and lib.find('-lm') == -1, ch_libs))
 
+        # Force-load the TTL validation functions so their REGISTER_FUNCTION static
+        # registrars run even though nothing references their symbols directly.
+        # DB::registerFunctions() (called from ValidateTTL.cpp) registers all of them.
+        ttl_functions_lib = os.path.join(CLICKHOUSE_BUILD_PATH, 'src/Functions/libclickhouse_ttl_functions.a')
+        if sys.platform == 'darwin':
+            ch_libs = [f'-Wl,-force_load,{ttl_functions_lib}'] + ch_libs
+        else:
+            ch_libs = ['-Wl,--whole-archive', ttl_functions_lib, '-Wl,--no-whole-archive'] + ch_libs
+
         if sys.platform == 'linux':
             ch_libs = ['-Wl,--no-whole-archive'] + ch_libs
 

{tinybird_toolset-2.4.13 → tinybird_toolset-2.5.0}/setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, Extension
 
 NAME = 'tinybird-toolset'
-VERSION = '2.4.13'
+VERSION = '2.5.0'
 
 try:
     from conf import *
@@ -20,6 +20,7 @@ try:
             "functions/ReplaceTables.h",
             "functions/Tables.h",
             "functions/TBQueryParser.h",
+            "functions/ValidateTTL.h",
             "functions/Validation.h",
             "functions/simdjsonHelpers.h",
             "functions/JSONPathQuery.h",

{tinybird_toolset-2.4.13 → tinybird_toolset-2.5.0}/src/chtoolset/query.py

@@ -20,6 +20,7 @@ from chtoolset._query import replace_tables, \
     set_row_binary_encoder_thread_buffer_size, \
     get_columns_from_create_query, \
     normalize_table_column, \
+    validate_ttl, \
     parse_create_materialized_view_target_table as _parse_create_materialized_view_target_table
 
 from typing import Optional, TypedDict

{tinybird_toolset-2.4.13 → tinybird_toolset-2.5.0/src/tinybird_toolset.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tinybird-toolset
-Version: 2.4.13
+Version: 2.5.0
 Home-page: https://gitlab.com/tinybird/clickhouse-toolset
 Author: Tinybird.co
 Author-email: support@tinybird.co

{tinybird_toolset-2.4.13 → tinybird_toolset-2.5.0}/src/tinybird_toolset.egg-info/SOURCES.txt

@@ -24,4 +24,5 @@ tests/test_parse_create_materialized_view_target_table.py
 tests/test_replace_tables.py
 tests/test_replace_tables_backward_compat.py
 tests/test_rewrite_aggregation_states.py
-tests/test_tables.py
+tests/test_tables.py
+tests/test_validate_ttl.py

{tinybird_toolset-2.4.13 → tinybird_toolset-2.5.0}/tests/test_convert_to_row_binary.py

@@ -8346,20 +8346,22 @@ TEST_CASES = [
         1, 0),
 
 
-    # schemaless JSON encoded as string can choose too-narrow integer types and overflow values
+    # schemaless JSON encoded as string widens too-narrow integer types instead of
+    # overflowing: y=22 first pins the path to Int8, then y=220 widens it to Int16
+    # rather than wrapping to -36 (CH-440, ColumnVector::tryInsert range check)
     (ConversionMode.ONLY_LEGACY,
         [('v', 'Map(String, JSON)', '$', False)],
         '{"a":{"x":11,"y":22},"b":{"x":110,"y":220}}',
-        b'\x00\x02\x01a\x0f{"x":11,"y":22}\x01b\x11{"x":110,"y":-36}',
+        b'\x00\x02\x01a\x0f{"x":11,"y":22}\x01b\x11{"x":110,"y":220}',
         b'',
         1, 0),
     (ConversionMode.ONLY_LEGACY,
         [('json', 'Map(String, JSON)', "$", False)],
         '{"a":"{\\"x\\":11,\\"y\\":22}","b":"{\\"x\\":110,\\"y\\":220}"}',
-        b'\x00\x02\x01a\x0f{"x":11,"y":22}\x01b\x11{"x":110,"y":-36}',
+        b'\x00\x02\x01a\x0f{"x":11,"y":22}\x01b\x11{"x":110,"y":220}',
         b'',
         1, 0),
-    # doesn't happen with binary JSON encoding
+    # binary JSON encoding stores per-value type tags, so it already preserved the value
     (ConversionMode.ONLY_NONLEGACY,
         [('v', 'Map(String, JSON)', '$', False)],
         '{"a":{"x":11,"y":22},"b":{"x":110,"y":220}}',

tinybird_toolset-2.5.0/tests/test_validate_ttl.py

@@ -0,0 +1,394 @@
+import unittest
+from chtoolset import query as chquery
+
+
+class TestValidateTTL(unittest.TestCase):
+    def test_empty_ttl_is_valid(self):
+        self.assertIsNone(
+            chquery.validate_ttl(
+                ttl="",
+                columns=[{"name": "ts", "type": "DateTime"}],
+                sorting_key="ts",
+                primary_key="ts",
+            )
+        )
+
+    def test_valid_no_primary_key_uses_sorting_key(self):
+        self.assertIsNone(
+            chquery.validate_ttl(
+                ttl="ts + INTERVAL 7 DAY",
+                columns=[{"name": "ts", "type": "DateTime"}],
+                sorting_key="ts",
+            )
+        )
+
+    def test_valid_datetime_plus_interval(self):
+        self.assertIsNone(
+            chquery.validate_ttl(
+                ttl="ts + INTERVAL 7 DAY",
+                columns=[{"name": "ts", "type": "DateTime"}, {"name": "v", "type": "Int32"}],
+                sorting_key="ts",
+                primary_key="ts",
+            )
+        )
+
+    def test_valid_datetime_plus_to_interval(self):
+        self.assertIsNone(
+            chquery.validate_ttl(
+                ttl="ts + toIntervalDay(7)",
+                columns=[{"name": "ts", "type": "DateTime"}],
+                sorting_key="ts",
+                primary_key="ts",
+            )
+        )
+
+    def test_valid_date_column(self):
+        self.assertIsNone(
+            chquery.validate_ttl(
+                ttl="d + INTERVAL 30 DAY",
+                columns=[{"name": "d", "type": "Date"}],
+                sorting_key="d",
+                primary_key="d",
+            )
+        )
+
+    def test_valid_wrapped_datetime64(self):
+        self.assertIsNone(
+            chquery.validate_ttl(
+                ttl="toDateTime(event_time) + toIntervalDay(7)",
+                columns=[{"name": "event_time", "type": "DateTime64(3)"}],
+                sorting_key="event_time",
+                primary_key="event_time",
+            )
+        )
+
+    def test_invalid_non_date_result_type(self):
+        err = chquery.validate_ttl(
+            ttl="counter + toIntervalDay(7)",
+            columns=[{"name": "counter", "type": "Int32"}],
+            sorting_key="counter",
+            primary_key="counter",
+        )
+        self.assertIsNotNone(err)
+        self.assertIn("Illegal types", err)
+
+    def test_valid_datetime_plus_integer_expression(self):
+        self.assertIsNone(
+            chquery.validate_ttl(
+                ttl="ts + 7",
+                columns=[{"name": "ts", "type": "DateTime"}],
+                sorting_key="ts",
+                primary_key="ts",
+            )
+        )
+
+    def test_invalid_unwrapped_datetime64(self):
+        err = chquery.validate_ttl(
+            ttl="event_time + toIntervalDay(7)",
+            columns=[{"name": "event_time", "type": "DateTime64(3)"}],
+            sorting_key="event_time",
+            primary_key="event_time",
+        )
+        self.assertIsNotNone(err)
+        self.assertIn("TTL expression result column should have DateTime or Date type", err)
+
+    def test_invalid_unknown_column(self):
+        err = chquery.validate_ttl(
+            ttl="missing + INTERVAL 7 DAY",
+            columns=[{"name": "ts", "type": "DateTime"}],
+            sorting_key="ts",
+            primary_key="ts",
+        )
+        self.assertIsNotNone(err)
+        self.assertIn("Missing columns: 'missing'", err)
+
+    def test_invalid_malformed_ttl(self):
+        err = chquery.validate_ttl(
+            ttl="this is not a valid ttl !!!",
+            columns=[{"name": "ts", "type": "DateTime"}],
+            sorting_key="ts",
+            primary_key="ts",
+        )
+        self.assertIsNotNone(err)
+        self.assertIn("Syntax error", err)
+
+    # --- Date/time functions supported by ClickHouse in TTL expressions ---
+    def test_valid_from_unix_timestamp(self):
+        self.assertIsNone(
+            chquery.validate_ttl(
+                ttl="fromUnixTimestamp(uts) + INTERVAL 7 DAY",
+                columns=[{"name": "uts", "type": "UInt32"}],
+                sorting_key="uts",
+                primary_key="uts",
+            )
+        )
+
+    def test_valid_from_unix_timestamp64_milli(self):
+        self.assertIsNone(
+            chquery.validate_ttl(
+                ttl="toDateTime(fromUnixTimestamp64Milli(uts)) + INTERVAL 7 DAY",
+                columns=[{"name": "uts", "type": "Int64"}],
+                sorting_key="uts",
+                primary_key="uts",
+            )
+        )
+
+    def test_valid_from_unix_timestamp64_micro_and_nano(self):
+        for fn in ("fromUnixTimestamp64Micro", "fromUnixTimestamp64Nano"):
+            with self.subTest(fn=fn):
+                self.assertIsNone(
+                    chquery.validate_ttl(
+                        ttl=f"toDateTime({fn}(uts)) + INTERVAL 1 DAY",
+                        columns=[{"name": "uts", "type": "Int64"}],
+                        sorting_key="uts",
+                        primary_key="uts",
+                    )
+                )
+
+    def test_valid_to_start_of_functions(self):
+        for fn in (
+            "toStartOfDay",
+            "toStartOfHour",
+            "toStartOfMinute",
+            "toStartOfFifteenMinutes",
+            "toStartOfMonth",
+            "toStartOfQuarter",
+            "toStartOfYear",
+            "toStartOfWeek",
+            "toMonday",
+            "toLastDayOfMonth",
+        ):
+            with self.subTest(fn=fn):
+                self.assertIsNone(
+                    chquery.validate_ttl(
+                        ttl=f"{fn}(ts) + INTERVAL 30 DAY",
+                        columns=[{"name": "ts", "type": "DateTime"}],
+                        sorting_key="ts",
+                        primary_key="ts",
+                    )
+                )
+
+    def test_valid_to_start_of_interval(self):
+        self.assertIsNone(
+            chquery.validate_ttl(
+                ttl="toStartOfInterval(ts, INTERVAL 1 DAY) + INTERVAL 90 DAY",
+                columns=[{"name": "ts", "type": "DateTime"}],
+                sorting_key="ts",
+                primary_key="ts",
+            )
+        )
+
+    def test_valid_date_trunc(self):
+        self.assertIsNone(
+            chquery.validate_ttl(
+                ttl="dateTrunc('month', ts) + INTERVAL 6 MONTH",
+                columns=[{"name": "ts", "type": "DateTime"}],
+                sorting_key="ts",
+                primary_key="ts",
+            )
+        )
+
+    def test_valid_to_timezone(self):
+        self.assertIsNone(
+            chquery.validate_ttl(
+                ttl="toTimezone(ts, 'UTC') + INTERVAL 7 DAY",
+                columns=[{"name": "ts", "type": "DateTime"}],
+                sorting_key="ts",
+                primary_key="ts",
+            )
+        )
+
+    def test_valid_make_date(self):
+        self.assertIsNone(
+            chquery.validate_ttl(
+                ttl="makeDate(y, m, d) + INTERVAL 7 DAY",
+                columns=[
+                    {"name": "y", "type": "UInt16"},
+                    {"name": "m", "type": "UInt8"},
+                    {"name": "d", "type": "UInt8"},
+                ],
+                sorting_key="y",
+                primary_key="y",
+            )
+        )
+
+    # --- Conditional / multi-clause TTLs ---
+    def test_valid_ttl_with_where(self):
+        self.assertIsNone(
+            chquery.validate_ttl(
+                ttl="ts + INTERVAL 7 DAY WHERE status = 'expired'",
+                columns=[
+                    {"name": "ts", "type": "DateTime"},
+                    {"name": "status", "type": "String"},
+                ],
+                sorting_key="ts",
+                primary_key="ts",
+            )
+        )
+
+    def test_valid_ttl_with_where_numeric(self):
+        self.assertIsNone(
+            chquery.validate_ttl(
+                ttl="ts + INTERVAL 30 DAY WHERE x % 10 == 0 AND y > 5",
+                columns=[
+                    {"name": "ts", "type": "DateTime"},
+                    {"name": "x", "type": "Int32"},
+                    {"name": "y", "type": "Int32"},
+                ],
+                sorting_key="ts",
+                primary_key="ts",
+            )
+        )
+
+    def test_invalid_ttl_where_unknown_column(self):
+        err = chquery.validate_ttl(
+            ttl="ts + INTERVAL 7 DAY WHERE missing = 1",
+            columns=[{"name": "ts", "type": "DateTime"}],
+            sorting_key="ts",
+            primary_key="ts",
+        )
+        self.assertIsNotNone(err)
+        self.assertIn("Missing columns: 'missing'", err)
+
+    def test_valid_ttl_where_string_expression(self):
+        self.assertIsNone(
+            chquery.validate_ttl(
+                ttl="ts + INTERVAL 7 DAY WHERE status",
+                columns=[
+                    {"name": "ts", "type": "DateTime"},
+                    {"name": "status", "type": "String"},
+                ],
+                sorting_key="ts",
+                primary_key="ts",
+            )
+        )
+
+    def test_valid_multiple_ttls_comma_separated(self):
+        self.assertIsNone(
+            chquery.validate_ttl(
+                ttl="ts + INTERVAL 7 DAY WHERE status = 'a', ts + INTERVAL 14 DAY WHERE status = 'b'",
+                columns=[
+                    {"name": "ts", "type": "DateTime"},
+                    {"name": "status", "type": "String"},
+                ],
+                sorting_key="ts",
+                primary_key="ts",
+            )
+        )
+
+    def test_valid_ttl_group_by(self):
+        # GROUP BY rollup; must group by a prefix of the primary key
+        self.assertIsNone(
+            chquery.validate_ttl(
+                ttl="ts + INTERVAL 1 DAY GROUP BY k SET v = sum(v)",
+                columns=[
+                    {"name": "k", "type": "String"},
+                    {"name": "ts", "type": "DateTime"},
+                    {"name": "v", "type": "Int64"},
+                ],
+                sorting_key="k",
+                primary_key="k",
+            )
+        )
+
+    def test_invalid_ttl_group_by_not_in_primary_key(self):
+        err = chquery.validate_ttl(
+            ttl="ts + INTERVAL 1 DAY GROUP BY missing SET v = sum(v)",
+            columns=[
+                {"name": "k", "type": "String"},
+                {"name": "ts", "type": "DateTime"},
+                {"name": "v", "type": "Int64"},
+            ],
+            sorting_key="k",
+            primary_key="k",
+        )
+        self.assertIsNotNone(err)
+        self.assertIn("GROUP BY key should be a prefix of primary key", err)
+
+    def test_valid_where_string_functions(self):
+        columns = [
+            {"name": "ts", "type": "DateTime"},
+            {"name": "s", "type": "String"},
+            {"name": "n", "type": "Int32"},
+        ]
+        predicates = {
+            "concat": "concat(s, 'x') = 'ax'",
+            "startsWith": "startsWith(s, 'a')",
+            "endsWith": "endsWith(s, 'z')",
+            "length": "length(s) > 3",
+            "lower": "lower(s) = 'a'",
+            "substring": "substring(s, 1, 2) = 'ab'",
+            "match": "match(s, '^a')",
+            "in": "n IN (1, 2, 3)",
+        }
+        for name, predicate in predicates.items():
+            with self.subTest(fn=name):
+                self.assertIsNone(
+                    chquery.validate_ttl(
+                        ttl=f"ts + INTERVAL 7 DAY WHERE {predicate}",
+                        columns=columns,
+                        sorting_key="ts",
+                        primary_key="ts",
+                    ),
+                    f"{name} should be a known function in TTL WHERE clauses",
+                )
+
+    # --- Input validation ---
+    def test_columns_must_be_list_of_dicts(self):
+        with self.assertRaises(TypeError):
+            chquery.validate_ttl(ttl="ts + INTERVAL 7 DAY", columns="ts DateTime", sorting_key="ts", primary_key="ts")
+
+    def test_column_dict_requires_name_and_type(self):
+        with self.assertRaises((TypeError, ValueError)):
+            chquery.validate_ttl(
+                ttl="ts + INTERVAL 7 DAY",
+                columns=[{"name": "ts"}],
+                sorting_key="ts",
+                primary_key="ts",
+            )
+
+
+    # --- Regression guard: sequential throwing calls must not crash ---
+    def test_sequential_throwing_calls_do_not_crash(self):
+        # A validate_ttl call that throws must not corrupt shared state and take
+        # down a subsequent call. validateTTL parses against a private per-call
+        # context copy (functions/ValidateTTL.cpp) precisely so this holds.
+        # Run in a subprocess so a hypothetical crash doesn't kill pytest; the
+        # child must exit 0.
+        import signal
+        import subprocess
+        import sys
+        import textwrap
+        repro = textwrap.dedent("""\
+            from chtoolset import query as q
+            err1 = q.validate_ttl(
+                ttl="counter + toIntervalDay(7)",
+                columns=[{"name": "counter", "type": "Int32"}],
+                sorting_key="counter",
+                primary_key="counter",
+            )
+            assert err1 is not None and "Illegal types" in err1, (
+                f"call 1 should return an 'Illegal types' error, got: {err1!r}")
+            q.validate_ttl(
+                ttl="ts + INTERVAL 7 DAY WHERE missing = 1",
+                columns=[{"name": "ts", "type": "DateTime"}],
+                sorting_key="ts",
+                primary_key="ts",
+            )
+        """)
+        result = subprocess.run([sys.executable, "-c", repro],
+                                capture_output=True, text=True, timeout=30)
+        if result.returncode != 0 and result.returncode != -signal.SIGABRT:
+            raise RuntimeError(
+                f"Unexpected subprocess exit {result.returncode} "
+                f"(expected 0 or SIGABRT {-signal.SIGABRT})."
+                f"\nSTDERR:\n{result.stderr}"
+                f"\nSTDOUT:\n{result.stdout}")
+        self.assertEqual(result.returncode, 0,
+                         f"Subprocess crashed with SIGABRT."
+                         f"\nSTDERR:\n{result.stderr}"
+                         f"\nSTDOUT:\n{result.stdout}")
+
+
+if __name__ == "__main__":
+    unittest.main()