tinybird-toolset 2.4.14__tar.gz → 2.5.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of tinybird-toolset might be problematic. Click here for more details.

Files changed (30) hide show
  1. {tinybird_toolset-2.4.14/src/tinybird_toolset.egg-info → tinybird_toolset-2.5.3}/PKG-INFO +1 -1
  2. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/conf.py +26 -3
  3. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/setup.py +18 -21
  4. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/src/chtoolset/query.py +1 -0
  5. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3/src/tinybird_toolset.egg-info}/PKG-INFO +1 -1
  6. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/src/tinybird_toolset.egg-info/SOURCES.txt +2 -1
  7. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_convert_to_row_binary.py +64 -0
  8. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_tables.py +1 -0
  9. tinybird_toolset-2.5.3/tests/test_validate_ttl.py +436 -0
  10. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/LICENSE +0 -0
  11. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/MANIFEST.in +0 -0
  12. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/README.md +0 -0
  13. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/setup.cfg +0 -0
  14. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/src/chtoolset/__init__.py +0 -0
  15. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/src/tinybird_toolset.egg-info/dependency_links.txt +0 -0
  16. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/src/tinybird_toolset.egg-info/requires.txt +0 -0
  17. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/src/tinybird_toolset.egg-info/top_level.txt +0 -0
  18. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_check_compatible_types.py +0 -0
  19. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_check_write_query.py +0 -0
  20. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_chquery.py +0 -0
  21. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_explain_ast.py +0 -0
  22. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_get_columns_from_create_query.py +0 -0
  23. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_get_left_table.py +0 -0
  24. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_internal_cache.py +0 -0
  25. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_normalize_query_keep_names.py +0 -0
  26. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_normalize_table_column.py +0 -0
  27. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_parse_create_materialized_view_target_table.py +0 -0
  28. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_replace_tables.py +0 -0
  29. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_replace_tables_backward_compat.py +0 -0
  30. {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_rewrite_aggregation_states.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tinybird-toolset
3
- Version: 2.4.14
3
+ Version: 2.5.3
4
4
  Home-page: https://gitlab.com/tinybird/clickhouse-toolset
5
5
  Author: Tinybird.co
6
6
  Author-email: support@tinybird.co
@@ -94,9 +94,11 @@ class ClickHouseBuildExt(build_ext):
94
94
  if not os.path.exists(os.path.join(CLICKHOUSE_PATH, 'CMakeLists.txt')):
95
95
  raise RuntimeError('Git submodules are not initialized. Run: `git submodule update --init --recursive`.')
96
96
 
97
- # One of the libraries, chosen randomly
97
+ # One of the libraries, chosen randomly (+ the TTL functions one, added later)
98
98
  if os.path.isfile(os.path.join(CLICKHOUSE_BUILD_PATH,
99
- 'src/AggregateFunctions/libclickhouse_aggregate_functions.a')):
99
+ 'src/AggregateFunctions/libclickhouse_aggregate_functions.a')) \
100
+ and os.path.isfile(os.path.join(CLICKHOUSE_BUILD_PATH,
101
+ 'src/Functions/libclickhouse_ttl_functions.a')):
100
102
  return
101
103
 
102
104
  # Avoid segfault caused by missing global context
@@ -138,7 +140,11 @@ class ClickHouseBuildExt(build_ext):
138
140
  # Removes TSA warnings (we don't compile with a libc++ with annotations)
139
141
  apply_patch("patches/tsa.patch")
140
142
 
141
- # Temporary for debugging: some functions are missing from libdbms.a, this is in the private repo pmaster already so we should include it from the repo
143
+ # Builds the functions needed by validate_ttl into libclickhouse_ttl_functions.a,
144
+ # which is linked with --whole-archive/-force_load (see extra_libs).
145
+ # NOTE: after editing this patch (or any other), the early-return above skips the
146
+ # ClickHouse build; delete ch_build/src/Functions/libclickhouse_ttl_functions.a
147
+ # (or the whole ch_build) to force it to run again.
142
148
  apply_patch("patches/dbmsfunctions.patch")
143
149
 
144
150
  # Range-check narrow-integer Field inserts so JSON/Dynamic paths widen instead of
@@ -248,6 +254,14 @@ class ClickHouseBuildExt(build_ext):
248
254
  ['--config', 'Release',
249
255
  '--target', 'src/AggregateFunctions/libclickhouse_aggregate_functions.a'])
250
256
 
257
+ # Functions for TTL validation, also with MinSize as performance is not a concern.
258
+ # This library is linked with --whole-archive/-force_load (see extra_libs) so that
259
+ # its REGISTER_FUNCTION static registrars run and DB::registerFunctions() can
260
+ # register all of them without listing them one by one.
261
+ subprocess.check_call([cmake_cmd, '--build', CLICKHOUSE_BUILD_PATH] +
262
+ ['--config', 'Release',
263
+ '--target', 'src/Functions/libclickhouse_ttl_functions.a'])
264
+
251
265
  # Now build with Release just the parser, which is where we want speed
252
266
  subprocess.check_call([cmake_cmd, CLICKHOUSE_PATH] + cmake_args + ['-DCMAKE_BUILD_TYPE=Release'],
253
267
  cwd=CLICKHOUSE_BUILD_PATH)
@@ -380,6 +394,15 @@ class CustomBuildWithFromCH(build_ext):
380
394
  # - Remove .o. These files only contain function declaration and we don't need them
381
395
  ch_libs = list(filter(lambda lib: lib and lib.find('whole-archive') == -1 and lib.find('-lm') == -1, ch_libs))
382
396
 
397
+ # Force-load the TTL validation functions so their REGISTER_FUNCTION static
398
+ # registrars run even though nothing references their symbols directly.
399
+ # DB::registerFunctions() (called from ValidateTTL.cpp) registers all of them.
400
+ ttl_functions_lib = os.path.join(CLICKHOUSE_BUILD_PATH, 'src/Functions/libclickhouse_ttl_functions.a')
401
+ if sys.platform == 'darwin':
402
+ ch_libs = [f'-Wl,-force_load,{ttl_functions_lib}'] + ch_libs
403
+ else:
404
+ ch_libs = ['-Wl,--whole-archive', ttl_functions_lib, '-Wl,--no-whole-archive'] + ch_libs
405
+
383
406
  if sys.platform == 'linux':
384
407
  ch_libs = ['-Wl,--no-whole-archive'] + ch_libs
385
408
 
@@ -1,7 +1,21 @@
1
1
  from setuptools import setup, Extension
2
2
 
3
3
  NAME = 'tinybird-toolset'
4
- VERSION = '2.4.14'
4
+ VERSION = '2.5.3'
5
+
6
+ # Shared metadata for both the full (extension) build and the metadata-only
7
+ # fallback below, so the two setup() calls can't drift apart.
8
+ common_kwargs = dict(
9
+ name=NAME,
10
+ version=VERSION,
11
+ url="https://gitlab.com/tinybird/clickhouse-toolset",
12
+ author="Tinybird.co",
13
+ author_email="support@tinybird.co",
14
+ packages=["chtoolset"],
15
+ package_dir={"": "src"},
16
+ python_requires=">=3.10, <3.15",
17
+ install_requires=[],
18
+ )
5
19
 
6
20
  try:
7
21
  from conf import *
@@ -20,6 +34,7 @@ try:
20
34
  "functions/ReplaceTables.h",
21
35
  "functions/Tables.h",
22
36
  "functions/TBQueryParser.h",
37
+ "functions/ValidateTTL.h",
23
38
  "functions/Validation.h",
24
39
  "functions/simdjsonHelpers.h",
25
40
  "functions/JSONPathQuery.h",
@@ -31,15 +46,7 @@ try:
31
46
  ],
32
47
  )
33
48
  setup(
34
- name=NAME,
35
- version=VERSION,
36
- url="https://gitlab.com/tinybird/clickhouse-toolset",
37
- author="Tinybird.co",
38
- author_email="support@tinybird.co",
39
- packages=["chtoolset"],
40
- package_dir={"": "src"},
41
- python_requires=">=3.10, <3.15",
42
- install_requires=[],
49
+ **common_kwargs,
43
50
  extras_require={"test": requirements_from_file("requirements-test.txt")},
44
51
  cmdclass={
45
52
  "clickhouse": ClickHouseBuildExt,
@@ -50,14 +57,4 @@ try:
50
57
  )
51
58
 
52
59
  except ModuleNotFoundError:
53
- setup(
54
- name=NAME,
55
- version=VERSION,
56
- url="https://gitlab.com/tinybird/clickhouse-toolset",
57
- author="Tinybird.co",
58
- author_email="support@tinybird.co",
59
- packages=["chtoolset"],
60
- package_dir={"": "src"},
61
- python_requires=">=3.10, <3.15",
62
- install_requires=[],
63
- )
60
+ setup(**common_kwargs)
@@ -20,6 +20,7 @@ from chtoolset._query import replace_tables, \
20
20
  set_row_binary_encoder_thread_buffer_size, \
21
21
  get_columns_from_create_query, \
22
22
  normalize_table_column, \
23
+ validate_ttl, \
23
24
  parse_create_materialized_view_target_table as _parse_create_materialized_view_target_table
24
25
 
25
26
  from typing import Optional, TypedDict
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tinybird-toolset
3
- Version: 2.4.14
3
+ Version: 2.5.3
4
4
  Home-page: https://gitlab.com/tinybird/clickhouse-toolset
5
5
  Author: Tinybird.co
6
6
  Author-email: support@tinybird.co
@@ -24,4 +24,5 @@ tests/test_parse_create_materialized_view_target_table.py
24
24
  tests/test_replace_tables.py
25
25
  tests/test_replace_tables_backward_compat.py
26
26
  tests/test_rewrite_aggregation_states.py
27
- tests/test_tables.py
27
+ tests/test_tables.py
28
+ tests/test_validate_ttl.py
@@ -8467,6 +8467,70 @@ TEST_CASES = [
8467
8467
 
8468
8468
 
8469
8469
  class TestRowBinaryEncoder:
8470
+ @pytest.mark.parametrize("legacy_conversion_mode", [True, False])
8471
+ def test_convert_ipv4_to_row_binary(self, legacy_conversion_mode):
8472
+ schema = json.dumps(get_schema([('address', 'IPv4', '$.address', False)]))
8473
+
8474
+ with chquery.RowBinaryEncoder(schema, legacy_conversion_mode=legacy_conversion_mode) as encoder:
8475
+ result, quarantine, nrows, nquarantine = encoder.encode('{"address": "192.0.2.1"}')
8476
+
8477
+ assert result == b'\x00\x01\x02\x00\xc0'
8478
+ assert quarantine == b''
8479
+ assert nrows == 1
8480
+ assert nquarantine == 0
8481
+
8482
+ @pytest.mark.parametrize("legacy_conversion_mode", [True, False])
8483
+ def test_convert_ipv6_to_row_binary(self, legacy_conversion_mode):
8484
+ schema = json.dumps(get_schema([('address', 'IPv6', '$.address', False)]))
8485
+
8486
+ with chquery.RowBinaryEncoder(schema, legacy_conversion_mode=legacy_conversion_mode) as encoder:
8487
+ result, quarantine, nrows, nquarantine = encoder.encode('{"address": "2001:db8::1"}')
8488
+
8489
+ assert result == b'\x00\x20\x01\x0d\xb8' + b'\x00' * 11 + b'\x01'
8490
+ assert quarantine == b''
8491
+ assert nrows == 1
8492
+ assert nquarantine == 0
8493
+
8494
+ @pytest.mark.parametrize("legacy_conversion_mode", [True, False])
8495
+ def test_convert_ipv4_formatted_address_to_ipv6_row_binary(self, legacy_conversion_mode):
8496
+ schema = json.dumps(get_schema([('address', 'IPv6', '$.address', False)]))
8497
+
8498
+ with chquery.RowBinaryEncoder(schema, legacy_conversion_mode=legacy_conversion_mode) as encoder:
8499
+ result, quarantine, nrows, nquarantine = encoder.encode('{"address": "1.1.1.1"}')
8500
+
8501
+ assert result == b'\x00' + b'\x00' * 10 + b'\xff\xff\x01\x01\x01\x01'
8502
+ assert quarantine == b''
8503
+ assert nrows == 1
8504
+ assert nquarantine == 0
8505
+
8506
+ @pytest.mark.parametrize("legacy_conversion_mode", [True, False])
8507
+ @pytest.mark.parametrize("address_type", ["IPv4", "IPv6"])
8508
+ def test_invalid_ip_address_is_quarantined(self, legacy_conversion_mode, address_type):
8509
+ schema = json.dumps(get_schema([('address', address_type, '$.address', False)]))
8510
+
8511
+ with chquery.RowBinaryEncoder(schema, legacy_conversion_mode=legacy_conversion_mode) as encoder:
8512
+ result, quarantine, nrows, nquarantine = encoder.encode('{"address": "invalid-ip"}')
8513
+
8514
+ assert result == b''
8515
+ assert quarantine != b''
8516
+ assert b"invalid-ip" in quarantine
8517
+ assert nrows == 0
8518
+ assert nquarantine == 1
8519
+
8520
+ @pytest.mark.parametrize("legacy_conversion_mode", [True, False])
8521
+ @pytest.mark.parametrize("address", ["1.1.1.1junk", "2001:db8::1junk"])
8522
+ def test_ipv6_with_trailing_junk_is_quarantined(self, legacy_conversion_mode, address):
8523
+ schema = json.dumps(get_schema([('address', 'IPv6', '$.address', False)]))
8524
+
8525
+ with chquery.RowBinaryEncoder(schema, legacy_conversion_mode=legacy_conversion_mode) as encoder:
8526
+ result, quarantine, nrows, nquarantine = encoder.encode(json.dumps({"address": address}))
8527
+
8528
+ assert result == b''
8529
+ assert quarantine != b''
8530
+ assert address.encode() in quarantine
8531
+ assert nrows == 0
8532
+ assert nquarantine == 1
8533
+
8470
8534
  @pytest.mark.parametrize("conversion_mode, schema, data, expected_result, expected_quarantine, expected_nrows, expected_nquarantine", TEST_CASES)
8471
8535
  def test_legacy_convert_to_row_binary(self, conversion_mode, schema, data, expected_result, expected_quarantine, expected_nrows, expected_nquarantine):
8472
8536
  schema_rowbinary = get_schema(schema)
@@ -2033,6 +2033,7 @@ SELECT *
2033
2033
  optimize_aggregation_in_order = 1,
2034
2034
  use_skip_indexes = 1,
2035
2035
  use_skip_indexes_if_final = 0,
2036
+ do_not_merge_across_partitions_select_final = 1,
2036
2037
  optimize_move_to_prewhere = 1,
2037
2038
  query_plan_optimize_lazy_materialization = 0,
2038
2039
  min_bytes_to_use_direct_io = 0,
@@ -0,0 +1,436 @@
1
+ import unittest
2
+ from chtoolset import query as chquery
3
+
4
+
5
+ class TestValidateTTL(unittest.TestCase):
6
+ def test_empty_ttl_is_valid(self):
7
+ self.assertIsNone(
8
+ chquery.validate_ttl(
9
+ ttl="",
10
+ columns=[{"name": "ts", "type": "DateTime"}],
11
+ sorting_key="ts",
12
+ primary_key="ts",
13
+ )
14
+ )
15
+
16
+ def test_valid_no_primary_key_uses_sorting_key(self):
17
+ self.assertIsNone(
18
+ chquery.validate_ttl(
19
+ ttl="ts + INTERVAL 7 DAY",
20
+ columns=[{"name": "ts", "type": "DateTime"}],
21
+ sorting_key="ts",
22
+ )
23
+ )
24
+
25
+ def test_valid_datetime_plus_interval(self):
26
+ self.assertIsNone(
27
+ chquery.validate_ttl(
28
+ ttl="ts + INTERVAL 7 DAY",
29
+ columns=[{"name": "ts", "type": "DateTime"}, {"name": "v", "type": "Int32"}],
30
+ sorting_key="ts",
31
+ primary_key="ts",
32
+ )
33
+ )
34
+
35
+ def test_valid_datetime_plus_to_interval(self):
36
+ self.assertIsNone(
37
+ chquery.validate_ttl(
38
+ ttl="ts + toIntervalDay(7)",
39
+ columns=[{"name": "ts", "type": "DateTime"}],
40
+ sorting_key="ts",
41
+ primary_key="ts",
42
+ )
43
+ )
44
+
45
+ def test_valid_date_column(self):
46
+ self.assertIsNone(
47
+ chquery.validate_ttl(
48
+ ttl="d + INTERVAL 30 DAY",
49
+ columns=[{"name": "d", "type": "Date"}],
50
+ sorting_key="d",
51
+ primary_key="d",
52
+ )
53
+ )
54
+
55
+ def test_valid_wrapped_datetime64(self):
56
+ self.assertIsNone(
57
+ chquery.validate_ttl(
58
+ ttl="toDateTime(event_time) + toIntervalDay(7)",
59
+ columns=[{"name": "event_time", "type": "DateTime64(3)"}],
60
+ sorting_key="event_time",
61
+ primary_key="event_time",
62
+ )
63
+ )
64
+
65
+ def test_invalid_non_date_result_type(self):
66
+ err = chquery.validate_ttl(
67
+ ttl="counter + toIntervalDay(7)",
68
+ columns=[{"name": "counter", "type": "Int32"}],
69
+ sorting_key="counter",
70
+ primary_key="counter",
71
+ )
72
+ self.assertIsNotNone(err)
73
+ self.assertIn("Illegal types", err)
74
+
75
+ def test_valid_datetime_plus_integer_expression(self):
76
+ self.assertIsNone(
77
+ chquery.validate_ttl(
78
+ ttl="ts + 7",
79
+ columns=[{"name": "ts", "type": "DateTime"}],
80
+ sorting_key="ts",
81
+ primary_key="ts",
82
+ )
83
+ )
84
+
85
+ def test_invalid_unwrapped_datetime64(self):
86
+ err = chquery.validate_ttl(
87
+ ttl="event_time + toIntervalDay(7)",
88
+ columns=[{"name": "event_time", "type": "DateTime64(3)"}],
89
+ sorting_key="event_time",
90
+ primary_key="event_time",
91
+ )
92
+ self.assertIsNotNone(err)
93
+ self.assertIn("TTL expression result column should have DateTime or Date type", err)
94
+
95
+ def test_invalid_unknown_column(self):
96
+ err = chquery.validate_ttl(
97
+ ttl="missing + INTERVAL 7 DAY",
98
+ columns=[{"name": "ts", "type": "DateTime"}],
99
+ sorting_key="ts",
100
+ primary_key="ts",
101
+ )
102
+ self.assertIsNotNone(err)
103
+ self.assertIn("Missing columns: 'missing'", err)
104
+
105
+ def test_invalid_malformed_ttl(self):
106
+ err = chquery.validate_ttl(
107
+ ttl="this is not a valid ttl !!!",
108
+ columns=[{"name": "ts", "type": "DateTime"}],
109
+ sorting_key="ts",
110
+ primary_key="ts",
111
+ )
112
+ self.assertIsNotNone(err)
113
+ self.assertIn("Syntax error", err)
114
+
115
+ # --- Date/time functions supported by ClickHouse in TTL expressions ---
116
+ def test_valid_from_unix_timestamp(self):
117
+ self.assertIsNone(
118
+ chquery.validate_ttl(
119
+ ttl="fromUnixTimestamp(uts) + INTERVAL 7 DAY",
120
+ columns=[{"name": "uts", "type": "UInt32"}],
121
+ sorting_key="uts",
122
+ primary_key="uts",
123
+ )
124
+ )
125
+
126
+ def test_valid_from_unix_timestamp64_milli(self):
127
+ self.assertIsNone(
128
+ chquery.validate_ttl(
129
+ ttl="toDateTime(fromUnixTimestamp64Milli(uts)) + INTERVAL 7 DAY",
130
+ columns=[{"name": "uts", "type": "Int64"}],
131
+ sorting_key="uts",
132
+ primary_key="uts",
133
+ )
134
+ )
135
+
136
+ def test_valid_from_unix_timestamp64_micro_and_nano(self):
137
+ for fn in ("fromUnixTimestamp64Micro", "fromUnixTimestamp64Nano"):
138
+ with self.subTest(fn=fn):
139
+ self.assertIsNone(
140
+ chquery.validate_ttl(
141
+ ttl=f"toDateTime({fn}(uts)) + INTERVAL 1 DAY",
142
+ columns=[{"name": "uts", "type": "Int64"}],
143
+ sorting_key="uts",
144
+ primary_key="uts",
145
+ )
146
+ )
147
+
148
+ def test_valid_to_start_of_functions(self):
149
+ for fn in (
150
+ "toStartOfDay",
151
+ "toStartOfHour",
152
+ "toStartOfMinute",
153
+ "toStartOfFifteenMinutes",
154
+ "toStartOfMonth",
155
+ "toStartOfQuarter",
156
+ "toStartOfYear",
157
+ "toStartOfWeek",
158
+ "toMonday",
159
+ "toLastDayOfMonth",
160
+ ):
161
+ with self.subTest(fn=fn):
162
+ self.assertIsNone(
163
+ chquery.validate_ttl(
164
+ ttl=f"{fn}(ts) + INTERVAL 30 DAY",
165
+ columns=[{"name": "ts", "type": "DateTime"}],
166
+ sorting_key="ts",
167
+ primary_key="ts",
168
+ )
169
+ )
170
+
171
+ def test_valid_to_start_of_interval(self):
172
+ self.assertIsNone(
173
+ chquery.validate_ttl(
174
+ ttl="toStartOfInterval(ts, INTERVAL 1 DAY) + INTERVAL 90 DAY",
175
+ columns=[{"name": "ts", "type": "DateTime"}],
176
+ sorting_key="ts",
177
+ primary_key="ts",
178
+ )
179
+ )
180
+
181
+ def test_valid_date_trunc(self):
182
+ self.assertIsNone(
183
+ chquery.validate_ttl(
184
+ ttl="dateTrunc('month', ts) + INTERVAL 6 MONTH",
185
+ columns=[{"name": "ts", "type": "DateTime"}],
186
+ sorting_key="ts",
187
+ primary_key="ts",
188
+ )
189
+ )
190
+
191
+ def test_valid_to_timezone(self):
192
+ self.assertIsNone(
193
+ chquery.validate_ttl(
194
+ ttl="toTimezone(ts, 'UTC') + INTERVAL 7 DAY",
195
+ columns=[{"name": "ts", "type": "DateTime"}],
196
+ sorting_key="ts",
197
+ primary_key="ts",
198
+ )
199
+ )
200
+
201
+ def test_valid_make_date(self):
202
+ self.assertIsNone(
203
+ chquery.validate_ttl(
204
+ ttl="makeDate(y, m, d) + INTERVAL 7 DAY",
205
+ columns=[
206
+ {"name": "y", "type": "UInt16"},
207
+ {"name": "m", "type": "UInt8"},
208
+ {"name": "d", "type": "UInt8"},
209
+ ],
210
+ sorting_key="y",
211
+ primary_key="y",
212
+ )
213
+ )
214
+
215
+ # --- Conditional / multi-clause TTLs ---
216
+ def test_valid_ttl_with_where(self):
217
+ self.assertIsNone(
218
+ chquery.validate_ttl(
219
+ ttl="ts + INTERVAL 7 DAY WHERE status = 'expired'",
220
+ columns=[
221
+ {"name": "ts", "type": "DateTime"},
222
+ {"name": "status", "type": "String"},
223
+ ],
224
+ sorting_key="ts",
225
+ primary_key="ts",
226
+ )
227
+ )
228
+
229
+ def test_valid_ttl_with_where_numeric(self):
230
+ self.assertIsNone(
231
+ chquery.validate_ttl(
232
+ ttl="ts + INTERVAL 30 DAY WHERE x % 10 == 0 AND y > 5",
233
+ columns=[
234
+ {"name": "ts", "type": "DateTime"},
235
+ {"name": "x", "type": "Int32"},
236
+ {"name": "y", "type": "Int32"},
237
+ ],
238
+ sorting_key="ts",
239
+ primary_key="ts",
240
+ )
241
+ )
242
+
243
+ def test_invalid_ttl_where_unknown_column(self):
244
+ err = chquery.validate_ttl(
245
+ ttl="ts + INTERVAL 7 DAY WHERE missing = 1",
246
+ columns=[{"name": "ts", "type": "DateTime"}],
247
+ sorting_key="ts",
248
+ primary_key="ts",
249
+ )
250
+ self.assertIsNotNone(err)
251
+ self.assertIn("Missing columns: 'missing'", err)
252
+
253
+ def test_valid_ttl_where_string_expression(self):
254
+ self.assertIsNone(
255
+ chquery.validate_ttl(
256
+ ttl="ts + INTERVAL 7 DAY WHERE status",
257
+ columns=[
258
+ {"name": "ts", "type": "DateTime"},
259
+ {"name": "status", "type": "String"},
260
+ ],
261
+ sorting_key="ts",
262
+ primary_key="ts",
263
+ )
264
+ )
265
+
266
+ def test_valid_multiple_ttls_comma_separated(self):
267
+ self.assertIsNone(
268
+ chquery.validate_ttl(
269
+ ttl="ts + INTERVAL 7 DAY WHERE status = 'a', ts + INTERVAL 14 DAY WHERE status = 'b'",
270
+ columns=[
271
+ {"name": "ts", "type": "DateTime"},
272
+ {"name": "status", "type": "String"},
273
+ ],
274
+ sorting_key="ts",
275
+ primary_key="ts",
276
+ )
277
+ )
278
+
279
+ def test_valid_ttl_group_by(self):
280
+ # GROUP BY rollup; must group by a prefix of the primary key
281
+ self.assertIsNone(
282
+ chquery.validate_ttl(
283
+ ttl="ts + INTERVAL 1 DAY GROUP BY k SET v = sum(v)",
284
+ columns=[
285
+ {"name": "k", "type": "String"},
286
+ {"name": "ts", "type": "DateTime"},
287
+ {"name": "v", "type": "Int64"},
288
+ ],
289
+ sorting_key="k",
290
+ primary_key="k",
291
+ )
292
+ )
293
+
294
+ def test_invalid_ttl_group_by_not_in_primary_key(self):
295
+ err = chquery.validate_ttl(
296
+ ttl="ts + INTERVAL 1 DAY GROUP BY missing SET v = sum(v)",
297
+ columns=[
298
+ {"name": "k", "type": "String"},
299
+ {"name": "ts", "type": "DateTime"},
300
+ {"name": "v", "type": "Int64"},
301
+ ],
302
+ sorting_key="k",
303
+ primary_key="k",
304
+ )
305
+ self.assertIsNotNone(err)
306
+ self.assertIn("GROUP BY key should be a prefix of primary key", err)
307
+
308
+ def test_valid_where_string_functions(self):
309
+ columns = [
310
+ {"name": "ts", "type": "DateTime"},
311
+ {"name": "s", "type": "String"},
312
+ {"name": "n", "type": "Int32"},
313
+ ]
314
+ predicates = {
315
+ "concat": "concat(s, 'x') = 'ax'",
316
+ "startsWith": "startsWith(s, 'a')",
317
+ "endsWith": "endsWith(s, 'z')",
318
+ "length": "length(s) > 3",
319
+ "lower": "lower(s) = 'a'",
320
+ "substring": "substring(s, 1, 2) = 'ab'",
321
+ "match": "match(s, '^a')",
322
+ "in": "n IN (1, 2, 3)",
323
+ }
324
+ for name, predicate in predicates.items():
325
+ with self.subTest(fn=name):
326
+ self.assertIsNone(
327
+ chquery.validate_ttl(
328
+ ttl=f"ts + INTERVAL 7 DAY WHERE {predicate}",
329
+ columns=columns,
330
+ sorting_key="ts",
331
+ primary_key="ts",
332
+ ),
333
+ f"{name} should be a known function in TTL WHERE clauses",
334
+ )
335
+
336
+ # --- Hash functions in sorting / primary keys (KeyDescription::parse) ---
337
+ def test_valid_sorting_key_with_cityhash64(self):
338
+ self.assertIsNone(
339
+ chquery.validate_ttl(
340
+ ttl="timestamp + INTERVAL 365 DAY",
341
+ columns=[
342
+ {"name": "organizationId", "type": "String"},
343
+ {"name": "surfaceId", "type": "String"},
344
+ {"name": "timestamp", "type": "DateTime"},
345
+ {"name": "name", "type": "String"},
346
+ {"name": "traceId", "type": "String"},
347
+ {"name": "spanId", "type": "String"},
348
+ {"name": "eventIndex", "type": "UInt32"},
349
+ ],
350
+ sorting_key="organizationId, surfaceId, toDate(timestamp), name, cityHash64(traceId), spanId, eventIndex",
351
+ )
352
+ )
353
+
354
+ # --- Arithmetic functions in TTL expressions ---
355
+ def test_valid_ttl_with_int_div(self):
356
+ # Reported failure: "Unknown function intDiv".
357
+ self.assertIsNone(
358
+ chquery.validate_ttl(
359
+ ttl="toDateTime(intDiv(ts_server, 1000)) + INTERVAL 13 MONTH",
360
+ columns=[{"name": "ts_server", "type": "Int64"}],
361
+ sorting_key="ts_server",
362
+ )
363
+ )
364
+
365
+ def test_valid_ttl_with_greatest(self):
366
+ # Reported failure: "Unknown function greatest".
367
+ self.assertIsNone(
368
+ chquery.validate_ttl(
369
+ ttl="greatest(departure_date, toDateTime(creation_date)) + toIntervalDay(2)",
370
+ columns=[
371
+ {"name": "departure_date", "type": "DateTime"},
372
+ {"name": "creation_date", "type": "Date"},
373
+ ],
374
+ sorting_key="departure_date",
375
+ )
376
+ )
377
+
378
+ # --- Input validation ---
379
+ def test_columns_must_be_list_of_dicts(self):
380
+ with self.assertRaises(TypeError):
381
+ chquery.validate_ttl(ttl="ts + INTERVAL 7 DAY", columns="ts DateTime", sorting_key="ts", primary_key="ts")
382
+
383
+ def test_column_dict_requires_name_and_type(self):
384
+ with self.assertRaises((TypeError, ValueError)):
385
+ chquery.validate_ttl(
386
+ ttl="ts + INTERVAL 7 DAY",
387
+ columns=[{"name": "ts"}],
388
+ sorting_key="ts",
389
+ primary_key="ts",
390
+ )
391
+
392
+
393
+ # --- Regression guard: sequential throwing calls must not crash ---
394
+ def test_sequential_throwing_calls_do_not_crash(self):
395
+ # A validate_ttl call that throws must not corrupt shared state and take
396
+ # down a subsequent call. validateTTL parses against a private per-call
397
+ # context copy (functions/ValidateTTL.cpp) precisely so this holds.
398
+ # Run in a subprocess so a hypothetical crash doesn't kill pytest; the
399
+ # child must exit 0.
400
+ import signal
401
+ import subprocess
402
+ import sys
403
+ import textwrap
404
+ repro = textwrap.dedent("""\
405
+ from chtoolset import query as q
406
+ err1 = q.validate_ttl(
407
+ ttl="counter + toIntervalDay(7)",
408
+ columns=[{"name": "counter", "type": "Int32"}],
409
+ sorting_key="counter",
410
+ primary_key="counter",
411
+ )
412
+ assert err1 is not None and "Illegal types" in err1, (
413
+ f"call 1 should return an 'Illegal types' error, got: {err1!r}")
414
+ q.validate_ttl(
415
+ ttl="ts + INTERVAL 7 DAY WHERE missing = 1",
416
+ columns=[{"name": "ts", "type": "DateTime"}],
417
+ sorting_key="ts",
418
+ primary_key="ts",
419
+ )
420
+ """)
421
+ result = subprocess.run([sys.executable, "-c", repro],
422
+ capture_output=True, text=True, timeout=30)
423
+ if result.returncode != 0 and result.returncode != -signal.SIGABRT:
424
+ raise RuntimeError(
425
+ f"Unexpected subprocess exit {result.returncode} "
426
+ f"(expected 0 or SIGABRT {-signal.SIGABRT})."
427
+ f"\nSTDERR:\n{result.stderr}"
428
+ f"\nSTDOUT:\n{result.stdout}")
429
+ self.assertEqual(result.returncode, 0,
430
+ f"Subprocess crashed with SIGABRT."
431
+ f"\nSTDERR:\n{result.stderr}"
432
+ f"\nSTDOUT:\n{result.stdout}")
433
+
434
+
435
+ if __name__ == "__main__":
436
+ unittest.main()