tinybird-toolset 2.4.14__tar.gz → 2.5.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird-toolset might be problematic. Click here for more details.
- {tinybird_toolset-2.4.14/src/tinybird_toolset.egg-info → tinybird_toolset-2.5.3}/PKG-INFO +1 -1
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/conf.py +26 -3
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/setup.py +18 -21
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/src/chtoolset/query.py +1 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3/src/tinybird_toolset.egg-info}/PKG-INFO +1 -1
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/src/tinybird_toolset.egg-info/SOURCES.txt +2 -1
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_convert_to_row_binary.py +64 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_tables.py +1 -0
- tinybird_toolset-2.5.3/tests/test_validate_ttl.py +436 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/LICENSE +0 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/MANIFEST.in +0 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/README.md +0 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/setup.cfg +0 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/src/chtoolset/__init__.py +0 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/src/tinybird_toolset.egg-info/dependency_links.txt +0 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/src/tinybird_toolset.egg-info/requires.txt +0 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/src/tinybird_toolset.egg-info/top_level.txt +0 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_check_compatible_types.py +0 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_check_write_query.py +0 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_chquery.py +0 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_explain_ast.py +0 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_get_columns_from_create_query.py +0 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_get_left_table.py +0 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_internal_cache.py +0 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_normalize_query_keep_names.py +0 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_normalize_table_column.py +0 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_parse_create_materialized_view_target_table.py +0 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_replace_tables.py +0 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_replace_tables_backward_compat.py +0 -0
- {tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_rewrite_aggregation_states.py +0 -0
|
@@ -94,9 +94,11 @@ class ClickHouseBuildExt(build_ext):
|
|
|
94
94
|
if not os.path.exists(os.path.join(CLICKHOUSE_PATH, 'CMakeLists.txt')):
|
|
95
95
|
raise RuntimeError('Git submodules are not initialized. Run: `git submodule update --init --recursive`.')
|
|
96
96
|
|
|
97
|
-
# One of the libraries, chosen randomly
|
|
97
|
+
# One of the libraries, chosen randomly (+ the TTL functions one, added later)
|
|
98
98
|
if os.path.isfile(os.path.join(CLICKHOUSE_BUILD_PATH,
|
|
99
|
-
'src/AggregateFunctions/libclickhouse_aggregate_functions.a'))
|
|
99
|
+
'src/AggregateFunctions/libclickhouse_aggregate_functions.a')) \
|
|
100
|
+
and os.path.isfile(os.path.join(CLICKHOUSE_BUILD_PATH,
|
|
101
|
+
'src/Functions/libclickhouse_ttl_functions.a')):
|
|
100
102
|
return
|
|
101
103
|
|
|
102
104
|
# Avoid segfault caused by missing global context
|
|
@@ -138,7 +140,11 @@ class ClickHouseBuildExt(build_ext):
|
|
|
138
140
|
# Removes TSA warnings (we don't compile with a libc++ with annotations)
|
|
139
141
|
apply_patch("patches/tsa.patch")
|
|
140
142
|
|
|
141
|
-
#
|
|
143
|
+
# Builds the functions needed by validate_ttl into libclickhouse_ttl_functions.a,
|
|
144
|
+
# which is linked with --whole-archive/-force_load (see extra_libs).
|
|
145
|
+
# NOTE: after editing this patch (or any other), the early-return above skips the
|
|
146
|
+
# ClickHouse build; delete ch_build/src/Functions/libclickhouse_ttl_functions.a
|
|
147
|
+
# (or the whole ch_build) to force it to run again.
|
|
142
148
|
apply_patch("patches/dbmsfunctions.patch")
|
|
143
149
|
|
|
144
150
|
# Range-check narrow-integer Field inserts so JSON/Dynamic paths widen instead of
|
|
@@ -248,6 +254,14 @@ class ClickHouseBuildExt(build_ext):
|
|
|
248
254
|
['--config', 'Release',
|
|
249
255
|
'--target', 'src/AggregateFunctions/libclickhouse_aggregate_functions.a'])
|
|
250
256
|
|
|
257
|
+
# Functions for TTL validation, also with MinSize as performance is not a concern.
|
|
258
|
+
# This library is linked with --whole-archive/-force_load (see extra_libs) so that
|
|
259
|
+
# its REGISTER_FUNCTION static registrars run and DB::registerFunctions() can
|
|
260
|
+
# register all of them without listing them one by one.
|
|
261
|
+
subprocess.check_call([cmake_cmd, '--build', CLICKHOUSE_BUILD_PATH] +
|
|
262
|
+
['--config', 'Release',
|
|
263
|
+
'--target', 'src/Functions/libclickhouse_ttl_functions.a'])
|
|
264
|
+
|
|
251
265
|
# Now build with Release just the parser, which is where we want speed
|
|
252
266
|
subprocess.check_call([cmake_cmd, CLICKHOUSE_PATH] + cmake_args + ['-DCMAKE_BUILD_TYPE=Release'],
|
|
253
267
|
cwd=CLICKHOUSE_BUILD_PATH)
|
|
@@ -380,6 +394,15 @@ class CustomBuildWithFromCH(build_ext):
|
|
|
380
394
|
# - Remove .o. These files only contain function declaration and we don't need them
|
|
381
395
|
ch_libs = list(filter(lambda lib: lib and lib.find('whole-archive') == -1 and lib.find('-lm') == -1, ch_libs))
|
|
382
396
|
|
|
397
|
+
# Force-load the TTL validation functions so their REGISTER_FUNCTION static
|
|
398
|
+
# registrars run even though nothing references their symbols directly.
|
|
399
|
+
# DB::registerFunctions() (called from ValidateTTL.cpp) registers all of them.
|
|
400
|
+
ttl_functions_lib = os.path.join(CLICKHOUSE_BUILD_PATH, 'src/Functions/libclickhouse_ttl_functions.a')
|
|
401
|
+
if sys.platform == 'darwin':
|
|
402
|
+
ch_libs = [f'-Wl,-force_load,{ttl_functions_lib}'] + ch_libs
|
|
403
|
+
else:
|
|
404
|
+
ch_libs = ['-Wl,--whole-archive', ttl_functions_lib, '-Wl,--no-whole-archive'] + ch_libs
|
|
405
|
+
|
|
383
406
|
if sys.platform == 'linux':
|
|
384
407
|
ch_libs = ['-Wl,--no-whole-archive'] + ch_libs
|
|
385
408
|
|
|
@@ -1,7 +1,21 @@
|
|
|
1
1
|
from setuptools import setup, Extension
|
|
2
2
|
|
|
3
3
|
NAME = 'tinybird-toolset'
|
|
4
|
-
VERSION = '2.
|
|
4
|
+
VERSION = '2.5.3'
|
|
5
|
+
|
|
6
|
+
# Shared metadata for both the full (extension) build and the metadata-only
|
|
7
|
+
# fallback below, so the two setup() calls can't drift apart.
|
|
8
|
+
common_kwargs = dict(
|
|
9
|
+
name=NAME,
|
|
10
|
+
version=VERSION,
|
|
11
|
+
url="https://gitlab.com/tinybird/clickhouse-toolset",
|
|
12
|
+
author="Tinybird.co",
|
|
13
|
+
author_email="support@tinybird.co",
|
|
14
|
+
packages=["chtoolset"],
|
|
15
|
+
package_dir={"": "src"},
|
|
16
|
+
python_requires=">=3.10, <3.15",
|
|
17
|
+
install_requires=[],
|
|
18
|
+
)
|
|
5
19
|
|
|
6
20
|
try:
|
|
7
21
|
from conf import *
|
|
@@ -20,6 +34,7 @@ try:
|
|
|
20
34
|
"functions/ReplaceTables.h",
|
|
21
35
|
"functions/Tables.h",
|
|
22
36
|
"functions/TBQueryParser.h",
|
|
37
|
+
"functions/ValidateTTL.h",
|
|
23
38
|
"functions/Validation.h",
|
|
24
39
|
"functions/simdjsonHelpers.h",
|
|
25
40
|
"functions/JSONPathQuery.h",
|
|
@@ -31,15 +46,7 @@ try:
|
|
|
31
46
|
],
|
|
32
47
|
)
|
|
33
48
|
setup(
|
|
34
|
-
|
|
35
|
-
version=VERSION,
|
|
36
|
-
url="https://gitlab.com/tinybird/clickhouse-toolset",
|
|
37
|
-
author="Tinybird.co",
|
|
38
|
-
author_email="support@tinybird.co",
|
|
39
|
-
packages=["chtoolset"],
|
|
40
|
-
package_dir={"": "src"},
|
|
41
|
-
python_requires=">=3.10, <3.15",
|
|
42
|
-
install_requires=[],
|
|
49
|
+
**common_kwargs,
|
|
43
50
|
extras_require={"test": requirements_from_file("requirements-test.txt")},
|
|
44
51
|
cmdclass={
|
|
45
52
|
"clickhouse": ClickHouseBuildExt,
|
|
@@ -50,14 +57,4 @@ try:
|
|
|
50
57
|
)
|
|
51
58
|
|
|
52
59
|
except ModuleNotFoundError:
|
|
53
|
-
setup(
|
|
54
|
-
name=NAME,
|
|
55
|
-
version=VERSION,
|
|
56
|
-
url="https://gitlab.com/tinybird/clickhouse-toolset",
|
|
57
|
-
author="Tinybird.co",
|
|
58
|
-
author_email="support@tinybird.co",
|
|
59
|
-
packages=["chtoolset"],
|
|
60
|
-
package_dir={"": "src"},
|
|
61
|
-
python_requires=">=3.10, <3.15",
|
|
62
|
-
install_requires=[],
|
|
63
|
-
)
|
|
60
|
+
setup(**common_kwargs)
|
|
@@ -20,6 +20,7 @@ from chtoolset._query import replace_tables, \
|
|
|
20
20
|
set_row_binary_encoder_thread_buffer_size, \
|
|
21
21
|
get_columns_from_create_query, \
|
|
22
22
|
normalize_table_column, \
|
|
23
|
+
validate_ttl, \
|
|
23
24
|
parse_create_materialized_view_target_table as _parse_create_materialized_view_target_table
|
|
24
25
|
|
|
25
26
|
from typing import Optional, TypedDict
|
|
@@ -8467,6 +8467,70 @@ TEST_CASES = [
|
|
|
8467
8467
|
|
|
8468
8468
|
|
|
8469
8469
|
class TestRowBinaryEncoder:
|
|
8470
|
+
@pytest.mark.parametrize("legacy_conversion_mode", [True, False])
|
|
8471
|
+
def test_convert_ipv4_to_row_binary(self, legacy_conversion_mode):
|
|
8472
|
+
schema = json.dumps(get_schema([('address', 'IPv4', '$.address', False)]))
|
|
8473
|
+
|
|
8474
|
+
with chquery.RowBinaryEncoder(schema, legacy_conversion_mode=legacy_conversion_mode) as encoder:
|
|
8475
|
+
result, quarantine, nrows, nquarantine = encoder.encode('{"address": "192.0.2.1"}')
|
|
8476
|
+
|
|
8477
|
+
assert result == b'\x00\x01\x02\x00\xc0'
|
|
8478
|
+
assert quarantine == b''
|
|
8479
|
+
assert nrows == 1
|
|
8480
|
+
assert nquarantine == 0
|
|
8481
|
+
|
|
8482
|
+
@pytest.mark.parametrize("legacy_conversion_mode", [True, False])
|
|
8483
|
+
def test_convert_ipv6_to_row_binary(self, legacy_conversion_mode):
|
|
8484
|
+
schema = json.dumps(get_schema([('address', 'IPv6', '$.address', False)]))
|
|
8485
|
+
|
|
8486
|
+
with chquery.RowBinaryEncoder(schema, legacy_conversion_mode=legacy_conversion_mode) as encoder:
|
|
8487
|
+
result, quarantine, nrows, nquarantine = encoder.encode('{"address": "2001:db8::1"}')
|
|
8488
|
+
|
|
8489
|
+
assert result == b'\x00\x20\x01\x0d\xb8' + b'\x00' * 11 + b'\x01'
|
|
8490
|
+
assert quarantine == b''
|
|
8491
|
+
assert nrows == 1
|
|
8492
|
+
assert nquarantine == 0
|
|
8493
|
+
|
|
8494
|
+
@pytest.mark.parametrize("legacy_conversion_mode", [True, False])
|
|
8495
|
+
def test_convert_ipv4_formatted_address_to_ipv6_row_binary(self, legacy_conversion_mode):
|
|
8496
|
+
schema = json.dumps(get_schema([('address', 'IPv6', '$.address', False)]))
|
|
8497
|
+
|
|
8498
|
+
with chquery.RowBinaryEncoder(schema, legacy_conversion_mode=legacy_conversion_mode) as encoder:
|
|
8499
|
+
result, quarantine, nrows, nquarantine = encoder.encode('{"address": "1.1.1.1"}')
|
|
8500
|
+
|
|
8501
|
+
assert result == b'\x00' + b'\x00' * 10 + b'\xff\xff\x01\x01\x01\x01'
|
|
8502
|
+
assert quarantine == b''
|
|
8503
|
+
assert nrows == 1
|
|
8504
|
+
assert nquarantine == 0
|
|
8505
|
+
|
|
8506
|
+
@pytest.mark.parametrize("legacy_conversion_mode", [True, False])
|
|
8507
|
+
@pytest.mark.parametrize("address_type", ["IPv4", "IPv6"])
|
|
8508
|
+
def test_invalid_ip_address_is_quarantined(self, legacy_conversion_mode, address_type):
|
|
8509
|
+
schema = json.dumps(get_schema([('address', address_type, '$.address', False)]))
|
|
8510
|
+
|
|
8511
|
+
with chquery.RowBinaryEncoder(schema, legacy_conversion_mode=legacy_conversion_mode) as encoder:
|
|
8512
|
+
result, quarantine, nrows, nquarantine = encoder.encode('{"address": "invalid-ip"}')
|
|
8513
|
+
|
|
8514
|
+
assert result == b''
|
|
8515
|
+
assert quarantine != b''
|
|
8516
|
+
assert b"invalid-ip" in quarantine
|
|
8517
|
+
assert nrows == 0
|
|
8518
|
+
assert nquarantine == 1
|
|
8519
|
+
|
|
8520
|
+
@pytest.mark.parametrize("legacy_conversion_mode", [True, False])
|
|
8521
|
+
@pytest.mark.parametrize("address", ["1.1.1.1junk", "2001:db8::1junk"])
|
|
8522
|
+
def test_ipv6_with_trailing_junk_is_quarantined(self, legacy_conversion_mode, address):
|
|
8523
|
+
schema = json.dumps(get_schema([('address', 'IPv6', '$.address', False)]))
|
|
8524
|
+
|
|
8525
|
+
with chquery.RowBinaryEncoder(schema, legacy_conversion_mode=legacy_conversion_mode) as encoder:
|
|
8526
|
+
result, quarantine, nrows, nquarantine = encoder.encode(json.dumps({"address": address}))
|
|
8527
|
+
|
|
8528
|
+
assert result == b''
|
|
8529
|
+
assert quarantine != b''
|
|
8530
|
+
assert address.encode() in quarantine
|
|
8531
|
+
assert nrows == 0
|
|
8532
|
+
assert nquarantine == 1
|
|
8533
|
+
|
|
8470
8534
|
@pytest.mark.parametrize("conversion_mode, schema, data, expected_result, expected_quarantine, expected_nrows, expected_nquarantine", TEST_CASES)
|
|
8471
8535
|
def test_legacy_convert_to_row_binary(self, conversion_mode, schema, data, expected_result, expected_quarantine, expected_nrows, expected_nquarantine):
|
|
8472
8536
|
schema_rowbinary = get_schema(schema)
|
|
@@ -2033,6 +2033,7 @@ SELECT *
|
|
|
2033
2033
|
optimize_aggregation_in_order = 1,
|
|
2034
2034
|
use_skip_indexes = 1,
|
|
2035
2035
|
use_skip_indexes_if_final = 0,
|
|
2036
|
+
do_not_merge_across_partitions_select_final = 1,
|
|
2036
2037
|
optimize_move_to_prewhere = 1,
|
|
2037
2038
|
query_plan_optimize_lazy_materialization = 0,
|
|
2038
2039
|
min_bytes_to_use_direct_io = 0,
|
|
@@ -0,0 +1,436 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
from chtoolset import query as chquery
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class TestValidateTTL(unittest.TestCase):
|
|
6
|
+
def test_empty_ttl_is_valid(self):
|
|
7
|
+
self.assertIsNone(
|
|
8
|
+
chquery.validate_ttl(
|
|
9
|
+
ttl="",
|
|
10
|
+
columns=[{"name": "ts", "type": "DateTime"}],
|
|
11
|
+
sorting_key="ts",
|
|
12
|
+
primary_key="ts",
|
|
13
|
+
)
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
def test_valid_no_primary_key_uses_sorting_key(self):
|
|
17
|
+
self.assertIsNone(
|
|
18
|
+
chquery.validate_ttl(
|
|
19
|
+
ttl="ts + INTERVAL 7 DAY",
|
|
20
|
+
columns=[{"name": "ts", "type": "DateTime"}],
|
|
21
|
+
sorting_key="ts",
|
|
22
|
+
)
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
def test_valid_datetime_plus_interval(self):
|
|
26
|
+
self.assertIsNone(
|
|
27
|
+
chquery.validate_ttl(
|
|
28
|
+
ttl="ts + INTERVAL 7 DAY",
|
|
29
|
+
columns=[{"name": "ts", "type": "DateTime"}, {"name": "v", "type": "Int32"}],
|
|
30
|
+
sorting_key="ts",
|
|
31
|
+
primary_key="ts",
|
|
32
|
+
)
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
def test_valid_datetime_plus_to_interval(self):
|
|
36
|
+
self.assertIsNone(
|
|
37
|
+
chquery.validate_ttl(
|
|
38
|
+
ttl="ts + toIntervalDay(7)",
|
|
39
|
+
columns=[{"name": "ts", "type": "DateTime"}],
|
|
40
|
+
sorting_key="ts",
|
|
41
|
+
primary_key="ts",
|
|
42
|
+
)
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
def test_valid_date_column(self):
|
|
46
|
+
self.assertIsNone(
|
|
47
|
+
chquery.validate_ttl(
|
|
48
|
+
ttl="d + INTERVAL 30 DAY",
|
|
49
|
+
columns=[{"name": "d", "type": "Date"}],
|
|
50
|
+
sorting_key="d",
|
|
51
|
+
primary_key="d",
|
|
52
|
+
)
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
def test_valid_wrapped_datetime64(self):
|
|
56
|
+
self.assertIsNone(
|
|
57
|
+
chquery.validate_ttl(
|
|
58
|
+
ttl="toDateTime(event_time) + toIntervalDay(7)",
|
|
59
|
+
columns=[{"name": "event_time", "type": "DateTime64(3)"}],
|
|
60
|
+
sorting_key="event_time",
|
|
61
|
+
primary_key="event_time",
|
|
62
|
+
)
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
def test_invalid_non_date_result_type(self):
|
|
66
|
+
err = chquery.validate_ttl(
|
|
67
|
+
ttl="counter + toIntervalDay(7)",
|
|
68
|
+
columns=[{"name": "counter", "type": "Int32"}],
|
|
69
|
+
sorting_key="counter",
|
|
70
|
+
primary_key="counter",
|
|
71
|
+
)
|
|
72
|
+
self.assertIsNotNone(err)
|
|
73
|
+
self.assertIn("Illegal types", err)
|
|
74
|
+
|
|
75
|
+
def test_valid_datetime_plus_integer_expression(self):
|
|
76
|
+
self.assertIsNone(
|
|
77
|
+
chquery.validate_ttl(
|
|
78
|
+
ttl="ts + 7",
|
|
79
|
+
columns=[{"name": "ts", "type": "DateTime"}],
|
|
80
|
+
sorting_key="ts",
|
|
81
|
+
primary_key="ts",
|
|
82
|
+
)
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
def test_invalid_unwrapped_datetime64(self):
|
|
86
|
+
err = chquery.validate_ttl(
|
|
87
|
+
ttl="event_time + toIntervalDay(7)",
|
|
88
|
+
columns=[{"name": "event_time", "type": "DateTime64(3)"}],
|
|
89
|
+
sorting_key="event_time",
|
|
90
|
+
primary_key="event_time",
|
|
91
|
+
)
|
|
92
|
+
self.assertIsNotNone(err)
|
|
93
|
+
self.assertIn("TTL expression result column should have DateTime or Date type", err)
|
|
94
|
+
|
|
95
|
+
def test_invalid_unknown_column(self):
|
|
96
|
+
err = chquery.validate_ttl(
|
|
97
|
+
ttl="missing + INTERVAL 7 DAY",
|
|
98
|
+
columns=[{"name": "ts", "type": "DateTime"}],
|
|
99
|
+
sorting_key="ts",
|
|
100
|
+
primary_key="ts",
|
|
101
|
+
)
|
|
102
|
+
self.assertIsNotNone(err)
|
|
103
|
+
self.assertIn("Missing columns: 'missing'", err)
|
|
104
|
+
|
|
105
|
+
def test_invalid_malformed_ttl(self):
|
|
106
|
+
err = chquery.validate_ttl(
|
|
107
|
+
ttl="this is not a valid ttl !!!",
|
|
108
|
+
columns=[{"name": "ts", "type": "DateTime"}],
|
|
109
|
+
sorting_key="ts",
|
|
110
|
+
primary_key="ts",
|
|
111
|
+
)
|
|
112
|
+
self.assertIsNotNone(err)
|
|
113
|
+
self.assertIn("Syntax error", err)
|
|
114
|
+
|
|
115
|
+
# --- Date/time functions supported by ClickHouse in TTL expressions ---
|
|
116
|
+
def test_valid_from_unix_timestamp(self):
|
|
117
|
+
self.assertIsNone(
|
|
118
|
+
chquery.validate_ttl(
|
|
119
|
+
ttl="fromUnixTimestamp(uts) + INTERVAL 7 DAY",
|
|
120
|
+
columns=[{"name": "uts", "type": "UInt32"}],
|
|
121
|
+
sorting_key="uts",
|
|
122
|
+
primary_key="uts",
|
|
123
|
+
)
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
def test_valid_from_unix_timestamp64_milli(self):
|
|
127
|
+
self.assertIsNone(
|
|
128
|
+
chquery.validate_ttl(
|
|
129
|
+
ttl="toDateTime(fromUnixTimestamp64Milli(uts)) + INTERVAL 7 DAY",
|
|
130
|
+
columns=[{"name": "uts", "type": "Int64"}],
|
|
131
|
+
sorting_key="uts",
|
|
132
|
+
primary_key="uts",
|
|
133
|
+
)
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
def test_valid_from_unix_timestamp64_micro_and_nano(self):
|
|
137
|
+
for fn in ("fromUnixTimestamp64Micro", "fromUnixTimestamp64Nano"):
|
|
138
|
+
with self.subTest(fn=fn):
|
|
139
|
+
self.assertIsNone(
|
|
140
|
+
chquery.validate_ttl(
|
|
141
|
+
ttl=f"toDateTime({fn}(uts)) + INTERVAL 1 DAY",
|
|
142
|
+
columns=[{"name": "uts", "type": "Int64"}],
|
|
143
|
+
sorting_key="uts",
|
|
144
|
+
primary_key="uts",
|
|
145
|
+
)
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
def test_valid_to_start_of_functions(self):
|
|
149
|
+
for fn in (
|
|
150
|
+
"toStartOfDay",
|
|
151
|
+
"toStartOfHour",
|
|
152
|
+
"toStartOfMinute",
|
|
153
|
+
"toStartOfFifteenMinutes",
|
|
154
|
+
"toStartOfMonth",
|
|
155
|
+
"toStartOfQuarter",
|
|
156
|
+
"toStartOfYear",
|
|
157
|
+
"toStartOfWeek",
|
|
158
|
+
"toMonday",
|
|
159
|
+
"toLastDayOfMonth",
|
|
160
|
+
):
|
|
161
|
+
with self.subTest(fn=fn):
|
|
162
|
+
self.assertIsNone(
|
|
163
|
+
chquery.validate_ttl(
|
|
164
|
+
ttl=f"{fn}(ts) + INTERVAL 30 DAY",
|
|
165
|
+
columns=[{"name": "ts", "type": "DateTime"}],
|
|
166
|
+
sorting_key="ts",
|
|
167
|
+
primary_key="ts",
|
|
168
|
+
)
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
def test_valid_to_start_of_interval(self):
|
|
172
|
+
self.assertIsNone(
|
|
173
|
+
chquery.validate_ttl(
|
|
174
|
+
ttl="toStartOfInterval(ts, INTERVAL 1 DAY) + INTERVAL 90 DAY",
|
|
175
|
+
columns=[{"name": "ts", "type": "DateTime"}],
|
|
176
|
+
sorting_key="ts",
|
|
177
|
+
primary_key="ts",
|
|
178
|
+
)
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
def test_valid_date_trunc(self):
|
|
182
|
+
self.assertIsNone(
|
|
183
|
+
chquery.validate_ttl(
|
|
184
|
+
ttl="dateTrunc('month', ts) + INTERVAL 6 MONTH",
|
|
185
|
+
columns=[{"name": "ts", "type": "DateTime"}],
|
|
186
|
+
sorting_key="ts",
|
|
187
|
+
primary_key="ts",
|
|
188
|
+
)
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
def test_valid_to_timezone(self):
|
|
192
|
+
self.assertIsNone(
|
|
193
|
+
chquery.validate_ttl(
|
|
194
|
+
ttl="toTimezone(ts, 'UTC') + INTERVAL 7 DAY",
|
|
195
|
+
columns=[{"name": "ts", "type": "DateTime"}],
|
|
196
|
+
sorting_key="ts",
|
|
197
|
+
primary_key="ts",
|
|
198
|
+
)
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
def test_valid_make_date(self):
|
|
202
|
+
self.assertIsNone(
|
|
203
|
+
chquery.validate_ttl(
|
|
204
|
+
ttl="makeDate(y, m, d) + INTERVAL 7 DAY",
|
|
205
|
+
columns=[
|
|
206
|
+
{"name": "y", "type": "UInt16"},
|
|
207
|
+
{"name": "m", "type": "UInt8"},
|
|
208
|
+
{"name": "d", "type": "UInt8"},
|
|
209
|
+
],
|
|
210
|
+
sorting_key="y",
|
|
211
|
+
primary_key="y",
|
|
212
|
+
)
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
# --- Conditional / multi-clause TTLs ---
|
|
216
|
+
def test_valid_ttl_with_where(self):
|
|
217
|
+
self.assertIsNone(
|
|
218
|
+
chquery.validate_ttl(
|
|
219
|
+
ttl="ts + INTERVAL 7 DAY WHERE status = 'expired'",
|
|
220
|
+
columns=[
|
|
221
|
+
{"name": "ts", "type": "DateTime"},
|
|
222
|
+
{"name": "status", "type": "String"},
|
|
223
|
+
],
|
|
224
|
+
sorting_key="ts",
|
|
225
|
+
primary_key="ts",
|
|
226
|
+
)
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
def test_valid_ttl_with_where_numeric(self):
|
|
230
|
+
self.assertIsNone(
|
|
231
|
+
chquery.validate_ttl(
|
|
232
|
+
ttl="ts + INTERVAL 30 DAY WHERE x % 10 == 0 AND y > 5",
|
|
233
|
+
columns=[
|
|
234
|
+
{"name": "ts", "type": "DateTime"},
|
|
235
|
+
{"name": "x", "type": "Int32"},
|
|
236
|
+
{"name": "y", "type": "Int32"},
|
|
237
|
+
],
|
|
238
|
+
sorting_key="ts",
|
|
239
|
+
primary_key="ts",
|
|
240
|
+
)
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
def test_invalid_ttl_where_unknown_column(self):
|
|
244
|
+
err = chquery.validate_ttl(
|
|
245
|
+
ttl="ts + INTERVAL 7 DAY WHERE missing = 1",
|
|
246
|
+
columns=[{"name": "ts", "type": "DateTime"}],
|
|
247
|
+
sorting_key="ts",
|
|
248
|
+
primary_key="ts",
|
|
249
|
+
)
|
|
250
|
+
self.assertIsNotNone(err)
|
|
251
|
+
self.assertIn("Missing columns: 'missing'", err)
|
|
252
|
+
|
|
253
|
+
def test_valid_ttl_where_string_expression(self):
|
|
254
|
+
self.assertIsNone(
|
|
255
|
+
chquery.validate_ttl(
|
|
256
|
+
ttl="ts + INTERVAL 7 DAY WHERE status",
|
|
257
|
+
columns=[
|
|
258
|
+
{"name": "ts", "type": "DateTime"},
|
|
259
|
+
{"name": "status", "type": "String"},
|
|
260
|
+
],
|
|
261
|
+
sorting_key="ts",
|
|
262
|
+
primary_key="ts",
|
|
263
|
+
)
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
def test_valid_multiple_ttls_comma_separated(self):
|
|
267
|
+
self.assertIsNone(
|
|
268
|
+
chquery.validate_ttl(
|
|
269
|
+
ttl="ts + INTERVAL 7 DAY WHERE status = 'a', ts + INTERVAL 14 DAY WHERE status = 'b'",
|
|
270
|
+
columns=[
|
|
271
|
+
{"name": "ts", "type": "DateTime"},
|
|
272
|
+
{"name": "status", "type": "String"},
|
|
273
|
+
],
|
|
274
|
+
sorting_key="ts",
|
|
275
|
+
primary_key="ts",
|
|
276
|
+
)
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
def test_valid_ttl_group_by(self):
|
|
280
|
+
# GROUP BY rollup; must group by a prefix of the primary key
|
|
281
|
+
self.assertIsNone(
|
|
282
|
+
chquery.validate_ttl(
|
|
283
|
+
ttl="ts + INTERVAL 1 DAY GROUP BY k SET v = sum(v)",
|
|
284
|
+
columns=[
|
|
285
|
+
{"name": "k", "type": "String"},
|
|
286
|
+
{"name": "ts", "type": "DateTime"},
|
|
287
|
+
{"name": "v", "type": "Int64"},
|
|
288
|
+
],
|
|
289
|
+
sorting_key="k",
|
|
290
|
+
primary_key="k",
|
|
291
|
+
)
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
def test_invalid_ttl_group_by_not_in_primary_key(self):
|
|
295
|
+
err = chquery.validate_ttl(
|
|
296
|
+
ttl="ts + INTERVAL 1 DAY GROUP BY missing SET v = sum(v)",
|
|
297
|
+
columns=[
|
|
298
|
+
{"name": "k", "type": "String"},
|
|
299
|
+
{"name": "ts", "type": "DateTime"},
|
|
300
|
+
{"name": "v", "type": "Int64"},
|
|
301
|
+
],
|
|
302
|
+
sorting_key="k",
|
|
303
|
+
primary_key="k",
|
|
304
|
+
)
|
|
305
|
+
self.assertIsNotNone(err)
|
|
306
|
+
self.assertIn("GROUP BY key should be a prefix of primary key", err)
|
|
307
|
+
|
|
308
|
+
def test_valid_where_string_functions(self):
|
|
309
|
+
columns = [
|
|
310
|
+
{"name": "ts", "type": "DateTime"},
|
|
311
|
+
{"name": "s", "type": "String"},
|
|
312
|
+
{"name": "n", "type": "Int32"},
|
|
313
|
+
]
|
|
314
|
+
predicates = {
|
|
315
|
+
"concat": "concat(s, 'x') = 'ax'",
|
|
316
|
+
"startsWith": "startsWith(s, 'a')",
|
|
317
|
+
"endsWith": "endsWith(s, 'z')",
|
|
318
|
+
"length": "length(s) > 3",
|
|
319
|
+
"lower": "lower(s) = 'a'",
|
|
320
|
+
"substring": "substring(s, 1, 2) = 'ab'",
|
|
321
|
+
"match": "match(s, '^a')",
|
|
322
|
+
"in": "n IN (1, 2, 3)",
|
|
323
|
+
}
|
|
324
|
+
for name, predicate in predicates.items():
|
|
325
|
+
with self.subTest(fn=name):
|
|
326
|
+
self.assertIsNone(
|
|
327
|
+
chquery.validate_ttl(
|
|
328
|
+
ttl=f"ts + INTERVAL 7 DAY WHERE {predicate}",
|
|
329
|
+
columns=columns,
|
|
330
|
+
sorting_key="ts",
|
|
331
|
+
primary_key="ts",
|
|
332
|
+
),
|
|
333
|
+
f"{name} should be a known function in TTL WHERE clauses",
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
# --- Hash functions in sorting / primary keys (KeyDescription::parse) ---
|
|
337
|
+
def test_valid_sorting_key_with_cityhash64(self):
|
|
338
|
+
self.assertIsNone(
|
|
339
|
+
chquery.validate_ttl(
|
|
340
|
+
ttl="timestamp + INTERVAL 365 DAY",
|
|
341
|
+
columns=[
|
|
342
|
+
{"name": "organizationId", "type": "String"},
|
|
343
|
+
{"name": "surfaceId", "type": "String"},
|
|
344
|
+
{"name": "timestamp", "type": "DateTime"},
|
|
345
|
+
{"name": "name", "type": "String"},
|
|
346
|
+
{"name": "traceId", "type": "String"},
|
|
347
|
+
{"name": "spanId", "type": "String"},
|
|
348
|
+
{"name": "eventIndex", "type": "UInt32"},
|
|
349
|
+
],
|
|
350
|
+
sorting_key="organizationId, surfaceId, toDate(timestamp), name, cityHash64(traceId), spanId, eventIndex",
|
|
351
|
+
)
|
|
352
|
+
)
|
|
353
|
+
|
|
354
|
+
# --- Arithmetic functions in TTL expressions ---
|
|
355
|
+
def test_valid_ttl_with_int_div(self):
|
|
356
|
+
# Reported failure: "Unknown function intDiv".
|
|
357
|
+
self.assertIsNone(
|
|
358
|
+
chquery.validate_ttl(
|
|
359
|
+
ttl="toDateTime(intDiv(ts_server, 1000)) + INTERVAL 13 MONTH",
|
|
360
|
+
columns=[{"name": "ts_server", "type": "Int64"}],
|
|
361
|
+
sorting_key="ts_server",
|
|
362
|
+
)
|
|
363
|
+
)
|
|
364
|
+
|
|
365
|
+
def test_valid_ttl_with_greatest(self):
|
|
366
|
+
# Reported failure: "Unknown function greatest".
|
|
367
|
+
self.assertIsNone(
|
|
368
|
+
chquery.validate_ttl(
|
|
369
|
+
ttl="greatest(departure_date, toDateTime(creation_date)) + toIntervalDay(2)",
|
|
370
|
+
columns=[
|
|
371
|
+
{"name": "departure_date", "type": "DateTime"},
|
|
372
|
+
{"name": "creation_date", "type": "Date"},
|
|
373
|
+
],
|
|
374
|
+
sorting_key="departure_date",
|
|
375
|
+
)
|
|
376
|
+
)
|
|
377
|
+
|
|
378
|
+
# --- Input validation ---
|
|
379
|
+
def test_columns_must_be_list_of_dicts(self):
|
|
380
|
+
with self.assertRaises(TypeError):
|
|
381
|
+
chquery.validate_ttl(ttl="ts + INTERVAL 7 DAY", columns="ts DateTime", sorting_key="ts", primary_key="ts")
|
|
382
|
+
|
|
383
|
+
def test_column_dict_requires_name_and_type(self):
|
|
384
|
+
with self.assertRaises((TypeError, ValueError)):
|
|
385
|
+
chquery.validate_ttl(
|
|
386
|
+
ttl="ts + INTERVAL 7 DAY",
|
|
387
|
+
columns=[{"name": "ts"}],
|
|
388
|
+
sorting_key="ts",
|
|
389
|
+
primary_key="ts",
|
|
390
|
+
)
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
# --- Regression guard: sequential throwing calls must not crash ---
|
|
394
|
+
def test_sequential_throwing_calls_do_not_crash(self):
|
|
395
|
+
# A validate_ttl call that throws must not corrupt shared state and take
|
|
396
|
+
# down a subsequent call. validateTTL parses against a private per-call
|
|
397
|
+
# context copy (functions/ValidateTTL.cpp) precisely so this holds.
|
|
398
|
+
# Run in a subprocess so a hypothetical crash doesn't kill pytest; the
|
|
399
|
+
# child must exit 0.
|
|
400
|
+
import signal
|
|
401
|
+
import subprocess
|
|
402
|
+
import sys
|
|
403
|
+
import textwrap
|
|
404
|
+
repro = textwrap.dedent("""\
|
|
405
|
+
from chtoolset import query as q
|
|
406
|
+
err1 = q.validate_ttl(
|
|
407
|
+
ttl="counter + toIntervalDay(7)",
|
|
408
|
+
columns=[{"name": "counter", "type": "Int32"}],
|
|
409
|
+
sorting_key="counter",
|
|
410
|
+
primary_key="counter",
|
|
411
|
+
)
|
|
412
|
+
assert err1 is not None and "Illegal types" in err1, (
|
|
413
|
+
f"call 1 should return an 'Illegal types' error, got: {err1!r}")
|
|
414
|
+
q.validate_ttl(
|
|
415
|
+
ttl="ts + INTERVAL 7 DAY WHERE missing = 1",
|
|
416
|
+
columns=[{"name": "ts", "type": "DateTime"}],
|
|
417
|
+
sorting_key="ts",
|
|
418
|
+
primary_key="ts",
|
|
419
|
+
)
|
|
420
|
+
""")
|
|
421
|
+
result = subprocess.run([sys.executable, "-c", repro],
|
|
422
|
+
capture_output=True, text=True, timeout=30)
|
|
423
|
+
if result.returncode != 0 and result.returncode != -signal.SIGABRT:
|
|
424
|
+
raise RuntimeError(
|
|
425
|
+
f"Unexpected subprocess exit {result.returncode} "
|
|
426
|
+
f"(expected 0 or SIGABRT {-signal.SIGABRT})."
|
|
427
|
+
f"\nSTDERR:\n{result.stderr}"
|
|
428
|
+
f"\nSTDOUT:\n{result.stdout}")
|
|
429
|
+
self.assertEqual(result.returncode, 0,
|
|
430
|
+
f"Subprocess crashed with SIGABRT."
|
|
431
|
+
f"\nSTDERR:\n{result.stderr}"
|
|
432
|
+
f"\nSTDOUT:\n{result.stdout}")
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
if __name__ == "__main__":
|
|
436
|
+
unittest.main()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/src/tinybird_toolset.egg-info/requires.txt
RENAMED
|
File without changes
|
{tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/src/tinybird_toolset.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_get_columns_from_create_query.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tinybird_toolset-2.4.14 → tinybird_toolset-2.5.3}/tests/test_replace_tables_backward_compat.py
RENAMED
|
File without changes
|
|
File without changes
|