clickhouse-driver 0.2.5__cp311-cp311-musllinux_1_1_aarch64.whl → 0.2.9__cp311-cp311-musllinux_1_1_aarch64.whl

clickhouse_driver/__init__.py

@@ -3,7 +3,7 @@ from .client import Client
 from .dbapi import connect
 
 
-VERSION = (0, 2, 5)
+VERSION = (0, 2, 9)
 __version__ = '.'.join(str(x) for x in VERSION)
 
 __all__ = ['Client', 'connect']

clickhouse_driver/block.py

@@ -1,7 +1,7 @@
+from .columns.util import get_inner_spec, get_inner_columns_with_types
 from .reader import read_varint, read_binary_uint8, read_binary_int32
 from .varint import write_varint
 from .writer import write_binary_uint8, write_binary_int32
-from .columns import nestedcolumn
 
 
 class BlockInfo(object):
@@ -172,7 +172,8 @@ class RowOrientedBlock(BaseBlock):
         for name, type_ in columns_with_types:
             cwt = None
             if type_.startswith('Nested'):
-                cwt = nestedcolumn.get_columns_with_types(type_)
+                inner_spec = get_inner_spec('Nested', type_)
+                cwt = get_inner_columns_with_types(inner_spec)
             columns_with_cwt.append((name, cwt))
 
         for i, row in enumerate(data):

clickhouse_driver/client.py

@@ -1,10 +1,9 @@
 import re
-import ssl
 from collections import deque
 from contextlib import contextmanager
 from time import time
 import types
-from urllib.parse import urlparse, parse_qs, unquote
+from urllib.parse import urlparse
 
 from . import errors, defines
 from .block import ColumnOrientedBlock, RowOrientedBlock
@@ -15,7 +14,7 @@ from .result import (
     IterQueryResult, ProgressQueryResult, QueryResult, QueryInfo
 )
 from .util.escape import escape_params
-from .util.helpers import column_chunks, chunks, asbool
+from .util.helpers import column_chunks, chunks, parse_url
 
 
 class Client(object):
@@ -57,6 +56,17 @@ class Client(object):
         * ``round_robin`` -- If ``alt_hosts`` are provided the query will be
                            executed on host picked with round-robin algorithm.
                            New in version *0.2.5*.
+        * ``namedtuple_as_json`` -- Controls named tuple and nested types
+                           deserialization. To interpret these column alongside
+                           with ``allow_experimental_object_type=1`` as Python
+                           tuple set ``namedtuple_as_json`` to ``False``.
+                           Default: True.
+                           New in version *0.2.6*.
+        * ``server_side_params`` -- Species on which side query parameters
+                           should be rendered into placeholders.
+                           Default: False. Means that parameters are rendered
+                           on driver's side.
+                           New in version *0.2.7*.
     """
 
     available_client_settings = (
@@ -67,7 +77,9 @@ class Client(object):
         'opentelemetry_traceparent',
         'opentelemetry_tracestate',
         'quota_key',
-        'input_format_null_as_default'
+        'input_format_null_as_default',
+        'namedtuple_as_json',
+        'server_side_params'
     )
 
     def __init__(self, *args, **kwargs):
@@ -97,6 +109,12 @@ class Client(object):
             ),
             'input_format_null_as_default': self.settings.pop(
                 'input_format_null_as_default', False
+            ),
+            'namedtuple_as_json': self.settings.pop(
+                'namedtuple_as_json', True
+            ),
+            'server_side_params': self.settings.pop(
+                'server_side_params', False
             )
         }
 
@@ -125,13 +143,19 @@ class Client(object):
                 url = urlparse('clickhouse://' + host)
 
                 connection_kwargs = kwargs.copy()
-                if len(args) > 2:
-                    # port as positional argument
+                num_args = len(args)
+                if num_args >= 2:
+                    # host and port as positional arguments
                     connection_args = (url.hostname, url.port) + args[2:]
-                else:
-                    # port as keyword argument
+                elif num_args >= 1:
+                    # host as positional and port as keyword argument
                     connection_args = (url.hostname, ) + args[1:]
                     connection_kwargs['port'] = url.port
+                else:
+                    # host and port as keyword arguments
+                    connection_args = tuple()
+                    connection_kwargs['host'] = url.hostname
+                    connection_kwargs['port'] = url.port
 
                 connection = Connection(*connection_args, **connection_kwargs)
                 self.connections.append(connection)
@@ -287,6 +311,7 @@ class Client(object):
     def disconnect_on_error(self, query, settings):
         try:
             self.establish_connection(settings)
+            self.connection.server_info.session_timezone = None
 
             yield
 
@@ -438,7 +463,7 @@ class Client(object):
 
     def query_dataframe(
             self, query, params=None, external_tables=None, query_id=None,
-            settings=None):
+            settings=None, replace_nonwords=True):
         """
         *New in version 0.2.0.*
 
@@ -453,6 +478,8 @@ class Client(object):
                          ClickHouse server will generate it.
         :param settings: dictionary of query settings.
                          Defaults to ``None`` (no additional settings).
+        :param replace_nonwords: boolean to replace non-words in column names
+                                 to underscores. Defaults to ``True``.
         :return: pandas DataFrame.
         """
 
@@ -467,7 +494,10 @@ class Client(object):
             settings=settings
         )
 
-        columns = [re.sub(r'\W', '_', name) for name, type_ in columns]
+        columns = [name for name, type_ in columns]
+        if replace_nonwords:
+            columns = [re.sub(r'\W', '_', x) for x in columns]
+
         return pd.DataFrame(
             {col: d for d, col in zip(data, columns)}, columns=columns
         )
@@ -529,7 +559,7 @@ class Client(object):
                 query, params, self.connection.context
             )
 
-        self.connection.send_query(query, query_id=query_id)
+        self.connection.send_query(query, query_id=query_id, params=params)
         self.connection.send_external_tables(external_tables,
                                              types_check=types_check)
         return self.receive_result(with_column_types=with_column_types,
@@ -544,8 +574,7 @@ class Client(object):
             query = self.substitute_params(
                 query, params, self.connection.context
             )
-
-        self.connection.send_query(query, query_id=query_id)
+        self.connection.send_query(query, query_id=query_id, params=params)
         self.connection.send_external_tables(external_tables,
                                              types_check=types_check)
         return self.receive_result(with_column_types=with_column_types,
@@ -561,7 +590,7 @@ class Client(object):
                 query, params, self.connection.context
             )
 
-        self.connection.send_query(query, query_id=query_id)
+        self.connection.send_query(query, query_id=query_id, params=params)
         self.connection.send_external_tables(external_tables,
                                              types_check=types_check)
         return self.iter_receive_result(with_column_types=with_column_types)
@@ -572,12 +601,12 @@ class Client(object):
         self.connection.send_query(query_without_data, query_id=query_id)
         self.connection.send_external_tables(external_tables,
                                              types_check=types_check)
-
         sample_block = self.receive_sample_block()
+
         if sample_block:
             rv = self.send_data(sample_block, data,
                                 types_check=types_check, columnar=columnar)
-            self.receive_end_of_query()
+            self.receive_end_of_insert_query()
             return rv
 
     def receive_sample_block(self):
@@ -631,8 +660,15 @@ class Client(object):
             self.connection.send_data(block)
             inserted_rows += block.num_rows
 
+            # Starting from the specific revision there are profile events
+            # sent by server in response to each inserted block
+            self.receive_profile_events()
+
         # Empty block means end of data.
         self.connection.send_data(block_cls())
+        # If enabled by revision profile events are also sent after empty block
+        self.receive_profile_events()
+
         return inserted_rows
 
     def receive_end_of_query(self):
@@ -643,7 +679,7 @@ class Client(object):
                 break
 
             elif packet.type == ServerPacketTypes.PROGRESS:
-                continue
+                self.last_query.store_progress(packet.progress)
 
             elif packet.type == ServerPacketTypes.EXCEPTION:
                 raise packet.exception
@@ -655,11 +691,68 @@ class Client(object):
                 pass
 
             elif packet.type == ServerPacketTypes.PROFILE_EVENTS:
+                self.last_query.store_profile(packet.profile_info)
+
+            else:
+                message = self.connection.unexpected_packet_message(
+                    'Exception, EndOfStream, Progress, TableColumns, '
+                    'ProfileEvents or Log', packet.type
+                )
+                raise errors.UnexpectedPacketFromServerError(message)
+
+    def receive_end_of_insert_query(self):
+        while True:
+            packet = self.connection.receive_packet()
+
+            if packet.type == ServerPacketTypes.END_OF_STREAM:
+                break
+
+            elif packet.type == ServerPacketTypes.LOG:
+                log_block(packet.block)
+
+            elif packet.type == ServerPacketTypes.PROGRESS:
+                self.last_query.store_progress(packet.progress)
+
+            elif packet.type == ServerPacketTypes.EXCEPTION:
+                raise packet.exception
+
+            else:
+                message = self.connection.unexpected_packet_message(
+                    'EndOfStream, Log, Progress or Exception', packet.type
+                )
+                raise errors.UnexpectedPacketFromServerError(message)
+
+    def receive_profile_events(self):
+        revision = self.connection.server_info.used_revision
+        if (
+            revision <
+            defines.DBMS_MIN_PROTOCOL_VERSION_WITH_PROFILE_EVENTS_IN_INSERT
+        ):
+            return None
+
+        while True:
+            packet = self.connection.receive_packet()
+
+            if packet.type == ServerPacketTypes.PROFILE_EVENTS:
+                self.last_query.store_profile(packet.profile_info)
+                break
+
+            elif packet.type == ServerPacketTypes.PROGRESS:
+                self.last_query.store_progress(packet.progress)
+
+            elif packet.type == ServerPacketTypes.LOG:
+                log_block(packet.block)
+
+            elif packet.type == ServerPacketTypes.EXCEPTION:
+                raise packet.exception
+
+            elif packet.type == ServerPacketTypes.TIMEZONE_UPDATE:
                 pass
 
             else:
                 message = self.connection.unexpected_packet_message(
-                    'Exception, EndOfStream or Log', packet.type
+                    'ProfileEvents, Progress, Log, Exception or '
+                    'TimezoneUpdate', packet.type
                 )
                 raise errors.UnexpectedPacketFromServerError(message)
 
@@ -686,6 +779,10 @@ class Client(object):
             # prints: SELECT 1234, 'bar'
             print(substituted_query)
         """
+        # In case of server side templating we don't substitute here.
+        if self.connection.context.client_settings['server_side_params']:
+            return query
+
         if not isinstance(params, dict):
             raise ValueError('Parameters are expected in dict form')
 
@@ -703,90 +800,13 @@ class Client(object):
             clickhouses://[user:password]@localhost:9440/default
 
         Three URL schemes are supported:
-            clickhouse:// creates a normal TCP socket connection
-            clickhouses:// creates a SSL wrapped TCP socket connection
+
+            * clickhouse:// creates a normal TCP socket connection
+            * clickhouses:// creates a SSL wrapped TCP socket connection
 
         Any additional querystring arguments will be passed along to
         the Connection class's initializer.
         """
-        url = urlparse(url)
-
-        settings = {}
-        kwargs = {}
-
-        host = url.hostname
-
-        if url.port is not None:
-            kwargs['port'] = url.port
-
-        path = url.path.replace('/', '', 1)
-        if path:
-            kwargs['database'] = path
-
-        if url.username is not None:
-            kwargs['user'] = unquote(url.username)
-
-        if url.password is not None:
-            kwargs['password'] = unquote(url.password)
-
-        if url.scheme == 'clickhouses':
-            kwargs['secure'] = True
-
-        compression_algs = {'lz4', 'lz4hc', 'zstd'}
-        timeouts = {
-            'connect_timeout',
-            'send_receive_timeout',
-            'sync_request_timeout'
-        }
-
-        for name, value in parse_qs(url.query).items():
-            if not value or not len(value):
-                continue
-
-            value = value[0]
-
-            if name == 'compression':
-                value = value.lower()
-                if value in compression_algs:
-                    kwargs[name] = value
-                else:
-                    kwargs[name] = asbool(value)
-
-            elif name == 'secure':
-                kwargs[name] = asbool(value)
-
-            elif name == 'use_numpy':
-                settings[name] = asbool(value)
-
-            elif name == 'round_robin':
-                kwargs[name] = asbool(value)
-
-            elif name == 'client_name':
-                kwargs[name] = value
-
-            elif name in timeouts:
-                kwargs[name] = float(value)
-
-            elif name == 'compress_block_size':
-                kwargs[name] = int(value)
-
-            elif name == 'settings_is_important':
-                kwargs[name] = asbool(value)
-
-            # ssl
-            elif name == 'verify':
-                kwargs[name] = asbool(value)
-            elif name == 'ssl_version':
-                kwargs[name] = getattr(ssl, value)
-            elif name in ['ca_certs', 'ciphers', 'keyfile', 'certfile',
-                          'server_hostname']:
-                kwargs[name] = value
-            elif name == 'alt_hosts':
-                kwargs['alt_hosts'] = value
-            else:
-                settings[name] = value
-
-        if settings:
-            kwargs['settings'] = settings
+        host, kwargs = parse_url(url)
 
         return cls(host, **kwargs)

clickhouse_driver/clientinfo.py

@@ -28,14 +28,13 @@ class ClientInfo(object):
     client_version_major = defines.CLIENT_VERSION_MAJOR
     client_version_minor = defines.CLIENT_VERSION_MINOR
     client_version_patch = defines.CLIENT_VERSION_PATCH
-    client_revision = defines.CLIENT_REVISION
     interface = Interface.TCP
 
     initial_user = ''
     initial_query_id = ''
     initial_address = '0.0.0.0:0'
 
-    def __init__(self, client_name, context):
+    def __init__(self, client_name, context, client_revision):
         self.query_kind = ClientInfo.QueryKind.NO_QUERY
 
         try:
@@ -44,6 +43,7 @@ class ClientInfo(object):
             self.os_user = ''
         self.client_hostname = socket.gethostname()
         self.client_name = client_name
+        self.client_revision = client_revision
 
         self.client_trace_context = OpenTelemetryTraceContext(
             context.client_settings['opentelemetry_traceparent'],

clickhouse_driver/columns/arraycolumn.py

@@ -28,7 +28,8 @@ class ArrayColumn(Column):
     py_types = (list, tuple)
 
     def __init__(self, nested_column, **kwargs):
-        self.size_column = UInt64Column()
+        self.init_kwargs = kwargs
+        self.size_column = UInt64Column(**kwargs)
         self.nested_column = nested_column
         self._write_depth_0_size = True
         super(ArrayColumn, self).__init__(**kwargs)
@@ -37,17 +38,21 @@ class ArrayColumn(Column):
     def write_data(self, data, buf):
         # Column of Array(T) is stored in "compact" format and passed to server
         # wrapped into another Array without size of wrapper array.
-        self.nested_column = ArrayColumn(self.nested_column)
+        self.nested_column = ArrayColumn(
+            self.nested_column, **self.init_kwargs
+        )
         self.nested_column.nullable = self.nullable
         self.nullable = False
         self._write_depth_0_size = False
         self._write(data, buf)
 
-    def read_data(self, rows, buf):
-        self.nested_column = ArrayColumn(self.nested_column)
+    def read_data(self, n_rows, buf):
+        self.nested_column = ArrayColumn(
+            self.nested_column, **self.init_kwargs
+        )
         self.nested_column.nullable = self.nullable
         self.nullable = False
-        return self._read(rows, buf)[0]
+        return self._read(n_rows, buf)[0]
 
     def _write_sizes(self, value, buf):
         nulls_map = []
@@ -106,9 +111,13 @@ class ArrayColumn(Column):
         self._write_data(value, buf)
 
     def read_state_prefix(self, buf):
-        return self.nested_column.read_state_prefix(buf)
+        super(ArrayColumn, self).read_state_prefix(buf)
+
+        self.nested_column.read_state_prefix(buf)
 
     def write_state_prefix(self, buf):
+        super(ArrayColumn, self).write_state_prefix(buf)
+
         self.nested_column.write_state_prefix(buf)
 
     def _read(self, size, buf):

clickhouse_driver/columns/base.py

@@ -1,6 +1,61 @@
 from struct import Struct, error as struct_error
 
 from . import exceptions
+from ..varint import read_varint
+
+
+class CommonSerialization(object):
+    def __init__(self, column):
+        self.column = column
+        super(CommonSerialization, self).__init__()
+
+    def read_sparse(self, n_items, buf):
+        return n_items
+
+    def apply_sparse(self, items):
+        return items
+
+
+class SparseSerialization(CommonSerialization):
+
+    def __init__(self, *args, **kwargs):
+        self.sparse_indexes = []
+        self.items_total = None
+        super(SparseSerialization, self).__init__(*args, **kwargs)
+
+    def read_sparse(self, n_items, buf):
+        sparse_indexes = []
+        items_total = 0
+        non_default_items = 0
+
+        END_OF_GRANULE_FLAG = 1 << 62
+        end_of_granule = False
+
+        while not end_of_granule:
+            group_size = read_varint(buf)
+            end_of_granule = group_size & END_OF_GRANULE_FLAG
+            group_size &= ~END_OF_GRANULE_FLAG
+
+            items_total += group_size + 1
+            if not end_of_granule:
+                non_default_items += 1
+                sparse_indexes.append(items_total)
+
+        self.sparse_indexes = sparse_indexes
+        self.items_total = items_total
+
+        return non_default_items
+
+    def apply_sparse(self, items):
+        default = self.column.null_value
+        if self.column.after_read_items:
+            default = self.column.after_read_items([default])[0]
+
+        rv = [default] * (self.items_total - 1)
+        for item_number, i in enumerate(self.sparse_indexes):
+            rv[i - 1] = items[item_number]
+
+        return rv
 
 
 class Column(object):
@@ -15,14 +70,17 @@ class Column(object):
 
     null_value = 0
 
-    def __init__(self, types_check=False, **kwargs):
+    def __init__(self, types_check=False, has_custom_serialization=False,
+                 **kwargs):
         self.nullable = False
         self.types_check_enabled = types_check
+        self.has_custom_serialization = has_custom_serialization
+        self.serialization = CommonSerialization(self)
         self.input_null_as_default = False
-        if 'context' in kwargs:
-            settings = kwargs['context'].client_settings
-            self.input_null_as_default = settings \
-                .get('input_format_null_as_default', False)
+
+        self.context = kwargs['context']
+        self.input_null_as_default = self.context.client_settings \
+            .get('input_format_null_as_default', False)
 
         super(Column, self).__init__()
 
@@ -94,12 +152,15 @@ class Column(object):
         raise NotImplementedError
 
     def read_data(self, n_items, buf):
+        n_items = self.serialization.read_sparse(n_items, buf)
+
         if self.nullable:
             nulls_map = self._read_nulls_map(n_items, buf)
         else:
             nulls_map = None
 
-        return self._read_data(n_items, buf, nulls_map=nulls_map)
+        items = self._read_data(n_items, buf, nulls_map=nulls_map)
+        return self.serialization.apply_sparse(items)
 
     def _read_data(self, n_items, buf, nulls_map=None):
         items = self.read_items(n_items, buf)
@@ -117,7 +178,10 @@ class Column(object):
         raise NotImplementedError
 
     def read_state_prefix(self, buf):
-        pass
+        if self.has_custom_serialization:
+            use_custom_serialization = read_varint(buf)
+            if use_custom_serialization:
+                self.serialization = SparseSerialization(self)
 
     def write_state_prefix(self, buf):
         pass

clickhouse_driver/columns/datecolumn.py

@@ -1,3 +1,4 @@
+from os import getenv
 from datetime import date, timedelta
 
 from .base import FormatColumn
@@ -6,8 +7,54 @@ from .base import FormatColumn
 epoch_start = date(1970, 1, 1)
 epoch_end = date(2149, 6, 6)
 
-epoch_start_date32 = date(1925, 1, 1)
-epoch_end_date32 = date(2283, 11, 11)
+epoch_start_date32 = date(1900, 1, 1)
+epoch_end_date32 = date(2299, 12, 31)
+
+
+class LazyLUT(dict):
+    def __init__(self, *args, _factory, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._default_factory = _factory
+
+    def __missing__(self, key):
+        return self.setdefault(key, self._default_factory(key))
+
+
+def make_date_lut_range(date_start, date_end):
+    return range(
+        (date_start - epoch_start).days,
+        (date_end - epoch_start).days + 1,
+    )
+
+
+enable_lazy_date_lut = getenv('CLICKHOUSE_DRIVER_LASY_DATE_LUT', False)
+if enable_lazy_date_lut:
+    try:
+        start, end = enable_lazy_date_lut.split(':')
+        start_date = date.fromisoformat(start)
+        end_date = date.fromisoformat(end)
+
+        date_range = make_date_lut_range(start_date, end_date)
+    except ValueError:
+        date_range = ()
+
+    # Since we initialize lazy lut with some initially warmed values,
+    # we use iterator and not dict comprehension for memory & time optimization
+    _date_lut = LazyLUT(
+        ((x, epoch_start + timedelta(days=x)) for x in date_range),
+        _factory=lambda x: epoch_start + timedelta(days=x),
+    )
+    _date_lut_reverse = LazyLUT(
+        ((value, key) for key, value in _date_lut.items()),
+        _factory=lambda x: (x - epoch_start).days,
+    )
+else:
+    # If lazy lut is not enabled, we fallback to static dict initialization
+    # In both cases, we use same lut for both data types,
+    # since one encompasses the other and we can avoid duplicating overlap
+    date_range = make_date_lut_range(epoch_start_date32, epoch_end_date32)
+    _date_lut = {x: epoch_start + timedelta(days=x) for x in date_range}
+    _date_lut_reverse = {value: key for key, value in _date_lut.items()}
 
 
 class DateColumn(FormatColumn):
@@ -18,9 +65,8 @@ class DateColumn(FormatColumn):
     min_value = epoch_start
     max_value = epoch_end
 
-    date_lut_days = (epoch_end - epoch_start).days + 1
-    date_lut = {x: epoch_start + timedelta(x) for x in range(date_lut_days)}
-    date_lut_reverse = {value: key for key, value in date_lut.items()}
+    date_lut = _date_lut
+    date_lut_reverse = _date_lut_reverse
 
     def before_write_items(self, items, nulls_map=None):
         null_value = self.null_value
@@ -34,7 +80,7 @@ class DateColumn(FormatColumn):
                 items[i] = null_value
                 continue
 
-            if type(item) != date:
+            if item is not date:
                 item = date(item.year, item.month, item.day)
 
             if min_value <= item <= max_value:
@@ -60,10 +106,3 @@ class Date32Column(DateColumn):
 
     min_value = epoch_start_date32
     max_value = epoch_end_date32
-
-    date_lut_days = (epoch_end_date32 - epoch_start).days + 1
-    date_lut = {
-        x: epoch_start + timedelta(x)
-        for x in range((epoch_start_date32 - epoch_start).days, date_lut_days)
-    }
-    date_lut_reverse = {value: key for key, value in date_lut.items()}

clickhouse_driver/columns/datetimecolumn.py

@@ -193,8 +193,9 @@ def create_datetime_column(spec, column_options):
     else:
         if not context.settings.get('use_client_time_zone', False):
             local_timezone = get_localzone_name_compat()
-            if local_timezone != context.server_info.timezone:
-                tz_name = context.server_info.timezone
+            remote_timezone = context.server_info.get_timezone()
+            if local_timezone != remote_timezone:
+                tz_name = remote_timezone
 
     if tz_name:
         timezone = get_timezone(tz_name)