clickhouse-driver 0.2.5__pp39-pypy39_pp73-win_amd64.whl → 0.2.7__pp39-pypy39_pp73-win_amd64.whl

clickhouse_driver/__init__.py

@@ -3,7 +3,7 @@ from .client import Client
 from .dbapi import connect
 
 
-VERSION = (0, 2, 5)
+VERSION = (0, 2, 7)
 __version__ = '.'.join(str(x) for x in VERSION)
 
 __all__ = ['Client', 'connect']

clickhouse_driver/block.py

@@ -1,7 +1,7 @@
+from .columns.util import get_inner_spec, get_inner_columns_with_types
 from .reader import read_varint, read_binary_uint8, read_binary_int32
 from .varint import write_varint
 from .writer import write_binary_uint8, write_binary_int32
-from .columns import nestedcolumn
 
 
 class BlockInfo(object):
@@ -172,7 +172,8 @@ class RowOrientedBlock(BaseBlock):
         for name, type_ in columns_with_types:
             cwt = None
             if type_.startswith('Nested'):
-                cwt = nestedcolumn.get_columns_with_types(type_)
+                inner_spec = get_inner_spec('Nested', type_)
+                cwt = get_inner_columns_with_types(inner_spec)
             columns_with_cwt.append((name, cwt))
 
         for i, row in enumerate(data):

clickhouse_driver/client.py

@@ -57,6 +57,17 @@ class Client(object):
         * ``round_robin`` -- If ``alt_hosts`` are provided the query will be
                            executed on host picked with round-robin algorithm.
                            New in version *0.2.5*.
+        * ``namedtuple_as_json`` -- Controls named tuple and nested types
+                           deserialization. To interpret these column alongside
+                           with ``allow_experimental_object_type=1`` as Python
+                           tuple set ``namedtuple_as_json`` to ``False``.
+                           Default: True.
+                           New in version *0.2.6*.
+        * ``server_side_params`` -- Species on which side query parameters
+                           should be rendered into placeholders.
+                           Default: False. Means that parameters are rendered
+                           on driver's side.
+                           New in version *0.2.7*.
     """
 
     available_client_settings = (
@@ -67,7 +78,9 @@ class Client(object):
         'opentelemetry_traceparent',
         'opentelemetry_tracestate',
         'quota_key',
-        'input_format_null_as_default'
+        'input_format_null_as_default',
+        'namedtuple_as_json',
+        'server_side_params'
     )
 
     def __init__(self, *args, **kwargs):
@@ -97,6 +110,12 @@ class Client(object):
             ),
             'input_format_null_as_default': self.settings.pop(
                 'input_format_null_as_default', False
+            ),
+            'namedtuple_as_json': self.settings.pop(
+                'namedtuple_as_json', True
+            ),
+            'server_side_params': self.settings.pop(
+                'server_side_params', False
             )
         }
 
@@ -125,13 +144,19 @@ class Client(object):
                 url = urlparse('clickhouse://' + host)
 
                 connection_kwargs = kwargs.copy()
-                if len(args) > 2:
-                    # port as positional argument
+                num_args = len(args)
+                if num_args >= 2:
+                    # host and port as positional arguments
                     connection_args = (url.hostname, url.port) + args[2:]
-                else:
-                    # port as keyword argument
+                elif num_args >= 1:
+                    # host as positional and port as keyword argument
                     connection_args = (url.hostname, ) + args[1:]
                     connection_kwargs['port'] = url.port
+                else:
+                    # host and port as keyword arguments
+                    connection_args = tuple()
+                    connection_kwargs['host'] = url.hostname
+                    connection_kwargs['port'] = url.port
 
                 connection = Connection(*connection_args, **connection_kwargs)
                 self.connections.append(connection)
@@ -438,7 +463,7 @@ class Client(object):
 
     def query_dataframe(
             self, query, params=None, external_tables=None, query_id=None,
-            settings=None):
+            settings=None, replace_nonwords=True):
         """
         *New in version 0.2.0.*
 
@@ -453,6 +478,8 @@ class Client(object):
                          ClickHouse server will generate it.
         :param settings: dictionary of query settings.
                          Defaults to ``None`` (no additional settings).
+        :param replace_nonwords: boolean to replace non-words in column names
+                                 to underscores. Defaults to ``True``.
         :return: pandas DataFrame.
         """
 
@@ -467,7 +494,10 @@ class Client(object):
             settings=settings
         )
 
-        columns = [re.sub(r'\W', '_', name) for name, type_ in columns]
+        columns = [name for name, type_ in columns]
+        if replace_nonwords:
+            columns = [re.sub(r'\W', '_', x) for x in columns]
+
         return pd.DataFrame(
             {col: d for d, col in zip(data, columns)}, columns=columns
         )
@@ -529,7 +559,7 @@ class Client(object):
                 query, params, self.connection.context
             )
 
-        self.connection.send_query(query, query_id=query_id)
+        self.connection.send_query(query, query_id=query_id, params=params)
         self.connection.send_external_tables(external_tables,
                                              types_check=types_check)
         return self.receive_result(with_column_types=with_column_types,
@@ -544,8 +574,7 @@ class Client(object):
             query = self.substitute_params(
                 query, params, self.connection.context
             )
-
-        self.connection.send_query(query, query_id=query_id)
+        self.connection.send_query(query, query_id=query_id, params=params)
         self.connection.send_external_tables(external_tables,
                                              types_check=types_check)
         return self.receive_result(with_column_types=with_column_types,
@@ -561,7 +590,7 @@ class Client(object):
                 query, params, self.connection.context
             )
 
-        self.connection.send_query(query, query_id=query_id)
+        self.connection.send_query(query, query_id=query_id, params=params)
         self.connection.send_external_tables(external_tables,
                                              types_check=types_check)
         return self.iter_receive_result(with_column_types=with_column_types)
@@ -572,12 +601,12 @@ class Client(object):
         self.connection.send_query(query_without_data, query_id=query_id)
         self.connection.send_external_tables(external_tables,
                                              types_check=types_check)
-
         sample_block = self.receive_sample_block()
+
         if sample_block:
             rv = self.send_data(sample_block, data,
                                 types_check=types_check, columnar=columnar)
-            self.receive_end_of_query()
+            self.receive_end_of_insert_query()
             return rv
 
     def receive_sample_block(self):
@@ -631,8 +660,15 @@ class Client(object):
             self.connection.send_data(block)
             inserted_rows += block.num_rows
 
+            # Starting from the specific revision there are profile events
+            # sent by server in response to each inserted block
+            self.receive_profile_events()
+
         # Empty block means end of data.
         self.connection.send_data(block_cls())
+        # If enabled by revision profile events are also sent after empty block
+        self.receive_profile_events()
+
         return inserted_rows
 
     def receive_end_of_query(self):
@@ -643,7 +679,7 @@ class Client(object):
                 break
 
             elif packet.type == ServerPacketTypes.PROGRESS:
-                continue
+                self.last_query.store_progress(packet.progress)
 
             elif packet.type == ServerPacketTypes.EXCEPTION:
                 raise packet.exception
@@ -655,11 +691,64 @@ class Client(object):
                 pass
 
             elif packet.type == ServerPacketTypes.PROFILE_EVENTS:
-                pass
+                self.last_query.store_profile(packet.profile_info)
 
             else:
                 message = self.connection.unexpected_packet_message(
-                    'Exception, EndOfStream or Log', packet.type
+                    'Exception, EndOfStream, Progress, TableColumns, '
+                    'ProfileEvents or Log', packet.type
+                )
+                raise errors.UnexpectedPacketFromServerError(message)
+
+    def receive_end_of_insert_query(self):
+        while True:
+            packet = self.connection.receive_packet()
+
+            if packet.type == ServerPacketTypes.END_OF_STREAM:
+                break
+
+            elif packet.type == ServerPacketTypes.LOG:
+                log_block(packet.block)
+
+            elif packet.type == ServerPacketTypes.PROGRESS:
+                self.last_query.store_progress(packet.progress)
+
+            elif packet.type == ServerPacketTypes.EXCEPTION:
+                raise packet.exception
+
+            else:
+                message = self.connection.unexpected_packet_message(
+                    'EndOfStream, Log, Progress or Exception', packet.type
+                )
+                raise errors.UnexpectedPacketFromServerError(message)
+
+    def receive_profile_events(self):
+        revision = self.connection.server_info.used_revision
+        if (
+            revision <
+            defines.DBMS_MIN_PROTOCOL_VERSION_WITH_PROFILE_EVENTS_IN_INSERT
+        ):
+            return None
+
+        while True:
+            packet = self.connection.receive_packet()
+
+            if packet.type == ServerPacketTypes.PROFILE_EVENTS:
+                self.last_query.store_profile(packet.profile_info)
+                break
+
+            elif packet.type == ServerPacketTypes.PROGRESS:
+                self.last_query.store_progress(packet.progress)
+
+            elif packet.type == ServerPacketTypes.LOG:
+                log_block(packet.block)
+
+            elif packet.type == ServerPacketTypes.EXCEPTION:
+                raise packet.exception
+
+            else:
+                message = self.connection.unexpected_packet_message(
+                    'ProfileEvents, Progress, Log or Exception', packet.type
                 )
                 raise errors.UnexpectedPacketFromServerError(message)
 
@@ -686,6 +775,10 @@ class Client(object):
             # prints: SELECT 1234, 'bar'
             print(substituted_query)
         """
+        # In case of server side templating we don't substitute here.
+        if self.connection.context.client_settings['server_side_params']:
+            return query
+
         if not isinstance(params, dict):
             raise ValueError('Parameters are expected in dict form')
 
@@ -773,6 +866,17 @@ class Client(object):
             elif name == 'settings_is_important':
                 kwargs[name] = asbool(value)
 
+            elif name == 'tcp_keepalive':
+                try:
+                    kwargs[name] = asbool(value)
+                except ValueError:
+                    parts = value.split(',')
+                    kwargs[name] = (
+                        float(parts[0]), float(parts[1]), int(parts[2])
+                    )
+            elif name == 'client_revision':
+                kwargs[name] = int(value)
+
             # ssl
             elif name == 'verify':
                 kwargs[name] = asbool(value)

clickhouse_driver/clientinfo.py

@@ -28,14 +28,13 @@ class ClientInfo(object):
     client_version_major = defines.CLIENT_VERSION_MAJOR
     client_version_minor = defines.CLIENT_VERSION_MINOR
     client_version_patch = defines.CLIENT_VERSION_PATCH
-    client_revision = defines.CLIENT_REVISION
     interface = Interface.TCP
 
     initial_user = ''
     initial_query_id = ''
     initial_address = '0.0.0.0:0'
 
-    def __init__(self, client_name, context):
+    def __init__(self, client_name, context, client_revision):
         self.query_kind = ClientInfo.QueryKind.NO_QUERY
 
         try:
@@ -44,6 +43,7 @@ class ClientInfo(object):
             self.os_user = ''
         self.client_hostname = socket.gethostname()
         self.client_name = client_name
+        self.client_revision = client_revision
 
         self.client_trace_context = OpenTelemetryTraceContext(
             context.client_settings['opentelemetry_traceparent'],

clickhouse_driver/columns/arraycolumn.py

@@ -28,7 +28,8 @@ class ArrayColumn(Column):
     py_types = (list, tuple)
 
     def __init__(self, nested_column, **kwargs):
-        self.size_column = UInt64Column()
+        self.init_kwargs = kwargs
+        self.size_column = UInt64Column(**kwargs)
         self.nested_column = nested_column
         self._write_depth_0_size = True
         super(ArrayColumn, self).__init__(**kwargs)
@@ -37,17 +38,21 @@ class ArrayColumn(Column):
     def write_data(self, data, buf):
         # Column of Array(T) is stored in "compact" format and passed to server
         # wrapped into another Array without size of wrapper array.
-        self.nested_column = ArrayColumn(self.nested_column)
+        self.nested_column = ArrayColumn(
+            self.nested_column, **self.init_kwargs
+        )
         self.nested_column.nullable = self.nullable
         self.nullable = False
         self._write_depth_0_size = False
         self._write(data, buf)
 
-    def read_data(self, rows, buf):
-        self.nested_column = ArrayColumn(self.nested_column)
+    def read_data(self, n_rows, buf):
+        self.nested_column = ArrayColumn(
+            self.nested_column, **self.init_kwargs
+        )
         self.nested_column.nullable = self.nullable
         self.nullable = False
-        return self._read(rows, buf)[0]
+        return self._read(n_rows, buf)[0]
 
     def _write_sizes(self, value, buf):
         nulls_map = []
@@ -106,9 +111,13 @@ class ArrayColumn(Column):
         self._write_data(value, buf)
 
     def read_state_prefix(self, buf):
-        return self.nested_column.read_state_prefix(buf)
+        super(ArrayColumn, self).read_state_prefix(buf)
+
+        self.nested_column.read_state_prefix(buf)
 
     def write_state_prefix(self, buf):
+        super(ArrayColumn, self).write_state_prefix(buf)
+
         self.nested_column.write_state_prefix(buf)
 
     def _read(self, size, buf):

clickhouse_driver/columns/base.py

@@ -1,6 +1,61 @@
 from struct import Struct, error as struct_error
 
 from . import exceptions
+from ..varint import read_varint
+
+
+class CommonSerialization(object):
+    def __init__(self, column):
+        self.column = column
+        super(CommonSerialization, self).__init__()
+
+    def read_sparse(self, n_items, buf):
+        return n_items
+
+    def apply_sparse(self, items):
+        return items
+
+
+class SparseSerialization(CommonSerialization):
+
+    def __init__(self, *args, **kwargs):
+        self.sparse_indexes = []
+        self.items_total = None
+        super(SparseSerialization, self).__init__(*args, **kwargs)
+
+    def read_sparse(self, n_items, buf):
+        sparse_indexes = []
+        items_total = 0
+        non_default_items = 0
+
+        END_OF_GRANULE_FLAG = 1 << 62
+        end_of_granule = False
+
+        while not end_of_granule:
+            group_size = read_varint(buf)
+            end_of_granule = group_size & END_OF_GRANULE_FLAG
+            group_size &= ~END_OF_GRANULE_FLAG
+
+            items_total += group_size + 1
+            if not end_of_granule:
+                non_default_items += 1
+                sparse_indexes.append(items_total)
+
+        self.sparse_indexes = sparse_indexes
+        self.items_total = items_total
+
+        return non_default_items
+
+    def apply_sparse(self, items):
+        default = self.column.null_value
+        if self.column.after_read_items:
+            default = self.column.after_read_items([default])[0]
+
+        rv = [default] * (self.items_total - 1)
+        for item_number, i in enumerate(self.sparse_indexes):
+            rv[i - 1] = items[item_number]
+
+        return rv
 
 
 class Column(object):
@@ -15,14 +70,17 @@ class Column(object):
 
     null_value = 0
 
-    def __init__(self, types_check=False, **kwargs):
+    def __init__(self, types_check=False, has_custom_serialization=False,
+                 **kwargs):
         self.nullable = False
         self.types_check_enabled = types_check
+        self.has_custom_serialization = has_custom_serialization
+        self.serialization = CommonSerialization(self)
         self.input_null_as_default = False
-        if 'context' in kwargs:
-            settings = kwargs['context'].client_settings
-            self.input_null_as_default = settings \
-                .get('input_format_null_as_default', False)
+
+        self.context = kwargs['context']
+        self.input_null_as_default = self.context.client_settings \
+            .get('input_format_null_as_default', False)
 
         super(Column, self).__init__()
 
@@ -94,12 +152,15 @@ class Column(object):
         raise NotImplementedError
 
     def read_data(self, n_items, buf):
+        n_items = self.serialization.read_sparse(n_items, buf)
+
         if self.nullable:
             nulls_map = self._read_nulls_map(n_items, buf)
         else:
             nulls_map = None
 
-        return self._read_data(n_items, buf, nulls_map=nulls_map)
+        items = self._read_data(n_items, buf, nulls_map=nulls_map)
+        return self.serialization.apply_sparse(items)
 
     def _read_data(self, n_items, buf, nulls_map=None):
         items = self.read_items(n_items, buf)
@@ -117,7 +178,10 @@ class Column(object):
         raise NotImplementedError
 
     def read_state_prefix(self, buf):
-        pass
+        if self.has_custom_serialization:
+            use_custom_serialization = read_varint(buf)
+            if use_custom_serialization:
+                self.serialization = SparseSerialization(self)
 
     def write_state_prefix(self, buf):
         pass

clickhouse_driver/columns/datecolumn.py

@@ -6,7 +6,7 @@ from .base import FormatColumn
 epoch_start = date(1970, 1, 1)
 epoch_end = date(2149, 6, 6)
 
-epoch_start_date32 = date(1925, 1, 1)
+epoch_start_date32 = date(1900, 1, 1)
 epoch_end_date32 = date(2283, 11, 11)
 
 
@@ -34,7 +34,7 @@ class DateColumn(FormatColumn):
                 items[i] = null_value
                 continue
 
-            if type(item) != date:
+            if item is not date:
                 item = date(item.year, item.month, item.day)
 
             if min_value <= item <= max_value:

clickhouse_driver/columns/jsoncolumn.py

@@ -0,0 +1,37 @@
+from .base import Column
+from .stringcolumn import String
+from ..reader import read_binary_uint8, read_binary_str
+from ..util.compat import json
+from ..writer import write_binary_uint8
+
+
+class JsonColumn(Column):
+    py_types = (dict, )
+
+    # No NULL value actually
+    null_value = {}
+
+    def __init__(self, column_by_spec_getter, **kwargs):
+        self.column_by_spec_getter = column_by_spec_getter
+        self.string_column = String(**kwargs)
+        super(JsonColumn, self).__init__(**kwargs)
+
+    def write_state_prefix(self, buf):
+        # Read in binary format.
+        # Write in text format.
+        write_binary_uint8(1, buf)
+
+    def read_items(self, n_items, buf):
+        read_binary_uint8(buf)
+        spec = read_binary_str(buf)
+        col = self.column_by_spec_getter(spec)
+        col.read_state_prefix(buf)
+        return col.read_data(n_items, buf)
+
+    def write_items(self, items, buf):
+        items = [x if isinstance(x, str) else json.dumps(x) for x in items]
+        self.string_column.write_items(items, buf)
+
+
+def create_json_column(spec, column_by_spec_getter, column_options):
+    return JsonColumn(column_by_spec_getter, **column_options)

clickhouse_driver/columns/lowcardinalitycolumn.py

@@ -35,25 +35,32 @@ class LowCardinalityColumn(Column):
     serialization_type = has_additional_keys_bit | need_update_dictionary
 
     def __init__(self, nested_column, **kwargs):
+        self.init_kwargs = kwargs
         self.nested_column = nested_column
         super(LowCardinalityColumn, self).__init__(**kwargs)
 
     def read_state_prefix(self, buf):
-        return read_binary_uint64(buf)
+        super(LowCardinalityColumn, self).read_state_prefix(buf)
+
+        read_binary_uint64(buf)
 
     def write_state_prefix(self, buf):
+        super(LowCardinalityColumn, self).write_state_prefix(buf)
+
         # KeysSerializationVersion. See ClickHouse docs.
         write_binary_int64(1, buf)
 
     def _write_data(self, items, buf):
         index, keys = [], []
         key_by_index_element = {}
+        nested_is_nullable = False
 
         if self.nested_column.nullable:
             # First element represents NULL if column is nullable.
             index.append(self.nested_column.null_value)
             # Prevent null map writing. Reset nested column nullable flag.
             self.nested_column.nullable = False
+            nested_is_nullable = True
 
             for x in items:
                 if x is None:
@@ -87,14 +94,26 @@ class LowCardinalityColumn(Column):
             return
 
         int_type = int(log(len(index), 2) / 8)
-        int_column = self.int_types[int_type]()
+        int_column = self.int_types[int_type](**self.init_kwargs)
 
         serialization_type = self.serialization_type | int_type
 
         write_binary_int64(serialization_type, buf)
         write_binary_int64(len(index), buf)
 
-        self.nested_column.write_data(index, buf)
+        if nested_is_nullable:
+            # Given we reset nested column nullable flag above,
+            # we need to write null map manually. If to invoke
+            # write_data method, it will cause an exception,
+            # because `prepare_data` may not be able to handle
+            # null value correctly.
+            self.nested_column.write_items(
+                [self.nested_column.null_value], buf)
+            # Remove null map from index, because it is already written.
+            index_to_write = index[1:]
+            self.nested_column.write_data(index_to_write, buf)
+        else:
+            self.nested_column.write_data(index, buf)
         write_binary_int64(len(items), buf)
         int_column.write_items(keys, buf)
 
@@ -106,7 +125,7 @@ class LowCardinalityColumn(Column):
 
         # Lowest byte contains info about key type.
         key_type = serialization_type & 0xf
-        keys_column = self.int_types[key_type]()
+        keys_column = self.int_types[key_type](**self.init_kwargs)
 
         nullable = self.nested_column.nullable
         # Prevent null map reading. Reset nested column nullable flag.

clickhouse_driver/columns/mapcolumn.py

@@ -13,16 +13,20 @@ class MapColumn(Column):
     null_value = {}
 
     def __init__(self, key_column, value_column, **kwargs):
-        self.offset_column = UInt64Column()
+        self.offset_column = UInt64Column(**kwargs)
         self.key_column = key_column
         self.value_column = value_column
         super(MapColumn, self).__init__(**kwargs)
 
     def read_state_prefix(self, buf):
+        super(MapColumn, self).read_state_prefix(buf)
+
         self.key_column.read_state_prefix(buf)
         self.value_column.read_state_prefix(buf)
 
     def write_state_prefix(self, buf):
+        super(MapColumn, self).write_state_prefix(buf)
+
         self.key_column.write_state_prefix(buf)
         self.value_column.write_state_prefix(buf)
 
@@ -57,7 +61,7 @@ class MapColumn(Column):
 
 
 def create_map_column(spec, column_by_spec_getter, column_options):
-    # Match commas outside of parentheses so we don't match the comma in
+    # Match commas outside of parentheses, so we don't match the comma in
     # Decimal types.
     key, value = comma_re.split(spec[4:-1])
     key_column = column_by_spec_getter(key.strip())

clickhouse_driver/columns/nestedcolumn.py

@@ -1,21 +1,10 @@
 
 from .arraycolumn import create_array_column
-from .util import get_inner_spec, get_inner_columns, \
-    get_inner_columns_with_types
+from .util import get_inner_spec
 
 
 def create_nested_column(spec, column_by_spec_getter, column_options):
     return create_array_column(
-        'Array(Tuple({}))'.format(','.join(get_nested_columns(spec))),
+        'Array(Tuple({}))'.format(get_inner_spec('Nested', spec)),
         column_by_spec_getter, column_options
     )
-
-
-def get_nested_columns(spec):
-    inner_spec = get_inner_spec('Nested', spec)
-    return get_inner_columns(inner_spec)
-
-
-def get_columns_with_types(spec):
-    inner_spec = get_inner_spec('Nested', spec)
-    return get_inner_columns_with_types(inner_spec)