clickhouse-driver 0.2.5__cp38-cp38-win32.whl → 0.2.7__cp38-cp38-win32.whl

clickhouse_driver/columns/numpy/datetimecolumn.py

@@ -21,12 +21,12 @@ class NumpyDateTimeColumnBase(NumpyColumn):
     def apply_timezones_after_read(self, dt):
         timezone = self.timezone if self.timezone else self.local_timezone
 
-        ts = pd.to_datetime(dt, utc=True).tz_convert(timezone)
-
-        if self.offset_naive:
+        if self.offset_naive and timezone.zone != 'UTC':
+            ts = pd.to_datetime(dt, utc=True).tz_convert(timezone)
             ts = ts.tz_localize(None)
+            return ts.to_numpy(self.datetime_dtype)
 
-        return ts.to_numpy(self.datetime_dtype)
+        return dt
 
     def apply_timezones_before_write(self, items):
         if isinstance(items, pd.DatetimeIndex):
@@ -65,7 +65,7 @@ class NumpyDateTimeColumn(NumpyDateTimeColumnBase):
 
 
 class NumpyDateTime64Column(NumpyDateTimeColumnBase):
-    dtype = np.dtype(np.uint64)
+    dtype = np.dtype(np.int64)
     datetime_dtype = 'datetime64[ns]'
 
     max_scale = 9
@@ -75,15 +75,15 @@ class NumpyDateTime64Column(NumpyDateTimeColumnBase):
         super(NumpyDateTime64Column, self).__init__(**kwargs)
 
     def read_items(self, n_items, buf):
-        scale = 10 ** self.scale
-        frac_scale = 10 ** (self.max_scale - self.scale)
-
+        # Clickhouse:     t seconds is represented as t * 10^scale.
+        # datetime64[ns]: t seconds is represented as t * 10^9.
+        # Since 0 <= scale <= 9, multiply by the integer 10^(9 - scale).
         items = super(NumpyDateTime64Column, self).read_items(n_items, buf)
 
-        seconds = (items // scale).astype('datetime64[s]')
-        microseconds = ((items % scale) * frac_scale).astype('timedelta64[ns]')
+        tmp = np.copy(items)
+        tmp *= 10 ** (9 - self.scale)
+        dt = tmp.view(dtype='datetime64[ns]')
 
-        dt = seconds + microseconds
         return self.apply_timezones_after_read(dt)
 
     def write_items(self, items, buf):
@@ -120,12 +120,12 @@ def create_numpy_datetime_column(spec, column_options):
 
     context = column_options['context']
 
-    tz_name = timezone = None
+    tz_name = None
     offset_naive = True
 
     # As Numpy do not use local timezone for converting timestamp to
     # datetime we need always detect local timezone for manual converting.
-    local_timezone = get_localzone_name_compat()
+    local_tz_name = get_localzone_name_compat()
 
     # Use column's timezone if it's specified.
     if spec and spec[-1] == ')':
@@ -133,11 +133,11 @@ def create_numpy_datetime_column(spec, column_options):
         offset_naive = False
     else:
         if not context.settings.get('use_client_time_zone', False):
-            if local_timezone != context.server_info.timezone:
+            if local_tz_name != context.server_info.timezone:
                 tz_name = context.server_info.timezone
 
-    if tz_name:
-        timezone = get_timezone(tz_name)
+    timezone = get_timezone(tz_name) if tz_name else None
+    local_timezone = get_timezone(local_tz_name) if local_tz_name else None
 
     return cls(timezone=timezone, offset_naive=offset_naive,
                local_timezone=local_timezone, **column_options)

clickhouse_driver/columns/numpy/lowcardinalitycolumn.py

@@ -37,7 +37,7 @@ class NumpyLowCardinalityColumn(LowCardinalityColumn):
         c = pd.Categorical(items)
 
         int_type = int(log(len(c.codes), 2) / 8)
-        int_column = self.int_types[int_type]()
+        int_column = self.int_types[int_type](**self.init_kwargs)
 
         serialization_type = self.serialization_type | int_type
 
@@ -66,7 +66,7 @@ class NumpyLowCardinalityColumn(LowCardinalityColumn):
 
         # Lowest byte contains info about key type.
         key_type = serialization_type & 0xf
-        keys_column = self.int_types[key_type]()
+        keys_column = self.int_types[key_type](**self.init_kwargs)
 
         nullable = self.nested_column.nullable
         # Prevent null map reading. Reset nested column nullable flag.

clickhouse_driver/columns/service.py

@@ -15,6 +15,7 @@ from .intcolumn import (
     UInt8Column, UInt16Column, UInt32Column, UInt64Column
 )
 from .lowcardinalitycolumn import create_low_cardinality_column
+from .jsoncolumn import create_json_column
 from .mapcolumn import create_map_column
 from .nothingcolumn import NothingColumn
 from .nullcolumn import NullColumn
@@ -122,6 +123,11 @@ def get_column_by_spec(spec, column_options, use_numpy=None):
             spec, create_column_with_options, column_options
         )
 
+    elif spec.startswith("Object('json')"):
+        return create_json_column(
+            spec, create_column_with_options, column_options
+        )
+
     else:
         for alias, primitive in aliases:
             if spec.startswith(alias):
@@ -137,8 +143,12 @@ def get_column_by_spec(spec, column_options, use_numpy=None):
             raise errors.UnknownTypeError('Unknown type {}'.format(spec))
 
 
-def read_column(context, column_spec, n_items, buf, use_numpy=None):
-    column_options = {'context': context}
+def read_column(context, column_spec, n_items, buf, use_numpy=None,
+                has_custom_serialization=False):
+    column_options = {
+        'context': context,
+        'has_custom_serialization': has_custom_serialization
+    }
     col = get_column_by_spec(column_spec, column_options, use_numpy=use_numpy)
     col.read_state_prefix(buf)
     return col.read_data(n_items, buf)

clickhouse_driver/columns/tuplecolumn.py

@@ -1,13 +1,21 @@
 
 from .base import Column
-from .util import get_inner_spec, get_inner_columns
+from .util import get_inner_spec, get_inner_columns_with_types
 
 
 class TupleColumn(Column):
     py_types = (list, tuple)
 
-    def __init__(self, nested_columns, **kwargs):
+    def __init__(self, names, nested_columns, **kwargs):
+        self.names = names
         self.nested_columns = nested_columns
+        client_settings = kwargs['context'].client_settings
+        settings = kwargs['context'].settings
+        self.namedtuple_as_json = (
+            settings.get('allow_experimental_object_type', False) and
+            client_settings.get('namedtuple_as_json', True)
+        )
+
         super(TupleColumn, self).__init__(**kwargs)
         self.null_value = tuple(x.null_value for x in nested_columns)
 
@@ -23,15 +31,33 @@ class TupleColumn(Column):
 
     def read_data(self, n_items, buf):
         rv = [x.read_data(n_items, buf) for x in self.nested_columns]
-        return list(zip(*rv))
+        rv = list(zip(*rv))
+
+        if self.names[0] and self.namedtuple_as_json:
+            return [dict(zip(self.names, x)) for x in rv]
+        else:
+            return rv
 
     def read_items(self, n_items, buf):
         return self.read_data(n_items, buf)
 
+    def read_state_prefix(self, buf):
+        super(TupleColumn, self).read_state_prefix(buf)
+
+        for x in self.nested_columns:
+            x.read_state_prefix(buf)
+
+    def write_state_prefix(self, buf):
+        super(TupleColumn, self).write_state_prefix(buf)
+
+        for x in self.nested_columns:
+            x.write_state_prefix(buf)
+
 
 def create_tuple_column(spec, column_by_spec_getter, column_options):
     inner_spec = get_inner_spec('Tuple', spec)
-    columns = get_inner_columns(inner_spec)
+    columns_with_types = get_inner_columns_with_types(inner_spec)
+    names, types = zip(*columns_with_types)
 
-    return TupleColumn([column_by_spec_getter(x) for x in columns],
+    return TupleColumn(names, [column_by_spec_getter(x) for x in types],
                        **column_options)

clickhouse_driver/columns/uuidcolumn.py

@@ -54,7 +54,7 @@ class UUIDColumn(FormatColumn):
 
             try:
                 if not isinstance(item, UUID):
-                    item = UUID(item)
+                    item = UUID(int=item) if item is null_value else UUID(item)
 
             except ValueError:
                 raise errors.CannotParseUuidError(

clickhouse_driver/connection.py

@@ -3,6 +3,7 @@ import socket
 import ssl
 from collections import deque
 from contextlib import contextmanager
+from sys import platform
 from time import time
 from urllib.parse import urlparse
 
@@ -19,11 +20,12 @@ from .log import log_block
 from .progress import Progress
 from .protocol import Compression, ClientPacketTypes, ServerPacketTypes
 from .queryprocessingstage import QueryProcessingStage
-from .reader import read_binary_str
+from .reader import read_binary_str, read_binary_uint64
 from .readhelpers import read_exception
-from .settings.writer import write_settings
+from .settings.writer import write_settings, SettingsFlags
 from .streams.native import BlockInputStream, BlockOutputStream
 from .util.compat import threading
+from .util.escape import escape_params
 from .varint import write_varint, read_varint
 from .writer import write_binary_str
 
@@ -44,7 +46,7 @@ class Packet(object):
 
 class ServerInfo(object):
     def __init__(self, name, version_major, version_minor, version_patch,
-                 revision, timezone, display_name):
+                 revision, timezone, display_name, used_revision):
         self.name = name
         self.version_major = version_major
         self.version_minor = version_minor
@@ -52,6 +54,7 @@ class ServerInfo(object):
         self.revision = revision
         self.timezone = timezone
         self.display_name = display_name
+        self.used_revision = used_revision
 
         super(ServerInfo, self).__init__()
 
@@ -66,6 +69,7 @@ class ServerInfo(object):
             ('name', self.name),
             ('version', version),
             ('revision', self.revision),
+            ('used revision', self.used_revision),
             ('timezone', self.timezone),
             ('display_name', self.display_name)
         ]
@@ -124,6 +128,15 @@ class Connection(object):
                                   ignored, ``True`` means that the query will
                                   fail with UNKNOWN_SETTING error.
                                   Defaults to ``False``.
+    :param tcp_keepalive: enables `TCP keepalive <https://tldp.org/HOWTO/
+                          TCP-Keepalive-HOWTO/overview.html>`_ on established
+                          connection. If is set to ``True``` system keepalive
+                          settings are used. You can also specify custom
+                          keepalive setting with tuple:
+                          ``(idle_time_sec, interval_sec, probes)``.
+                          Defaults to ``False``.
+    :param client_revision: can be used for client version downgrading.
+                          Defaults to ``None``.
     """
 
     def __init__(
@@ -143,6 +156,8 @@ class Connection(object):
             server_hostname=None,
             alt_hosts=None,
             settings_is_important=False,
+            tcp_keepalive=False,
+            client_revision=None
     ):
         if secure:
             default_port = defines.DEFAULT_SECURE_PORT
@@ -164,6 +179,10 @@ class Connection(object):
         self.send_receive_timeout = send_receive_timeout
         self.sync_request_timeout = sync_request_timeout
         self.settings_is_important = settings_is_important
+        self.tcp_keepalive = tcp_keepalive
+        self.client_revision = min(
+            client_revision or defines.CLIENT_REVISION, defines.CLIENT_REVISION
+        )
 
         self.secure_socket = secure
         self.verify_cert = verify
@@ -284,6 +303,7 @@ class Connection(object):
 
         version = ssl_options.get('ssl_version', ssl.PROTOCOL_TLS)
         context = ssl.SSLContext(version)
+        context.check_hostname = self.verify_cert
 
         if 'ca_certs' in ssl_options:
             context.load_verify_locations(ssl_options['ca_certs'])
@@ -310,6 +330,8 @@ class Connection(object):
 
         # performance tweak
         self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+        if self.tcp_keepalive:
+            self._set_keepalive()
 
         self.fin = BufferedSocketReader(self.socket, defines.BUFFER_SIZE)
         self.fout = BufferedSocketWriter(self.socket, defines.BUFFER_SIZE)
@@ -317,10 +339,42 @@ class Connection(object):
         self.send_hello()
         self.receive_hello()
 
+        revision = self.server_info.used_revision
+        if revision >= defines.DBMS_MIN_PROTOCOL_VERSION_WITH_ADDENDUM:
+            self.send_addendum()
+
         self.block_in = self.get_block_in_stream()
         self.block_in_raw = BlockInputStream(self.fin, self.context)
         self.block_out = self.get_block_out_stream()
 
+    def _set_keepalive(self):
+        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+
+        if not isinstance(self.tcp_keepalive, tuple):
+            return
+
+        idle_time_sec, interval_sec, probes = self.tcp_keepalive
+
+        if platform == 'linux' or platform == 'win32':
+            # This should also work for Windows
+            # starting with Windows 10, version 1709.
+            self.socket.setsockopt(
+                socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, idle_time_sec
+            )
+            self.socket.setsockopt(
+                socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, interval_sec
+            )
+            self.socket.setsockopt(
+                socket.IPPROTO_TCP, socket.TCP_KEEPCNT, probes
+            )
+
+        elif platform == 'darwin':
+            TCP_KEEPALIVE = 0x10
+            # Only interval is available in mac os.
+            self.socket.setsockopt(
+                socket.IPPROTO_TCP, TCP_KEEPALIVE, interval_sec
+            )
+
     def _format_connection_error(self, e, host, port):
         err = (e.strerror + ' ') if e.strerror else ''
         return err + '({}:{})'.format(host, port)
@@ -410,7 +464,7 @@ class Connection(object):
         write_varint(defines.CLIENT_VERSION_MINOR, self.fout)
         # NOTE For backward compatibility of the protocol,
         # client cannot send its version_patch.
-        write_varint(defines.CLIENT_REVISION, self.fout)
+        write_varint(self.client_revision, self.fout)
         write_binary_str(self.database, self.fout)
         write_binary_str(self.user, self.fout)
         write_binary_str(self.password, self.fout)
@@ -426,25 +480,38 @@ class Connection(object):
             server_version_minor = read_varint(self.fin)
             server_revision = read_varint(self.fin)
 
+            used_revision = min(self.client_revision, server_revision)
+
             server_timezone = None
-            if server_revision >= \
+            if used_revision >= \
                     defines.DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE:
                 server_timezone = read_binary_str(self.fin)
 
             server_display_name = ''
-            if server_revision >= \
+            if used_revision >= \
                     defines.DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME:
                 server_display_name = read_binary_str(self.fin)
 
             server_version_patch = server_revision
-            if server_revision >= \
+            if used_revision >= \
                     defines.DBMS_MIN_REVISION_WITH_VERSION_PATCH:
                 server_version_patch = read_varint(self.fin)
 
+            if used_revision >= defines. \
+                    DBMS_MIN_PROTOCOL_VERSION_WITH_PASSWORD_COMPLEXITY_RULES:
+                rules_size = read_varint(self.fin)
+                for _i in range(rules_size):
+                    read_binary_str(self.fin)  # original_pattern
+                    read_binary_str(self.fin)  # exception_message
+
+            if used_revision >= defines. \
+                    DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2:
+                read_binary_uint64(self.fin)  # read_nonce
+
             self.server_info = ServerInfo(
                 server_name, server_version_major, server_version_minor,
                 server_version_patch, server_revision,
-                server_timezone, server_display_name
+                server_timezone, server_display_name, used_revision
             )
             self.context.server_info = self.server_info
 
@@ -463,6 +530,14 @@ class Connection(object):
             self.disconnect()
             raise errors.UnexpectedPacketFromServerError(message)
 
+    def send_addendum(self):
+        revision = self.server_info.used_revision
+
+        if revision >= defines.DBMS_MIN_PROTOCOL_VERSION_WITH_QUOTA_KEY:
+            write_binary_str(
+                self.context.client_settings['quota_key'], self.fout
+            )
+
     def ping(self):
         timeout = self.sync_request_timeout
 
@@ -567,7 +642,7 @@ class Connection(object):
             return BlockOutputStream(self.fout, self.context)
 
     def receive_data(self, may_be_compressed=True, may_be_use_numpy=False):
-        revision = self.server_info.revision
+        revision = self.server_info.used_revision
 
         if revision >= defines.DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES:
             read_binary_str(self.fin)
@@ -581,7 +656,7 @@ class Connection(object):
 
     def receive_progress(self):
         progress = Progress()
-        progress.read(self.server_info.revision, self.fin)
+        progress.read(self.server_info, self.fin)
         return progress
 
     def receive_profile_info(self):
@@ -597,14 +672,14 @@ class Connection(object):
         start = time()
         write_varint(ClientPacketTypes.DATA, self.fout)
 
-        revision = self.server_info.revision
+        revision = self.server_info.used_revision
         if revision >= defines.DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES:
             write_binary_str(table_name, self.fout)
 
         self.block_out.write(block)
         logger.debug('Block "%s" send time: %f', table_name, time() - start)
 
-    def send_query(self, query, query_id=None):
+    def send_query(self, query, query_id=None, params=None):
         if not self.connected:
             self.connect()
 
@@ -612,9 +687,10 @@ class Connection(object):
 
         write_binary_str(query_id or '', self.fout)
 
-        revision = self.server_info.revision
+        revision = self.server_info.used_revision
         if revision >= defines.DBMS_MIN_REVISION_WITH_CLIENT_INFO:
-            client_info = ClientInfo(self.client_name, self.context)
+            client_info = ClientInfo(self.client_name, self.context,
+                                     client_revision=self.client_revision)
             client_info.query_kind = ClientInfo.QueryKind.INITIAL_QUERY
 
             client_info.write(revision, self.fout)
@@ -623,8 +699,11 @@ class Connection(object):
             revision >= defines
             .DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS
         )
+        settings_flags = 0
+        if self.settings_is_important:
+            settings_flags |= SettingsFlags.IMPORTANT
         write_settings(self.context.settings, self.fout, settings_as_strings,
-                       self.settings_is_important)
+                       settings_flags)
 
         if revision >= defines.DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET:
             write_binary_str('', self.fout)
@@ -634,6 +713,16 @@ class Connection(object):
 
         write_binary_str(query, self.fout)
 
+        if revision >= defines.DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS:
+            if self.context.client_settings['server_side_params']:
+                # Always settings_as_strings = True
+                escaped = escape_params(
+                    params or {}, self.context, for_server=True
+                )
+            else:
+                escaped = {}
+            write_settings(escaped, self.fout, True, SettingsFlags.CUSTOM)
+
         logger.debug('Query: %s', query)
 
         self.fout.flush()

clickhouse_driver/defines.py

@@ -25,6 +25,14 @@ DBMS_MIN_PROTOCOL_VERSION_WITH_DISTRIBUTED_DEPTH = 54448
 DBMS_MIN_PROTOCOL_VERSION_WITH_INITIAL_QUERY_START_TIME = 54449
 DBMS_MIN_PROTOCOL_VERSION_WITH_INCREMENTAL_PROFILE_EVENTS = 54451
 DBMS_MIN_REVISION_WITH_PARALLEL_REPLICAS = 54453
+DBMS_MIN_REVISION_WITH_CUSTOM_SERIALIZATION = 54454
+DBMS_MIN_PROTOCOL_VERSION_WITH_PROFILE_EVENTS_IN_INSERT = 54456
+DBMS_MIN_PROTOCOL_VERSION_WITH_ADDENDUM = 54458
+DBMS_MIN_PROTOCOL_VERSION_WITH_QUOTA_KEY = 54458
+DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS = 54459
+DBMS_MIN_PROTOCOL_VERSION_WITH_SERVER_QUERY_TIME_IN_PROGRESS = 54460
+DBMS_MIN_PROTOCOL_VERSION_WITH_PASSWORD_COMPLEXITY_RULES = 54461
+DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2 = 54462
 
 # Timeouts
 DBMS_DEFAULT_CONNECT_TIMEOUT_SEC = 10
@@ -40,7 +48,7 @@ CLIENT_NAME = 'python-driver'
 CLIENT_VERSION_MAJOR = 20
 CLIENT_VERSION_MINOR = 10
 CLIENT_VERSION_PATCH = 2
-CLIENT_REVISION = 54453
+CLIENT_REVISION = DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2
 
 BUFFER_SIZE = 1048576
 

clickhouse_driver/log.py

@@ -2,7 +2,8 @@ import logging
 
 logger = logging.getLogger(__name__)
 
-
+# Keep in sync with ClickHouse priorities
+# https://github.com/ClickHouse/ClickHouse/blob/master/src/Interpreters/InternalTextLogsQueue.cpp
 log_priorities = (
     'Unknown',
     'Fatal',
@@ -12,9 +13,12 @@ log_priorities = (
     'Notice',
     'Information',
     'Debug',
-    'Trace'
+    'Trace',
+    'Test',
 )
 
+num_priorities = len(log_priorities)
+
 
 def log_block(block):
     if block is None:
@@ -25,7 +29,7 @@ def log_block(block):
     for row in block.get_rows():
         row = dict(zip(column_names, row))
 
-        if 1 <= row['priority'] <= 8:
+        if 1 <= row['priority'] <= num_priorities:
             priority = log_priorities[row['priority']]
         else:
             priority = row[0]

clickhouse_driver/progress.py

@@ -9,14 +9,15 @@ class Progress(object):
         self.total_rows = 0
         self.written_rows = 0
         self.written_bytes = 0
+        self.elapsed_ns = 0
 
         super(Progress, self).__init__()
 
-    def read(self, server_revision, fin):
+    def read(self, server_info, fin):
         self.rows = read_varint(fin)
         self.bytes = read_varint(fin)
 
-        revision = server_revision
+        revision = server_info.used_revision
         if revision >= defines.DBMS_MIN_REVISION_WITH_TOTAL_ROWS_IN_PROGRESS:
             self.total_rows = read_varint(fin)
 
@@ -24,9 +25,14 @@ class Progress(object):
             self.written_rows = read_varint(fin)
             self.written_bytes = read_varint(fin)
 
+        if revision >= defines. \
+                DBMS_MIN_PROTOCOL_VERSION_WITH_SERVER_QUERY_TIME_IN_PROGRESS:
+            self.elapsed_ns = read_varint(fin)
+
     def increment(self, another_progress):
         self.rows += another_progress.rows
         self.bytes += another_progress.bytes
         self.total_rows += another_progress.total_rows
         self.written_rows += another_progress.written_rows
         self.written_bytes += another_progress.written_bytes
+        self.elapsed_ns += another_progress.elapsed_ns

clickhouse_driver/settings/writer.py

@@ -7,13 +7,18 @@ from .available import settings as available_settings
 logger = logging.getLogger(__name__)
 
 
-def write_settings(settings, buf, settings_as_strings, is_important=False):
+class SettingsFlags:
+    IMPORTANT = 0x1
+    CUSTOM = 0x2
+
+
+def write_settings(settings, buf, settings_as_strings, flags):
     for setting, value in (settings or {}).items():
         # If the server support settings as string we do not need to know
         # anything about them, so we can write any setting.
         if settings_as_strings:
             write_binary_str(setting, buf)
-            write_binary_uint8(int(is_important), buf)
+            write_binary_uint8(flags, buf)
             write_binary_str(str(value), buf)
 
         else:

clickhouse_driver/streams/native.py

@@ -1,8 +1,8 @@
 from ..block import ColumnOrientedBlock, BlockInfo
 from ..columns.service import read_column, write_column
-from ..reader import read_binary_str
+from ..reader import read_binary_str, read_binary_uint8
 from ..varint import write_varint, read_varint
-from ..writer import write_binary_str
+from ..writer import write_binary_str, write_binary_uint8
 from .. import defines
 
 
@@ -14,7 +14,7 @@ class BlockOutputStream(object):
         super(BlockOutputStream, self).__init__()
 
     def write(self, block):
-        revision = self.context.server_info.revision
+        revision = self.context.server_info.used_revision
         if revision >= defines.DBMS_MIN_REVISION_WITH_BLOCK_INFO:
             block.info.write(self.fout)
 
@@ -35,6 +35,11 @@ class BlockOutputStream(object):
                 except IndexError:
                     raise ValueError('Different rows length')
 
+                if revision >= \
+                        defines.DBMS_MIN_REVISION_WITH_CUSTOM_SERIALIZATION:
+                    # We write always sparse data without custom serialization.
+                    write_binary_uint8(0, self.fout)
+
                 write_column(self.context, col_name, col_type, items,
                              self.fout, types_check=block.types_check)
 
@@ -54,7 +59,7 @@ class BlockInputStream(object):
     def read(self, use_numpy=None):
         info = BlockInfo()
 
-        revision = self.context.server_info.revision
+        revision = self.context.server_info.used_revision
         if revision >= defines.DBMS_MIN_REVISION_WITH_BLOCK_INFO:
             info.read(self.fin)
 
@@ -70,9 +75,16 @@ class BlockInputStream(object):
             names.append(column_name)
             types.append(column_type)
 
+            has_custom_serialization = False
+            if revision >= defines.DBMS_MIN_REVISION_WITH_CUSTOM_SERIALIZATION:
+                has_custom_serialization = bool(read_binary_uint8(self.fin))
+
             if n_rows:
-                column = read_column(self.context, column_type, n_rows,
-                                     self.fin, use_numpy=use_numpy)
+                column = read_column(
+                    self.context, column_type, n_rows,
+                    self.fin, use_numpy=use_numpy,
+                    has_custom_serialization=has_custom_serialization
+                )
                 data.append(column)
 
         if self.context.client_settings['use_numpy']:

clickhouse_driver/util/compat.py

@@ -5,6 +5,18 @@ try:
 except ImportError:
     import dummy_threading as threading  # noqa: F401
 
+import json  # noqa: F401
+try:
+    import orjson as json  # noqa: F811
+except ImportError:
+    pass
+
+try:
+    import ujson as json  # noqa: F811,F401
+except ImportError:
+    pass
+
+
 try:
     # since tzlocal 4.0+
     # this will avoid warning for get_localzone().key