clickhouse-driver 0.2.3__cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl → 0.2.9__cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl

clickhouse_driver/__init__.py

@@ -3,7 +3,7 @@ from .client import Client
 from .dbapi import connect
 
 
-VERSION = (0, 2, 3)
+VERSION = (0, 2, 9)
 __version__ = '.'.join(str(x) for x in VERSION)
 
 __all__ = ['Client', 'connect']

clickhouse_driver/block.py

@@ -1,7 +1,7 @@
+from .columns.util import get_inner_spec, get_inner_columns_with_types
 from .reader import read_varint, read_binary_uint8, read_binary_int32
 from .varint import write_varint
 from .writer import write_binary_uint8, write_binary_int32
-from .columns import nestedcolumn
 
 
 class BlockInfo(object):
@@ -172,7 +172,8 @@ class RowOrientedBlock(BaseBlock):
         for name, type_ in columns_with_types:
             cwt = None
             if type_.startswith('Nested'):
-                cwt = nestedcolumn.get_columns_with_types(type_)
+                inner_spec = get_inner_spec('Nested', type_)
+                cwt = get_inner_columns_with_types(inner_spec)
             columns_with_cwt.append((name, cwt))
 
         for i, row in enumerate(data):

clickhouse_driver/client.py

@@ -1,9 +1,9 @@
 import re
-import ssl
+from collections import deque
 from contextlib import contextmanager
 from time import time
 import types
-from urllib.parse import urlparse, parse_qs, unquote
+from urllib.parse import urlparse
 
 from . import errors, defines
 from .block import ColumnOrientedBlock, RowOrientedBlock
@@ -14,7 +14,7 @@ from .result import (
     IterQueryResult, ProgressQueryResult, QueryResult, QueryInfo
 )
 from .util.escape import escape_params
-from .util.helpers import column_chunks, chunks, asbool
+from .util.helpers import column_chunks, chunks, parse_url
 
 
 class Client(object):
@@ -26,7 +26,7 @@ class Client(object):
                      for the client settings, see below). Defaults to ``None``
                      (no additional settings). See all available settings in
                      `ClickHouse docs
-                     <https://clickhouse.tech/docs/en/operations/settings/settings/>`_.
+                     <https://clickhouse.com/docs/en/operations/settings/settings/>`_.
     :param \\**kwargs: All other args are passed to the
                        :py:class:`~clickhouse_driver.connection.Connection`
                        constructor.
@@ -38,7 +38,7 @@ class Client(object):
           Defaults to ``1048576``.
         * ``strings_as_bytes`` -- turns off string column encoding/decoding.
         * ``strings_encoding`` -- specifies string encoding. UTF-8 by default.
-        * ``use_numpy`` -- Use numpy for columns reading. New in version
+        * ``use_numpy`` -- Use NumPy for columns reading. New in version
                            *0.2.0*.
         * ``opentelemetry_traceparent`` -- OpenTelemetry traceparent header as
                            described by W3C Trace Context recommendation.
@@ -49,6 +49,24 @@ class Client(object):
         * ``quota_key`` -- A string to differentiate quotas when the user have
                            keyed quotas configured on server.
                            New in version *0.2.3*.
+        * ``input_format_null_as_default`` -- Initialize null fields with
+                           default values if data type of this field is not
+                           nullable. Does not work for NumPy. Default: False.
+                           New in version *0.2.4*.
+        * ``round_robin`` -- If ``alt_hosts`` are provided the query will be
+                           executed on host picked with round-robin algorithm.
+                           New in version *0.2.5*.
+        * ``namedtuple_as_json`` -- Controls named tuple and nested types
+                           deserialization. To interpret these column alongside
+                           with ``allow_experimental_object_type=1`` as Python
+                           tuple set ``namedtuple_as_json`` to ``False``.
+                           Default: True.
+                           New in version *0.2.6*.
+        * ``server_side_params`` -- Species on which side query parameters
+                           should be rendered into placeholders.
+                           Default: False. Means that parameters are rendered
+                           on driver's side.
+                           New in version *0.2.7*.
     """
 
     available_client_settings = (
@@ -58,7 +76,10 @@ class Client(object):
         'use_numpy',
         'opentelemetry_traceparent',
         'opentelemetry_tracestate',
-        'quota_key'
+        'quota_key',
+        'input_format_null_as_default',
+        'namedtuple_as_json',
+        'server_side_params'
     )
 
     def __init__(self, *args, **kwargs):
@@ -85,6 +106,15 @@ class Client(object):
             ),
             'quota_key': self.settings.pop(
                 'quota_key', ''
+            ),
+            'input_format_null_as_default': self.settings.pop(
+                'input_format_null_as_default', False
+            ),
+            'namedtuple_as_json': self.settings.pop(
+                'namedtuple_as_json', True
+            ),
+            'server_side_params': self.settings.pop(
+                'server_side_params', False
             )
         }
 
@@ -104,9 +134,33 @@ class Client(object):
             self.iter_query_result_cls = IterQueryResult
             self.progress_query_result_cls = ProgressQueryResult
 
-        self.connection = Connection(*args, **kwargs)
-        self.connection.context.settings = self.settings
-        self.connection.context.client_settings = self.client_settings
+        round_robin = kwargs.pop('round_robin', False)
+        self.connections = deque([Connection(*args, **kwargs)])
+
+        if round_robin and 'alt_hosts' in kwargs:
+            alt_hosts = kwargs.pop('alt_hosts')
+            for host in alt_hosts.split(','):
+                url = urlparse('clickhouse://' + host)
+
+                connection_kwargs = kwargs.copy()
+                num_args = len(args)
+                if num_args >= 2:
+                    # host and port as positional arguments
+                    connection_args = (url.hostname, url.port) + args[2:]
+                elif num_args >= 1:
+                    # host as positional and port as keyword argument
+                    connection_args = (url.hostname, ) + args[1:]
+                    connection_kwargs['port'] = url.port
+                else:
+                    # host and port as keyword arguments
+                    connection_args = tuple()
+                    connection_kwargs['host'] = url.hostname
+                    connection_kwargs['port'] = url.port
+
+                connection = Connection(*connection_args, **connection_kwargs)
+                self.connections.append(connection)
+
+        self.connection = self.get_connection()
         self.reset_last_query()
         super(Client, self).__init__()
 
@@ -116,7 +170,22 @@ class Client(object):
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.disconnect()
 
+    def get_connection(self):
+        if hasattr(self, 'connection'):
+            self.connections.append(self.connection)
+
+        connection = self.connections.popleft()
+
+        connection.context.settings = self.settings
+        connection.context.client_settings = self.client_settings
+        return connection
+
     def disconnect(self):
+        self.disconnect_connection()
+        for connection in self.connections:
+            connection.disconnect()
+
+    def disconnect_connection(self):
         """
         Disconnects from the server.
         """
@@ -219,13 +288,30 @@ class Client(object):
         if query.lower().startswith('use '):
             self.connection.database = query[4:].strip()
 
+    def establish_connection(self, settings):
+        num_connections = len(self.connections)
+        if hasattr(self, 'connection'):
+            num_connections += 1
+
+        for i in range(num_connections):
+            try:
+                self.connection = self.get_connection()
+                self.make_query_settings(settings)
+                self.connection.force_connect()
+                self.last_query = QueryInfo()
+
+            except (errors.SocketTimeoutError, errors.NetworkError):
+                if i < num_connections - 1:
+                    continue
+                raise
+
+            return
+
     @contextmanager
     def disconnect_on_error(self, query, settings):
-        self.make_query_settings(settings)
-
         try:
-            self.connection.force_connect()
-            self.last_query = QueryInfo()
+            self.establish_connection(settings)
+            self.connection.server_info.session_timezone = None
 
             yield
 
@@ -342,7 +428,7 @@ class Client(object):
     def execute_iter(
             self, query, params=None, with_column_types=False,
             external_tables=None, query_id=None, settings=None,
-            types_check=False):
+            types_check=False, chunk_size=1):
         """
         *New in version 0.0.14.*
 
@@ -364,19 +450,20 @@ class Client(object):
                          Defaults to ``None`` (no additional settings).
         :param types_check: enables type checking of data for INSERT queries.
                             Causes additional overhead. Defaults to ``False``.
+        :param chunk_size: chunk query results.
         :return: :ref:`iter-query-result` proxy.
         """
-
         with self.disconnect_on_error(query, settings):
-            return self.iter_process_ordinary_query(
+            rv = self.iter_process_ordinary_query(
                 query, params=params, with_column_types=with_column_types,
                 external_tables=external_tables,
                 query_id=query_id, types_check=types_check
             )
+            return chunks(rv, chunk_size) if chunk_size > 1 else rv
 
     def query_dataframe(
             self, query, params=None, external_tables=None, query_id=None,
-            settings=None):
+            settings=None, replace_nonwords=True):
         """
         *New in version 0.2.0.*
 
@@ -391,6 +478,8 @@ class Client(object):
                          ClickHouse server will generate it.
         :param settings: dictionary of query settings.
                          Defaults to ``None`` (no additional settings).
+        :param replace_nonwords: boolean to replace non-words in column names
+                                 to underscores. Defaults to ``True``.
         :return: pandas DataFrame.
         """
 
@@ -405,8 +494,12 @@ class Client(object):
             settings=settings
         )
 
+        columns = [name for name, type_ in columns]
+        if replace_nonwords:
+            columns = [re.sub(r'\W', '_', x) for x in columns]
+
         return pd.DataFrame(
-            {re.sub(r'\W', '_', col[0]): d for d, col in zip(data, columns)}
+            {col: d for d, col in zip(data, columns)}, columns=columns
         )
 
     def insert_dataframe(
@@ -443,6 +536,12 @@ class Client(object):
             rv = None
             if sample_block:
                 columns = [x[0] for x in sample_block.columns_with_types]
+                # raise if any columns are missing from the dataframe
+                diff = set(columns) - set(dataframe.columns)
+                if len(diff):
+                    msg = "DataFrame missing required columns: {}"
+                    raise ValueError(msg.format(list(diff)))
+
                 data = [dataframe[column].values for column in columns]
                 rv = self.send_data(sample_block, data, columnar=True)
                 self.receive_end_of_query()
@@ -460,7 +559,7 @@ class Client(object):
                 query, params, self.connection.context
             )
 
-        self.connection.send_query(query, query_id=query_id)
+        self.connection.send_query(query, query_id=query_id, params=params)
         self.connection.send_external_tables(external_tables,
                                              types_check=types_check)
         return self.receive_result(with_column_types=with_column_types,
@@ -475,8 +574,7 @@ class Client(object):
             query = self.substitute_params(
                 query, params, self.connection.context
             )
-
-        self.connection.send_query(query, query_id=query_id)
+        self.connection.send_query(query, query_id=query_id, params=params)
         self.connection.send_external_tables(external_tables,
                                              types_check=types_check)
         return self.receive_result(with_column_types=with_column_types,
@@ -492,7 +590,7 @@ class Client(object):
                 query, params, self.connection.context
             )
 
-        self.connection.send_query(query, query_id=query_id)
+        self.connection.send_query(query, query_id=query_id, params=params)
         self.connection.send_external_tables(external_tables,
                                              types_check=types_check)
         return self.iter_receive_result(with_column_types=with_column_types)
@@ -503,12 +601,12 @@ class Client(object):
         self.connection.send_query(query_without_data, query_id=query_id)
         self.connection.send_external_tables(external_tables,
                                              types_check=types_check)
-
         sample_block = self.receive_sample_block()
+
         if sample_block:
             rv = self.send_data(sample_block, data,
                                 types_check=types_check, columnar=columnar)
-            self.receive_end_of_query()
+            self.receive_end_of_insert_query()
             return rv
 
     def receive_sample_block(self):
@@ -562,8 +660,15 @@ class Client(object):
             self.connection.send_data(block)
             inserted_rows += block.num_rows
 
+            # Starting from the specific revision there are profile events
+            # sent by server in response to each inserted block
+            self.receive_profile_events()
+
         # Empty block means end of data.
         self.connection.send_data(block_cls())
+        # If enabled by revision profile events are also sent after empty block
+        self.receive_profile_events()
+
         return inserted_rows
 
     def receive_end_of_query(self):
@@ -573,6 +678,9 @@ class Client(object):
             if packet.type == ServerPacketTypes.END_OF_STREAM:
                 break
 
+            elif packet.type == ServerPacketTypes.PROGRESS:
+                self.last_query.store_progress(packet.progress)
+
             elif packet.type == ServerPacketTypes.EXCEPTION:
                 raise packet.exception
 
@@ -582,9 +690,69 @@ class Client(object):
             elif packet.type == ServerPacketTypes.TABLE_COLUMNS:
                 pass
 
+            elif packet.type == ServerPacketTypes.PROFILE_EVENTS:
+                self.last_query.store_profile(packet.profile_info)
+
+            else:
+                message = self.connection.unexpected_packet_message(
+                    'Exception, EndOfStream, Progress, TableColumns, '
+                    'ProfileEvents or Log', packet.type
+                )
+                raise errors.UnexpectedPacketFromServerError(message)
+
+    def receive_end_of_insert_query(self):
+        while True:
+            packet = self.connection.receive_packet()
+
+            if packet.type == ServerPacketTypes.END_OF_STREAM:
+                break
+
+            elif packet.type == ServerPacketTypes.LOG:
+                log_block(packet.block)
+
+            elif packet.type == ServerPacketTypes.PROGRESS:
+                self.last_query.store_progress(packet.progress)
+
+            elif packet.type == ServerPacketTypes.EXCEPTION:
+                raise packet.exception
+
+            else:
+                message = self.connection.unexpected_packet_message(
+                    'EndOfStream, Log, Progress or Exception', packet.type
+                )
+                raise errors.UnexpectedPacketFromServerError(message)
+
+    def receive_profile_events(self):
+        revision = self.connection.server_info.used_revision
+        if (
+            revision <
+            defines.DBMS_MIN_PROTOCOL_VERSION_WITH_PROFILE_EVENTS_IN_INSERT
+        ):
+            return None
+
+        while True:
+            packet = self.connection.receive_packet()
+
+            if packet.type == ServerPacketTypes.PROFILE_EVENTS:
+                self.last_query.store_profile(packet.profile_info)
+                break
+
+            elif packet.type == ServerPacketTypes.PROGRESS:
+                self.last_query.store_progress(packet.progress)
+
+            elif packet.type == ServerPacketTypes.LOG:
+                log_block(packet.block)
+
+            elif packet.type == ServerPacketTypes.EXCEPTION:
+                raise packet.exception
+
+            elif packet.type == ServerPacketTypes.TIMEZONE_UPDATE:
+                pass
+
             else:
                 message = self.connection.unexpected_packet_message(
-                    'Exception, EndOfStream or Log', packet.type
+                    'ProfileEvents, Progress, Log, Exception or '
+                    'TimezoneUpdate', packet.type
                 )
                 raise errors.UnexpectedPacketFromServerError(message)
 
@@ -595,6 +763,26 @@ class Client(object):
         return self.receive_result(with_column_types=with_column_types)
 
     def substitute_params(self, query, params, context):
+        """
+        Substitutes parameters into a provided query.
+
+        For example::
+
+            client = Client(...)
+
+            substituted_query = client.substitute_params(
+                query='SELECT 1234, %(foo)s',
+                params={'foo': 'bar'},
+                context=client.connection.context
+            )
+
+            # prints: SELECT 1234, 'bar'
+            print(substituted_query)
+        """
+        # In case of server side templating we don't substitute here.
+        if self.connection.context.client_settings['server_side_params']:
+            return query
+
         if not isinstance(params, dict):
             raise ValueError('Parameters are expected in dict form')
 
@@ -612,86 +800,13 @@ class Client(object):
             clickhouses://[user:password]@localhost:9440/default
 
         Three URL schemes are supported:
-            clickhouse:// creates a normal TCP socket connection
-            clickhouses:// creates a SSL wrapped TCP socket connection
+
+            * clickhouse:// creates a normal TCP socket connection
+            * clickhouses:// creates a SSL wrapped TCP socket connection
 
         Any additional querystring arguments will be passed along to
         the Connection class's initializer.
         """
-        url = urlparse(url)
-
-        settings = {}
-        kwargs = {}
-
-        host = url.hostname
-
-        if url.port is not None:
-            kwargs['port'] = url.port
-
-        path = url.path.replace('/', '', 1)
-        if path:
-            kwargs['database'] = path
-
-        if url.username is not None:
-            kwargs['user'] = unquote(url.username)
-
-        if url.password is not None:
-            kwargs['password'] = unquote(url.password)
-
-        if url.scheme == 'clickhouses':
-            kwargs['secure'] = True
-
-        compression_algs = {'lz4', 'lz4hc', 'zstd'}
-        timeouts = {
-            'connect_timeout',
-            'send_receive_timeout',
-            'sync_request_timeout'
-        }
-
-        for name, value in parse_qs(url.query).items():
-            if not value or not len(value):
-                continue
-
-            value = value[0]
-
-            if name == 'compression':
-                value = value.lower()
-                if value in compression_algs:
-                    kwargs[name] = value
-                else:
-                    kwargs[name] = asbool(value)
-
-            elif name == 'secure':
-                kwargs[name] = asbool(value)
-
-            elif name == 'use_numpy':
-                settings[name] = asbool(value)
-
-            elif name == 'client_name':
-                kwargs[name] = value
-
-            elif name in timeouts:
-                kwargs[name] = float(value)
-
-            elif name == 'compress_block_size':
-                kwargs[name] = int(value)
-
-            elif name == 'settings_is_important':
-                kwargs[name] = asbool(value)
-
-            # ssl
-            elif name == 'verify':
-                kwargs[name] = asbool(value)
-            elif name == 'ssl_version':
-                kwargs[name] = getattr(ssl, value)
-            elif name in ['ca_certs', 'ciphers', 'keyfile', 'certfile']:
-                kwargs[name] = value
-            elif name == 'alt_hosts':
-                kwargs['alt_hosts'] = value
-            else:
-                settings[name] = value
-
-        if settings:
-            kwargs['settings'] = settings
+        host, kwargs = parse_url(url)
 
         return cls(host, **kwargs)

clickhouse_driver/clientinfo.py

@@ -1,5 +1,6 @@
 import socket
 import getpass
+from time import time
 
 from . import defines
 from . import errors
@@ -27,14 +28,13 @@ class ClientInfo(object):
     client_version_major = defines.CLIENT_VERSION_MAJOR
     client_version_minor = defines.CLIENT_VERSION_MINOR
     client_version_patch = defines.CLIENT_VERSION_PATCH
-    client_revision = defines.CLIENT_REVISION
     interface = Interface.TCP
 
     initial_user = ''
     initial_query_id = ''
     initial_address = '0.0.0.0:0'
 
-    def __init__(self, client_name, context):
+    def __init__(self, client_name, context, client_revision):
         self.query_kind = ClientInfo.QueryKind.NO_QUERY
 
         try:
@@ -43,6 +43,7 @@ class ClientInfo(object):
             self.os_user = ''
         self.client_hostname = socket.gethostname()
         self.client_name = client_name
+        self.client_revision = client_revision
 
         self.client_trace_context = OpenTelemetryTraceContext(
             context.client_settings['opentelemetry_traceparent'],
@@ -50,6 +51,8 @@ class ClientInfo(object):
         )
 
         self.quota_key = context.client_settings['quota_key']
+        self.distributed_depth = 0
+        self.initial_query_start_time_microseconds = int(time() * 1000000)
 
         super(ClientInfo, self).__init__()
 
@@ -71,6 +74,14 @@ class ClientInfo(object):
         write_binary_str(self.initial_query_id, fout)
         write_binary_str(self.initial_address, fout)
 
+        if (
+            revision >=
+            defines.DBMS_MIN_PROTOCOL_VERSION_WITH_INITIAL_QUERY_START_TIME
+        ):
+            write_binary_uint64(
+                self.initial_query_start_time_microseconds, fout
+            )
+
         write_binary_uint8(self.interface, fout)
 
         write_binary_str(self.os_user, fout)
@@ -83,6 +94,10 @@ class ClientInfo(object):
         if revision >= defines.DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO:
             write_binary_str(self.quota_key, fout)
 
+        if revision >= \
+                defines.DBMS_MIN_PROTOCOL_VERSION_WITH_DISTRIBUTED_DEPTH:
+            write_varint(self.distributed_depth, fout)
+
         if revision >= defines.DBMS_MIN_REVISION_WITH_VERSION_PATCH:
             write_varint(self.client_version_patch, fout)
 
@@ -97,3 +112,8 @@ class ClientInfo(object):
             else:
                 # Don't have OpenTelemetry header.
                 write_binary_uint8(0, fout)
+
+        if revision >= defines.DBMS_MIN_REVISION_WITH_PARALLEL_REPLICAS:
+            write_varint(0, fout)  # collaborate_with_initiator
+            write_varint(0, fout)  # count_participating_replicas
+            write_varint(0, fout)  # number_of_current_replica

clickhouse_driver/columns/arraycolumn.py

@@ -28,25 +28,31 @@ class ArrayColumn(Column):
     py_types = (list, tuple)
 
     def __init__(self, nested_column, **kwargs):
-        self.size_column = UInt64Column()
+        self.init_kwargs = kwargs
+        self.size_column = UInt64Column(**kwargs)
         self.nested_column = nested_column
         self._write_depth_0_size = True
         super(ArrayColumn, self).__init__(**kwargs)
+        self.null_value = []
 
     def write_data(self, data, buf):
         # Column of Array(T) is stored in "compact" format and passed to server
         # wrapped into another Array without size of wrapper array.
-        self.nested_column = ArrayColumn(self.nested_column)
+        self.nested_column = ArrayColumn(
+            self.nested_column, **self.init_kwargs
+        )
         self.nested_column.nullable = self.nullable
         self.nullable = False
         self._write_depth_0_size = False
         self._write(data, buf)
 
-    def read_data(self, rows, buf):
-        self.nested_column = ArrayColumn(self.nested_column)
+    def read_data(self, n_rows, buf):
+        self.nested_column = ArrayColumn(
+            self.nested_column, **self.init_kwargs
+        )
         self.nested_column.nullable = self.nullable
         self.nullable = False
-        return self._read(rows, buf)[0]
+        return self._read(n_rows, buf)[0]
 
     def _write_sizes(self, value, buf):
         nulls_map = []
@@ -99,14 +105,19 @@ class ArrayColumn(Column):
                 self.nested_column._write_nulls_map(value, buf)
 
     def _write(self, value, buf):
+        value = self.prepare_items(value)
         self._write_sizes(value, buf)
         self._write_nulls_data(value, buf)
         self._write_data(value, buf)
 
     def read_state_prefix(self, buf):
-        return self.nested_column.read_state_prefix(buf)
+        super(ArrayColumn, self).read_state_prefix(buf)
+
+        self.nested_column.read_state_prefix(buf)
 
     def write_state_prefix(self, buf):
+        super(ArrayColumn, self).write_state_prefix(buf)
+
         self.nested_column.write_state_prefix(buf)
 
     def _read(self, size, buf):
@@ -145,6 +156,6 @@ class ArrayColumn(Column):
         return tuple(data)
 
 
-def create_array_column(spec, column_by_spec_getter):
+def create_array_column(spec, column_by_spec_getter, column_options):
     inner = spec[6:-1]
-    return ArrayColumn(column_by_spec_getter(inner))
+    return ArrayColumn(column_by_spec_getter(inner), **column_options)