clickhouse-driver 0.2.2__cp38-cp38-macosx_10_9_x86_64.whl → 0.2.9__cp38-cp38-macosx_10_9_x86_64.whl

clickhouse_driver/__init__.py

@@ -3,7 +3,7 @@ from .client import Client
 from .dbapi import connect
 
 
-VERSION = (0, 2, 2)
+VERSION = (0, 2, 9)
 __version__ = '.'.join(str(x) for x in VERSION)
 
 __all__ = ['Client', 'connect']

clickhouse_driver/block.py

@@ -1,3 +1,4 @@
+from .columns.util import get_inner_spec, get_inner_columns_with_types
 from .reader import read_varint, read_binary_uint8, read_binary_int32
 from .varint import write_varint
 from .writer import write_binary_uint8, write_binary_int32
@@ -151,17 +152,45 @@ class RowOrientedBlock(BaseBlock):
         return [row[index] for row in self.data]
 
     def _mutate_dicts_to_rows(self, data):
-        column_names = [x[0] for x in self.columns_with_types]
-
         check_row_type = False
         if self.types_check:
             check_row_type = self._check_dict_row_type
 
+        return self._pure_mutate_dicts_to_rows(
+            data,
+            self.columns_with_types,
+            check_row_type,
+        )
+
+    def _pure_mutate_dicts_to_rows(
+        self,
+        data,
+        columns_with_types,
+        check_row_type,
+    ):
+        columns_with_cwt = []
+        for name, type_ in columns_with_types:
+            cwt = None
+            if type_.startswith('Nested'):
+                inner_spec = get_inner_spec('Nested', type_)
+                cwt = get_inner_columns_with_types(inner_spec)
+            columns_with_cwt.append((name, cwt))
+
         for i, row in enumerate(data):
             if check_row_type:
                 check_row_type(row)
 
-            data[i] = [row[name] for name in column_names]
+            new_data = []
+            for name, cwt in columns_with_cwt:
+                if cwt is None:
+                    new_data.append(row[name])
+                else:
+                    new_data.append(self._pure_mutate_dicts_to_rows(
+                        row[name], cwt, check_row_type
+                    ))
+            data[i] = new_data
+        # return for recursion
+        return data
 
     def _check_rows(self, data):
         expected_row_len = len(self.columns_with_types)

clickhouse_driver/client.py

@@ -1,9 +1,9 @@
 import re
-import ssl
+from collections import deque
 from contextlib import contextmanager
 from time import time
 import types
-from urllib.parse import urlparse, parse_qs, unquote
+from urllib.parse import urlparse
 
 from . import errors, defines
 from .block import ColumnOrientedBlock, RowOrientedBlock
@@ -14,7 +14,7 @@ from .result import (
     IterQueryResult, ProgressQueryResult, QueryResult, QueryInfo
 )
 from .util.escape import escape_params
-from .util.helpers import column_chunks, chunks, asbool
+from .util.helpers import column_chunks, chunks, parse_url
 
 
 class Client(object):
@@ -26,7 +26,7 @@ class Client(object):
                      for the client settings, see below). Defaults to ``None``
                      (no additional settings). See all available settings in
                      `ClickHouse docs
-                     <https://clickhouse.tech/docs/en/operations/settings/settings/>`_.
+                     <https://clickhouse.com/docs/en/operations/settings/settings/>`_.
     :param \\**kwargs: All other args are passed to the
                        :py:class:`~clickhouse_driver.connection.Connection`
                        constructor.
@@ -38,7 +38,7 @@ class Client(object):
           Defaults to ``1048576``.
         * ``strings_as_bytes`` -- turns off string column encoding/decoding.
         * ``strings_encoding`` -- specifies string encoding. UTF-8 by default.
-        * ``use_numpy`` -- Use numpy for columns reading. New in version
+        * ``use_numpy`` -- Use NumPy for columns reading. New in version
                            *0.2.0*.
         * ``opentelemetry_traceparent`` -- OpenTelemetry traceparent header as
                            described by W3C Trace Context recommendation.
@@ -46,6 +46,27 @@ class Client(object):
         * ``opentelemetry_tracestate`` -- OpenTelemetry tracestate header as
                            described by W3C Trace Context recommendation.
                            New in version *0.2.2*.
+        * ``quota_key`` -- A string to differentiate quotas when the user have
+                           keyed quotas configured on server.
+                           New in version *0.2.3*.
+        * ``input_format_null_as_default`` -- Initialize null fields with
+                           default values if data type of this field is not
+                           nullable. Does not work for NumPy. Default: False.
+                           New in version *0.2.4*.
+        * ``round_robin`` -- If ``alt_hosts`` are provided the query will be
+                           executed on host picked with round-robin algorithm.
+                           New in version *0.2.5*.
+        * ``namedtuple_as_json`` -- Controls named tuple and nested types
+                           deserialization. To interpret these column alongside
+                           with ``allow_experimental_object_type=1`` as Python
+                           tuple set ``namedtuple_as_json`` to ``False``.
+                           Default: True.
+                           New in version *0.2.6*.
+        * ``server_side_params`` -- Species on which side query parameters
+                           should be rendered into placeholders.
+                           Default: False. Means that parameters are rendered
+                           on driver's side.
+                           New in version *0.2.7*.
     """
 
     available_client_settings = (
@@ -54,11 +75,15 @@ class Client(object):
         'strings_encoding',
         'use_numpy',
         'opentelemetry_traceparent',
-        'opentelemetry_tracestate'
+        'opentelemetry_tracestate',
+        'quota_key',
+        'input_format_null_as_default',
+        'namedtuple_as_json',
+        'server_side_params'
     )
 
     def __init__(self, *args, **kwargs):
-        self.settings = kwargs.pop('settings', {}).copy()
+        self.settings = (kwargs.pop('settings', None) or {}).copy()
 
         self.client_settings = {
             'insert_block_size': int(self.settings.pop(
@@ -78,6 +103,18 @@ class Client(object):
             ),
             'opentelemetry_tracestate': self.settings.pop(
                 'opentelemetry_tracestate', ''
+            ),
+            'quota_key': self.settings.pop(
+                'quota_key', ''
+            ),
+            'input_format_null_as_default': self.settings.pop(
+                'input_format_null_as_default', False
+            ),
+            'namedtuple_as_json': self.settings.pop(
+                'namedtuple_as_json', True
+            ),
+            'server_side_params': self.settings.pop(
+                'server_side_params', False
             )
         }
 
@@ -97,9 +134,33 @@ class Client(object):
             self.iter_query_result_cls = IterQueryResult
             self.progress_query_result_cls = ProgressQueryResult
 
-        self.connection = Connection(*args, **kwargs)
-        self.connection.context.settings = self.settings
-        self.connection.context.client_settings = self.client_settings
+        round_robin = kwargs.pop('round_robin', False)
+        self.connections = deque([Connection(*args, **kwargs)])
+
+        if round_robin and 'alt_hosts' in kwargs:
+            alt_hosts = kwargs.pop('alt_hosts')
+            for host in alt_hosts.split(','):
+                url = urlparse('clickhouse://' + host)
+
+                connection_kwargs = kwargs.copy()
+                num_args = len(args)
+                if num_args >= 2:
+                    # host and port as positional arguments
+                    connection_args = (url.hostname, url.port) + args[2:]
+                elif num_args >= 1:
+                    # host as positional and port as keyword argument
+                    connection_args = (url.hostname, ) + args[1:]
+                    connection_kwargs['port'] = url.port
+                else:
+                    # host and port as keyword arguments
+                    connection_args = tuple()
+                    connection_kwargs['host'] = url.hostname
+                    connection_kwargs['port'] = url.port
+
+                connection = Connection(*connection_args, **connection_kwargs)
+                self.connections.append(connection)
+
+        self.connection = self.get_connection()
         self.reset_last_query()
         super(Client, self).__init__()
 
@@ -109,7 +170,22 @@ class Client(object):
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.disconnect()
 
+    def get_connection(self):
+        if hasattr(self, 'connection'):
+            self.connections.append(self.connection)
+
+        connection = self.connections.popleft()
+
+        connection.context.settings = self.settings
+        connection.context.client_settings = self.client_settings
+        return connection
+
     def disconnect(self):
+        self.disconnect_connection()
+        for connection in self.connections:
+            connection.disconnect()
+
+    def disconnect_connection(self):
         """
         Disconnects from the server.
         """
@@ -212,10 +288,33 @@ class Client(object):
         if query.lower().startswith('use '):
             self.connection.database = query[4:].strip()
 
+    def establish_connection(self, settings):
+        num_connections = len(self.connections)
+        if hasattr(self, 'connection'):
+            num_connections += 1
+
+        for i in range(num_connections):
+            try:
+                self.connection = self.get_connection()
+                self.make_query_settings(settings)
+                self.connection.force_connect()
+                self.last_query = QueryInfo()
+
+            except (errors.SocketTimeoutError, errors.NetworkError):
+                if i < num_connections - 1:
+                    continue
+                raise
+
+            return
+
     @contextmanager
-    def disconnect_on_error(self, query):
+    def disconnect_on_error(self, query, settings):
         try:
+            self.establish_connection(settings)
+            self.connection.server_info.session_timezone = None
+
             yield
+
             self.track_current_database(query)
 
         except (Exception, KeyboardInterrupt):
@@ -267,11 +366,8 @@ class Client(object):
         """
 
         start_time = time()
-        self.make_query_settings(settings)
-        self.connection.force_connect()
-        self.last_query = QueryInfo()
 
-        with self.disconnect_on_error(query):
+        with self.disconnect_on_error(query, settings):
             # INSERT queries can use list/tuple/generator of list/tuples/dicts.
             # For SELECT parameters can be passed in only in dict right now.
             is_insert = isinstance(params, (list, tuple, types.GeneratorType))
@@ -322,11 +418,7 @@ class Client(object):
         :return: :ref:`progress-query-result` proxy.
         """
 
-        self.make_query_settings(settings)
-        self.connection.force_connect()
-        self.last_query = QueryInfo()
-
-        with self.disconnect_on_error(query):
+        with self.disconnect_on_error(query, settings):
             return self.process_ordinary_query_with_progress(
                 query, params=params, with_column_types=with_column_types,
                 external_tables=external_tables, query_id=query_id,
@@ -336,7 +428,7 @@ class Client(object):
     def execute_iter(
             self, query, params=None, with_column_types=False,
             external_tables=None, query_id=None, settings=None,
-            types_check=False):
+            types_check=False, chunk_size=1):
         """
         *New in version 0.0.14.*
 
@@ -358,23 +450,20 @@ class Client(object):
                          Defaults to ``None`` (no additional settings).
         :param types_check: enables type checking of data for INSERT queries.
                             Causes additional overhead. Defaults to ``False``.
+        :param chunk_size: chunk query results.
         :return: :ref:`iter-query-result` proxy.
         """
-
-        self.make_query_settings(settings)
-        self.connection.force_connect()
-        self.last_query = QueryInfo()
-
-        with self.disconnect_on_error(query):
-            return self.iter_process_ordinary_query(
+        with self.disconnect_on_error(query, settings):
+            rv = self.iter_process_ordinary_query(
                 query, params=params, with_column_types=with_column_types,
                 external_tables=external_tables,
                 query_id=query_id, types_check=types_check
             )
+            return chunks(rv, chunk_size) if chunk_size > 1 else rv
 
     def query_dataframe(
             self, query, params=None, external_tables=None, query_id=None,
-            settings=None):
+            settings=None, replace_nonwords=True):
         """
         *New in version 0.2.0.*
 
@@ -389,6 +478,8 @@ class Client(object):
                          ClickHouse server will generate it.
         :param settings: dictionary of query settings.
                          Defaults to ``None`` (no additional settings).
+        :param replace_nonwords: boolean to replace non-words in column names
+                                 to underscores. Defaults to ``True``.
         :return: pandas DataFrame.
         """
 
@@ -403,8 +494,12 @@ class Client(object):
             settings=settings
         )
 
+        columns = [name for name, type_ in columns]
+        if replace_nonwords:
+            columns = [re.sub(r'\W', '_', x) for x in columns]
+
         return pd.DataFrame(
-            {re.sub(r'\W', '_', col[0]): d for d, col in zip(data, columns)}
+            {col: d for d, col in zip(data, columns)}, columns=columns
         )
 
     def insert_dataframe(
@@ -432,11 +527,8 @@ class Client(object):
             raise RuntimeError('Extras for NumPy must be installed')
 
         start_time = time()
-        self.make_query_settings(settings)
-        self.connection.force_connect()
-        self.last_query = QueryInfo()
 
-        with self.disconnect_on_error(query):
+        with self.disconnect_on_error(query, settings):
             self.connection.send_query(query, query_id=query_id)
             self.connection.send_external_tables(external_tables)
 
@@ -444,6 +536,12 @@ class Client(object):
             rv = None
             if sample_block:
                 columns = [x[0] for x in sample_block.columns_with_types]
+                # raise if any columns are missing from the dataframe
+                diff = set(columns) - set(dataframe.columns)
+                if len(diff):
+                    msg = "DataFrame missing required columns: {}"
+                    raise ValueError(msg.format(list(diff)))
+
                 data = [dataframe[column].values for column in columns]
                 rv = self.send_data(sample_block, data, columnar=True)
                 self.receive_end_of_query()
@@ -457,9 +555,11 @@ class Client(object):
             types_check=False, columnar=False):
 
         if params is not None:
-            query = self.substitute_params(query, params)
+            query = self.substitute_params(
+                query, params, self.connection.context
+            )
 
-        self.connection.send_query(query, query_id=query_id)
+        self.connection.send_query(query, query_id=query_id, params=params)
         self.connection.send_external_tables(external_tables,
                                              types_check=types_check)
         return self.receive_result(with_column_types=with_column_types,
@@ -471,9 +571,10 @@ class Client(object):
             types_check=False, columnar=False):
 
         if params is not None:
-            query = self.substitute_params(query, params)
-
-        self.connection.send_query(query, query_id=query_id)
+            query = self.substitute_params(
+                query, params, self.connection.context
+            )
+        self.connection.send_query(query, query_id=query_id, params=params)
         self.connection.send_external_tables(external_tables,
                                              types_check=types_check)
         return self.receive_result(with_column_types=with_column_types,
@@ -485,9 +586,11 @@ class Client(object):
             types_check=False):
 
         if params is not None:
-            query = self.substitute_params(query, params)
+            query = self.substitute_params(
+                query, params, self.connection.context
+            )
 
-        self.connection.send_query(query, query_id=query_id)
+        self.connection.send_query(query, query_id=query_id, params=params)
         self.connection.send_external_tables(external_tables,
                                              types_check=types_check)
         return self.iter_receive_result(with_column_types=with_column_types)
@@ -498,12 +601,12 @@ class Client(object):
         self.connection.send_query(query_without_data, query_id=query_id)
         self.connection.send_external_tables(external_tables,
                                              types_check=types_check)
-
         sample_block = self.receive_sample_block()
+
         if sample_block:
             rv = self.send_data(sample_block, data,
                                 types_check=types_check, columnar=columnar)
-            self.receive_end_of_query()
+            self.receive_end_of_insert_query()
             return rv
 
     def receive_sample_block(self):
@@ -557,8 +660,15 @@ class Client(object):
             self.connection.send_data(block)
             inserted_rows += block.num_rows
 
+            # Starting from the specific revision there are profile events
+            # sent by server in response to each inserted block
+            self.receive_profile_events()
+
         # Empty block means end of data.
         self.connection.send_data(block_cls())
+        # If enabled by revision profile events are also sent after empty block
+        self.receive_profile_events()
+
         return inserted_rows
 
     def receive_end_of_query(self):
@@ -568,6 +678,9 @@ class Client(object):
             if packet.type == ServerPacketTypes.END_OF_STREAM:
                 break
 
+            elif packet.type == ServerPacketTypes.PROGRESS:
+                self.last_query.store_progress(packet.progress)
+
             elif packet.type == ServerPacketTypes.EXCEPTION:
                 raise packet.exception
 
@@ -577,116 +690,123 @@ class Client(object):
             elif packet.type == ServerPacketTypes.TABLE_COLUMNS:
                 pass
 
+            elif packet.type == ServerPacketTypes.PROFILE_EVENTS:
+                self.last_query.store_profile(packet.profile_info)
+
             else:
                 message = self.connection.unexpected_packet_message(
-                    'Exception, EndOfStream or Log', packet.type
+                    'Exception, EndOfStream, Progress, TableColumns, '
+                    'ProfileEvents or Log', packet.type
                 )
                 raise errors.UnexpectedPacketFromServerError(message)
 
-    def cancel(self, with_column_types=False):
-        # TODO: Add warning if already cancelled.
-        self.connection.send_cancel()
-        # Client must still read until END_OF_STREAM packet.
-        return self.receive_result(with_column_types=with_column_types)
+    def receive_end_of_insert_query(self):
+        while True:
+            packet = self.connection.receive_packet()
 
-    def substitute_params(self, query, params):
-        if not isinstance(params, dict):
-            raise ValueError('Parameters are expected in dict form')
+            if packet.type == ServerPacketTypes.END_OF_STREAM:
+                break
 
-        escaped = escape_params(params)
-        return query % escaped
+            elif packet.type == ServerPacketTypes.LOG:
+                log_block(packet.block)
 
-    @classmethod
-    def from_url(cls, url):
-        """
-        Return a client configured from the given URL.
+            elif packet.type == ServerPacketTypes.PROGRESS:
+                self.last_query.store_progress(packet.progress)
 
-        For example::
+            elif packet.type == ServerPacketTypes.EXCEPTION:
+                raise packet.exception
 
-            clickhouse://[user:password]@localhost:9000/default
-            clickhouses://[user:password]@localhost:9440/default
+            else:
+                message = self.connection.unexpected_packet_message(
+                    'EndOfStream, Log, Progress or Exception', packet.type
+                )
+                raise errors.UnexpectedPacketFromServerError(message)
 
-        Three URL schemes are supported:
-            clickhouse:// creates a normal TCP socket connection
-            clickhouses:// creates a SSL wrapped TCP socket connection
+    def receive_profile_events(self):
+        revision = self.connection.server_info.used_revision
+        if (
+            revision <
+            defines.DBMS_MIN_PROTOCOL_VERSION_WITH_PROFILE_EVENTS_IN_INSERT
+        ):
+            return None
 
-        Any additional querystring arguments will be passed along to
-        the Connection class's initializer.
-        """
-        url = urlparse(url)
+        while True:
+            packet = self.connection.receive_packet()
 
-        settings = {}
-        kwargs = {}
+            if packet.type == ServerPacketTypes.PROFILE_EVENTS:
+                self.last_query.store_profile(packet.profile_info)
+                break
 
-        host = url.hostname
+            elif packet.type == ServerPacketTypes.PROGRESS:
+                self.last_query.store_progress(packet.progress)
 
-        if url.port is not None:
-            kwargs['port'] = url.port
+            elif packet.type == ServerPacketTypes.LOG:
+                log_block(packet.block)
 
-        path = url.path.replace('/', '', 1)
-        if path:
-            kwargs['database'] = path
+            elif packet.type == ServerPacketTypes.EXCEPTION:
+                raise packet.exception
 
-        if url.username is not None:
-            kwargs['user'] = unquote(url.username)
+            elif packet.type == ServerPacketTypes.TIMEZONE_UPDATE:
+                pass
 
-        if url.password is not None:
-            kwargs['password'] = unquote(url.password)
+            else:
+                message = self.connection.unexpected_packet_message(
+                    'ProfileEvents, Progress, Log, Exception or '
+                    'TimezoneUpdate', packet.type
+                )
+                raise errors.UnexpectedPacketFromServerError(message)
 
-        if url.scheme == 'clickhouses':
-            kwargs['secure'] = True
+    def cancel(self, with_column_types=False):
+        # TODO: Add warning if already cancelled.
+        self.connection.send_cancel()
+        # Client must still read until END_OF_STREAM packet.
+        return self.receive_result(with_column_types=with_column_types)
 
-        compression_algs = {'lz4', 'lz4hc', 'zstd'}
-        timeouts = {
-            'connect_timeout',
-            'send_receive_timeout',
-            'sync_request_timeout'
-        }
+    def substitute_params(self, query, params, context):
+        """
+        Substitutes parameters into a provided query.
 
-        for name, value in parse_qs(url.query).items():
-            if not value or not len(value):
-                continue
+        For example::
 
-            value = value[0]
+            client = Client(...)
 
-            if name == 'compression':
-                value = value.lower()
-                if value in compression_algs:
-                    kwargs[name] = value
-                else:
-                    kwargs[name] = asbool(value)
+            substituted_query = client.substitute_params(
+                query='SELECT 1234, %(foo)s',
+                params={'foo': 'bar'},
+                context=client.connection.context
+            )
+
+            # prints: SELECT 1234, 'bar'
+            print(substituted_query)
+        """
+        # In case of server side templating we don't substitute here.
+        if self.connection.context.client_settings['server_side_params']:
+            return query
 
-            elif name == 'secure':
-                kwargs[name] = asbool(value)
+        if not isinstance(params, dict):
+            raise ValueError('Parameters are expected in dict form')
 
-            elif name == 'use_numpy':
-                settings[name] = asbool(value)
+        escaped = escape_params(params, context)
+        return query % escaped
 
-            elif name == 'client_name':
-                kwargs[name] = value
+    @classmethod
+    def from_url(cls, url):
+        """
+        Return a client configured from the given URL.
 
-            elif name in timeouts:
-                kwargs[name] = float(value)
+        For example::
 
-            elif name == 'compress_block_size':
-                kwargs[name] = int(value)
+            clickhouse://[user:password]@localhost:9000/default
+            clickhouses://[user:password]@localhost:9440/default
 
-            elif name == 'settings_is_important':
-                kwargs[name] = asbool(value)
+        Three URL schemes are supported:
 
-            # ssl
-            elif name == 'verify':
-                kwargs[name] = asbool(value)
-            elif name == 'ssl_version':
-                kwargs[name] = getattr(ssl, value)
-            elif name in ['ca_certs', 'ciphers', 'keyfile', 'certfile']:
-                kwargs[name] = value
-            elif name == 'alt_hosts':
-                kwargs['alt_hosts'] = value
-            else:
-                settings[name] = value
+            * clickhouse:// creates a normal TCP socket connection
+            * clickhouses:// creates a SSL wrapped TCP socket connection
 
-        if settings:
-            kwargs['settings'] = settings
+        Any additional querystring arguments will be passed along to
+        the Connection class's initializer.
+        """
+        host, kwargs = parse_url(url)
 
         return cls(host, **kwargs)