clickhouse-driver 0.2.4__cp38-cp38-musllinux_1_1_i686.whl → 0.2.8__cp38-cp38-musllinux_1_1_i686.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- clickhouse_driver/__init__.py +1 -1
- clickhouse_driver/block.py +3 -2
- clickhouse_driver/bufferedreader.cpython-38-i386-linux-gnu.so +0 -0
- clickhouse_driver/bufferedwriter.cpython-38-i386-linux-gnu.so +0 -0
- clickhouse_driver/client.py +209 -19
- clickhouse_driver/clientinfo.py +2 -2
- clickhouse_driver/columns/arraycolumn.py +16 -7
- clickhouse_driver/columns/base.py +71 -7
- clickhouse_driver/columns/datecolumn.py +52 -13
- clickhouse_driver/columns/jsoncolumn.py +37 -0
- clickhouse_driver/columns/largeint.cpython-38-i386-linux-gnu.so +0 -0
- clickhouse_driver/columns/lowcardinalitycolumn.py +23 -4
- clickhouse_driver/columns/mapcolumn.py +15 -2
- clickhouse_driver/columns/nestedcolumn.py +2 -13
- clickhouse_driver/columns/numpy/boolcolumn.py +8 -0
- clickhouse_driver/columns/numpy/datetimecolumn.py +18 -18
- clickhouse_driver/columns/numpy/lowcardinalitycolumn.py +2 -2
- clickhouse_driver/columns/numpy/service.py +3 -1
- clickhouse_driver/columns/service.py +12 -2
- clickhouse_driver/columns/tuplecolumn.py +31 -5
- clickhouse_driver/columns/uuidcolumn.py +1 -1
- clickhouse_driver/connection.py +123 -17
- clickhouse_driver/defines.py +9 -1
- clickhouse_driver/log.py +7 -3
- clickhouse_driver/progress.py +8 -2
- clickhouse_driver/settings/writer.py +7 -2
- clickhouse_driver/streams/native.py +18 -6
- clickhouse_driver/util/compat.py +12 -0
- clickhouse_driver/util/escape.py +35 -7
- clickhouse_driver/varint.cpython-38-i386-linux-gnu.so +0 -0
- {clickhouse_driver-0.2.4.dist-info → clickhouse_driver-0.2.8.dist-info}/METADATA +8 -13
- {clickhouse_driver-0.2.4.dist-info → clickhouse_driver-0.2.8.dist-info}/RECORD +71 -69
- {clickhouse_driver-0.2.4.dist-info → clickhouse_driver-0.2.8.dist-info}/WHEEL +1 -1
- {clickhouse_driver-0.2.4.dist-info → clickhouse_driver-0.2.8.dist-info}/LICENSE +0 -0
- {clickhouse_driver-0.2.4.dist-info → clickhouse_driver-0.2.8.dist-info}/top_level.txt +0 -0
clickhouse_driver/connection.py
CHANGED
|
@@ -3,6 +3,7 @@ import socket
|
|
|
3
3
|
import ssl
|
|
4
4
|
from collections import deque
|
|
5
5
|
from contextlib import contextmanager
|
|
6
|
+
from sys import platform
|
|
6
7
|
from time import time
|
|
7
8
|
from urllib.parse import urlparse
|
|
8
9
|
|
|
@@ -19,11 +20,12 @@ from .log import log_block
|
|
|
19
20
|
from .progress import Progress
|
|
20
21
|
from .protocol import Compression, ClientPacketTypes, ServerPacketTypes
|
|
21
22
|
from .queryprocessingstage import QueryProcessingStage
|
|
22
|
-
from .reader import read_binary_str
|
|
23
|
+
from .reader import read_binary_str, read_binary_uint64
|
|
23
24
|
from .readhelpers import read_exception
|
|
24
|
-
from .settings.writer import write_settings
|
|
25
|
+
from .settings.writer import write_settings, SettingsFlags
|
|
25
26
|
from .streams.native import BlockInputStream, BlockOutputStream
|
|
26
27
|
from .util.compat import threading
|
|
28
|
+
from .util.escape import escape_params
|
|
27
29
|
from .varint import write_varint, read_varint
|
|
28
30
|
from .writer import write_binary_str
|
|
29
31
|
|
|
@@ -44,7 +46,7 @@ class Packet(object):
|
|
|
44
46
|
|
|
45
47
|
class ServerInfo(object):
|
|
46
48
|
def __init__(self, name, version_major, version_minor, version_patch,
|
|
47
|
-
revision, timezone, display_name):
|
|
49
|
+
revision, timezone, display_name, used_revision):
|
|
48
50
|
self.name = name
|
|
49
51
|
self.version_major = version_major
|
|
50
52
|
self.version_minor = version_minor
|
|
@@ -52,6 +54,7 @@ class ServerInfo(object):
|
|
|
52
54
|
self.revision = revision
|
|
53
55
|
self.timezone = timezone
|
|
54
56
|
self.display_name = display_name
|
|
57
|
+
self.used_revision = used_revision
|
|
55
58
|
|
|
56
59
|
super(ServerInfo, self).__init__()
|
|
57
60
|
|
|
@@ -66,6 +69,7 @@ class ServerInfo(object):
|
|
|
66
69
|
('name', self.name),
|
|
67
70
|
('version', version),
|
|
68
71
|
('revision', self.revision),
|
|
72
|
+
('used revision', self.used_revision),
|
|
69
73
|
('timezone', self.timezone),
|
|
70
74
|
('display_name', self.display_name)
|
|
71
75
|
]
|
|
@@ -113,12 +117,26 @@ class Connection(object):
|
|
|
113
117
|
:param ciphers: see :func:`ssl.wrap_socket` docs.
|
|
114
118
|
:param keyfile: see :func:`ssl.wrap_socket` docs.
|
|
115
119
|
:param certfile: see :func:`ssl.wrap_socket` docs.
|
|
120
|
+
:param server_hostname: Hostname to use in SSL Wrapper construction.
|
|
121
|
+
Defaults to `None` which will send the passed
|
|
122
|
+
host param during SSL initialization. This param
|
|
123
|
+
may be used when connecting over an SSH tunnel
|
|
124
|
+
to correctly identify the desired server via SNI.
|
|
116
125
|
:param alt_hosts: list of alternative hosts for connection.
|
|
117
126
|
Example: alt_hosts=host1:port1,host2:port2.
|
|
118
127
|
:param settings_is_important: ``False`` means unknown settings will be
|
|
119
128
|
ignored, ``True`` means that the query will
|
|
120
129
|
fail with UNKNOWN_SETTING error.
|
|
121
130
|
Defaults to ``False``.
|
|
131
|
+
:param tcp_keepalive: enables `TCP keepalive <https://tldp.org/HOWTO/
|
|
132
|
+
TCP-Keepalive-HOWTO/overview.html>`_ on established
|
|
133
|
+
connection. If is set to ``True``` system keepalive
|
|
134
|
+
settings are used. You can also specify custom
|
|
135
|
+
keepalive setting with tuple:
|
|
136
|
+
``(idle_time_sec, interval_sec, probes)``.
|
|
137
|
+
Defaults to ``False``.
|
|
138
|
+
:param client_revision: can be used for client version downgrading.
|
|
139
|
+
Defaults to ``None``.
|
|
122
140
|
"""
|
|
123
141
|
|
|
124
142
|
def __init__(
|
|
@@ -135,8 +153,11 @@ class Connection(object):
|
|
|
135
153
|
# Secure socket parameters.
|
|
136
154
|
verify=True, ssl_version=None, ca_certs=None, ciphers=None,
|
|
137
155
|
keyfile=None, certfile=None,
|
|
156
|
+
server_hostname=None,
|
|
138
157
|
alt_hosts=None,
|
|
139
158
|
settings_is_important=False,
|
|
159
|
+
tcp_keepalive=False,
|
|
160
|
+
client_revision=None
|
|
140
161
|
):
|
|
141
162
|
if secure:
|
|
142
163
|
default_port = defines.DEFAULT_SECURE_PORT
|
|
@@ -158,6 +179,10 @@ class Connection(object):
|
|
|
158
179
|
self.send_receive_timeout = send_receive_timeout
|
|
159
180
|
self.sync_request_timeout = sync_request_timeout
|
|
160
181
|
self.settings_is_important = settings_is_important
|
|
182
|
+
self.tcp_keepalive = tcp_keepalive
|
|
183
|
+
self.client_revision = min(
|
|
184
|
+
client_revision or defines.CLIENT_REVISION, defines.CLIENT_REVISION
|
|
185
|
+
)
|
|
161
186
|
|
|
162
187
|
self.secure_socket = secure
|
|
163
188
|
self.verify_cert = verify
|
|
@@ -176,6 +201,8 @@ class Connection(object):
|
|
|
176
201
|
|
|
177
202
|
self.ssl_options = ssl_options
|
|
178
203
|
|
|
204
|
+
self.server_hostname = server_hostname
|
|
205
|
+
|
|
179
206
|
# Use LZ4 compression by default.
|
|
180
207
|
if compression is True:
|
|
181
208
|
compression = 'lz4'
|
|
@@ -209,6 +236,14 @@ class Connection(object):
|
|
|
209
236
|
|
|
210
237
|
super(Connection, self).__init__()
|
|
211
238
|
|
|
239
|
+
def __repr__(self):
|
|
240
|
+
dsn = '%s://%s:***@%s:%s/%s' % (
|
|
241
|
+
'clickhouses' if self.secure_socket else 'clickhouse',
|
|
242
|
+
self.user, self.host, self.port, self.database
|
|
243
|
+
) if self.connected else '(not connected)'
|
|
244
|
+
|
|
245
|
+
return '<Connection(dsn=%s, compression=%s)>' % (dsn, self.compression)
|
|
246
|
+
|
|
212
247
|
def get_description(self):
|
|
213
248
|
return '{}:{}'.format(self.host, self.port)
|
|
214
249
|
|
|
@@ -247,7 +282,8 @@ class Connection(object):
|
|
|
247
282
|
|
|
248
283
|
if self.secure_socket:
|
|
249
284
|
ssl_context = self._create_ssl_context(ssl_options)
|
|
250
|
-
sock = ssl_context.wrap_socket(
|
|
285
|
+
sock = ssl_context.wrap_socket(
|
|
286
|
+
sock, server_hostname=self.server_hostname or host)
|
|
251
287
|
|
|
252
288
|
sock.connect(sa)
|
|
253
289
|
return sock
|
|
@@ -267,6 +303,7 @@ class Connection(object):
|
|
|
267
303
|
|
|
268
304
|
version = ssl_options.get('ssl_version', ssl.PROTOCOL_TLS)
|
|
269
305
|
context = ssl.SSLContext(version)
|
|
306
|
+
context.check_hostname = self.verify_cert
|
|
270
307
|
|
|
271
308
|
if 'ca_certs' in ssl_options:
|
|
272
309
|
context.load_verify_locations(ssl_options['ca_certs'])
|
|
@@ -277,7 +314,7 @@ class Connection(object):
|
|
|
277
314
|
context.set_ciphers(ssl_options['ciphers'])
|
|
278
315
|
|
|
279
316
|
if 'cert_reqs' in ssl_options:
|
|
280
|
-
context.
|
|
317
|
+
context.verify_mode = ssl_options['cert_reqs']
|
|
281
318
|
|
|
282
319
|
if 'certfile' in ssl_options:
|
|
283
320
|
keyfile = ssl_options.get('keyfile')
|
|
@@ -293,6 +330,8 @@ class Connection(object):
|
|
|
293
330
|
|
|
294
331
|
# performance tweak
|
|
295
332
|
self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
|
333
|
+
if self.tcp_keepalive:
|
|
334
|
+
self._set_keepalive()
|
|
296
335
|
|
|
297
336
|
self.fin = BufferedSocketReader(self.socket, defines.BUFFER_SIZE)
|
|
298
337
|
self.fout = BufferedSocketWriter(self.socket, defines.BUFFER_SIZE)
|
|
@@ -300,10 +339,42 @@ class Connection(object):
|
|
|
300
339
|
self.send_hello()
|
|
301
340
|
self.receive_hello()
|
|
302
341
|
|
|
342
|
+
revision = self.server_info.used_revision
|
|
343
|
+
if revision >= defines.DBMS_MIN_PROTOCOL_VERSION_WITH_ADDENDUM:
|
|
344
|
+
self.send_addendum()
|
|
345
|
+
|
|
303
346
|
self.block_in = self.get_block_in_stream()
|
|
304
347
|
self.block_in_raw = BlockInputStream(self.fin, self.context)
|
|
305
348
|
self.block_out = self.get_block_out_stream()
|
|
306
349
|
|
|
350
|
+
def _set_keepalive(self):
|
|
351
|
+
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
|
|
352
|
+
|
|
353
|
+
if not isinstance(self.tcp_keepalive, tuple):
|
|
354
|
+
return
|
|
355
|
+
|
|
356
|
+
idle_time_sec, interval_sec, probes = self.tcp_keepalive
|
|
357
|
+
|
|
358
|
+
if platform == 'linux' or platform == 'win32':
|
|
359
|
+
# This should also work for Windows
|
|
360
|
+
# starting with Windows 10, version 1709.
|
|
361
|
+
self.socket.setsockopt(
|
|
362
|
+
socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, idle_time_sec
|
|
363
|
+
)
|
|
364
|
+
self.socket.setsockopt(
|
|
365
|
+
socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, interval_sec
|
|
366
|
+
)
|
|
367
|
+
self.socket.setsockopt(
|
|
368
|
+
socket.IPPROTO_TCP, socket.TCP_KEEPCNT, probes
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
elif platform == 'darwin':
|
|
372
|
+
TCP_KEEPALIVE = 0x10
|
|
373
|
+
# Only interval is available in mac os.
|
|
374
|
+
self.socket.setsockopt(
|
|
375
|
+
socket.IPPROTO_TCP, TCP_KEEPALIVE, interval_sec
|
|
376
|
+
)
|
|
377
|
+
|
|
307
378
|
def _format_connection_error(self, e, host, port):
|
|
308
379
|
err = (e.strerror + ' ') if e.strerror else ''
|
|
309
380
|
return err + '({}:{})'.format(host, port)
|
|
@@ -393,7 +464,7 @@ class Connection(object):
|
|
|
393
464
|
write_varint(defines.CLIENT_VERSION_MINOR, self.fout)
|
|
394
465
|
# NOTE For backward compatibility of the protocol,
|
|
395
466
|
# client cannot send its version_patch.
|
|
396
|
-
write_varint(
|
|
467
|
+
write_varint(self.client_revision, self.fout)
|
|
397
468
|
write_binary_str(self.database, self.fout)
|
|
398
469
|
write_binary_str(self.user, self.fout)
|
|
399
470
|
write_binary_str(self.password, self.fout)
|
|
@@ -409,25 +480,38 @@ class Connection(object):
|
|
|
409
480
|
server_version_minor = read_varint(self.fin)
|
|
410
481
|
server_revision = read_varint(self.fin)
|
|
411
482
|
|
|
483
|
+
used_revision = min(self.client_revision, server_revision)
|
|
484
|
+
|
|
412
485
|
server_timezone = None
|
|
413
|
-
if
|
|
486
|
+
if used_revision >= \
|
|
414
487
|
defines.DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE:
|
|
415
488
|
server_timezone = read_binary_str(self.fin)
|
|
416
489
|
|
|
417
490
|
server_display_name = ''
|
|
418
|
-
if
|
|
491
|
+
if used_revision >= \
|
|
419
492
|
defines.DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME:
|
|
420
493
|
server_display_name = read_binary_str(self.fin)
|
|
421
494
|
|
|
422
495
|
server_version_patch = server_revision
|
|
423
|
-
if
|
|
496
|
+
if used_revision >= \
|
|
424
497
|
defines.DBMS_MIN_REVISION_WITH_VERSION_PATCH:
|
|
425
498
|
server_version_patch = read_varint(self.fin)
|
|
426
499
|
|
|
500
|
+
if used_revision >= defines. \
|
|
501
|
+
DBMS_MIN_PROTOCOL_VERSION_WITH_PASSWORD_COMPLEXITY_RULES:
|
|
502
|
+
rules_size = read_varint(self.fin)
|
|
503
|
+
for _i in range(rules_size):
|
|
504
|
+
read_binary_str(self.fin) # original_pattern
|
|
505
|
+
read_binary_str(self.fin) # exception_message
|
|
506
|
+
|
|
507
|
+
if used_revision >= defines. \
|
|
508
|
+
DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2:
|
|
509
|
+
read_binary_uint64(self.fin) # read_nonce
|
|
510
|
+
|
|
427
511
|
self.server_info = ServerInfo(
|
|
428
512
|
server_name, server_version_major, server_version_minor,
|
|
429
513
|
server_version_patch, server_revision,
|
|
430
|
-
server_timezone, server_display_name
|
|
514
|
+
server_timezone, server_display_name, used_revision
|
|
431
515
|
)
|
|
432
516
|
self.context.server_info = self.server_info
|
|
433
517
|
|
|
@@ -446,6 +530,14 @@ class Connection(object):
|
|
|
446
530
|
self.disconnect()
|
|
447
531
|
raise errors.UnexpectedPacketFromServerError(message)
|
|
448
532
|
|
|
533
|
+
def send_addendum(self):
|
|
534
|
+
revision = self.server_info.used_revision
|
|
535
|
+
|
|
536
|
+
if revision >= defines.DBMS_MIN_PROTOCOL_VERSION_WITH_QUOTA_KEY:
|
|
537
|
+
write_binary_str(
|
|
538
|
+
self.context.client_settings['quota_key'], self.fout
|
|
539
|
+
)
|
|
540
|
+
|
|
449
541
|
def ping(self):
|
|
450
542
|
timeout = self.sync_request_timeout
|
|
451
543
|
|
|
@@ -550,7 +642,7 @@ class Connection(object):
|
|
|
550
642
|
return BlockOutputStream(self.fout, self.context)
|
|
551
643
|
|
|
552
644
|
def receive_data(self, may_be_compressed=True, may_be_use_numpy=False):
|
|
553
|
-
revision = self.server_info.
|
|
645
|
+
revision = self.server_info.used_revision
|
|
554
646
|
|
|
555
647
|
if revision >= defines.DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES:
|
|
556
648
|
read_binary_str(self.fin)
|
|
@@ -564,7 +656,7 @@ class Connection(object):
|
|
|
564
656
|
|
|
565
657
|
def receive_progress(self):
|
|
566
658
|
progress = Progress()
|
|
567
|
-
progress.read(self.server_info
|
|
659
|
+
progress.read(self.server_info, self.fin)
|
|
568
660
|
return progress
|
|
569
661
|
|
|
570
662
|
def receive_profile_info(self):
|
|
@@ -580,14 +672,14 @@ class Connection(object):
|
|
|
580
672
|
start = time()
|
|
581
673
|
write_varint(ClientPacketTypes.DATA, self.fout)
|
|
582
674
|
|
|
583
|
-
revision = self.server_info.
|
|
675
|
+
revision = self.server_info.used_revision
|
|
584
676
|
if revision >= defines.DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES:
|
|
585
677
|
write_binary_str(table_name, self.fout)
|
|
586
678
|
|
|
587
679
|
self.block_out.write(block)
|
|
588
680
|
logger.debug('Block "%s" send time: %f', table_name, time() - start)
|
|
589
681
|
|
|
590
|
-
def send_query(self, query, query_id=None):
|
|
682
|
+
def send_query(self, query, query_id=None, params=None):
|
|
591
683
|
if not self.connected:
|
|
592
684
|
self.connect()
|
|
593
685
|
|
|
@@ -595,9 +687,10 @@ class Connection(object):
|
|
|
595
687
|
|
|
596
688
|
write_binary_str(query_id or '', self.fout)
|
|
597
689
|
|
|
598
|
-
revision = self.server_info.
|
|
690
|
+
revision = self.server_info.used_revision
|
|
599
691
|
if revision >= defines.DBMS_MIN_REVISION_WITH_CLIENT_INFO:
|
|
600
|
-
client_info = ClientInfo(self.client_name, self.context
|
|
692
|
+
client_info = ClientInfo(self.client_name, self.context,
|
|
693
|
+
client_revision=self.client_revision)
|
|
601
694
|
client_info.query_kind = ClientInfo.QueryKind.INITIAL_QUERY
|
|
602
695
|
|
|
603
696
|
client_info.write(revision, self.fout)
|
|
@@ -606,8 +699,11 @@ class Connection(object):
|
|
|
606
699
|
revision >= defines
|
|
607
700
|
.DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS
|
|
608
701
|
)
|
|
702
|
+
settings_flags = 0
|
|
703
|
+
if self.settings_is_important:
|
|
704
|
+
settings_flags |= SettingsFlags.IMPORTANT
|
|
609
705
|
write_settings(self.context.settings, self.fout, settings_as_strings,
|
|
610
|
-
|
|
706
|
+
settings_flags)
|
|
611
707
|
|
|
612
708
|
if revision >= defines.DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET:
|
|
613
709
|
write_binary_str('', self.fout)
|
|
@@ -617,6 +713,16 @@ class Connection(object):
|
|
|
617
713
|
|
|
618
714
|
write_binary_str(query, self.fout)
|
|
619
715
|
|
|
716
|
+
if revision >= defines.DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS:
|
|
717
|
+
if self.context.client_settings['server_side_params']:
|
|
718
|
+
# Always settings_as_strings = True
|
|
719
|
+
escaped = escape_params(
|
|
720
|
+
params or {}, self.context, for_server=True
|
|
721
|
+
)
|
|
722
|
+
else:
|
|
723
|
+
escaped = {}
|
|
724
|
+
write_settings(escaped, self.fout, True, SettingsFlags.CUSTOM)
|
|
725
|
+
|
|
620
726
|
logger.debug('Query: %s', query)
|
|
621
727
|
|
|
622
728
|
self.fout.flush()
|
clickhouse_driver/defines.py
CHANGED
|
@@ -25,6 +25,14 @@ DBMS_MIN_PROTOCOL_VERSION_WITH_DISTRIBUTED_DEPTH = 54448
|
|
|
25
25
|
DBMS_MIN_PROTOCOL_VERSION_WITH_INITIAL_QUERY_START_TIME = 54449
|
|
26
26
|
DBMS_MIN_PROTOCOL_VERSION_WITH_INCREMENTAL_PROFILE_EVENTS = 54451
|
|
27
27
|
DBMS_MIN_REVISION_WITH_PARALLEL_REPLICAS = 54453
|
|
28
|
+
DBMS_MIN_REVISION_WITH_CUSTOM_SERIALIZATION = 54454
|
|
29
|
+
DBMS_MIN_PROTOCOL_VERSION_WITH_PROFILE_EVENTS_IN_INSERT = 54456
|
|
30
|
+
DBMS_MIN_PROTOCOL_VERSION_WITH_ADDENDUM = 54458
|
|
31
|
+
DBMS_MIN_PROTOCOL_VERSION_WITH_QUOTA_KEY = 54458
|
|
32
|
+
DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS = 54459
|
|
33
|
+
DBMS_MIN_PROTOCOL_VERSION_WITH_SERVER_QUERY_TIME_IN_PROGRESS = 54460
|
|
34
|
+
DBMS_MIN_PROTOCOL_VERSION_WITH_PASSWORD_COMPLEXITY_RULES = 54461
|
|
35
|
+
DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2 = 54462
|
|
28
36
|
|
|
29
37
|
# Timeouts
|
|
30
38
|
DBMS_DEFAULT_CONNECT_TIMEOUT_SEC = 10
|
|
@@ -40,7 +48,7 @@ CLIENT_NAME = 'python-driver'
|
|
|
40
48
|
CLIENT_VERSION_MAJOR = 20
|
|
41
49
|
CLIENT_VERSION_MINOR = 10
|
|
42
50
|
CLIENT_VERSION_PATCH = 2
|
|
43
|
-
CLIENT_REVISION =
|
|
51
|
+
CLIENT_REVISION = DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2
|
|
44
52
|
|
|
45
53
|
BUFFER_SIZE = 1048576
|
|
46
54
|
|
clickhouse_driver/log.py
CHANGED
|
@@ -2,7 +2,8 @@ import logging
|
|
|
2
2
|
|
|
3
3
|
logger = logging.getLogger(__name__)
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
# Keep in sync with ClickHouse priorities
|
|
6
|
+
# https://github.com/ClickHouse/ClickHouse/blob/master/src/Interpreters/InternalTextLogsQueue.cpp
|
|
6
7
|
log_priorities = (
|
|
7
8
|
'Unknown',
|
|
8
9
|
'Fatal',
|
|
@@ -12,9 +13,12 @@ log_priorities = (
|
|
|
12
13
|
'Notice',
|
|
13
14
|
'Information',
|
|
14
15
|
'Debug',
|
|
15
|
-
'Trace'
|
|
16
|
+
'Trace',
|
|
17
|
+
'Test',
|
|
16
18
|
)
|
|
17
19
|
|
|
20
|
+
num_priorities = len(log_priorities)
|
|
21
|
+
|
|
18
22
|
|
|
19
23
|
def log_block(block):
|
|
20
24
|
if block is None:
|
|
@@ -25,7 +29,7 @@ def log_block(block):
|
|
|
25
29
|
for row in block.get_rows():
|
|
26
30
|
row = dict(zip(column_names, row))
|
|
27
31
|
|
|
28
|
-
if 1 <= row['priority'] <=
|
|
32
|
+
if 1 <= row['priority'] <= num_priorities:
|
|
29
33
|
priority = log_priorities[row['priority']]
|
|
30
34
|
else:
|
|
31
35
|
priority = row[0]
|
clickhouse_driver/progress.py
CHANGED
|
@@ -9,14 +9,15 @@ class Progress(object):
|
|
|
9
9
|
self.total_rows = 0
|
|
10
10
|
self.written_rows = 0
|
|
11
11
|
self.written_bytes = 0
|
|
12
|
+
self.elapsed_ns = 0
|
|
12
13
|
|
|
13
14
|
super(Progress, self).__init__()
|
|
14
15
|
|
|
15
|
-
def read(self,
|
|
16
|
+
def read(self, server_info, fin):
|
|
16
17
|
self.rows = read_varint(fin)
|
|
17
18
|
self.bytes = read_varint(fin)
|
|
18
19
|
|
|
19
|
-
revision =
|
|
20
|
+
revision = server_info.used_revision
|
|
20
21
|
if revision >= defines.DBMS_MIN_REVISION_WITH_TOTAL_ROWS_IN_PROGRESS:
|
|
21
22
|
self.total_rows = read_varint(fin)
|
|
22
23
|
|
|
@@ -24,9 +25,14 @@ class Progress(object):
|
|
|
24
25
|
self.written_rows = read_varint(fin)
|
|
25
26
|
self.written_bytes = read_varint(fin)
|
|
26
27
|
|
|
28
|
+
if revision >= defines. \
|
|
29
|
+
DBMS_MIN_PROTOCOL_VERSION_WITH_SERVER_QUERY_TIME_IN_PROGRESS:
|
|
30
|
+
self.elapsed_ns = read_varint(fin)
|
|
31
|
+
|
|
27
32
|
def increment(self, another_progress):
|
|
28
33
|
self.rows += another_progress.rows
|
|
29
34
|
self.bytes += another_progress.bytes
|
|
30
35
|
self.total_rows += another_progress.total_rows
|
|
31
36
|
self.written_rows += another_progress.written_rows
|
|
32
37
|
self.written_bytes += another_progress.written_bytes
|
|
38
|
+
self.elapsed_ns += another_progress.elapsed_ns
|
|
@@ -7,13 +7,18 @@ from .available import settings as available_settings
|
|
|
7
7
|
logger = logging.getLogger(__name__)
|
|
8
8
|
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
class SettingsFlags:
|
|
11
|
+
IMPORTANT = 0x1
|
|
12
|
+
CUSTOM = 0x2
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def write_settings(settings, buf, settings_as_strings, flags):
|
|
11
16
|
for setting, value in (settings or {}).items():
|
|
12
17
|
# If the server support settings as string we do not need to know
|
|
13
18
|
# anything about them, so we can write any setting.
|
|
14
19
|
if settings_as_strings:
|
|
15
20
|
write_binary_str(setting, buf)
|
|
16
|
-
write_binary_uint8(
|
|
21
|
+
write_binary_uint8(flags, buf)
|
|
17
22
|
write_binary_str(str(value), buf)
|
|
18
23
|
|
|
19
24
|
else:
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
from ..block import ColumnOrientedBlock, BlockInfo
|
|
2
2
|
from ..columns.service import read_column, write_column
|
|
3
|
-
from ..reader import read_binary_str
|
|
3
|
+
from ..reader import read_binary_str, read_binary_uint8
|
|
4
4
|
from ..varint import write_varint, read_varint
|
|
5
|
-
from ..writer import write_binary_str
|
|
5
|
+
from ..writer import write_binary_str, write_binary_uint8
|
|
6
6
|
from .. import defines
|
|
7
7
|
|
|
8
8
|
|
|
@@ -14,7 +14,7 @@ class BlockOutputStream(object):
|
|
|
14
14
|
super(BlockOutputStream, self).__init__()
|
|
15
15
|
|
|
16
16
|
def write(self, block):
|
|
17
|
-
revision = self.context.server_info.
|
|
17
|
+
revision = self.context.server_info.used_revision
|
|
18
18
|
if revision >= defines.DBMS_MIN_REVISION_WITH_BLOCK_INFO:
|
|
19
19
|
block.info.write(self.fout)
|
|
20
20
|
|
|
@@ -35,6 +35,11 @@ class BlockOutputStream(object):
|
|
|
35
35
|
except IndexError:
|
|
36
36
|
raise ValueError('Different rows length')
|
|
37
37
|
|
|
38
|
+
if revision >= \
|
|
39
|
+
defines.DBMS_MIN_REVISION_WITH_CUSTOM_SERIALIZATION:
|
|
40
|
+
# We write always sparse data without custom serialization.
|
|
41
|
+
write_binary_uint8(0, self.fout)
|
|
42
|
+
|
|
38
43
|
write_column(self.context, col_name, col_type, items,
|
|
39
44
|
self.fout, types_check=block.types_check)
|
|
40
45
|
|
|
@@ -54,7 +59,7 @@ class BlockInputStream(object):
|
|
|
54
59
|
def read(self, use_numpy=None):
|
|
55
60
|
info = BlockInfo()
|
|
56
61
|
|
|
57
|
-
revision = self.context.server_info.
|
|
62
|
+
revision = self.context.server_info.used_revision
|
|
58
63
|
if revision >= defines.DBMS_MIN_REVISION_WITH_BLOCK_INFO:
|
|
59
64
|
info.read(self.fin)
|
|
60
65
|
|
|
@@ -70,9 +75,16 @@ class BlockInputStream(object):
|
|
|
70
75
|
names.append(column_name)
|
|
71
76
|
types.append(column_type)
|
|
72
77
|
|
|
78
|
+
has_custom_serialization = False
|
|
79
|
+
if revision >= defines.DBMS_MIN_REVISION_WITH_CUSTOM_SERIALIZATION:
|
|
80
|
+
has_custom_serialization = bool(read_binary_uint8(self.fin))
|
|
81
|
+
|
|
73
82
|
if n_rows:
|
|
74
|
-
column = read_column(
|
|
75
|
-
|
|
83
|
+
column = read_column(
|
|
84
|
+
self.context, column_type, n_rows,
|
|
85
|
+
self.fin, use_numpy=use_numpy,
|
|
86
|
+
has_custom_serialization=has_custom_serialization
|
|
87
|
+
)
|
|
76
88
|
data.append(column)
|
|
77
89
|
|
|
78
90
|
if self.context.client_settings['use_numpy']:
|
clickhouse_driver/util/compat.py
CHANGED
|
@@ -5,6 +5,18 @@ try:
|
|
|
5
5
|
except ImportError:
|
|
6
6
|
import dummy_threading as threading # noqa: F401
|
|
7
7
|
|
|
8
|
+
import json # noqa: F401
|
|
9
|
+
try:
|
|
10
|
+
import orjson as json # noqa: F811
|
|
11
|
+
except ImportError:
|
|
12
|
+
pass
|
|
13
|
+
|
|
14
|
+
try:
|
|
15
|
+
import ujson as json # noqa: F811,F401
|
|
16
|
+
except ImportError:
|
|
17
|
+
pass
|
|
18
|
+
|
|
19
|
+
|
|
8
20
|
try:
|
|
9
21
|
# since tzlocal 4.0+
|
|
10
22
|
# this will avoid warning for get_localzone().key
|
clickhouse_driver/util/escape.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
from datetime import date, datetime
|
|
1
|
+
from datetime import date, datetime, time
|
|
2
2
|
from enum import Enum
|
|
3
|
+
from functools import wraps
|
|
3
4
|
from uuid import UUID
|
|
4
5
|
|
|
5
6
|
from pytz import timezone
|
|
@@ -28,7 +29,24 @@ def escape_datetime(item, context):
|
|
|
28
29
|
return "'%s'" % item.strftime('%Y-%m-%d %H:%M:%S')
|
|
29
30
|
|
|
30
31
|
|
|
31
|
-
def
|
|
32
|
+
def maybe_enquote_for_server(f):
|
|
33
|
+
@wraps(f)
|
|
34
|
+
def wrapper(*args, **kwargs):
|
|
35
|
+
rv = f(*args, **kwargs)
|
|
36
|
+
|
|
37
|
+
if kwargs.get('for_server'):
|
|
38
|
+
is_str = isinstance(rv, str)
|
|
39
|
+
|
|
40
|
+
if not is_str or (is_str and not rv.startswith("'")):
|
|
41
|
+
rv = "'%s'" % rv
|
|
42
|
+
|
|
43
|
+
return rv
|
|
44
|
+
|
|
45
|
+
return wrapper
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@maybe_enquote_for_server
|
|
49
|
+
def escape_param(item, context, for_server=False):
|
|
32
50
|
if item is None:
|
|
33
51
|
return 'NULL'
|
|
34
52
|
|
|
@@ -38,17 +56,27 @@ def escape_param(item, context):
|
|
|
38
56
|
elif isinstance(item, date):
|
|
39
57
|
return "'%s'" % item.strftime('%Y-%m-%d')
|
|
40
58
|
|
|
59
|
+
elif isinstance(item, time):
|
|
60
|
+
return "'%s'" % item.strftime('%H:%M:%S')
|
|
61
|
+
|
|
41
62
|
elif isinstance(item, str):
|
|
63
|
+
# We need double escaping for server-side parameters.
|
|
64
|
+
if for_server:
|
|
65
|
+
item = ''.join(escape_chars_map.get(c, c) for c in item)
|
|
42
66
|
return "'%s'" % ''.join(escape_chars_map.get(c, c) for c in item)
|
|
43
67
|
|
|
44
68
|
elif isinstance(item, list):
|
|
45
|
-
return "[%s]" % ', '.join(
|
|
69
|
+
return "[%s]" % ', '.join(
|
|
70
|
+
str(escape_param(x, context, for_server=for_server)) for x in item
|
|
71
|
+
)
|
|
46
72
|
|
|
47
73
|
elif isinstance(item, tuple):
|
|
48
|
-
return "(%s)" % ', '.join(
|
|
74
|
+
return "(%s)" % ', '.join(
|
|
75
|
+
str(escape_param(x, context, for_server=for_server)) for x in item
|
|
76
|
+
)
|
|
49
77
|
|
|
50
78
|
elif isinstance(item, Enum):
|
|
51
|
-
return escape_param(item.value, context)
|
|
79
|
+
return escape_param(item.value, context, for_server=for_server)
|
|
52
80
|
|
|
53
81
|
elif isinstance(item, UUID):
|
|
54
82
|
return "'%s'" % str(item)
|
|
@@ -57,10 +85,10 @@ def escape_param(item, context):
|
|
|
57
85
|
return item
|
|
58
86
|
|
|
59
87
|
|
|
60
|
-
def escape_params(params, context):
|
|
88
|
+
def escape_params(params, context, for_server=False):
|
|
61
89
|
escaped = {}
|
|
62
90
|
|
|
63
91
|
for key, value in params.items():
|
|
64
|
-
escaped[key] = escape_param(value, context)
|
|
92
|
+
escaped[key] = escape_param(value, context, for_server=for_server)
|
|
65
93
|
|
|
66
94
|
return escaped
|
|
Binary file
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: clickhouse-driver
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.8
|
|
4
4
|
Summary: Python driver with native interface for ClickHouse
|
|
5
5
|
Home-page: https://github.com/mymarilyn/clickhouse-driver
|
|
6
6
|
Author: Konstantin Lebedev
|
|
@@ -9,7 +9,6 @@ License: MIT
|
|
|
9
9
|
Project-URL: Documentation, https://clickhouse-driver.readthedocs.io
|
|
10
10
|
Project-URL: Changes, https://github.com/mymarilyn/clickhouse-driver/blob/master/CHANGELOG.md
|
|
11
11
|
Keywords: ClickHouse db database cloud analytics
|
|
12
|
-
Platform: UNKNOWN
|
|
13
12
|
Classifier: Development Status :: 4 - Beta
|
|
14
13
|
Classifier: Environment :: Console
|
|
15
14
|
Classifier: Intended Audience :: Developers
|
|
@@ -18,12 +17,11 @@ Classifier: License :: OSI Approved :: MIT License
|
|
|
18
17
|
Classifier: Operating System :: OS Independent
|
|
19
18
|
Classifier: Programming Language :: SQL
|
|
20
19
|
Classifier: Programming Language :: Python :: 3
|
|
21
|
-
Classifier: Programming Language :: Python :: 3.5
|
|
22
|
-
Classifier: Programming Language :: Python :: 3.6
|
|
23
20
|
Classifier: Programming Language :: Python :: 3.7
|
|
24
21
|
Classifier: Programming Language :: Python :: 3.8
|
|
25
22
|
Classifier: Programming Language :: Python :: 3.9
|
|
26
23
|
Classifier: Programming Language :: Python :: 3.10
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
27
25
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
28
26
|
Classifier: Topic :: Database
|
|
29
27
|
Classifier: Topic :: Software Development
|
|
@@ -31,21 +29,20 @@ Classifier: Topic :: Software Development :: Libraries
|
|
|
31
29
|
Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
32
30
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
33
31
|
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
34
|
-
Requires-Python: >=3.
|
|
32
|
+
Requires-Python: >=3.7, <4
|
|
35
33
|
License-File: LICENSE
|
|
36
34
|
Requires-Dist: pytz
|
|
37
35
|
Requires-Dist: tzlocal
|
|
38
|
-
Requires-Dist: tzlocal (<2.1) ; python_version == "3.5"
|
|
39
36
|
Provides-Extra: lz4
|
|
40
|
-
Requires-Dist: clickhouse-cityhash
|
|
37
|
+
Requires-Dist: clickhouse-cityhash >=1.0.2.1 ; extra == 'lz4'
|
|
41
38
|
Requires-Dist: lz4 ; (implementation_name != "pypy") and extra == 'lz4'
|
|
42
|
-
Requires-Dist: lz4
|
|
39
|
+
Requires-Dist: lz4 <=3.0.1 ; (implementation_name == "pypy") and extra == 'lz4'
|
|
43
40
|
Provides-Extra: numpy
|
|
44
|
-
Requires-Dist: numpy
|
|
45
|
-
Requires-Dist: pandas
|
|
41
|
+
Requires-Dist: numpy >=1.12.0 ; extra == 'numpy'
|
|
42
|
+
Requires-Dist: pandas >=0.24.0 ; extra == 'numpy'
|
|
46
43
|
Provides-Extra: zstd
|
|
47
44
|
Requires-Dist: zstd ; extra == 'zstd'
|
|
48
|
-
Requires-Dist: clickhouse-cityhash
|
|
45
|
+
Requires-Dist: clickhouse-cityhash >=1.0.2.1 ; extra == 'zstd'
|
|
49
46
|
|
|
50
47
|
ClickHouse Python Driver
|
|
51
48
|
========================
|
|
@@ -202,5 +199,3 @@ License
|
|
|
202
199
|
|
|
203
200
|
ClickHouse Python Driver is distributed under the `MIT license
|
|
204
201
|
<http://www.opensource.org/licenses/mit-license.php>`_.
|
|
205
|
-
|
|
206
|
-
|