clickhouse-driver 0.2.7__tar.gz → 0.2.9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. {clickhouse-driver-0.2.7/clickhouse_driver.egg-info → clickhouse-driver-0.2.9}/PKG-INFO +2 -1
  2. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/__init__.py +1 -1
  3. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/client.py +12 -96
  4. clickhouse-driver-0.2.9/clickhouse_driver/columns/datecolumn.py +108 -0
  5. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/datetimecolumn.py +3 -2
  6. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/enumcolumn.py +27 -17
  7. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/mapcolumn.py +3 -0
  8. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/numpy/datetimecolumn.py +6 -3
  9. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/util.py +2 -1
  10. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/connection.py +13 -4
  11. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/defines.py +4 -1
  12. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/numpy/helpers.py +5 -2
  13. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/progress.py +7 -1
  14. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/protocol.py +19 -3
  15. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/streams/native.py +6 -0
  16. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/util/escape.py +1 -1
  17. clickhouse-driver-0.2.9/clickhouse_driver/util/helpers.py +171 -0
  18. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9/clickhouse_driver.egg-info}/PKG-INFO +2 -1
  19. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/setup.cfg +1 -0
  20. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/setup.py +1 -0
  21. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/tests/test_client.py +2 -2
  22. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/tests/test_connect.py +20 -1
  23. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/tests/test_dbapi.py +2 -2
  24. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/tests/test_insert.py +1 -1
  25. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/tests/test_settings.py +2 -2
  26. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/tests/test_substitution.py +1 -1
  27. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/tests/testcase.py +1 -0
  28. clickhouse-driver-0.2.7/clickhouse_driver/columns/datecolumn.py +0 -69
  29. clickhouse-driver-0.2.7/clickhouse_driver/util/helpers.py +0 -57
  30. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/LICENSE +0 -0
  31. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/MANIFEST.in +0 -0
  32. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/README.rst +0 -0
  33. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/block.py +0 -0
  34. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/blockstreamprofileinfo.py +0 -0
  35. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/bufferedreader.c +0 -0
  36. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/bufferedreader.pyx +0 -0
  37. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/bufferedwriter.c +0 -0
  38. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/bufferedwriter.pyx +0 -0
  39. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/clientinfo.py +0 -0
  40. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/__init__.py +0 -0
  41. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/arraycolumn.py +0 -0
  42. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/base.py +0 -0
  43. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/boolcolumn.py +0 -0
  44. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/decimalcolumn.py +0 -0
  45. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/exceptions.py +0 -0
  46. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/floatcolumn.py +0 -0
  47. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/intcolumn.py +0 -0
  48. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/intervalcolumn.py +0 -0
  49. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/ipcolumn.py +0 -0
  50. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/jsoncolumn.py +0 -0
  51. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/largeint.c +0 -0
  52. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/largeint.pyx +0 -0
  53. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/lowcardinalitycolumn.py +0 -0
  54. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/nestedcolumn.py +0 -0
  55. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/nothingcolumn.py +0 -0
  56. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/nullablecolumn.py +0 -0
  57. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/nullcolumn.py +0 -0
  58. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/numpy/__init__.py +0 -0
  59. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/numpy/base.py +0 -0
  60. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/numpy/boolcolumn.py +0 -0
  61. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/numpy/datecolumn.py +0 -0
  62. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/numpy/floatcolumn.py +0 -0
  63. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/numpy/intcolumn.py +0 -0
  64. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/numpy/lowcardinalitycolumn.py +0 -0
  65. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/numpy/service.py +0 -0
  66. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/numpy/stringcolumn.py +0 -0
  67. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/numpy/tuplecolumn.py +0 -0
  68. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/service.py +0 -0
  69. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/simpleaggregatefunctioncolumn.py +0 -0
  70. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/stringcolumn.py +0 -0
  71. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/tuplecolumn.py +0 -0
  72. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/columns/uuidcolumn.py +0 -0
  73. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/compression/__init__.py +0 -0
  74. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/compression/base.py +0 -0
  75. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/compression/lz4.py +0 -0
  76. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/compression/lz4hc.py +0 -0
  77. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/compression/zstd.py +0 -0
  78. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/context.py +0 -0
  79. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/dbapi/__init__.py +0 -0
  80. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/dbapi/connection.py +0 -0
  81. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/dbapi/cursor.py +0 -0
  82. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/dbapi/errors.py +0 -0
  83. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/dbapi/extras.py +0 -0
  84. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/errors.py +0 -0
  85. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/log.py +0 -0
  86. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/numpy/__init__.py +0 -0
  87. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/numpy/block.py +0 -0
  88. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/numpy/result.py +0 -0
  89. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/opentelemetry.py +0 -0
  90. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/queryprocessingstage.py +0 -0
  91. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/reader.py +0 -0
  92. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/readhelpers.py +0 -0
  93. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/result.py +0 -0
  94. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/settings/__init__.py +0 -0
  95. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/settings/available.py +0 -0
  96. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/settings/types.py +0 -0
  97. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/settings/writer.py +0 -0
  98. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/streams/__init__.py +0 -0
  99. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/streams/compressed.py +0 -0
  100. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/util/__init__.py +0 -0
  101. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/util/compat.py +0 -0
  102. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/varint.c +0 -0
  103. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/varint.pyx +0 -0
  104. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver/writer.py +0 -0
  105. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver.egg-info/SOURCES.txt +0 -0
  106. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver.egg-info/dependency_links.txt +0 -0
  107. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver.egg-info/requires.txt +0 -0
  108. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/clickhouse_driver.egg-info/top_level.txt +0 -0
  109. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/tests/test_blocks.py +0 -0
  110. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/tests/test_buffered_reader.py +0 -0
  111. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/tests/test_compression.py +0 -0
  112. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/tests/test_errors.py +0 -0
  113. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/tests/test_external_tables.py +0 -0
  114. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/tests/test_opentelemetry.py +0 -0
  115. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/tests/test_query_info.py +0 -0
  116. {clickhouse-driver-0.2.7 → clickhouse-driver-0.2.9}/tests/test_varint.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: clickhouse-driver
3
- Version: 0.2.7
3
+ Version: 0.2.9
4
4
  Summary: Python driver with native interface for ClickHouse
5
5
  Home-page: https://github.com/mymarilyn/clickhouse-driver
6
6
  Author: Konstantin Lebedev
@@ -179,6 +179,7 @@ Classifier: Programming Language :: Python :: 3.8
179
179
  Classifier: Programming Language :: Python :: 3.9
180
180
  Classifier: Programming Language :: Python :: 3.10
181
181
  Classifier: Programming Language :: Python :: 3.11
182
+ Classifier: Programming Language :: Python :: 3.12
182
183
  Classifier: Programming Language :: Python :: Implementation :: PyPy
183
184
  Classifier: Topic :: Database
184
185
  Classifier: Topic :: Software Development
@@ -3,7 +3,7 @@ from .client import Client
3
3
  from .dbapi import connect
4
4
 
5
5
 
6
- VERSION = (0, 2, 7)
6
+ VERSION = (0, 2, 9)
7
7
  __version__ = '.'.join(str(x) for x in VERSION)
8
8
 
9
9
  __all__ = ['Client', 'connect']
@@ -1,10 +1,9 @@
1
1
  import re
2
- import ssl
3
2
  from collections import deque
4
3
  from contextlib import contextmanager
5
4
  from time import time
6
5
  import types
7
- from urllib.parse import urlparse, parse_qs, unquote
6
+ from urllib.parse import urlparse
8
7
 
9
8
  from . import errors, defines
10
9
  from .block import ColumnOrientedBlock, RowOrientedBlock
@@ -15,7 +14,7 @@ from .result import (
15
14
  IterQueryResult, ProgressQueryResult, QueryResult, QueryInfo
16
15
  )
17
16
  from .util.escape import escape_params
18
- from .util.helpers import column_chunks, chunks, asbool
17
+ from .util.helpers import column_chunks, chunks, parse_url
19
18
 
20
19
 
21
20
  class Client(object):
@@ -312,6 +311,7 @@ class Client(object):
312
311
  def disconnect_on_error(self, query, settings):
313
312
  try:
314
313
  self.establish_connection(settings)
314
+ self.connection.server_info.session_timezone = None
315
315
 
316
316
  yield
317
317
 
@@ -746,9 +746,13 @@ class Client(object):
746
746
  elif packet.type == ServerPacketTypes.EXCEPTION:
747
747
  raise packet.exception
748
748
 
749
+ elif packet.type == ServerPacketTypes.TIMEZONE_UPDATE:
750
+ pass
751
+
749
752
  else:
750
753
  message = self.connection.unexpected_packet_message(
751
- 'ProfileEvents, Progress, Log or Exception', packet.type
754
+ 'ProfileEvents, Progress, Log, Exception or '
755
+ 'TimezoneUpdate', packet.type
752
756
  )
753
757
  raise errors.UnexpectedPacketFromServerError(message)
754
758
 
@@ -796,101 +800,13 @@ class Client(object):
796
800
  clickhouses://[user:password]@localhost:9440/default
797
801
 
798
802
  Three URL schemes are supported:
799
- clickhouse:// creates a normal TCP socket connection
800
- clickhouses:// creates a SSL wrapped TCP socket connection
803
+
804
+ * clickhouse:// creates a normal TCP socket connection
805
+ * clickhouses:// creates a SSL wrapped TCP socket connection
801
806
 
802
807
  Any additional querystring arguments will be passed along to
803
808
  the Connection class's initializer.
804
809
  """
805
- url = urlparse(url)
806
-
807
- settings = {}
808
- kwargs = {}
809
-
810
- host = url.hostname
811
-
812
- if url.port is not None:
813
- kwargs['port'] = url.port
814
-
815
- path = url.path.replace('/', '', 1)
816
- if path:
817
- kwargs['database'] = path
818
-
819
- if url.username is not None:
820
- kwargs['user'] = unquote(url.username)
821
-
822
- if url.password is not None:
823
- kwargs['password'] = unquote(url.password)
824
-
825
- if url.scheme == 'clickhouses':
826
- kwargs['secure'] = True
827
-
828
- compression_algs = {'lz4', 'lz4hc', 'zstd'}
829
- timeouts = {
830
- 'connect_timeout',
831
- 'send_receive_timeout',
832
- 'sync_request_timeout'
833
- }
834
-
835
- for name, value in parse_qs(url.query).items():
836
- if not value or not len(value):
837
- continue
838
-
839
- value = value[0]
840
-
841
- if name == 'compression':
842
- value = value.lower()
843
- if value in compression_algs:
844
- kwargs[name] = value
845
- else:
846
- kwargs[name] = asbool(value)
847
-
848
- elif name == 'secure':
849
- kwargs[name] = asbool(value)
850
-
851
- elif name == 'use_numpy':
852
- settings[name] = asbool(value)
853
-
854
- elif name == 'round_robin':
855
- kwargs[name] = asbool(value)
856
-
857
- elif name == 'client_name':
858
- kwargs[name] = value
859
-
860
- elif name in timeouts:
861
- kwargs[name] = float(value)
862
-
863
- elif name == 'compress_block_size':
864
- kwargs[name] = int(value)
865
-
866
- elif name == 'settings_is_important':
867
- kwargs[name] = asbool(value)
868
-
869
- elif name == 'tcp_keepalive':
870
- try:
871
- kwargs[name] = asbool(value)
872
- except ValueError:
873
- parts = value.split(',')
874
- kwargs[name] = (
875
- float(parts[0]), float(parts[1]), int(parts[2])
876
- )
877
- elif name == 'client_revision':
878
- kwargs[name] = int(value)
879
-
880
- # ssl
881
- elif name == 'verify':
882
- kwargs[name] = asbool(value)
883
- elif name == 'ssl_version':
884
- kwargs[name] = getattr(ssl, value)
885
- elif name in ['ca_certs', 'ciphers', 'keyfile', 'certfile',
886
- 'server_hostname']:
887
- kwargs[name] = value
888
- elif name == 'alt_hosts':
889
- kwargs['alt_hosts'] = value
890
- else:
891
- settings[name] = value
892
-
893
- if settings:
894
- kwargs['settings'] = settings
810
+ host, kwargs = parse_url(url)
895
811
 
896
812
  return cls(host, **kwargs)
@@ -0,0 +1,108 @@
1
+ from os import getenv
2
+ from datetime import date, timedelta
3
+
4
+ from .base import FormatColumn
5
+
6
+
7
+ epoch_start = date(1970, 1, 1)
8
+ epoch_end = date(2149, 6, 6)
9
+
10
+ epoch_start_date32 = date(1900, 1, 1)
11
+ epoch_end_date32 = date(2299, 12, 31)
12
+
13
+
14
+ class LazyLUT(dict):
15
+ def __init__(self, *args, _factory, **kwargs):
16
+ super().__init__(*args, **kwargs)
17
+ self._default_factory = _factory
18
+
19
+ def __missing__(self, key):
20
+ return self.setdefault(key, self._default_factory(key))
21
+
22
+
23
+ def make_date_lut_range(date_start, date_end):
24
+ return range(
25
+ (date_start - epoch_start).days,
26
+ (date_end - epoch_start).days + 1,
27
+ )
28
+
29
+
30
+ enable_lazy_date_lut = getenv('CLICKHOUSE_DRIVER_LASY_DATE_LUT', False)
31
+ if enable_lazy_date_lut:
32
+ try:
33
+ start, end = enable_lazy_date_lut.split(':')
34
+ start_date = date.fromisoformat(start)
35
+ end_date = date.fromisoformat(end)
36
+
37
+ date_range = make_date_lut_range(start_date, end_date)
38
+ except ValueError:
39
+ date_range = ()
40
+
41
+ # Since we initialize lazy lut with some initially warmed values,
42
+ # we use iterator and not dict comprehension for memory & time optimization
43
+ _date_lut = LazyLUT(
44
+ ((x, epoch_start + timedelta(days=x)) for x in date_range),
45
+ _factory=lambda x: epoch_start + timedelta(days=x),
46
+ )
47
+ _date_lut_reverse = LazyLUT(
48
+ ((value, key) for key, value in _date_lut.items()),
49
+ _factory=lambda x: (x - epoch_start).days,
50
+ )
51
+ else:
52
+ # If lazy lut is not enabled, we fallback to static dict initialization
53
+ # In both cases, we use same lut for both data types,
54
+ # since one encompasses the other and we can avoid duplicating overlap
55
+ date_range = make_date_lut_range(epoch_start_date32, epoch_end_date32)
56
+ _date_lut = {x: epoch_start + timedelta(days=x) for x in date_range}
57
+ _date_lut_reverse = {value: key for key, value in _date_lut.items()}
58
+
59
+
60
+ class DateColumn(FormatColumn):
61
+ ch_type = 'Date'
62
+ py_types = (date, )
63
+ format = 'H'
64
+
65
+ min_value = epoch_start
66
+ max_value = epoch_end
67
+
68
+ date_lut = _date_lut
69
+ date_lut_reverse = _date_lut_reverse
70
+
71
+ def before_write_items(self, items, nulls_map=None):
72
+ null_value = self.null_value
73
+
74
+ date_lut_reverse = self.date_lut_reverse
75
+ min_value = self.min_value
76
+ max_value = self.max_value
77
+
78
+ for i, item in enumerate(items):
79
+ if nulls_map and nulls_map[i]:
80
+ items[i] = null_value
81
+ continue
82
+
83
+ if item is not date:
84
+ item = date(item.year, item.month, item.day)
85
+
86
+ if min_value <= item <= max_value:
87
+ items[i] = date_lut_reverse[item]
88
+ else:
89
+ items[i] = 0
90
+
91
+ def after_read_items(self, items, nulls_map=None):
92
+ date_lut = self.date_lut
93
+
94
+ if nulls_map is None:
95
+ return tuple(date_lut[item] for item in items)
96
+ else:
97
+ return tuple(
98
+ (None if is_null else date_lut[items[i]])
99
+ for i, is_null in enumerate(nulls_map)
100
+ )
101
+
102
+
103
+ class Date32Column(DateColumn):
104
+ ch_type = 'Date32'
105
+ format = 'i'
106
+
107
+ min_value = epoch_start_date32
108
+ max_value = epoch_end_date32
@@ -193,8 +193,9 @@ def create_datetime_column(spec, column_options):
193
193
  else:
194
194
  if not context.settings.get('use_client_time_zone', False):
195
195
  local_timezone = get_localzone_name_compat()
196
- if local_timezone != context.server_info.timezone:
197
- tz_name = context.server_info.timezone
196
+ remote_timezone = context.server_info.get_timezone()
197
+ if local_timezone != remote_timezone:
198
+ tz_name = remote_timezone
198
199
 
199
200
  if tz_name:
200
201
  timezone = get_timezone(tz_name)
@@ -1,20 +1,24 @@
1
1
  from enum import Enum
2
+ from collections import OrderedDict
2
3
 
3
4
  from .. import errors
4
5
  from .intcolumn import IntColumn
5
6
 
7
+ invalid_names_for_python_enum = frozenset(['mro', ''])
8
+
6
9
 
7
10
  class EnumColumn(IntColumn):
8
11
  py_types = (Enum, int, str)
9
12
 
10
- def __init__(self, enum_cls, **kwargs):
11
- self.enum_cls = enum_cls
13
+ def __init__(self, name_by_value, value_by_name, **kwargs):
14
+ self.name_by_value = name_by_value
15
+ self.value_by_name = value_by_name
12
16
  super(EnumColumn, self).__init__(**kwargs)
13
17
 
14
18
  def before_write_items(self, items, nulls_map=None):
15
19
  null_value = self.null_value
16
-
17
- enum_cls = self.enum_cls
20
+ name_by_value = self.name_by_value
21
+ value_by_name = self.value_by_name
18
22
 
19
23
  for i, item in enumerate(items):
20
24
  if nulls_map and nulls_map[i]:
@@ -26,15 +30,15 @@ class EnumColumn(IntColumn):
26
30
  # Check real enum value
27
31
  try:
28
32
  if isinstance(source_value, str):
29
- items[i] = enum_cls[source_value].value
33
+ items[i] = value_by_name[source_value]
30
34
  else:
31
- items[i] = enum_cls(source_value).value
35
+ items[i] = value_by_name[name_by_value[source_value]]
32
36
  except (ValueError, KeyError):
33
37
  choices = ', '.join(
34
- "'{}' = {}".format(x.name.replace("'", r"\'"), x.value)
35
- for x in enum_cls
38
+ "'{}' = {}".format(name.replace("'", r"\'"), value)
39
+ for name, value in value_by_name.items()
36
40
  )
37
- enum_str = '{}({})'.format(enum_cls.__name__, choices)
41
+ enum_str = '{}({})'.format(self.ch_type, choices)
38
42
 
39
43
  raise errors.LogicalError(
40
44
  "Unknown element '{}' for type {}"
@@ -42,13 +46,13 @@ class EnumColumn(IntColumn):
42
46
  )
43
47
 
44
48
  def after_read_items(self, items, nulls_map=None):
45
- enum_cls = self.enum_cls
49
+ name_by_value = self.name_by_value
46
50
 
47
51
  if nulls_map is None:
48
- return tuple(enum_cls(item).name for item in items)
52
+ return tuple(name_by_value[item] for item in items)
49
53
  else:
50
54
  return tuple(
51
- (None if is_null else enum_cls(items[i]).name)
55
+ (None if is_null else name_by_value[items[i]])
52
56
  for i, is_null in enumerate(nulls_map)
53
57
  )
54
58
 
@@ -73,11 +77,13 @@ def create_enum_column(spec, column_options):
73
77
  params = spec[7:-1]
74
78
  cls = Enum16Column
75
79
 
76
- return cls(Enum(cls.ch_type, _parse_options(params)), **column_options)
80
+ name_by_value, value_by_name = _parse_options(params)
81
+
82
+ return cls(name_by_value, value_by_name, **column_options)
77
83
 
78
84
 
79
85
  def _parse_options(option_string):
80
- options = dict()
86
+ name_by_value, value_by_name = {}, OrderedDict()
81
87
  after_name = False
82
88
  escaped = False
83
89
  quote_character = None
@@ -93,7 +99,9 @@ def _parse_options(option_string):
93
99
  if ch in (' ', '='):
94
100
  pass
95
101
  elif ch == ',':
96
- options[name] = int(value)
102
+ value = int(value)
103
+ name_by_value[value] = name
104
+ value_by_name[name] = value
97
105
  after_name = False
98
106
  name = ''
99
107
  value = '' # reset before collecting new option
@@ -114,6 +122,8 @@ def _parse_options(option_string):
114
122
  quote_character = ch
115
123
 
116
124
  if after_name:
117
- options.setdefault(name, int(value)) # append word after last comma
125
+ value = int(value)
126
+ name_by_value[value] = name
127
+ value_by_name[name] = value
118
128
 
119
- return options
129
+ return name_by_value, value_by_name
@@ -31,6 +31,9 @@ class MapColumn(Column):
31
31
  self.value_column.write_state_prefix(buf)
32
32
 
33
33
  def read_items(self, n_items, buf):
34
+ if not n_items:
35
+ return [{}]
36
+
34
37
  offsets = list(self.offset_column.read_items(n_items, buf))
35
38
  last_offset = offsets[-1]
36
39
  keys = self.key_column.read_data(last_offset, buf)
@@ -33,7 +33,9 @@ class NumpyDateTimeColumnBase(NumpyColumn):
33
33
  ts = items
34
34
  else:
35
35
  timezone = self.timezone if self.timezone else self.local_timezone
36
- ts = pd.to_datetime(items).tz_localize(timezone)
36
+ ts = pd.to_datetime(items)
37
+ if not getattr(ts.dtype, 'tz', None):
38
+ ts = ts.tz_localize(timezone)
37
39
 
38
40
  ts = ts.tz_convert('UTC')
39
41
  return ts.tz_localize(None).to_numpy(self.datetime_dtype)
@@ -133,8 +135,9 @@ def create_numpy_datetime_column(spec, column_options):
133
135
  offset_naive = False
134
136
  else:
135
137
  if not context.settings.get('use_client_time_zone', False):
136
- if local_tz_name != context.server_info.timezone:
137
- tz_name = context.server_info.timezone
138
+ remote_timezone = context.server_info.get_timezone()
139
+ if local_tz_name != remote_timezone:
140
+ tz_name = remote_timezone
138
141
 
139
142
  timezone = get_timezone(tz_name) if tz_name else None
140
143
  local_timezone = get_timezone(local_tz_name) if local_tz_name else None
@@ -37,12 +37,13 @@ def get_inner_columns(spec):
37
37
 
38
38
 
39
39
  def get_inner_columns_with_types(spec):
40
+ spec = spec.strip()
40
41
  brackets = 0
41
42
  prev_comma = 0
42
43
  prev_space = 0
43
44
 
44
45
  columns = []
45
- for i, x in enumerate(spec + ','):
46
+ for i, x in enumerate(spec.strip() + ','):
46
47
  if x == ',':
47
48
  if brackets == 0:
48
49
  columns.append((
@@ -53,11 +53,15 @@ class ServerInfo(object):
53
53
  self.version_patch = version_patch
54
54
  self.revision = revision
55
55
  self.timezone = timezone
56
+ self.session_timezone = None
56
57
  self.display_name = display_name
57
58
  self.used_revision = used_revision
58
59
 
59
60
  super(ServerInfo, self).__init__()
60
61
 
62
+ def get_timezone(self):
63
+ return self.session_timezone or self.timezone
64
+
61
65
  def version_tuple(self):
62
66
  return self.version_major, self.version_minor, self.version_patch
63
67
 
@@ -301,20 +305,19 @@ class Connection(object):
301
305
  def _create_ssl_context(self, ssl_options):
302
306
  purpose = ssl.Purpose.SERVER_AUTH
303
307
 
304
- version = ssl_options.get('ssl_version', ssl.PROTOCOL_TLS)
308
+ version = ssl_options.get('ssl_version', ssl.PROTOCOL_TLS_CLIENT)
305
309
  context = ssl.SSLContext(version)
306
310
  context.check_hostname = self.verify_cert
307
311
 
308
312
  if 'ca_certs' in ssl_options:
309
313
  context.load_verify_locations(ssl_options['ca_certs'])
310
314
  elif ssl_options.get('cert_reqs') != ssl.CERT_NONE:
311
- context.load_default_certs(purpose
312
- )
315
+ context.load_default_certs(purpose)
313
316
  if 'ciphers' in ssl_options:
314
317
  context.set_ciphers(ssl_options['ciphers'])
315
318
 
316
319
  if 'cert_reqs' in ssl_options:
317
- context.options = ssl_options['cert_reqs']
320
+ context.verify_mode = ssl_options['cert_reqs']
318
321
 
319
322
  if 'certfile' in ssl_options:
320
323
  keyfile = ssl_options.get('keyfile')
@@ -613,6 +616,12 @@ class Connection(object):
613
616
  elif packet_type == ServerPacketTypes.PROFILE_EVENTS:
614
617
  packet.block = self.receive_data(may_be_compressed=False)
615
618
 
619
+ elif packet_type == ServerPacketTypes.TIMEZONE_UPDATE:
620
+ timezone = read_binary_str(self.fin)
621
+ if timezone:
622
+ logger.info('Server timezone changed to %s', timezone)
623
+ self.server_info.session_timezone = timezone
624
+
616
625
  else:
617
626
  message = 'Unknown packet {} from server {}'.format(
618
627
  packet_type, self.get_description()
@@ -33,6 +33,9 @@ DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS = 54459
33
33
  DBMS_MIN_PROTOCOL_VERSION_WITH_SERVER_QUERY_TIME_IN_PROGRESS = 54460
34
34
  DBMS_MIN_PROTOCOL_VERSION_WITH_PASSWORD_COMPLEXITY_RULES = 54461
35
35
  DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2 = 54462
36
+ DBMS_MIN_PROTOCOL_VERSION_WITH_TOTAL_BYTES_IN_PROGRESS = 54463
37
+ DBMS_MIN_PROTOCOL_VERSION_WITH_TIMEZONE_UPDATES = 54464
38
+ DBMS_MIN_REVISION_WITH_SYSTEM_KEYWORDS_TABLE = 54468
36
39
 
37
40
  # Timeouts
38
41
  DBMS_DEFAULT_CONNECT_TIMEOUT_SEC = 10
@@ -48,7 +51,7 @@ CLIENT_NAME = 'python-driver'
48
51
  CLIENT_VERSION_MAJOR = 20
49
52
  CLIENT_VERSION_MINOR = 10
50
53
  CLIENT_VERSION_PATCH = 2
51
- CLIENT_REVISION = DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2
54
+ CLIENT_REVISION = DBMS_MIN_REVISION_WITH_SYSTEM_KEYWORDS_TABLE
52
55
 
53
56
  BUFFER_SIZE = 1048576
54
57
 
@@ -1,13 +1,16 @@
1
1
  import numpy as np
2
2
  import pandas as pd
3
+ from pandas.core.arrays import ExtensionArray
3
4
 
4
5
 
5
6
  def column_chunks(columns, n):
6
7
  for column in columns:
7
- if not isinstance(column, (np.ndarray, pd.DatetimeIndex)):
8
+ if not isinstance(
9
+ column, (np.ndarray, pd.DatetimeIndex, ExtensionArray)
10
+ ):
8
11
  raise TypeError(
9
12
  'Unsupported column type: {}. '
10
- 'ndarray/DatetimeIndex is expected.'
13
+ 'ndarray/DatetimeIndex/ExtensionArray is expected.'
11
14
  .format(type(column))
12
15
  )
13
16
 
@@ -6,7 +6,8 @@ class Progress(object):
6
6
  def __init__(self):
7
7
  self.rows = 0
8
8
  self.bytes = 0
9
- self.total_rows = 0
9
+ self.total_rows = 0 # total_rows_to_read
10
+ self.total_bytes = 0 # total_bytes_to_read
10
11
  self.written_rows = 0
11
12
  self.written_bytes = 0
12
13
  self.elapsed_ns = 0
@@ -21,6 +22,10 @@ class Progress(object):
21
22
  if revision >= defines.DBMS_MIN_REVISION_WITH_TOTAL_ROWS_IN_PROGRESS:
22
23
  self.total_rows = read_varint(fin)
23
24
 
25
+ if revision >= defines. \
26
+ DBMS_MIN_PROTOCOL_VERSION_WITH_TOTAL_BYTES_IN_PROGRESS:
27
+ self.total_bytes = read_varint(fin)
28
+
24
29
  if revision >= defines.DBMS_MIN_REVISION_WITH_CLIENT_WRITE_INFO:
25
30
  self.written_rows = read_varint(fin)
26
31
  self.written_bytes = read_varint(fin)
@@ -33,6 +38,7 @@ class Progress(object):
33
38
  self.rows += another_progress.rows
34
39
  self.bytes += another_progress.bytes
35
40
  self.total_rows += another_progress.total_rows
41
+ self.total_bytes += another_progress.total_bytes
36
42
  self.written_rows += another_progress.written_rows
37
43
  self.written_bytes += another_progress.written_bytes
38
44
  self.elapsed_ns += another_progress.elapsed_ns
@@ -29,7 +29,10 @@ class ClientPacketTypes(object):
29
29
 
30
30
  @classmethod
31
31
  def to_str(cls, packet):
32
- return 'Unknown packet' if packet > 5 else cls._types_str[packet]
32
+ try:
33
+ return cls._types_str[packet]
34
+ except IndexError:
35
+ return 'Unknown packet'
33
36
 
34
37
 
35
38
  class ServerPacketTypes(object):
@@ -81,15 +84,28 @@ class ServerPacketTypes(object):
81
84
  # Packet with profile events from server.
82
85
  PROFILE_EVENTS = 14
83
86
 
87
+ MERGE_TREE_ALL_RANGES_ANNOUNCEMENT = 15
88
+
89
+ # Request from a MergeTree replica to a coordinator
90
+ MERGE_TREE_READ_TASK_REQUEST = 16
91
+
92
+ # Receive server's (session-wide) default timezone
93
+ TIMEZONE_UPDATE = 17
94
+
84
95
  _types_str = [
85
96
  'Hello', 'Data', 'Exception', 'Progress', 'Pong', 'EndOfStream',
86
97
  'ProfileInfo', 'Totals', 'Extremes', 'TablesStatusResponse', 'Log',
87
- 'TableColumns', 'PartUUIDs', 'ReadTaskRequest', 'ProfileEvents'
98
+ 'TableColumns', 'PartUUIDs', 'ReadTaskRequest', 'ProfileEvents',
99
+ 'MergeTreeAllRangesAnnouncement', 'MergeTreeReadTaskRequest',
100
+ 'TimezoneUpdate'
88
101
  ]
89
102
 
90
103
  @classmethod
91
104
  def to_str(cls, packet):
92
- return 'Unknown packet' if packet > 14 else cls._types_str[packet]
105
+ try:
106
+ return cls._types_str[packet]
107
+ except IndexError:
108
+ return 'Unknown packet'
93
109
 
94
110
  @classmethod
95
111
  def strings_in_message(cls, packet):
@@ -1,3 +1,5 @@
1
+ import logging
2
+
1
3
  from ..block import ColumnOrientedBlock, BlockInfo
2
4
  from ..columns.service import read_column, write_column
3
5
  from ..reader import read_binary_str, read_binary_uint8
@@ -5,6 +7,8 @@ from ..varint import write_varint, read_varint
5
7
  from ..writer import write_binary_str, write_binary_uint8
6
8
  from .. import defines
7
9
 
10
+ logger = logging.getLogger(__name__)
11
+
8
12
 
9
13
  class BlockOutputStream(object):
10
14
  def __init__(self, fout, context):
@@ -40,6 +44,7 @@ class BlockOutputStream(object):
40
44
  # We write always sparse data without custom serialization.
41
45
  write_binary_uint8(0, self.fout)
42
46
 
47
+ logger.debug('Writing column %s', col_name)
43
48
  write_column(self.context, col_name, col_type, items,
44
49
  self.fout, types_check=block.types_check)
45
50
 
@@ -80,6 +85,7 @@ class BlockInputStream(object):
80
85
  has_custom_serialization = bool(read_binary_uint8(self.fin))
81
86
 
82
87
  if n_rows:
88
+ logger.debug('Reading column %s', column_name)
83
89
  column = read_column(
84
90
  self.context, column_type, n_rows,
85
91
  self.fin, use_numpy=use_numpy,
@@ -21,7 +21,7 @@ escape_chars_map = {
21
21
 
22
22
 
23
23
  def escape_datetime(item, context):
24
- server_tz = timezone(context.server_info.timezone)
24
+ server_tz = timezone(context.server_info.get_timezone())
25
25
 
26
26
  if item.tzinfo is not None:
27
27
  item = item.astimezone(server_tz)