clickhouse-driver 0.2.4__cp38-cp38-musllinux_1_1_i686.whl → 0.2.8__cp38-cp38-musllinux_1_1_i686.whl

clickhouse_driver/columns/datecolumn.py

@@ -1,3 +1,4 @@
+from os import getenv
 from datetime import date, timedelta
 
 from .base import FormatColumn
@@ -6,8 +7,54 @@ from .base import FormatColumn
 epoch_start = date(1970, 1, 1)
 epoch_end = date(2149, 6, 6)
 
-epoch_start_date32 = date(1925, 1, 1)
-epoch_end_date32 = date(2283, 11, 11)
+epoch_start_date32 = date(1900, 1, 1)
+epoch_end_date32 = date(2299, 12, 31)
+
+
+class LazyLUT(dict):
+    def __init__(self, *args, _factory, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._default_factory = _factory
+
+    def __missing__(self, key):
+        return self.setdefault(key, self._default_factory(key))
+
+
+def make_date_lut_range(date_start, date_end):
+    return range(
+        (date_start - epoch_start).days,
+        (date_end - epoch_start).days + 1,
+    )
+
+
+enable_lazy_date_lut = getenv('CLICKHOUSE_DRIVER_LASY_DATE_LUT', False)
+if enable_lazy_date_lut:
+    try:
+        start, end = enable_lazy_date_lut.split(':')
+        start_date = date.fromisoformat(start)
+        end_date = date.fromisoformat(end)
+
+        date_range = make_date_lut_range(start_date, end_date)
+    except ValueError:
+        date_range = ()
+
+    # Since we initialize lazy lut with some initially warmed values,
+    # we use iterator and not dict comprehension for memory & time optimization
+    _date_lut = LazyLUT(
+        ((x, epoch_start + timedelta(days=x)) for x in date_range),
+        _factory=lambda x: epoch_start + timedelta(days=x),
+    )
+    _date_lut_reverse = LazyLUT(
+        ((value, key) for key, value in _date_lut.items()),
+        _factory=lambda x: (x - epoch_start).days,
+    )
+else:
+    # If lazy lut is not enabled, we fallback to static dict initialization
+    # In both cases, we use same lut for both data types,
+    # since one encompasses the other and we can avoid duplicating overlap
+    date_range = make_date_lut_range(epoch_start_date32, epoch_end_date32)
+    _date_lut = {x: epoch_start + timedelta(days=x) for x in date_range}
+    _date_lut_reverse = {value: key for key, value in _date_lut.items()}
 
 
 class DateColumn(FormatColumn):
@@ -18,9 +65,8 @@ class DateColumn(FormatColumn):
     min_value = epoch_start
     max_value = epoch_end
 
-    date_lut_days = (epoch_end - epoch_start).days + 1
-    date_lut = {x: epoch_start + timedelta(x) for x in range(date_lut_days)}
-    date_lut_reverse = {value: key for key, value in date_lut.items()}
+    date_lut = _date_lut
+    date_lut_reverse = _date_lut_reverse
 
     def before_write_items(self, items, nulls_map=None):
         null_value = self.null_value
@@ -34,7 +80,7 @@ class DateColumn(FormatColumn):
                 items[i] = null_value
                 continue
 
-            if type(item) != date:
+            if item is not date:
                 item = date(item.year, item.month, item.day)
 
             if min_value <= item <= max_value:
@@ -60,10 +106,3 @@ class Date32Column(DateColumn):
 
     min_value = epoch_start_date32
     max_value = epoch_end_date32
-
-    date_lut_days = (epoch_end_date32 - epoch_start).days + 1
-    date_lut = {
-        x: epoch_start + timedelta(x)
-        for x in range((epoch_start_date32 - epoch_start).days, date_lut_days)
-    }
-    date_lut_reverse = {value: key for key, value in date_lut.items()}

clickhouse_driver/columns/jsoncolumn.py

@@ -0,0 +1,37 @@
+from .base import Column
+from .stringcolumn import String
+from ..reader import read_binary_uint8, read_binary_str
+from ..util.compat import json
+from ..writer import write_binary_uint8
+
+
+class JsonColumn(Column):
+    py_types = (dict, )
+
+    # No NULL value actually
+    null_value = {}
+
+    def __init__(self, column_by_spec_getter, **kwargs):
+        self.column_by_spec_getter = column_by_spec_getter
+        self.string_column = String(**kwargs)
+        super(JsonColumn, self).__init__(**kwargs)
+
+    def write_state_prefix(self, buf):
+        # Read in binary format.
+        # Write in text format.
+        write_binary_uint8(1, buf)
+
+    def read_items(self, n_items, buf):
+        read_binary_uint8(buf)
+        spec = read_binary_str(buf)
+        col = self.column_by_spec_getter(spec)
+        col.read_state_prefix(buf)
+        return col.read_data(n_items, buf)
+
+    def write_items(self, items, buf):
+        items = [x if isinstance(x, str) else json.dumps(x) for x in items]
+        self.string_column.write_items(items, buf)
+
+
+def create_json_column(spec, column_by_spec_getter, column_options):
+    return JsonColumn(column_by_spec_getter, **column_options)

clickhouse_driver/columns/lowcardinalitycolumn.py

@@ -35,25 +35,32 @@ class LowCardinalityColumn(Column):
     serialization_type = has_additional_keys_bit | need_update_dictionary
 
     def __init__(self, nested_column, **kwargs):
+        self.init_kwargs = kwargs
         self.nested_column = nested_column
         super(LowCardinalityColumn, self).__init__(**kwargs)
 
     def read_state_prefix(self, buf):
-        return read_binary_uint64(buf)
+        super(LowCardinalityColumn, self).read_state_prefix(buf)
+
+        read_binary_uint64(buf)
 
     def write_state_prefix(self, buf):
+        super(LowCardinalityColumn, self).write_state_prefix(buf)
+
         # KeysSerializationVersion. See ClickHouse docs.
         write_binary_int64(1, buf)
 
     def _write_data(self, items, buf):
         index, keys = [], []
         key_by_index_element = {}
+        nested_is_nullable = False
 
         if self.nested_column.nullable:
             # First element represents NULL if column is nullable.
             index.append(self.nested_column.null_value)
             # Prevent null map writing. Reset nested column nullable flag.
             self.nested_column.nullable = False
+            nested_is_nullable = True
 
             for x in items:
                 if x is None:
@@ -87,14 +94,26 @@ class LowCardinalityColumn(Column):
             return
 
         int_type = int(log(len(index), 2) / 8)
-        int_column = self.int_types[int_type]()
+        int_column = self.int_types[int_type](**self.init_kwargs)
 
         serialization_type = self.serialization_type | int_type
 
         write_binary_int64(serialization_type, buf)
         write_binary_int64(len(index), buf)
 
-        self.nested_column.write_data(index, buf)
+        if nested_is_nullable:
+            # Given we reset nested column nullable flag above,
+            # we need to write null map manually. If to invoke
+            # write_data method, it will cause an exception,
+            # because `prepare_data` may not be able to handle
+            # null value correctly.
+            self.nested_column.write_items(
+                [self.nested_column.null_value], buf)
+            # Remove null map from index, because it is already written.
+            index_to_write = index[1:]
+            self.nested_column.write_data(index_to_write, buf)
+        else:
+            self.nested_column.write_data(index, buf)
         write_binary_int64(len(items), buf)
         int_column.write_items(keys, buf)
 
@@ -106,7 +125,7 @@ class LowCardinalityColumn(Column):
 
         # Lowest byte contains info about key type.
         key_type = serialization_type & 0xf
-        keys_column = self.int_types[key_type]()
+        keys_column = self.int_types[key_type](**self.init_kwargs)
 
         nullable = self.nested_column.nullable
         # Prevent null map reading. Reset nested column nullable flag.

clickhouse_driver/columns/mapcolumn.py

@@ -1,28 +1,39 @@
+import re
 from .base import Column
 from .intcolumn import UInt64Column
 from ..util.helpers import pairwise
 
 
+comma_re = re.compile(r',(?![^()]*\))')
+
+
 class MapColumn(Column):
     py_types = (dict, )
 
     null_value = {}
 
     def __init__(self, key_column, value_column, **kwargs):
-        self.offset_column = UInt64Column()
+        self.offset_column = UInt64Column(**kwargs)
         self.key_column = key_column
         self.value_column = value_column
         super(MapColumn, self).__init__(**kwargs)
 
     def read_state_prefix(self, buf):
+        super(MapColumn, self).read_state_prefix(buf)
+
         self.key_column.read_state_prefix(buf)
         self.value_column.read_state_prefix(buf)
 
     def write_state_prefix(self, buf):
+        super(MapColumn, self).write_state_prefix(buf)
+
         self.key_column.write_state_prefix(buf)
         self.value_column.write_state_prefix(buf)
 
     def read_items(self, n_items, buf):
+        if not n_items:
+            return [{}]
+
         offsets = list(self.offset_column.read_items(n_items, buf))
         last_offset = offsets[-1]
         keys = self.key_column.read_data(last_offset, buf)
@@ -53,7 +64,9 @@ class MapColumn(Column):
 
 
 def create_map_column(spec, column_by_spec_getter, column_options):
-    key, value = spec[4:-1].split(',')
+    # Match commas outside of parentheses, so we don't match the comma in
+    # Decimal types.
+    key, value = comma_re.split(spec[4:-1])
     key_column = column_by_spec_getter(key.strip())
     value_column = column_by_spec_getter(value.strip())
 

clickhouse_driver/columns/nestedcolumn.py

@@ -1,21 +1,10 @@
 
 from .arraycolumn import create_array_column
-from .util import get_inner_spec, get_inner_columns, \
-    get_inner_columns_with_types
+from .util import get_inner_spec
 
 
 def create_nested_column(spec, column_by_spec_getter, column_options):
     return create_array_column(
-        'Array(Tuple({}))'.format(','.join(get_nested_columns(spec))),
+        'Array(Tuple({}))'.format(get_inner_spec('Nested', spec)),
         column_by_spec_getter, column_options
     )
-
-
-def get_nested_columns(spec):
-    inner_spec = get_inner_spec('Nested', spec)
-    return get_inner_columns(inner_spec)
-
-
-def get_columns_with_types(spec):
-    inner_spec = get_inner_spec('Nested', spec)
-    return get_inner_columns_with_types(inner_spec)

clickhouse_driver/columns/numpy/boolcolumn.py

@@ -0,0 +1,8 @@
+import numpy as np
+
+from .base import NumpyColumn
+
+
+class NumpyBoolColumn(NumpyColumn):
+    dtype = np.dtype(np.bool_)
+    ch_type = 'Bool'

clickhouse_driver/columns/numpy/datetimecolumn.py

@@ -21,12 +21,12 @@ class NumpyDateTimeColumnBase(NumpyColumn):
     def apply_timezones_after_read(self, dt):
         timezone = self.timezone if self.timezone else self.local_timezone
 
-        ts = pd.to_datetime(dt, utc=True).tz_convert(timezone)
-
-        if self.offset_naive:
+        if self.offset_naive and timezone.zone != 'UTC':
+            ts = pd.to_datetime(dt, utc=True).tz_convert(timezone)
             ts = ts.tz_localize(None)
+            return ts.to_numpy(self.datetime_dtype)
 
-        return ts.to_numpy(self.datetime_dtype)
+        return dt
 
     def apply_timezones_before_write(self, items):
         if isinstance(items, pd.DatetimeIndex):
@@ -65,25 +65,25 @@ class NumpyDateTimeColumn(NumpyDateTimeColumnBase):
 
 
 class NumpyDateTime64Column(NumpyDateTimeColumnBase):
-    dtype = np.dtype(np.uint64)
+    dtype = np.dtype(np.int64)
     datetime_dtype = 'datetime64[ns]'
 
-    max_scale = 6
+    max_scale = 9
 
     def __init__(self, scale=0, **kwargs):
         self.scale = scale
         super(NumpyDateTime64Column, self).__init__(**kwargs)
 
     def read_items(self, n_items, buf):
-        scale = 10 ** self.scale
-        frac_scale = 10 ** (self.max_scale - self.scale)
-
+        # Clickhouse:     t seconds is represented as t * 10^scale.
+        # datetime64[ns]: t seconds is represented as t * 10^9.
+        # Since 0 <= scale <= 9, multiply by the integer 10^(9 - scale).
         items = super(NumpyDateTime64Column, self).read_items(n_items, buf)
 
-        seconds = (items // scale).astype('datetime64[s]')
-        microseconds = ((items % scale) * frac_scale).astype('timedelta64[us]')
+        tmp = np.copy(items)
+        tmp *= 10 ** (9 - self.scale)
+        dt = tmp.view(dtype='datetime64[ns]')
 
-        dt = seconds + microseconds
         return self.apply_timezones_after_read(dt)
 
     def write_items(self, items, buf):
@@ -98,7 +98,7 @@ class NumpyDateTime64Column(NumpyDateTimeColumnBase):
         items = self.apply_timezones_before_write(items)
 
         seconds = items.astype('datetime64[s]')
-        microseconds = (items - seconds).astype(dtype='timedelta64[us]') \
+        microseconds = (items - seconds).astype(dtype='timedelta64[ns]') \
             .astype(np.uint32) // frac_scale
 
         items = seconds.astype(self.dtype) * scale + microseconds
@@ -120,12 +120,12 @@ def create_numpy_datetime_column(spec, column_options):
 
     context = column_options['context']
 
-    tz_name = timezone = None
+    tz_name = None
     offset_naive = True
 
     # As Numpy do not use local timezone for converting timestamp to
     # datetime we need always detect local timezone for manual converting.
-    local_timezone = get_localzone_name_compat()
+    local_tz_name = get_localzone_name_compat()
 
     # Use column's timezone if it's specified.
     if spec and spec[-1] == ')':
@@ -133,11 +133,11 @@ def create_numpy_datetime_column(spec, column_options):
         offset_naive = False
     else:
         if not context.settings.get('use_client_time_zone', False):
-            if local_timezone != context.server_info.timezone:
+            if local_tz_name != context.server_info.timezone:
                 tz_name = context.server_info.timezone
 
-    if tz_name:
-        timezone = get_timezone(tz_name)
+    timezone = get_timezone(tz_name) if tz_name else None
+    local_timezone = get_timezone(local_tz_name) if local_tz_name else None
 
     return cls(timezone=timezone, offset_naive=offset_naive,
                local_timezone=local_timezone, **column_options)

clickhouse_driver/columns/numpy/lowcardinalitycolumn.py

@@ -37,7 +37,7 @@ class NumpyLowCardinalityColumn(LowCardinalityColumn):
         c = pd.Categorical(items)
 
         int_type = int(log(len(c.codes), 2) / 8)
-        int_column = self.int_types[int_type]()
+        int_column = self.int_types[int_type](**self.init_kwargs)
 
         serialization_type = self.serialization_type | int_type
 
@@ -66,7 +66,7 @@ class NumpyLowCardinalityColumn(LowCardinalityColumn):
 
         # Lowest byte contains info about key type.
         key_type = serialization_type & 0xf
-        keys_column = self.int_types[key_type]()
+        keys_column = self.int_types[key_type](**self.init_kwargs)
 
         nullable = self.nested_column.nullable
         # Prevent null map reading. Reset nested column nullable flag.

clickhouse_driver/columns/numpy/service.py

@@ -7,6 +7,7 @@ from .intcolumn import (
     NumpyInt8Column, NumpyInt16Column, NumpyInt32Column, NumpyInt64Column,
     NumpyUInt8Column, NumpyUInt16Column, NumpyUInt32Column, NumpyUInt64Column
 )
+from .boolcolumn import NumpyBoolColumn
 from .lowcardinalitycolumn import create_numpy_low_cardinality_column
 from .stringcolumn import create_string_column
 from .tuplecolumn import create_tuple_column
@@ -16,7 +17,8 @@ column_by_type = {c.ch_type: c for c in [
     NumpyDateColumn,
     NumpyFloat32Column, NumpyFloat64Column,
     NumpyInt8Column, NumpyInt16Column, NumpyInt32Column, NumpyInt64Column,
-    NumpyUInt8Column, NumpyUInt16Column, NumpyUInt32Column, NumpyUInt64Column
+    NumpyUInt8Column, NumpyUInt16Column, NumpyUInt32Column, NumpyUInt64Column,
+    NumpyBoolColumn
 ]}
 
 

clickhouse_driver/columns/service.py

@@ -15,6 +15,7 @@ from .intcolumn import (
     UInt8Column, UInt16Column, UInt32Column, UInt64Column
 )
 from .lowcardinalitycolumn import create_low_cardinality_column
+from .jsoncolumn import create_json_column
 from .mapcolumn import create_map_column
 from .nothingcolumn import NothingColumn
 from .nullcolumn import NullColumn
@@ -122,6 +123,11 @@ def get_column_by_spec(spec, column_options, use_numpy=None):
             spec, create_column_with_options, column_options
         )
 
+    elif spec.startswith("Object('json')"):
+        return create_json_column(
+            spec, create_column_with_options, column_options
+        )
+
     else:
         for alias, primitive in aliases:
             if spec.startswith(alias):
@@ -137,8 +143,12 @@ def get_column_by_spec(spec, column_options, use_numpy=None):
             raise errors.UnknownTypeError('Unknown type {}'.format(spec))
 
 
-def read_column(context, column_spec, n_items, buf, use_numpy=None):
-    column_options = {'context': context}
+def read_column(context, column_spec, n_items, buf, use_numpy=None,
+                has_custom_serialization=False):
+    column_options = {
+        'context': context,
+        'has_custom_serialization': has_custom_serialization
+    }
     col = get_column_by_spec(column_spec, column_options, use_numpy=use_numpy)
     col.read_state_prefix(buf)
     return col.read_data(n_items, buf)

clickhouse_driver/columns/tuplecolumn.py

@@ -1,13 +1,21 @@
 
 from .base import Column
-from .util import get_inner_spec, get_inner_columns
+from .util import get_inner_spec, get_inner_columns_with_types
 
 
 class TupleColumn(Column):
     py_types = (list, tuple)
 
-    def __init__(self, nested_columns, **kwargs):
+    def __init__(self, names, nested_columns, **kwargs):
+        self.names = names
         self.nested_columns = nested_columns
+        client_settings = kwargs['context'].client_settings
+        settings = kwargs['context'].settings
+        self.namedtuple_as_json = (
+            settings.get('allow_experimental_object_type', False) and
+            client_settings.get('namedtuple_as_json', True)
+        )
+
         super(TupleColumn, self).__init__(**kwargs)
         self.null_value = tuple(x.null_value for x in nested_columns)
 
@@ -23,15 +31,33 @@ class TupleColumn(Column):
 
     def read_data(self, n_items, buf):
         rv = [x.read_data(n_items, buf) for x in self.nested_columns]
-        return list(zip(*rv))
+        rv = list(zip(*rv))
+
+        if self.names[0] and self.namedtuple_as_json:
+            return [dict(zip(self.names, x)) for x in rv]
+        else:
+            return rv
 
     def read_items(self, n_items, buf):
         return self.read_data(n_items, buf)
 
+    def read_state_prefix(self, buf):
+        super(TupleColumn, self).read_state_prefix(buf)
+
+        for x in self.nested_columns:
+            x.read_state_prefix(buf)
+
+    def write_state_prefix(self, buf):
+        super(TupleColumn, self).write_state_prefix(buf)
+
+        for x in self.nested_columns:
+            x.write_state_prefix(buf)
+
 
 def create_tuple_column(spec, column_by_spec_getter, column_options):
     inner_spec = get_inner_spec('Tuple', spec)
-    columns = get_inner_columns(inner_spec)
+    columns_with_types = get_inner_columns_with_types(inner_spec)
+    names, types = zip(*columns_with_types)
 
-    return TupleColumn([column_by_spec_getter(x) for x in columns],
+    return TupleColumn(names, [column_by_spec_getter(x) for x in types],
                        **column_options)

clickhouse_driver/columns/uuidcolumn.py

@@ -54,7 +54,7 @@ class UUIDColumn(FormatColumn):
 
             try:
                 if not isinstance(item, UUID):
-                    item = UUID(item)
+                    item = UUID(int=item) if item is null_value else UUID(item)
 
             except ValueError:
                 raise errors.CannotParseUuidError(