clickhouse-driver 0.2.1__cp39-cp39-win_amd64.whl → 0.2.10__cp39-cp39-win_amd64.whl

clickhouse_driver/columns/mapcolumn.py

@@ -1,58 +1,73 @@
-from .base import Column
-from .intcolumn import UInt64Column
-from ..util.helpers import pairwise
-
-
-class MapColumn(Column):
-    py_types = (dict, )
-
-    def __init__(self, key_column, value_column, **kwargs):
-        self.offset_column = UInt64Column()
-        self.key_column = key_column
-        self.value_column = value_column
-        super(MapColumn, self).__init__(**kwargs)
-
-    def read_state_prefix(self, buf):
-        self.key_column.read_state_prefix(buf)
-        self.value_column.read_state_prefix(buf)
-
-    def write_state_prefix(self, buf):
-        self.key_column.write_state_prefix(buf)
-        self.value_column.write_state_prefix(buf)
-
-    def read_items(self, n_items, buf):
-        offsets = list(self.offset_column.read_items(n_items, buf))
-        last_offset = offsets[-1]
-        keys = self.key_column.read_data(last_offset, buf)
-        values = self.value_column.read_data(last_offset, buf)
-
-        offsets.insert(0, 0)
-
-        return [
-            dict(zip(keys[begin:end], values[begin:end]))
-            for begin, end in pairwise(offsets)
-        ]
-
-    def write_items(self, items, buf):
-        offsets = []
-        keys = []
-        values = []
-
-        total = 0
-        for x in items:
-            total += len(x)
-            offsets.append(total)
-            keys.extend(x.keys())
-            values.extend(x.values())
-
-        self.offset_column.write_items(offsets, buf)
-        self.key_column.write_data(keys, buf)
-        self.value_column.write_data(values, buf)
-
-
-def create_map_column(spec, column_by_spec_getter):
-    key, value = spec[4:-1].split(',')
-    key_column = column_by_spec_getter(key.strip())
-    value_column = column_by_spec_getter(value.strip())
-
-    return MapColumn(key_column, value_column)
+import re
+from .base import Column
+from .intcolumn import UInt64Column
+from ..util.helpers import pairwise
+
+
+comma_re = re.compile(r',(?![^()]*\))')
+
+
+class MapColumn(Column):
+    py_types = (dict, )
+
+    null_value = {}
+
+    def __init__(self, key_column, value_column, **kwargs):
+        self.offset_column = UInt64Column(**kwargs)
+        self.key_column = key_column
+        self.value_column = value_column
+        super(MapColumn, self).__init__(**kwargs)
+
+    def read_state_prefix(self, buf):
+        super(MapColumn, self).read_state_prefix(buf)
+
+        self.key_column.read_state_prefix(buf)
+        self.value_column.read_state_prefix(buf)
+
+    def write_state_prefix(self, buf):
+        super(MapColumn, self).write_state_prefix(buf)
+
+        self.key_column.write_state_prefix(buf)
+        self.value_column.write_state_prefix(buf)
+
+    def read_items(self, n_items, buf):
+        if not n_items:
+            return [{}]
+
+        offsets = list(self.offset_column.read_items(n_items, buf))
+        last_offset = offsets[-1]
+        keys = self.key_column.read_data(last_offset, buf)
+        values = self.value_column.read_data(last_offset, buf)
+
+        offsets.insert(0, 0)
+
+        return [
+            dict(zip(keys[begin:end], values[begin:end]))
+            for begin, end in pairwise(offsets)
+        ]
+
+    def write_items(self, items, buf):
+        offsets = []
+        keys = []
+        values = []
+
+        total = 0
+        for x in items:
+            total += len(x)
+            offsets.append(total)
+            keys.extend(x.keys())
+            values.extend(x.values())
+
+        self.offset_column.write_items(offsets, buf)
+        self.key_column.write_data(keys, buf)
+        self.value_column.write_data(values, buf)
+
+
+def create_map_column(spec, column_by_spec_getter, column_options):
+    # Match commas outside of parentheses, so we don't match the comma in
+    # Decimal types.
+    key, value = comma_re.split(spec[4:-1])
+    key_column = column_by_spec_getter(key.strip())
+    value_column = column_by_spec_getter(value.strip())
+
+    return MapColumn(key_column, value_column, **column_options)

clickhouse_driver/columns/nestedcolumn.py

@@ -0,0 +1,10 @@
+
+from .arraycolumn import create_array_column
+from .util import get_inner_spec
+
+
+def create_nested_column(spec, column_by_spec_getter, column_options):
+    return create_array_column(
+        'Array(Tuple({}))'.format(get_inner_spec('Nested', spec)),
+        column_by_spec_getter, column_options
+    )

clickhouse_driver/columns/nothingcolumn.py

@@ -1,13 +1,13 @@
-from .intcolumn import FormatColumn
-
-
-class NothingColumn(FormatColumn):
-    ch_type = 'Nothing'
-    format = 'B'
-
-    @property
-    def size(self):
-        return 1
-
-    def after_read_items(self, items, nulls_map=None):
-        return (None, ) * len(items)
+from .intcolumn import FormatColumn
+
+
+class NothingColumn(FormatColumn):
+    ch_type = 'Nothing'
+    format = 'B'
+
+    @property
+    def size(self):
+        return 1
+
+    def after_read_items(self, items, nulls_map=None):
+        return (None, ) * len(items)

clickhouse_driver/columns/nullablecolumn.py

@@ -1,7 +1,7 @@
-
-
-def create_nullable_column(spec, column_by_spec_getter):
-    inner = spec[9:-1]
-    nested = column_by_spec_getter(inner)
-    nested.nullable = True
-    return nested
+
+
+def create_nullable_column(spec, column_by_spec_getter):
+    inner = spec[9:-1]
+    nested = column_by_spec_getter(inner)
+    nested.nullable = True
+    return nested

clickhouse_driver/columns/nullcolumn.py

@@ -1,15 +1,15 @@
-from .intcolumn import FormatColumn
-
-
-# TODO: Drop Null column support in future.
-# Compatibility with old servers.
-class NullColumn(FormatColumn):
-    ch_type = 'Null'
-    format = 'B'
-
-    @property
-    def size(self):
-        return 1
-
-    def after_read_items(self, items, nulls_map=None):
-        return (None, ) * len(items)
+from .intcolumn import FormatColumn
+
+
+# TODO: Drop Null column support in future.
+# Compatibility with old servers.
+class NullColumn(FormatColumn):
+    ch_type = 'Null'
+    format = 'B'
+
+    @property
+    def size(self):
+        return 1
+
+    def after_read_items(self, items, nulls_map=None):
+        return (None, ) * len(items)

clickhouse_driver/columns/numpy/base.py

@@ -1,14 +1,47 @@
-import numpy as np
-
-from ..base import Column
-
-
-class NumpyColumn(Column):
-    dtype = None
-
-    def read_items(self, n_items, buf):
-        data = buf.read(n_items * self.dtype.itemsize)
-        return np.frombuffer(data, self.dtype, n_items)
-
-    def write_items(self, items, buf):
-        buf.write(items.astype(self.dtype).tobytes())
+import numpy as np
+import pandas as pd
+
+from ..base import Column
+
+
+class NumpyColumn(Column):
+    dtype = None
+
+    normalize_null_value = True
+
+    def read_items(self, n_items, buf):
+        data = buf.read(n_items * self.dtype.itemsize)
+        return np.frombuffer(data, self.dtype.newbyteorder('<'), n_items)
+
+    def write_items(self, items, buf):
+        buf.write(items.astype(self.dtype.newbyteorder('<')).tobytes())
+
+    def _write_nulls_map(self, items, buf):
+        s = self.make_null_struct(len(items))
+        nulls_map = self._get_nulls_map(items)
+        buf.write(s.pack(*nulls_map))
+
+    def _get_nulls_map(self, items):
+        return [bool(x) for x in pd.isnull(items)]
+
+    def _read_data(self, n_items, buf, nulls_map=None):
+        items = self.read_items(n_items, buf)
+
+        if self.after_read_items:
+            return self.after_read_items(items, nulls_map)
+        elif nulls_map is not None:
+            items = np.array(items, dtype=object)
+            np.place(items, nulls_map, None)
+
+        return items
+
+    def prepare_items(self, items):
+        nulls_map = pd.isnull(items)
+
+        # Always replace null values to null_value for proper inserts into
+        # non-nullable columns.
+        if isinstance(items, np.ndarray) and self.normalize_null_value:
+            items = np.array(items)
+            np.place(items, nulls_map, self.null_value)
+
+        return items

clickhouse_driver/columns/numpy/boolcolumn.py

@@ -0,0 +1,8 @@
+import numpy as np
+
+from .base import NumpyColumn
+
+
+class NumpyBoolColumn(NumpyColumn):
+    dtype = np.dtype(np.bool_)
+    ch_type = 'Bool'

clickhouse_driver/columns/numpy/datecolumn.py

@@ -1,12 +1,19 @@
-import numpy as np
-
-from .base import NumpyColumn
-
-
-class NumpyDateColumn(NumpyColumn):
-    dtype = np.dtype(np.uint16)
-    ch_type = 'Date'
-
-    def read_items(self, n_items, buf):
-        data = super(NumpyDateColumn, self).read_items(n_items, buf)
-        return data.astype('datetime64[D]')
+import numpy as np
+
+from .base import NumpyColumn
+
+
+class NumpyDateColumn(NumpyColumn):
+    dtype = np.dtype(np.uint16)
+    ch_type = 'Date'
+
+    null_value = np.datetime64(0, 'Y')
+
+    def read_items(self, n_items, buf):
+        data = super(NumpyDateColumn, self).read_items(n_items, buf)
+        return data.astype('datetime64[D]')
+
+    def write_items(self, items, buf):
+        super(NumpyDateColumn, self).write_items(
+            items.astype('datetime64[D]'), buf
+        )

clickhouse_driver/columns/numpy/datetimecolumn.py

@@ -1,145 +1,146 @@
-import numpy as np
-import pandas as pd
-from pytz import timezone as get_timezone
-from tzlocal import get_localzone
-
-from .base import NumpyColumn
-
-
-class NumpyDateTimeColumnBase(NumpyColumn):
-    datetime_dtype = None
-
-    def __init__(self, timezone=None, offset_naive=True, local_timezone=None,
-                 **kwargs):
-        self.timezone = timezone
-        self.offset_naive = offset_naive
-        self.local_timezone = local_timezone
-        super(NumpyDateTimeColumnBase, self).__init__(**kwargs)
-
-    def apply_timezones_after_read(self, dt):
-        timezone = self.timezone if self.timezone else self.local_timezone
-
-        ts = pd.to_datetime(dt, utc=True).tz_convert(timezone)
-
-        if self.offset_naive:
-            ts = ts.tz_localize(None)
-
-        return ts.to_numpy(self.datetime_dtype)
-
-    def apply_timezones_before_write(self, items):
-        if isinstance(items, pd.DatetimeIndex):
-            ts = items
-        else:
-            timezone = self.timezone if self.timezone else self.local_timezone
-            ts = pd.to_datetime(items).tz_localize(timezone)
-
-        ts = ts.tz_convert('UTC')
-        return ts.tz_localize(None).to_numpy(self.datetime_dtype)
-
-    def is_items_integer(self, items):
-        return (
-            isinstance(items, np.ndarray) and
-            np.issubdtype(items.dtype, np.integer)
-        )
-
-
-class NumpyDateTimeColumn(NumpyDateTimeColumnBase):
-    dtype = np.dtype(np.uint32)
-    datetime_dtype = 'datetime64[s]'
-
-    def write_items(self, items, buf):
-        # write int 'as is'.
-        if self.is_items_integer(items):
-            super(NumpyDateTimeColumn, self).write_items(items, buf)
-            return
-
-        items = self.apply_timezones_before_write(items)
-
-        super(NumpyDateTimeColumn, self).write_items(items, buf)
-
-    def read_items(self, n_items, buf):
-        items = super(NumpyDateTimeColumn, self).read_items(n_items, buf)
-        return self.apply_timezones_after_read(items.astype('datetime64[s]'))
-
-
-class NumpyDateTime64Column(NumpyDateTimeColumnBase):
-    dtype = np.dtype(np.uint64)
-    datetime_dtype = 'datetime64[ns]'
-
-    max_scale = 6
-
-    def __init__(self, scale=0, **kwargs):
-        self.scale = scale
-        super(NumpyDateTime64Column, self).__init__(**kwargs)
-
-    def read_items(self, n_items, buf):
-        scale = 10 ** self.scale
-        frac_scale = 10 ** (self.max_scale - self.scale)
-
-        items = super(NumpyDateTime64Column, self).read_items(n_items, buf)
-
-        seconds = (items // scale).astype('datetime64[s]')
-        microseconds = ((items % scale) * frac_scale).astype('timedelta64[us]')
-
-        dt = seconds + microseconds
-        return self.apply_timezones_after_read(dt)
-
-    def write_items(self, items, buf):
-        # write int 'as is'.
-        if self.is_items_integer(items):
-            super(NumpyDateTime64Column, self).write_items(items, buf)
-            return
-
-        scale = 10 ** self.scale
-        frac_scale = 10 ** (self.max_scale - self.scale)
-
-        items = self.apply_timezones_before_write(items)
-
-        seconds = items.astype('datetime64[s]')
-        microseconds = (items - seconds).astype(dtype='timedelta64[us]') \
-            .astype(np.uint32) // frac_scale
-
-        items = seconds.astype(self.dtype) * scale + microseconds
-
-        super(NumpyDateTime64Column, self).write_items(items, buf)
-
-
-def create_numpy_datetime_column(spec, column_options):
-    if spec.startswith('DateTime64'):
-        cls = NumpyDateTime64Column
-        spec = spec[11:-1]
-        params = spec.split(',', 1)
-        column_options['scale'] = int(params[0])
-        if len(params) > 1:
-            spec = params[1].strip() + ')'
-    else:
-        cls = NumpyDateTimeColumn
-        spec = spec[9:]
-
-    context = column_options['context']
-
-    tz_name = timezone = None
-    offset_naive = True
-    local_timezone = None
-
-    # As Numpy do not use local timezone for converting timestamp to
-    # datetime we need always detect local timezone for manual converting.
-    try:
-        local_timezone = get_localzone().zone
-    except Exception:
-        pass
-
-    # Use column's timezone if it's specified.
-    if spec and spec[-1] == ')':
-        tz_name = spec[1:-2]
-        offset_naive = False
-    else:
-        if not context.settings.get('use_client_time_zone', False):
-            if local_timezone != context.server_info.timezone:
-                tz_name = context.server_info.timezone
-
-    if tz_name:
-        timezone = get_timezone(tz_name)
-
-    return cls(timezone=timezone, offset_naive=offset_naive,
-               local_timezone=local_timezone, **column_options)
+import numpy as np
+import pandas as pd
+from pytz import timezone as get_timezone
+
+from .base import NumpyColumn
+from ...util.compat import get_localzone_name_compat
+
+
+class NumpyDateTimeColumnBase(NumpyColumn):
+    datetime_dtype = None
+
+    null_value = np.datetime64(0, 'Y')
+
+    def __init__(self, timezone=None, offset_naive=True, local_timezone=None,
+                 **kwargs):
+        self.timezone = timezone
+        self.offset_naive = offset_naive
+        self.local_timezone = local_timezone
+        super(NumpyDateTimeColumnBase, self).__init__(**kwargs)
+
+    def apply_timezones_after_read(self, dt):
+        timezone = self.timezone if self.timezone else self.local_timezone
+
+        if self.offset_naive and timezone.zone != 'UTC':
+            ts = pd.to_datetime(dt, utc=True).tz_convert(timezone)
+            ts = ts.tz_localize(None)
+            return ts.to_numpy(self.datetime_dtype)
+
+        return dt
+
+    def apply_timezones_before_write(self, items):
+        if isinstance(items, pd.DatetimeIndex):
+            ts = items
+        else:
+            timezone = self.timezone if self.timezone else self.local_timezone
+            ts = pd.to_datetime(items)
+            if not getattr(ts.dtype, 'tz', None):
+                ts = ts.tz_localize(timezone)
+
+        ts = ts.tz_convert('UTC')
+        return ts.tz_localize(None).to_numpy(self.datetime_dtype)
+
+    def is_items_integer(self, items):
+        return (
+            isinstance(items, np.ndarray) and
+            np.issubdtype(items.dtype, np.integer)
+        )
+
+
+class NumpyDateTimeColumn(NumpyDateTimeColumnBase):
+    dtype = np.dtype(np.uint32)
+    datetime_dtype = 'datetime64[s]'
+
+    def write_items(self, items, buf):
+        # write int 'as is'.
+        if self.is_items_integer(items):
+            super(NumpyDateTimeColumn, self).write_items(items, buf)
+            return
+
+        items = self.apply_timezones_before_write(items)
+
+        super(NumpyDateTimeColumn, self).write_items(items, buf)
+
+    def read_items(self, n_items, buf):
+        items = super(NumpyDateTimeColumn, self).read_items(n_items, buf)
+        return self.apply_timezones_after_read(items.astype('datetime64[s]'))
+
+
+class NumpyDateTime64Column(NumpyDateTimeColumnBase):
+    dtype = np.dtype(np.int64)
+    datetime_dtype = 'datetime64[ns]'
+
+    max_scale = 9
+
+    def __init__(self, scale=0, **kwargs):
+        self.scale = scale
+        super(NumpyDateTime64Column, self).__init__(**kwargs)
+
+    def read_items(self, n_items, buf):
+        # Clickhouse:     t seconds is represented as t * 10^scale.
+        # datetime64[ns]: t seconds is represented as t * 10^9.
+        # Since 0 <= scale <= 9, multiply by the integer 10^(9 - scale).
+        items = super(NumpyDateTime64Column, self).read_items(n_items, buf)
+
+        tmp = np.copy(items)
+        tmp *= 10 ** (9 - self.scale)
+        dt = tmp.view(dtype='datetime64[ns]')
+
+        return self.apply_timezones_after_read(dt)
+
+    def write_items(self, items, buf):
+        # write int 'as is'.
+        if self.is_items_integer(items):
+            super(NumpyDateTime64Column, self).write_items(items, buf)
+            return
+
+        scale = 10 ** self.scale
+        frac_scale = 10 ** (self.max_scale - self.scale)
+
+        items = self.apply_timezones_before_write(items)
+
+        seconds = items.astype('datetime64[s]')
+        microseconds = (items - seconds).astype(dtype='timedelta64[ns]') \
+            .astype(np.uint32) // frac_scale
+
+        items = seconds.astype(self.dtype) * scale + microseconds
+
+        super(NumpyDateTime64Column, self).write_items(items, buf)
+
+
+def create_numpy_datetime_column(spec, column_options):
+    if spec.startswith('DateTime64'):
+        cls = NumpyDateTime64Column
+        spec = spec[11:-1]
+        params = spec.split(',', 1)
+        column_options['scale'] = int(params[0])
+        if len(params) > 1:
+            spec = params[1].strip() + ')'
+    else:
+        cls = NumpyDateTimeColumn
+        spec = spec[9:]
+
+    context = column_options['context']
+
+    tz_name = None
+    offset_naive = True
+
+    # As Numpy do not use local timezone for converting timestamp to
+    # datetime we need always detect local timezone for manual converting.
+    local_tz_name = get_localzone_name_compat()
+
+    # Use column's timezone if it's specified.
+    if spec and spec[-1] == ')':
+        tz_name = spec[1:-2]
+        offset_naive = False
+    else:
+        if not context.settings.get('use_client_time_zone', False):
+            remote_timezone = context.server_info.get_timezone()
+            if local_tz_name != remote_timezone:
+                tz_name = remote_timezone
+
+    timezone = get_timezone(tz_name) if tz_name else None
+    local_timezone = get_timezone(local_tz_name) if local_tz_name else None
+
+    return cls(timezone=timezone, offset_naive=offset_naive,
+               local_timezone=local_timezone, **column_options)

clickhouse_driver/columns/numpy/floatcolumn.py

@@ -1,13 +1,24 @@
-import numpy as np
-
-from .base import NumpyColumn
-
-
-class NumpyFloat32Column(NumpyColumn):
-    dtype = np.dtype(np.float32)
-    ch_type = 'Float32'
-
-
-class NumpyFloat64Column(NumpyColumn):
-    dtype = np.dtype(np.float64)
-    ch_type = 'Float64'
+import numpy as np
+
+from .base import NumpyColumn
+
+# normalize_null_value = False due to float('nan')
+# With normalization pandas.isnull will threat float('nan') as NULL value.
+
+
+class NumpyFloat32Column(NumpyColumn):
+    dtype = np.dtype(np.float32)
+    ch_type = 'Float32'
+    normalize_null_value = False
+
+    def _get_nulls_map(self, items):
+        return [x is None for x in items]
+
+
+class NumpyFloat64Column(NumpyColumn):
+    dtype = np.dtype(np.float64)
+    ch_type = 'Float64'
+    normalize_null_value = False
+
+    def _get_nulls_map(self, items):
+        return [x is None for x in items]