clickhouse-driver 0.2.5__cp311-cp311-musllinux_1_1_aarch64.whl → 0.2.9__cp311-cp311-musllinux_1_1_aarch64.whl

clickhouse_driver/columns/enumcolumn.py

@@ -1,20 +1,24 @@
 from enum import Enum
+from collections import OrderedDict
 
 from .. import errors
 from .intcolumn import IntColumn
 
+invalid_names_for_python_enum = frozenset(['mro', ''])
+
 
 class EnumColumn(IntColumn):
     py_types = (Enum, int, str)
 
-    def __init__(self, enum_cls, **kwargs):
-        self.enum_cls = enum_cls
+    def __init__(self, name_by_value, value_by_name, **kwargs):
+        self.name_by_value = name_by_value
+        self.value_by_name = value_by_name
         super(EnumColumn, self).__init__(**kwargs)
 
     def before_write_items(self, items, nulls_map=None):
         null_value = self.null_value
-
-        enum_cls = self.enum_cls
+        name_by_value = self.name_by_value
+        value_by_name = self.value_by_name
 
         for i, item in enumerate(items):
             if nulls_map and nulls_map[i]:
@@ -26,15 +30,15 @@ class EnumColumn(IntColumn):
             # Check real enum value
             try:
                 if isinstance(source_value, str):
-                    items[i] = enum_cls[source_value].value
+                    items[i] = value_by_name[source_value]
                 else:
-                    items[i] = enum_cls(source_value).value
+                    items[i] = value_by_name[name_by_value[source_value]]
             except (ValueError, KeyError):
                 choices = ', '.join(
-                    "'{}' = {}".format(x.name.replace("'", r"\'"), x.value)
-                    for x in enum_cls
+                    "'{}' = {}".format(name.replace("'", r"\'"), value)
+                    for name, value in value_by_name.items()
                 )
-                enum_str = '{}({})'.format(enum_cls.__name__, choices)
+                enum_str = '{}({})'.format(self.ch_type, choices)
 
                 raise errors.LogicalError(
                     "Unknown element '{}' for type {}"
@@ -42,13 +46,13 @@ class EnumColumn(IntColumn):
                 )
 
     def after_read_items(self, items, nulls_map=None):
-        enum_cls = self.enum_cls
+        name_by_value = self.name_by_value
 
         if nulls_map is None:
-            return tuple(enum_cls(item).name for item in items)
+            return tuple(name_by_value[item] for item in items)
         else:
             return tuple(
-                (None if is_null else enum_cls(items[i]).name)
+                (None if is_null else name_by_value[items[i]])
                 for i, is_null in enumerate(nulls_map)
             )
 
@@ -73,11 +77,13 @@ def create_enum_column(spec, column_options):
         params = spec[7:-1]
         cls = Enum16Column
 
-    return cls(Enum(cls.ch_type, _parse_options(params)), **column_options)
+    name_by_value, value_by_name = _parse_options(params)
+
+    return cls(name_by_value, value_by_name, **column_options)
 
 
 def _parse_options(option_string):
-    options = dict()
+    name_by_value, value_by_name = {}, OrderedDict()
     after_name = False
     escaped = False
     quote_character = None
@@ -93,7 +99,9 @@ def _parse_options(option_string):
             if ch in (' ', '='):
                 pass
             elif ch == ',':
-                options[name] = int(value)
+                value = int(value)
+                name_by_value[value] = name
+                value_by_name[name] = value
                 after_name = False
                 name = ''
                 value = ''  # reset before collecting new option
@@ -114,6 +122,8 @@ def _parse_options(option_string):
                 quote_character = ch
 
     if after_name:
-        options.setdefault(name, int(value))  # append word after last comma
+        value = int(value)
+        name_by_value[value] = name
+        value_by_name[name] = value
 
-    return options
+    return name_by_value, value_by_name

clickhouse_driver/columns/jsoncolumn.py

@@ -0,0 +1,37 @@
+from .base import Column
+from .stringcolumn import String
+from ..reader import read_binary_uint8, read_binary_str
+from ..util.compat import json
+from ..writer import write_binary_uint8
+
+
+class JsonColumn(Column):
+    py_types = (dict, )
+
+    # No NULL value actually
+    null_value = {}
+
+    def __init__(self, column_by_spec_getter, **kwargs):
+        self.column_by_spec_getter = column_by_spec_getter
+        self.string_column = String(**kwargs)
+        super(JsonColumn, self).__init__(**kwargs)
+
+    def write_state_prefix(self, buf):
+        # Read in binary format.
+        # Write in text format.
+        write_binary_uint8(1, buf)
+
+    def read_items(self, n_items, buf):
+        read_binary_uint8(buf)
+        spec = read_binary_str(buf)
+        col = self.column_by_spec_getter(spec)
+        col.read_state_prefix(buf)
+        return col.read_data(n_items, buf)
+
+    def write_items(self, items, buf):
+        items = [x if isinstance(x, str) else json.dumps(x) for x in items]
+        self.string_column.write_items(items, buf)
+
+
+def create_json_column(spec, column_by_spec_getter, column_options):
+    return JsonColumn(column_by_spec_getter, **column_options)

clickhouse_driver/columns/lowcardinalitycolumn.py

@@ -35,25 +35,32 @@ class LowCardinalityColumn(Column):
     serialization_type = has_additional_keys_bit | need_update_dictionary
 
     def __init__(self, nested_column, **kwargs):
+        self.init_kwargs = kwargs
         self.nested_column = nested_column
         super(LowCardinalityColumn, self).__init__(**kwargs)
 
     def read_state_prefix(self, buf):
-        return read_binary_uint64(buf)
+        super(LowCardinalityColumn, self).read_state_prefix(buf)
+
+        read_binary_uint64(buf)
 
     def write_state_prefix(self, buf):
+        super(LowCardinalityColumn, self).write_state_prefix(buf)
+
         # KeysSerializationVersion. See ClickHouse docs.
         write_binary_int64(1, buf)
 
     def _write_data(self, items, buf):
         index, keys = [], []
         key_by_index_element = {}
+        nested_is_nullable = False
 
         if self.nested_column.nullable:
             # First element represents NULL if column is nullable.
             index.append(self.nested_column.null_value)
             # Prevent null map writing. Reset nested column nullable flag.
             self.nested_column.nullable = False
+            nested_is_nullable = True
 
             for x in items:
                 if x is None:
@@ -87,14 +94,26 @@ class LowCardinalityColumn(Column):
             return
 
         int_type = int(log(len(index), 2) / 8)
-        int_column = self.int_types[int_type]()
+        int_column = self.int_types[int_type](**self.init_kwargs)
 
         serialization_type = self.serialization_type | int_type
 
         write_binary_int64(serialization_type, buf)
         write_binary_int64(len(index), buf)
 
-        self.nested_column.write_data(index, buf)
+        if nested_is_nullable:
+            # Given we reset nested column nullable flag above,
+            # we need to write null map manually. If to invoke
+            # write_data method, it will cause an exception,
+            # because `prepare_data` may not be able to handle
+            # null value correctly.
+            self.nested_column.write_items(
+                [self.nested_column.null_value], buf)
+            # Remove null map from index, because it is already written.
+            index_to_write = index[1:]
+            self.nested_column.write_data(index_to_write, buf)
+        else:
+            self.nested_column.write_data(index, buf)
         write_binary_int64(len(items), buf)
         int_column.write_items(keys, buf)
 
@@ -106,7 +125,7 @@ class LowCardinalityColumn(Column):
 
         # Lowest byte contains info about key type.
         key_type = serialization_type & 0xf
-        keys_column = self.int_types[key_type]()
+        keys_column = self.int_types[key_type](**self.init_kwargs)
 
         nullable = self.nested_column.nullable
         # Prevent null map reading. Reset nested column nullable flag.

clickhouse_driver/columns/mapcolumn.py

@@ -13,20 +13,27 @@ class MapColumn(Column):
     null_value = {}
 
     def __init__(self, key_column, value_column, **kwargs):
-        self.offset_column = UInt64Column()
+        self.offset_column = UInt64Column(**kwargs)
         self.key_column = key_column
         self.value_column = value_column
         super(MapColumn, self).__init__(**kwargs)
 
     def read_state_prefix(self, buf):
+        super(MapColumn, self).read_state_prefix(buf)
+
         self.key_column.read_state_prefix(buf)
         self.value_column.read_state_prefix(buf)
 
     def write_state_prefix(self, buf):
+        super(MapColumn, self).write_state_prefix(buf)
+
         self.key_column.write_state_prefix(buf)
         self.value_column.write_state_prefix(buf)
 
     def read_items(self, n_items, buf):
+        if not n_items:
+            return [{}]
+
         offsets = list(self.offset_column.read_items(n_items, buf))
         last_offset = offsets[-1]
         keys = self.key_column.read_data(last_offset, buf)
@@ -57,7 +64,7 @@ class MapColumn(Column):
 
 
 def create_map_column(spec, column_by_spec_getter, column_options):
-    # Match commas outside of parentheses so we don't match the comma in
+    # Match commas outside of parentheses, so we don't match the comma in
     # Decimal types.
     key, value = comma_re.split(spec[4:-1])
     key_column = column_by_spec_getter(key.strip())

clickhouse_driver/columns/nestedcolumn.py

@@ -1,21 +1,10 @@
 
 from .arraycolumn import create_array_column
-from .util import get_inner_spec, get_inner_columns, \
-    get_inner_columns_with_types
+from .util import get_inner_spec
 
 
 def create_nested_column(spec, column_by_spec_getter, column_options):
     return create_array_column(
-        'Array(Tuple({}))'.format(','.join(get_nested_columns(spec))),
+        'Array(Tuple({}))'.format(get_inner_spec('Nested', spec)),
         column_by_spec_getter, column_options
     )
-
-
-def get_nested_columns(spec):
-    inner_spec = get_inner_spec('Nested', spec)
-    return get_inner_columns(inner_spec)
-
-
-def get_columns_with_types(spec):
-    inner_spec = get_inner_spec('Nested', spec)
-    return get_inner_columns_with_types(inner_spec)

clickhouse_driver/columns/numpy/datetimecolumn.py

@@ -21,19 +21,21 @@ class NumpyDateTimeColumnBase(NumpyColumn):
     def apply_timezones_after_read(self, dt):
         timezone = self.timezone if self.timezone else self.local_timezone
 
-        ts = pd.to_datetime(dt, utc=True).tz_convert(timezone)
-
-        if self.offset_naive:
+        if self.offset_naive and timezone.zone != 'UTC':
+            ts = pd.to_datetime(dt, utc=True).tz_convert(timezone)
             ts = ts.tz_localize(None)
+            return ts.to_numpy(self.datetime_dtype)
 
-        return ts.to_numpy(self.datetime_dtype)
+        return dt
 
     def apply_timezones_before_write(self, items):
         if isinstance(items, pd.DatetimeIndex):
             ts = items
         else:
             timezone = self.timezone if self.timezone else self.local_timezone
-            ts = pd.to_datetime(items).tz_localize(timezone)
+            ts = pd.to_datetime(items)
+            if not getattr(ts.dtype, 'tz', None):
+                ts = ts.tz_localize(timezone)
 
         ts = ts.tz_convert('UTC')
         return ts.tz_localize(None).to_numpy(self.datetime_dtype)
@@ -65,7 +67,7 @@ class NumpyDateTimeColumn(NumpyDateTimeColumnBase):
 
 
 class NumpyDateTime64Column(NumpyDateTimeColumnBase):
-    dtype = np.dtype(np.uint64)
+    dtype = np.dtype(np.int64)
     datetime_dtype = 'datetime64[ns]'
 
     max_scale = 9
@@ -75,15 +77,15 @@ class NumpyDateTime64Column(NumpyDateTimeColumnBase):
         super(NumpyDateTime64Column, self).__init__(**kwargs)
 
     def read_items(self, n_items, buf):
-        scale = 10 ** self.scale
-        frac_scale = 10 ** (self.max_scale - self.scale)
-
+        # Clickhouse:     t seconds is represented as t * 10^scale.
+        # datetime64[ns]: t seconds is represented as t * 10^9.
+        # Since 0 <= scale <= 9, multiply by the integer 10^(9 - scale).
         items = super(NumpyDateTime64Column, self).read_items(n_items, buf)
 
-        seconds = (items // scale).astype('datetime64[s]')
-        microseconds = ((items % scale) * frac_scale).astype('timedelta64[ns]')
+        tmp = np.copy(items)
+        tmp *= 10 ** (9 - self.scale)
+        dt = tmp.view(dtype='datetime64[ns]')
 
-        dt = seconds + microseconds
         return self.apply_timezones_after_read(dt)
 
     def write_items(self, items, buf):
@@ -120,12 +122,12 @@ def create_numpy_datetime_column(spec, column_options):
 
     context = column_options['context']
 
-    tz_name = timezone = None
+    tz_name = None
     offset_naive = True
 
     # As Numpy do not use local timezone for converting timestamp to
     # datetime we need always detect local timezone for manual converting.
-    local_timezone = get_localzone_name_compat()
+    local_tz_name = get_localzone_name_compat()
 
     # Use column's timezone if it's specified.
     if spec and spec[-1] == ')':
@@ -133,11 +135,12 @@ def create_numpy_datetime_column(spec, column_options):
         offset_naive = False
     else:
         if not context.settings.get('use_client_time_zone', False):
-            if local_timezone != context.server_info.timezone:
-                tz_name = context.server_info.timezone
+            remote_timezone = context.server_info.get_timezone()
+            if local_tz_name != remote_timezone:
+                tz_name = remote_timezone
 
-    if tz_name:
-        timezone = get_timezone(tz_name)
+    timezone = get_timezone(tz_name) if tz_name else None
+    local_timezone = get_timezone(local_tz_name) if local_tz_name else None
 
     return cls(timezone=timezone, offset_naive=offset_naive,
                local_timezone=local_timezone, **column_options)

clickhouse_driver/columns/numpy/lowcardinalitycolumn.py

@@ -37,7 +37,7 @@ class NumpyLowCardinalityColumn(LowCardinalityColumn):
         c = pd.Categorical(items)
 
         int_type = int(log(len(c.codes), 2) / 8)
-        int_column = self.int_types[int_type]()
+        int_column = self.int_types[int_type](**self.init_kwargs)
 
         serialization_type = self.serialization_type | int_type
 
@@ -66,7 +66,7 @@ class NumpyLowCardinalityColumn(LowCardinalityColumn):
 
         # Lowest byte contains info about key type.
         key_type = serialization_type & 0xf
-        keys_column = self.int_types[key_type]()
+        keys_column = self.int_types[key_type](**self.init_kwargs)
 
         nullable = self.nested_column.nullable
         # Prevent null map reading. Reset nested column nullable flag.

clickhouse_driver/columns/service.py

@@ -15,6 +15,7 @@ from .intcolumn import (
     UInt8Column, UInt16Column, UInt32Column, UInt64Column
 )
 from .lowcardinalitycolumn import create_low_cardinality_column
+from .jsoncolumn import create_json_column
 from .mapcolumn import create_map_column
 from .nothingcolumn import NothingColumn
 from .nullcolumn import NullColumn
@@ -122,6 +123,11 @@ def get_column_by_spec(spec, column_options, use_numpy=None):
             spec, create_column_with_options, column_options
         )
 
+    elif spec.startswith("Object('json')"):
+        return create_json_column(
+            spec, create_column_with_options, column_options
+        )
+
     else:
         for alias, primitive in aliases:
             if spec.startswith(alias):
@@ -137,8 +143,12 @@ def get_column_by_spec(spec, column_options, use_numpy=None):
             raise errors.UnknownTypeError('Unknown type {}'.format(spec))
 
 
-def read_column(context, column_spec, n_items, buf, use_numpy=None):
-    column_options = {'context': context}
+def read_column(context, column_spec, n_items, buf, use_numpy=None,
+                has_custom_serialization=False):
+    column_options = {
+        'context': context,
+        'has_custom_serialization': has_custom_serialization
+    }
     col = get_column_by_spec(column_spec, column_options, use_numpy=use_numpy)
     col.read_state_prefix(buf)
     return col.read_data(n_items, buf)

clickhouse_driver/columns/tuplecolumn.py

@@ -1,13 +1,21 @@
 
 from .base import Column
-from .util import get_inner_spec, get_inner_columns
+from .util import get_inner_spec, get_inner_columns_with_types
 
 
 class TupleColumn(Column):
     py_types = (list, tuple)
 
-    def __init__(self, nested_columns, **kwargs):
+    def __init__(self, names, nested_columns, **kwargs):
+        self.names = names
         self.nested_columns = nested_columns
+        client_settings = kwargs['context'].client_settings
+        settings = kwargs['context'].settings
+        self.namedtuple_as_json = (
+            settings.get('allow_experimental_object_type', False) and
+            client_settings.get('namedtuple_as_json', True)
+        )
+
         super(TupleColumn, self).__init__(**kwargs)
         self.null_value = tuple(x.null_value for x in nested_columns)
 
@@ -23,15 +31,33 @@ class TupleColumn(Column):
 
     def read_data(self, n_items, buf):
         rv = [x.read_data(n_items, buf) for x in self.nested_columns]
-        return list(zip(*rv))
+        rv = list(zip(*rv))
+
+        if self.names[0] and self.namedtuple_as_json:
+            return [dict(zip(self.names, x)) for x in rv]
+        else:
+            return rv
 
     def read_items(self, n_items, buf):
         return self.read_data(n_items, buf)
 
+    def read_state_prefix(self, buf):
+        super(TupleColumn, self).read_state_prefix(buf)
+
+        for x in self.nested_columns:
+            x.read_state_prefix(buf)
+
+    def write_state_prefix(self, buf):
+        super(TupleColumn, self).write_state_prefix(buf)
+
+        for x in self.nested_columns:
+            x.write_state_prefix(buf)
+
 
 def create_tuple_column(spec, column_by_spec_getter, column_options):
     inner_spec = get_inner_spec('Tuple', spec)
-    columns = get_inner_columns(inner_spec)
+    columns_with_types = get_inner_columns_with_types(inner_spec)
+    names, types = zip(*columns_with_types)
 
-    return TupleColumn([column_by_spec_getter(x) for x in columns],
+    return TupleColumn(names, [column_by_spec_getter(x) for x in types],
                        **column_options)

clickhouse_driver/columns/util.py

@@ -37,12 +37,13 @@ def get_inner_columns(spec):
 
 
 def get_inner_columns_with_types(spec):
+    spec = spec.strip()
     brackets = 0
     prev_comma = 0
     prev_space = 0
 
     columns = []
-    for i, x in enumerate(spec + ','):
+    for i, x in enumerate(spec.strip() + ','):
         if x == ',':
             if brackets == 0:
                 columns.append((

clickhouse_driver/columns/uuidcolumn.py

@@ -54,7 +54,7 @@ class UUIDColumn(FormatColumn):
 
             try:
                 if not isinstance(item, UUID):
-                    item = UUID(item)
+                    item = UUID(int=item) if item is null_value else UUID(item)
 
             except ValueError:
                 raise errors.CannotParseUuidError(