clickhouse-driver 0.2.1__cp39-cp39-win_amd64.whl → 0.2.10__cp39-cp39-win_amd64.whl

clickhouse_driver/columns/intervalcolumn.py

@@ -1,33 +1,33 @@
-from .intcolumn import Int64Column
-
-
-class IntervalColumn(Int64Column):
-    pass
-
-
-class IntervalDayColumn(IntervalColumn):
-    ch_type = 'IntervalDay'
-
-
-class IntervalWeekColumn(IntervalColumn):
-    ch_type = 'IntervalWeek'
-
-
-class IntervalMonthColumn(IntervalColumn):
-    ch_type = 'IntervalMonth'
-
-
-class IntervalYearColumn(IntervalColumn):
-    ch_type = 'IntervalYear'
-
-
-class IntervalHourColumn(IntervalColumn):
-    ch_type = 'IntervalHour'
-
-
-class IntervalMinuteColumn(IntervalColumn):
-    ch_type = 'IntervalMinute'
-
-
-class IntervalSecondColumn(IntervalColumn):
-    ch_type = 'IntervalSecond'
+from .intcolumn import Int64Column
+
+
+class IntervalColumn(Int64Column):
+    pass
+
+
+class IntervalDayColumn(IntervalColumn):
+    ch_type = 'IntervalDay'
+
+
+class IntervalWeekColumn(IntervalColumn):
+    ch_type = 'IntervalWeek'
+
+
+class IntervalMonthColumn(IntervalColumn):
+    ch_type = 'IntervalMonth'
+
+
+class IntervalYearColumn(IntervalColumn):
+    ch_type = 'IntervalYear'
+
+
+class IntervalHourColumn(IntervalColumn):
+    ch_type = 'IntervalHour'
+
+
+class IntervalMinuteColumn(IntervalColumn):
+    ch_type = 'IntervalMinute'
+
+
+class IntervalSecondColumn(IntervalColumn):
+    ch_type = 'IntervalSecond'

clickhouse_driver/columns/ipcolumn.py

@@ -1,118 +1,118 @@
-from ipaddress import IPv4Address, IPv6Address, AddressValueError
-
-from .. import errors
-from .exceptions import ColumnTypeMismatchException
-from .stringcolumn import ByteFixedString
-from .intcolumn import UInt32Column
-
-
-class IPv4Column(UInt32Column):
-    ch_type = "IPv4"
-    py_types = (str, IPv4Address, int)
-
-    def __init__(self, types_check=False, **kwargs):
-        # UIntColumn overrides before_write_item and check_item
-        # in its __init__ when types_check is True so we force
-        # __init__ without it then add the appropriate check method for IPv4
-        super(UInt32Column, self).__init__(types_check=False, **kwargs)
-
-        self.types_check_enabled = types_check
-        if types_check:
-
-            def check_item(value):
-                if isinstance(value, int) and value < 0:
-                    raise ColumnTypeMismatchException(value)
-
-                if not isinstance(value, IPv4Address):
-                    try:
-                        value = IPv4Address(value)
-                    except AddressValueError:
-                        # Cannot parse input in a valid IPv4
-                        raise ColumnTypeMismatchException(value)
-
-            self.check_item = check_item
-
-    def after_read_items(self, items, nulls_map=None):
-        if nulls_map is None:
-            return tuple(IPv4Address(item) for item in items)
-        else:
-            return tuple(
-                (None if is_null else IPv4Address(items[i]))
-                for i, is_null in enumerate(nulls_map)
-            )
-
-    def before_write_items(self, items, nulls_map=None):
-        null_value = self.null_value
-
-        for i, item in enumerate(items):
-            if nulls_map and nulls_map[i]:
-                items[i] = null_value
-                continue
-
-            # allow Ipv4 in integer, string or IPv4Address object
-            try:
-                if isinstance(item, int):
-                    continue
-
-                if not isinstance(item, IPv4Address):
-                    item = IPv4Address(item)
-
-                items[i] = int(item)
-            except AddressValueError:
-                raise errors.CannotParseDomainError(
-                    "Cannot parse IPv4 '{}'".format(item)
-                )
-
-
-class IPv6Column(ByteFixedString):
-    ch_type = "IPv6"
-    py_types = (str, IPv6Address, bytes)
-
-    def __init__(self, types_check=False, **kwargs):
-        super(IPv6Column, self).__init__(16, types_check=types_check, **kwargs)
-
-        if types_check:
-
-            def check_item(value):
-                if isinstance(value, bytes) and len(value) != 16:
-                    raise ColumnTypeMismatchException(value)
-
-                if not isinstance(value, IPv6Address):
-                    try:
-                        value = IPv6Address(value)
-                    except AddressValueError:
-                        # Cannot parse input in a valid IPv6
-                        raise ColumnTypeMismatchException(value)
-
-            self.check_item = check_item
-
-    def after_read_items(self, items, nulls_map=None):
-        if nulls_map is None:
-            return tuple(IPv6Address(item) for item in items)
-        else:
-            return tuple(
-                (None if is_null else IPv6Address(items[i]))
-                for i, is_null in enumerate(nulls_map)
-            )
-
-    def before_write_items(self, items, nulls_map=None):
-        null_value = self.null_value
-
-        for i, item in enumerate(items):
-            if nulls_map and nulls_map[i]:
-                items[i] = null_value
-                continue
-
-            # allow Ipv6 in bytes or python IPv6Address
-            # this is raw bytes (not encoded) in order to fit FixedString(16)
-            try:
-                if isinstance(item, bytes):
-                    continue
-
-                if not isinstance(item, IPv6Address):
-                    item = IPv6Address(item)
-                items[i] = item.packed
-            except AddressValueError:
-                raise errors.CannotParseDomainError(
-                    "Cannot parse IPv6 '{}'".format(item)
-                )
+from ipaddress import IPv4Address, IPv6Address, AddressValueError
+
+from .. import errors
+from .exceptions import ColumnTypeMismatchException
+from .stringcolumn import ByteFixedString
+from .intcolumn import UInt32Column
+
+
+class IPv4Column(UInt32Column):
+    ch_type = "IPv4"
+    py_types = (str, IPv4Address, int)
+
+    def __init__(self, types_check=False, **kwargs):
+        # UIntColumn overrides before_write_item and check_item
+        # in its __init__ when types_check is True so we force
+        # __init__ without it then add the appropriate check method for IPv4
+        super(UInt32Column, self).__init__(types_check=False, **kwargs)
+
+        self.types_check_enabled = types_check
+        if types_check:
+
+            def check_item(value):
+                if isinstance(value, int) and value < 0:
+                    raise ColumnTypeMismatchException(value)
+
+                if not isinstance(value, IPv4Address):
+                    try:
+                        value = IPv4Address(value)
+                    except AddressValueError:
+                        # Cannot parse input in a valid IPv4
+                        raise ColumnTypeMismatchException(value)
+
+            self.check_item = check_item
+
+    def after_read_items(self, items, nulls_map=None):
+        if nulls_map is None:
+            return tuple(IPv4Address(item) for item in items)
+        else:
+            return tuple(
+                (None if is_null else IPv4Address(items[i]))
+                for i, is_null in enumerate(nulls_map)
+            )
+
+    def before_write_items(self, items, nulls_map=None):
+        null_value = self.null_value
+
+        for i, item in enumerate(items):
+            if nulls_map and nulls_map[i]:
+                items[i] = null_value
+                continue
+
+            # allow Ipv4 in integer, string or IPv4Address object
+            try:
+                if isinstance(item, int):
+                    continue
+
+                if not isinstance(item, IPv4Address):
+                    item = IPv4Address(item)
+
+                items[i] = int(item)
+            except AddressValueError:
+                raise errors.CannotParseDomainError(
+                    "Cannot parse IPv4 '{}'".format(item)
+                )
+
+
+class IPv6Column(ByteFixedString):
+    ch_type = "IPv6"
+    py_types = (str, IPv6Address, bytes)
+
+    def __init__(self, types_check=False, **kwargs):
+        super(IPv6Column, self).__init__(16, types_check=types_check, **kwargs)
+
+        if types_check:
+
+            def check_item(value):
+                if isinstance(value, bytes) and len(value) != 16:
+                    raise ColumnTypeMismatchException(value)
+
+                if not isinstance(value, IPv6Address):
+                    try:
+                        value = IPv6Address(value)
+                    except AddressValueError:
+                        # Cannot parse input in a valid IPv6
+                        raise ColumnTypeMismatchException(value)
+
+            self.check_item = check_item
+
+    def after_read_items(self, items, nulls_map=None):
+        if nulls_map is None:
+            return tuple(IPv6Address(item) for item in items)
+        else:
+            return tuple(
+                (None if is_null else IPv6Address(items[i]))
+                for i, is_null in enumerate(nulls_map)
+            )
+
+    def before_write_items(self, items, nulls_map=None):
+        null_value = self.null_value
+
+        for i, item in enumerate(items):
+            if nulls_map and nulls_map[i]:
+                items[i] = null_value
+                continue
+
+            # allow Ipv6 in bytes or python IPv6Address
+            # this is raw bytes (not encoded) in order to fit FixedString(16)
+            try:
+                if isinstance(item, bytes):
+                    continue
+
+                if not isinstance(item, IPv6Address):
+                    item = IPv6Address(item)
+                items[i] = item.packed
+            except AddressValueError:
+                raise errors.CannotParseDomainError(
+                    "Cannot parse IPv6 '{}'".format(item)
+                )

clickhouse_driver/columns/jsoncolumn.py

@@ -0,0 +1,37 @@
+from .base import Column
+from .stringcolumn import String
+from ..reader import read_binary_uint8, read_binary_str
+from ..util.compat import json
+from ..writer import write_binary_uint8
+
+
+class JsonColumn(Column):
+    py_types = (dict, )
+
+    # No NULL value actually
+    null_value = {}
+
+    def __init__(self, column_by_spec_getter, **kwargs):
+        self.column_by_spec_getter = column_by_spec_getter
+        self.string_column = String(**kwargs)
+        super(JsonColumn, self).__init__(**kwargs)
+
+    def write_state_prefix(self, buf):
+        # Read in binary format.
+        # Write in text format.
+        write_binary_uint8(1, buf)
+
+    def read_items(self, n_items, buf):
+        read_binary_uint8(buf)
+        spec = read_binary_str(buf)
+        col = self.column_by_spec_getter(spec)
+        col.read_state_prefix(buf)
+        return col.read_data(n_items, buf)
+
+    def write_items(self, items, buf):
+        items = [x if isinstance(x, str) else json.dumps(x) for x in items]
+        self.string_column.write_items(items, buf)
+
+
+def create_json_column(spec, column_by_spec_getter, column_options):
+    return JsonColumn(column_by_spec_getter, **column_options)

clickhouse_driver/columns/lowcardinalitycolumn.py

@@ -1,123 +1,142 @@
-from math import log
-
-from ..reader import read_binary_uint64
-from ..writer import write_binary_int64
-from .base import Column
-from .intcolumn import UInt8Column, UInt16Column, UInt32Column, UInt64Column
-
-
-def create_low_cardinality_column(spec, column_by_spec_getter):
-    inner = spec[15:-1]
-    nested = column_by_spec_getter(inner)
-    return LowCardinalityColumn(nested)
-
-
-class LowCardinalityColumn(Column):
-    """
-    Stores column as index (unique elements) and keys.
-    Good for de-duplication of large values with low cardinality.
-    """
-    int_types = {
-        0: UInt8Column,
-        1: UInt16Column,
-        2: UInt32Column,
-        3: UInt64Column
-    }
-
-    # Need to read additional keys.
-    # Additional keys are stored before indexes as value N and N keys
-    # after them.
-    has_additional_keys_bit = 1 << 9
-    # Need to update dictionary.
-    # It means that previous granule has different dictionary.
-    need_update_dictionary = 1 << 10
-
-    serialization_type = has_additional_keys_bit | need_update_dictionary
-
-    def __init__(self, nested_column, **kwargs):
-        self.nested_column = nested_column
-        super(LowCardinalityColumn, self).__init__(**kwargs)
-
-    def read_state_prefix(self, buf):
-        return read_binary_uint64(buf)
-
-    def write_state_prefix(self, buf):
-        # KeysSerializationVersion. See ClickHouse docs.
-        write_binary_int64(1, buf)
-
-    def _write_data(self, items, buf):
-        index, keys = [], []
-        key_by_index_element = {}
-
-        if self.nested_column.nullable:
-            # First element represents NULL if column is nullable.
-            index.append(self.nested_column.null_value)
-            # Prevent null map writing. Reset nested column nullable flag.
-            self.nested_column.nullable = False
-
-            for x in items:
-                if x is None:
-                    # Zero element for null.
-                    keys.append(0)
-
-                else:
-                    key = key_by_index_element.get(x)
-                    # Get key from index or add it to index.
-                    if key is None:
-                        key = len(key_by_index_element)
-                        key_by_index_element[x] = key
-                        index.append(x)
-
-                    keys.append(key + 1)
-        else:
-            for x in items:
-                key = key_by_index_element.get(x)
-
-                # Get key from index or add it to index.
-                if key is None:
-                    key = len(key_by_index_element)
-                    key_by_index_element[x] = len(key_by_index_element)
-                    index.append(x)
-
-                keys.append(key)
-
-        # Do not write anything for empty column.
-        # May happen while writing empty arrays.
-        if not len(index):
-            return
-
-        int_type = int(log(len(index), 2) / 8)
-        int_column = self.int_types[int_type]()
-
-        serialization_type = self.serialization_type | int_type
-
-        write_binary_int64(serialization_type, buf)
-        write_binary_int64(len(index), buf)
-
-        self.nested_column.write_data(index, buf)
-        write_binary_int64(len(items), buf)
-        int_column.write_data(keys, buf)
-
-    def _read_data(self, n_items, buf, nulls_map=None):
-        if not n_items:
-            return tuple()
-
-        serialization_type = read_binary_uint64(buf)
-
-        # Lowest byte contains info about key type.
-        key_type = serialization_type & 0xf
-        keys_column = self.int_types[key_type]()
-
-        nullable = self.nested_column.nullable
-        # Prevent null map reading. Reset nested column nullable flag.
-        self.nested_column.nullable = False
-
-        index_size = read_binary_uint64(buf)
-        index = self.nested_column.read_data(index_size, buf)
-        if nullable:
-            index = (None, ) + index[1:]
-
-        read_binary_uint64(buf)  # number of keys
-        keys = keys_column.read_data(n_items, buf)
-
-        return tuple(index[x] for x in keys)
+from math import log
+
+from ..reader import read_binary_uint64
+from ..writer import write_binary_int64
+from .base import Column
+from .intcolumn import UInt8Column, UInt16Column, UInt32Column, UInt64Column
+
+
+def create_low_cardinality_column(spec, column_by_spec_getter, column_options):
+    inner = spec[15:-1]
+    nested = column_by_spec_getter(inner)
+    return LowCardinalityColumn(nested, **column_options)
+
+
+class LowCardinalityColumn(Column):
+    """
+    Stores column as index (unique elements) and keys.
+    Good for de-duplication of large values with low cardinality.
+    """
+    int_types = {
+        0: UInt8Column,
+        1: UInt16Column,
+        2: UInt32Column,
+        3: UInt64Column
+    }
+
+    # Need to read additional keys.
+    # Additional keys are stored before indexes as value N and N keys
+    # after them.
+    has_additional_keys_bit = 1 << 9
+    # Need to update dictionary.
+    # It means that previous granule has different dictionary.
+    need_update_dictionary = 1 << 10
+
+    serialization_type = has_additional_keys_bit | need_update_dictionary
+
+    def __init__(self, nested_column, **kwargs):
+        self.init_kwargs = kwargs
+        self.nested_column = nested_column
+        super(LowCardinalityColumn, self).__init__(**kwargs)
+
+    def read_state_prefix(self, buf):
+        super(LowCardinalityColumn, self).read_state_prefix(buf)
+
+        read_binary_uint64(buf)
+
+    def write_state_prefix(self, buf):
+        super(LowCardinalityColumn, self).write_state_prefix(buf)
+
+        # KeysSerializationVersion. See ClickHouse docs.
+        write_binary_int64(1, buf)
+
+    def _write_data(self, items, buf):
+        index, keys = [], []
+        key_by_index_element = {}
+        nested_is_nullable = False
+
+        if self.nested_column.nullable:
+            # First element represents NULL if column is nullable.
+            index.append(self.nested_column.null_value)
+            # Prevent null map writing. Reset nested column nullable flag.
+            self.nested_column.nullable = False
+            nested_is_nullable = True
+
+            for x in items:
+                if x is None:
+                    # Zero element for null.
+                    keys.append(0)
+
+                else:
+                    key = key_by_index_element.get(x)
+                    # Get key from index or add it to index.
+                    if key is None:
+                        key = len(key_by_index_element)
+                        key_by_index_element[x] = key
+                        index.append(x)
+
+                    keys.append(key + 1)
+        else:
+            for x in items:
+                key = key_by_index_element.get(x)
+
+                # Get key from index or add it to index.
+                if key is None:
+                    key = len(key_by_index_element)
+                    key_by_index_element[x] = len(key_by_index_element)
+                    index.append(x)
+
+                keys.append(key)
+
+        # Do not write anything for empty column.
+        # May happen while writing empty arrays.
+        if not len(index):
+            return
+
+        int_type = int(log(len(index), 2) / 8)
+        int_column = self.int_types[int_type](**self.init_kwargs)
+
+        serialization_type = self.serialization_type | int_type
+
+        write_binary_int64(serialization_type, buf)
+        write_binary_int64(len(index), buf)
+
+        if nested_is_nullable:
+            # Given we reset nested column nullable flag above,
+            # we need to write null map manually. If to invoke
+            # write_data method, it will cause an exception,
+            # because `prepare_data` may not be able to handle
+            # null value correctly.
+            self.nested_column.write_items(
+                [self.nested_column.null_value], buf)
+            # Remove null map from index, because it is already written.
+            index_to_write = index[1:]
+            self.nested_column.write_data(index_to_write, buf)
+        else:
+            self.nested_column.write_data(index, buf)
+        write_binary_int64(len(items), buf)
+        int_column.write_items(keys, buf)
+
+    def _read_data(self, n_items, buf, nulls_map=None):
+        if not n_items:
+            return tuple()
+
+        serialization_type = read_binary_uint64(buf)
+
+        # Lowest byte contains info about key type.
+        key_type = serialization_type & 0xf
+        keys_column = self.int_types[key_type](**self.init_kwargs)
+
+        nullable = self.nested_column.nullable
+        # Prevent null map reading. Reset nested column nullable flag.
+        self.nested_column.nullable = False
+
+        index_size = read_binary_uint64(buf)
+        index = self.nested_column.read_data(index_size, buf)
+        if nullable:
+            index = (None, ) + index[1:]
+
+        read_binary_uint64(buf)  # number of keys
+        keys = keys_column.read_data(n_items, buf)
+
+        return tuple(index[x] for x in keys)