another-ayx-python-sdk 2.4.2__py3-none-any.whl → 2.4.22__py3-none-any.whl

build/lib/another_ayx_python_sdk/providers/amp_provider/builders/packers.py

@@ -0,0 +1,458 @@
+# Copyright (C) 2022 Alteryx, Inc. All rights reserved.
+#
+# Licensed under the ALTERYX SDK AND API LICENSE AGREEMENT;
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.alteryx.com/alteryx-sdk-and-api-license-agreement
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utility classes for converting between byte representations."""
+import datetime
+import struct
+from abc import ABC, abstractmethod
+from decimal import Decimal
+from functools import lru_cache
+from typing import Any, Callable, Tuple
+
+
+class _Packer(ABC):
+    def pack(self, element: Any) -> bytes:
+        try:
+            element = self._caster(element)
+        except Exception:
+            raise ValueError(
+                f"Could not cast value {element} with caster {self._caster}."
+            )
+
+        self._validator(element)
+        return struct.pack(self._format_code, self._caster(element))
+
+    def unpack(self, raw_bytes: bytes, start_idx: int) -> Tuple[Any, int]:
+        element_bytes = raw_bytes[start_idx : start_idx + self._element_size]
+        [parsed_element] = struct.unpack(self._format_code, element_bytes)
+
+        self._validator(parsed_element)
+        return self._caster(parsed_element), self._element_size
+
+    @property
+    @abstractmethod
+    def _format_code(self) -> str:
+        """Get the format code for struct package."""
+        raise NotImplementedError()
+
+    @property
+    @abstractmethod
+    def _element_size(self) -> int:
+        """Get the element size (in bytes) of the packer."""
+        raise NotImplementedError()
+
+    @property
+    @abstractmethod
+    def _caster(self) -> Callable:
+        """Get the method for casting the element."""
+        raise NotImplementedError()
+
+    @staticmethod
+    def _unpack_sized_blob(blob: bytes, start_idx: int) -> Tuple[bytes, int]:
+        element_size, integer_size = _UnsignedIntegerPacker().unpack(blob, start_idx)
+        return (
+            blob[start_idx + integer_size : start_idx + integer_size + element_size],
+            element_size + integer_size,
+        )
+
+    @abstractmethod
+    def _validator(self, element: Any) -> None:
+        """Validate the data."""
+        raise NotImplementedError()
+
+
+class _BoolPacker(_Packer):
+    _element_size = 1
+    _format_code = "?"
+    _caster = bool
+
+    def _validator(self, element: Any) -> None:
+        # No additional validation needed for booleans
+        pass
+
+
+class _IntegerPacker(_Packer):
+    _element_size = 4
+    _format_code = "i"
+    _caster = int
+    _signed = True
+
+    def _validator(self, element: Any) -> None:
+        """Validate the data."""
+        _validate_integer(element, self._element_size * 8, self._signed)
+
+
+class _BytePacker(_IntegerPacker):
+    _element_size = 1
+    _format_code = "B"
+    _caster = int
+    _signed = False
+
+
+class _Int16Packer(_IntegerPacker):
+    _element_size = 2
+    _format_code = "h"
+    _caster = int
+
+
+class _UnsignedIntegerPacker(_IntegerPacker):
+    _element_size = 4
+    _format_code = "I"
+    _caster = int
+    _signed = False
+
+
+class _LongIntegerPacker(_IntegerPacker):
+    _element_size = 8
+    _format_code = "q"
+    _caster = int
+
+
+class _FloatPacker(_Packer):
+    _element_size = 4
+    _format_code = "f"
+    _caster = float
+
+    def _validator(self, element: Any) -> None:
+        _validate_float(element, 1e38, 3.4)
+
+
+class _DoublePacker(_Packer):
+    _element_size = 8
+    _format_code = "d"
+    _caster = float
+
+    def _validator(self, element: Any) -> None:
+        _validate_float(element, 1e308, 1.7)
+
+
+class _StringPacker(_Packer):
+    def pack(self, string: str) -> bytes:
+        string = str(string)
+        self._validator(string)
+
+        string_bytes = string.encode("utf-8")
+        string_size = len(string_bytes)
+
+        return _UnsignedIntegerPacker().pack(string_size) + string_bytes
+
+    def unpack(self, raw_bytes: bytes, start_idx: int) -> Tuple[str, int]:
+        blob, element_size = self._unpack_sized_blob(raw_bytes, start_idx)
+
+        string = blob.decode("utf-8")
+        self._validator(string)
+        return string, element_size
+
+    @property
+    def _format_code(self) -> str:
+        """Get the format code for struct package."""
+        raise NotImplementedError()
+
+    @property
+    def _element_size(self) -> int:
+        """Get the element size (in bytes) of the packer."""
+        raise NotImplementedError()
+
+    @property
+    def _caster(self) -> int:
+        """Get the method for casting the element."""
+        raise NotImplementedError()
+
+    def _validator(self, element: Any) -> None:
+        # No additional validation needed for strings
+        pass
+
+
+class _DatePacker(_StringPacker):
+    def _validator(self, element: Any) -> None:
+        super()._validator(element)
+        datetime.datetime.strptime(element, "%Y-%m-%d")
+
+
+class _DatetimePacker(_StringPacker):
+    def _validator(self, element: Any) -> None:
+        super()._validator(element)
+        datetime.datetime.strptime(element, "%Y-%m-%d %H:%M:%S")
+
+
+class _BlobPacker(_Packer):
+    def pack(self, blob: bytes) -> bytes:
+        blob = bytes(blob)
+        self._validator(blob)
+        return _UnsignedIntegerPacker().pack(len(blob)) + blob
+
+    def unpack(self, raw_bytes: bytes, start_idx: int) -> Tuple[bytes, int]:
+        blob, size = self._unpack_sized_blob(raw_bytes, start_idx)
+        self._validator(blob)
+        return blob, size
+
+    @property
+    def _format_code(self) -> str:
+        """Get the format code for struct package."""
+        raise NotImplementedError()
+
+    @property
+    def _element_size(self) -> int:
+        """Get the element size (in bytes) of the packer."""
+        raise NotImplementedError()
+
+    @property
+    def _caster(self) -> int:
+        """Get the method for casting the element."""
+        raise NotImplementedError()
+
+    def _validator(self, element: Any) -> None:
+        # No additional validation needed for blobs
+        pass
+
+
+class _IndirectStringPacker(_Packer):
+    @classmethod
+    def pack(cls, string: str) -> bytes:
+        return bytes()
+
+    @classmethod
+    def unpack(cls, raw_bytes: bytes, start_idx: int) -> Tuple[str, int]:
+        return "", 0
+
+    @property
+    def _format_code(self) -> str:
+        """Get the format code for struct package."""
+        raise NotImplementedError()
+
+    @property
+    def _element_size(self) -> int:
+        """Get the element size (in bytes) of the packer."""
+        raise NotImplementedError()
+
+    @property
+    def _caster(self) -> int:
+        """Get the method for casting the element."""
+        raise NotImplementedError()
+
+    def _validator(self, element: Any) -> None:
+        # No additional validation needed for indirect strings
+        pass
+
+
+class _IndirectBlobPacker(_Packer):
+    @classmethod
+    def pack(cls, blob: bytes) -> bytes:
+        return bytes()
+
+    @classmethod
+    def unpack(cls, raw_bytes: bytes, start_idx: int) -> Tuple[bytes, int]:
+        return bytes(), 0
+
+    @property
+    def _format_code(self) -> str:
+        """Get the format code for struct package."""
+        raise NotImplementedError()
+
+    @property
+    def _element_size(self) -> int:
+        """Get the element size (in bytes) of the packer."""
+        raise NotImplementedError()
+
+    @property
+    def _caster(self) -> int:
+        """Get the method for casting the element."""
+        raise NotImplementedError()
+
+    def _validator(self, element: Any) -> None:
+        # No additional validation needed for indirect blobs
+        pass
+
+
+class _BoolTruePacker(_Packer):
+    @classmethod
+    def pack(cls, element: bool) -> bytes:
+        return bytes()
+
+    @classmethod
+    def unpack(cls, raw_bytes: bytes, start_idx: int) -> Tuple[bool, int]:
+        return True, 0
+
+    @property
+    def _format_code(self) -> str:
+        """Get the format code for struct package."""
+        raise NotImplementedError()
+
+    @property
+    def _element_size(self) -> int:
+        """Get the element size (in bytes) of the packer."""
+        raise NotImplementedError()
+
+    @property
+    def _caster(self) -> int:
+        """Get the method for casting the element."""
+        raise NotImplementedError()
+
+    def _validator(self, element: Any) -> None:
+        # No additional validation needed for boolean true types
+        pass
+
+
+class _BoolFalsePacker(_Packer):
+    @classmethod
+    def pack(cls, element: bool) -> bytes:
+        return bytes()
+
+    @classmethod
+    def unpack(cls, raw_bytes: bytes, start_idx: int) -> Tuple[bool, int]:
+        return False, 0
+
+    @property
+    def _format_code(self) -> str:
+        """Get the format code for struct package."""
+        raise NotImplementedError()
+
+    @property
+    def _element_size(self) -> int:
+        """Get the element size (in bytes) of the packer."""
+        raise NotImplementedError()
+
+    @property
+    def _caster(self) -> int:
+        """Get the method for casting the element."""
+        raise NotImplementedError()
+
+    def _validator(self, element: Any) -> None:
+        # No additional validation needed for boolean false types
+        pass
+
+
+class _EmptyStringPacker(_Packer):
+    @classmethod
+    def pack(cls, element: str) -> bytes:
+        return bytes()
+
+    @classmethod
+    def unpack(cls, raw_bytes: bytes, start_idx: int) -> Tuple[str, int]:
+        return "", 0
+
+    @property
+    def _format_code(self) -> str:
+        """Get the format code for struct package."""
+        raise NotImplementedError()
+
+    @property
+    def _element_size(self) -> int:
+        """Get the element size (in bytes) of the packer."""
+        raise NotImplementedError()
+
+    @property
+    def _caster(self) -> int:
+        """Get the method for casting the element."""
+        raise NotImplementedError()
+
+    def _validator(self, element: Any) -> None:
+        # No additional validation needed for empty strings
+        pass
+
+
+class _NullPacker(_Packer):
+    @classmethod
+    def pack(cls, element: str) -> bytes:
+        return bytes()
+
+    @classmethod
+    def unpack(cls, raw_bytes: bytes, start_idx: int) -> Tuple[None, int]:
+        return None, 0
+
+    @property
+    def _format_code(self) -> str:
+        """Get the format code for struct package."""
+        raise NotImplementedError()
+
+    @property
+    def _element_size(self) -> int:
+        """Get the element size (in bytes) of the packer."""
+        raise NotImplementedError()
+
+    @property
+    def _caster(self) -> int:
+        """Get the method for casting the element."""
+        raise NotImplementedError()
+
+    def _validator(self, element: Any) -> None:
+        # No additional validation needed for nulls
+        pass
+
+
+class _Int0Packer(_Packer):
+    @classmethod
+    def pack(cls, element: str) -> bytes:
+        return bytes()
+
+    @classmethod
+    def unpack(cls, raw_bytes: bytes, start_idx: int) -> Tuple[int, int]:
+        return 0, 0
+
+    @property
+    def _format_code(self) -> str:
+        """Get the format code for struct package."""
+        raise NotImplementedError()
+
+    @property
+    def _element_size(self) -> int:
+        """Get the element size (in bytes) of the packer."""
+        raise NotImplementedError()
+
+    @property
+    def _caster(self) -> int:
+        """Get the method for casting the element."""
+        raise NotImplementedError()
+
+    def _validator(self, element: Any) -> None:
+        # No additional validation needed for int 0
+        pass
+
+
+def _validate_integer(element: Any, num_bits: int, signed: bool) -> None:
+    """Validate whether this integer is in bounds given it's AMPFieldType."""
+    max_val = max_int(num_bits, signed)
+    min_val = min_int(num_bits, signed)
+
+    if element < min_val or element > max_val:
+        raise ValueError(
+            f"{'Signed' if signed else 'Unsigned'} integer with {num_bits} bits must fall in the range of {min_val} to {max_val}."
+        )
+
+
+def _validate_float(element: Any, exp, digit) -> None:
+    """Validate whether this float is in bounds given it's AMPFieldType."""
+    # TODO potentially validate for floats with exponent less than -38 and for doubles with exponent less than -308
+    limit = float_limit(exp, digit)
+    dec_element = Decimal(str(element))
+
+    if dec_element < -limit or dec_element > limit:
+        raise ValueError(f"Float must fall in the range of -{limit} to {limit}.")
+
+
+@lru_cache()
+def min_int(num_bits: int, signed: bool) -> int:
+    """Calculate the minimum value that an integer can be considering it's bit length and whether it is signed."""
+    return signed * -1 * (2 ** (num_bits - 1))
+
+
+@lru_cache()
+def max_int(num_bits: int, signed: bool) -> int:
+    """Calculate the maximum value that an integer can be considering it's bit length and whether it is signed."""
+    return 2 ** (num_bits - (1 * signed)) - 1
+
+
+@lru_cache()
+def float_limit(exp: float, digit: float) -> "Decimal":
+    """Calculate the limit of a float given it's exponent and digit."""
+    return Decimal(str(exp)) * Decimal(str(digit))

build/lib/another_ayx_python_sdk/providers/amp_provider/builders/record_builder.py

@@ -0,0 +1,204 @@
+# Copyright (C) 2022 Alteryx, Inc. All rights reserved.
+#
+# Licensed under the ALTERYX SDK AND API LICENSE AGREEMENT;
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.alteryx.com/alteryx-sdk-and-api-license-agreement
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Record building utilities for converting between core and protobuf."""
+from typing import Any, Dict, List, Sequence
+
+from another_ayx_python_sdk.core.constants import NULL_VALUE_PLACEHOLDER
+from another_ayx_python_sdk.core.field import FieldType
+from another_ayx_python_sdk.core.metadata import Metadata
+from another_ayx_python_sdk.providers.amp_provider.builders.packers import (
+    _BlobPacker,
+    _BoolFalsePacker,
+    _BoolPacker,
+    _BoolTruePacker,
+    _BytePacker,
+    _DatePacker,
+    _DatetimePacker,
+    _DoublePacker,
+    _EmptyStringPacker,
+    _FloatPacker,
+    _IndirectBlobPacker,
+    _IndirectStringPacker,
+    _Int0Packer,
+    _Int16Packer,
+    _IntegerPacker,
+    _LongIntegerPacker,
+    _NullPacker,
+    _Packer,
+    _StringPacker,
+)
+from another_ayx_python_sdk.providers.amp_provider.resources.generated.record_packet_pb2 import (
+    AMPFieldType,
+    Record as ProtobufRecord,
+)
+
+
+class RecordBuilder:
+    """Utilities for converting records between protobuf and core format objects."""
+
+    _field_type_to_packer: Dict[Any, _Packer] = {
+        AMPFieldType.STRING: _StringPacker(),
+        AMPFieldType.BLOB: _BlobPacker(),
+        AMPFieldType.SPATIALOBJ: _BlobPacker(),
+        AMPFieldType.BCD: _BlobPacker(),
+        AMPFieldType.BOOL: _BoolPacker(),
+        AMPFieldType.INT0: _Int0Packer(),
+        AMPFieldType.UINT8: _BytePacker(),
+        AMPFieldType.INT16: _Int16Packer(),
+        AMPFieldType.INT32: _IntegerPacker(),
+        AMPFieldType.INT64: _LongIntegerPacker(),
+        AMPFieldType.FLOAT: _FloatPacker(),
+        AMPFieldType.DOUBLE: _DoublePacker(),
+        AMPFieldType.DATE: _DatePacker(),
+        AMPFieldType.TIME: _IntegerPacker(),
+        AMPFieldType.DATETIME: _DatetimePacker(),
+        AMPFieldType.INDIRECTSTRING: _IndirectStringPacker(),
+        AMPFieldType.INDIRECTBLOB: _IndirectBlobPacker(),
+        AMPFieldType.INDIRECTSPATIAL: _IndirectBlobPacker(),
+        AMPFieldType.BOOLFALSE: _BoolFalsePacker(),
+        AMPFieldType.BOOLTRUE: _BoolTruePacker(),
+        AMPFieldType.EMPTYSTRING: _EmptyStringPacker(),
+        AMPFieldType.NULL: _NullPacker(),
+    }
+
+    _blob_cosmetic_types_to_amp_type = {
+        FieldType.blob: AMPFieldType.BLOB,
+        FieldType.spatialobj: AMPFieldType.SPATIALOBJ,
+        FieldType.fixeddecimal: AMPFieldType.BCD,
+    }
+
+    _datetime_cosmetic_types_to_amp_type = {
+        FieldType.date: AMPFieldType.DATE,
+        FieldType.datetime: AMPFieldType.DATETIME,
+        FieldType.time: AMPFieldType.TIME,
+    }
+
+    _float_cosmetic_types_to_amp_type = {
+        FieldType.float: AMPFieldType.FLOAT,
+        FieldType.double: AMPFieldType.DOUBLE,
+    }
+
+    _int_cosmetic_types_to_amp_type = {
+        FieldType.byte: AMPFieldType.UINT8,
+        FieldType.int16: AMPFieldType.INT16,
+        FieldType.int32: AMPFieldType.INT32,
+        FieldType.int64: AMPFieldType.INT64,
+    }
+
+    _null_values = {NULL_VALUE_PLACEHOLDER, None}
+
+    @classmethod
+    def from_protobuf(cls, record: ProtobufRecord) -> List:
+        """
+        Convert a protobuf to a list of values.
+
+        Parameters
+        ----------
+        record
+            Protobuf representation of a record.
+
+        Returns
+        -------
+        List
+            All elements of the record as a list.
+        """
+        start_byte_idx = 0
+        parsed_record = []
+
+        for amp_type in record.types:
+            try:
+                packer = cls._field_type_to_packer[amp_type]
+            except KeyError:
+                raise ValueError(f"Packer not found for type: {amp_type}")
+
+            element, stride = packer.unpack(record.data, start_byte_idx)
+
+            parsed_record.append(element)
+            start_byte_idx += stride
+
+        return parsed_record
+
+    @classmethod
+    def to_protobuf(cls, record: Sequence, metadata: Metadata) -> ProtobufRecord:
+        """
+        Convert a sequence of values to a protobuf.
+
+        Parameters
+        ----------
+        record
+            A sequence of values.
+        metadata
+            Metadata associated with the values.
+
+        Returns
+        -------
+        ProtobufRecord
+            The protobuf representation of the passed in record.
+        """
+        assert len(record) == len(metadata)
+
+        raw_data = bytes()
+        amp_types = []
+        for field, element in zip(metadata, record):
+            try:
+                amp_type = cls._get_amp_type(field.type, element)
+                packer = cls._field_type_to_packer[amp_type]
+            except KeyError:
+                raise ValueError(f"Packer not found for type: {field.type}")
+
+            raw_data += packer.pack(element)
+            amp_types.append(amp_type)
+
+        return ProtobufRecord(data=raw_data, types=amp_types)
+
+    @classmethod
+    def _get_amp_type(cls, cosmetic_type: FieldType, element: Any) -> AMPFieldType:
+        if element in cls._null_values:
+            return AMPFieldType.NULL
+
+        if cosmetic_type in [
+            FieldType.string,
+            FieldType.v_string,
+            FieldType.v_wstring,
+            FieldType.wstring,
+        ]:
+            if element == "":
+                return AMPFieldType.EMPTYSTRING
+
+            return AMPFieldType.STRING
+
+        if cosmetic_type in cls._blob_cosmetic_types_to_amp_type:
+            return cls._blob_cosmetic_types_to_amp_type[cosmetic_type]
+
+        if cosmetic_type == FieldType.bool:
+            if element:
+                return AMPFieldType.BOOLTRUE
+
+            return AMPFieldType.BOOLFALSE
+
+        if element == 0:
+            return AMPFieldType.INT0
+
+        if cosmetic_type in cls._float_cosmetic_types_to_amp_type:
+            return cls._float_cosmetic_types_to_amp_type[cosmetic_type]
+
+        if cosmetic_type in cls._datetime_cosmetic_types_to_amp_type:
+            return cls._datetime_cosmetic_types_to_amp_type[cosmetic_type]
+
+        if cosmetic_type in cls._int_cosmetic_types_to_amp_type:
+            return cls._int_cosmetic_types_to_amp_type[cosmetic_type]
+
+        raise ValueError(
+            f"AMP field type not found for cosmetic type {cosmetic_type} and value {element}."
+        )