maxframe 0.1.0b5__cp311-cp311-macosx_11_0_arm64.whl

maxframe/serialization/exception.py

@@ -0,0 +1,86 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import traceback
+from typing import Dict, List
+
+from ..errors import MaxFrameError
+from ..lib import wrapped_pickle as pickle
+from .core import Serializer, buffered, pickle_buffers, unpickle_buffers
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteException(MaxFrameError):
+    def __init__(
+        self, messages: List[str], tracebacks: List[List[str]], buffers: List[bytes]
+    ):
+        self.messages = messages
+        self.tracebacks = tracebacks
+        self.buffers = buffers
+
+    @classmethod
+    def from_exception(cls, exc: Exception):
+        try:
+            buffers = pickle_buffers(exc)
+        except (TypeError, pickle.PicklingError):
+            logger.exception("Cannot pickle exception %s", exc)
+            buffers = []
+
+        messages, tracebacks = [], []
+        while exc is not None:
+            messages.append(str(exc))
+            tracebacks.append(traceback.format_tb(exc.__traceback__))
+            exc = exc.__cause__
+        return RemoteException(messages, tracebacks, buffers)
+
+    def get_buffers(self) -> List[bytes]:
+        return self.buffers
+
+    def get(self) -> Exception:
+        return unpickle_buffers(self.buffers) if self.buffers else self
+
+    def __str__(self):
+        tbs = []
+        for msg, tb in zip(self.messages, self.tracebacks):
+            tbs.append("".join([msg + "\n"] + tb))
+        return "\nCaused by:\n".join(tbs)
+
+
+class ExceptionSerializer(Serializer):
+    @buffered
+    def serial(self, obj: Exception, context: Dict):
+        if isinstance(obj, RemoteException):
+            messages, tracebacks, buffers = obj.messages, obj.tracebacks, obj.buffers
+        else:
+            remote_exc = RemoteException.from_exception(obj)
+            messages, tracebacks = remote_exc.messages, remote_exc.tracebacks
+            buffers = remote_exc.get_buffers()
+        return [messages, tracebacks], buffers, True
+
+    def deserial(self, serialized: List, context: Dict, subs: List):
+        messages, tracebacks = serialized[:2]
+        if subs and not pickle.is_unpickle_forbidden():
+            try:
+                return unpickle_buffers(subs)
+            except ImportError as ex:
+                logger.info(
+                    "Failed to load error from module %s, will raise a normal error",
+                    ex.name,
+                )
+        return RemoteException(messages, tracebacks, subs)
+
+
+ExceptionSerializer.register(Exception)

maxframe/serialization/maxframe_objects.py

@@ -0,0 +1,39 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, List
+
+try:
+    import scipy.sparse as sps
+except ImportError:  # pragma: no cover
+    sps = None
+
+from ..lib.sparse.array import SparseNDArray
+from .core import Serializer, buffered, deserialize, serialize
+
+
+class SparseNDArraySerializer(Serializer):
+    @buffered
+    def serial(self, obj: Any, context: Dict):
+        raw_header, raw_buffers = serialize(obj.raw, context)
+        return [raw_header, obj.shape], raw_buffers, True
+
+    def deserial(self, serialized: Dict, context: Dict, subs: List):
+        raw_header, obj_shape = serialized
+        raw_csr = deserialize(raw_header, subs)
+        return SparseNDArray(raw_csr, shape=tuple(obj_shape))
+
+
+if sps:  # pragma: no branch
+    SparseNDArraySerializer.register(SparseNDArray)

maxframe/serialization/numpy.py

@@ -0,0 +1,91 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, List
+
+import numpy as np
+
+from .core import Serializer, buffered
+
+_TYPE_CHAR_NP_GENERIC = "G"
+
+
+class NDArraySerializer(Serializer):
+    @buffered
+    def serial(self, obj: np.generic, context: Dict):
+        order = "C"
+        if obj.flags.f_contiguous:
+            order = "F"
+        elif not obj.flags.c_contiguous:
+            obj = np.ascontiguousarray(obj)
+        try:
+            desc = np.lib.format.dtype_to_descr(obj.dtype)
+            dtype_new_order = None
+        except ValueError:
+            # for structured dtype, array[[field2, field1]] will create a view,
+            # and dtype_to_desc will fail due to the order
+            fields = obj.dtype.fields
+            new_fields = sorted(fields, key=lambda k: fields[k][1])
+            desc = np.lib.format.dtype_to_descr(obj.dtype[new_fields])
+            dtype_new_order = list(fields)
+        type_char = _TYPE_CHAR_NP_GENERIC if not isinstance(obj, np.ndarray) else None
+
+        header = dict(
+            type=type_char,
+            descr=desc,
+            dtype_new_order=dtype_new_order,
+            shape=list(obj.shape),
+            strides=list(obj.strides),
+            order=order,
+        )
+        flattened = obj.ravel(order=order)
+        if obj.dtype.hasobject:
+            is_leaf = False
+            data = flattened.tolist()
+        else:
+            is_leaf = True
+            data = memoryview(flattened.view("uint8").data)
+        return [header], [data], is_leaf
+
+    def deserial(self, serialized: List, context: Dict, subs: List[Any]):
+        header = serialized[0]
+        try:
+            dtype = np.lib.format.descr_to_dtype(header["descr"])
+        except AttributeError:  # pragma: no cover
+            # for older numpy versions, descr_to_dtype is not implemented
+            dtype = np.dtype(header["descr"])
+
+        dtype_new_order = header["dtype_new_order"]
+        if dtype_new_order:
+            dtype = dtype[dtype_new_order]
+        if dtype.hasobject:
+            shape = header["shape"]
+            # fill empty object array
+            val = np.empty(shape, dtype=dtype)
+            val[(slice(None),) * len(shape)] = subs[0]
+        else:
+            val = np.ndarray(
+                shape=tuple(header["shape"]),
+                dtype=dtype,
+                buffer=subs[0],
+                strides=tuple(header["strides"]),
+                order=header["order"],
+            )
+        if header.get("type") == _TYPE_CHAR_NP_GENERIC:
+            return np.take(val, 0)
+        return val
+
+
+NDArraySerializer.register(np.generic)
+NDArraySerializer.register(np.ndarray)

maxframe/serialization/pandas.py

@@ -0,0 +1,202 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+import enum
+from typing import Any, Dict, List
+
+import pandas as pd
+from pandas.api.extensions import ExtensionArray
+from pandas.arrays import IntervalArray
+
+from ..utils import no_default
+from .core import Serializer, buffered
+
+
+class DataFrameSerializer(Serializer):
+    @buffered
+    def serial(self, obj: pd.DataFrame, context: Dict):
+        col_data = []
+        for _, col in obj.items():
+            if getattr(col.dtype, "hasobject", False):
+                col_data.append(col.tolist())
+            else:
+                col_data.append(col.values)
+        return [], [obj.dtypes, obj.index] + col_data, False
+
+    def deserial(
+        self, serialized: List, context: Dict, subs: List[Any]
+    ) -> pd.DataFrame:
+        dtypes, idx = subs[:2]
+        df = pd.concat([pd.Series(d, index=idx) for d in subs[2:]], axis=1)
+        df.columns = dtypes.index
+        df.index = idx
+        return df.astype(dtypes)
+
+
+class SeriesSerializer(Serializer):
+    @buffered
+    def serial(self, obj: pd.Series, context: Dict):
+        if getattr(obj.dtype, "hasobject", False):
+            data = obj.tolist()
+        else:
+            data = obj.values
+        return [], [obj.index, obj.name, obj.dtype, data], False
+
+    def deserial(self, serialized: List, context: Dict, subs: List[Any]) -> pd.Series:
+        idx, name, dtype, data = subs
+        return pd.Series(data, index=idx, name=name).astype(dtype)
+
+
+_TYPE_CHAR_MULTI_INDEX = "M"
+_TYPE_CHAR_RANGE_INDEX = "R"
+_TYPE_CHAR_CATEGORICAL_INDEX = "C"
+
+
+class IndexSerializer(Serializer):
+    @buffered
+    def serial(self, obj: pd.Index, context: Dict):
+        if isinstance(obj, pd.MultiIndex):
+            data = [obj.get_level_values(idx) for idx in range(obj.nlevels)]
+            header = [_TYPE_CHAR_MULTI_INDEX]
+        elif isinstance(obj, pd.RangeIndex):
+            data = [obj.name, obj.dtype]
+            header = [_TYPE_CHAR_RANGE_INDEX, obj.start, obj.stop, obj.step]
+        elif isinstance(obj, pd.CategoricalIndex):
+            data = [obj.name, obj.values]
+            header = [_TYPE_CHAR_CATEGORICAL_INDEX]
+        else:
+            if getattr(obj.dtype, "hasobject", False):
+                values = obj.tolist()
+            else:
+                values = obj.values
+            data = [obj.dtype, obj.name, values]
+            header = [None]
+        return header, data, len(data) == 0
+
+    def deserial(self, serialized: List, context: Dict, subs: List[Any]) -> pd.Index:
+        header = serialized
+        if header[0] == _TYPE_CHAR_MULTI_INDEX:
+            return pd.MultiIndex.from_arrays(subs)
+        elif header[0] == _TYPE_CHAR_RANGE_INDEX:
+            name, dtype = subs[:2]
+            start, stop, step = header[1:]
+            return pd.RangeIndex(start, stop, step, dtype=dtype, name=name)
+        elif header[0] == _TYPE_CHAR_CATEGORICAL_INDEX:
+            name, data = subs[:2]
+            return pd.CategoricalIndex(data, name=name)
+        elif header[0] is None:  # Normal index
+            dtype, name, values = subs
+            return pd.Index(values, dtype=dtype, name=name)
+        else:  # pragma: no cover
+            raise NotImplementedError(
+                f"Deserialization for index header label {header[0]} not implemented"
+            )
+
+
+class CategoricalSerializer(Serializer):
+    @buffered
+    def serial(self, obj: pd.Categorical, context: Dict):
+        return [obj.ordered], [obj.codes, obj.dtype, obj.categories], False
+
+    def deserial(
+        self, serialized: List, context: Dict, subs: List[Any]
+    ) -> pd.Categorical:
+        codes, dtype, categories = subs
+        ordered = serialized[0]
+        return pd.Categorical.from_codes(codes, categories, ordered=ordered)
+
+
+_TYPE_CHAR_INTERVAL_ARRAY = "I"
+
+
+class ArraySerializer(Serializer):
+    @buffered
+    def serial(self, obj: ExtensionArray, context: Dict):
+        ser_type = None
+        dtype = obj.dtype
+        if isinstance(obj.dtype, pd.IntervalDtype):
+            ser_type = _TYPE_CHAR_INTERVAL_ARRAY
+            data_parts = [obj.left, obj.right]
+        elif isinstance(obj.dtype, pd.StringDtype):
+            if hasattr(obj, "tolist"):
+                data_parts = [obj.tolist()]
+            else:
+                data_parts = [obj.to_numpy().tolist()]
+        else:
+            data_parts = [getattr(obj, "_data")]
+        return [ser_type], [dtype] + data_parts, False
+
+    def deserial(self, serialized: List, context: Dict, subs: List):
+        if serialized[0] == _TYPE_CHAR_INTERVAL_ARRAY:
+            dtype, left, right = subs
+            return IntervalArray.from_arrays(left, right, dtype=dtype)
+        else:
+            dtype, data = subs
+            return pd.array(data, dtype)
+
+
+class PdTimestampSerializer(Serializer):
+    def serial(self, obj: pd.Timestamp, context: Dict):
+        if obj.tz:
+            zone_info = [obj.tz]
+            ts = obj.timestamp()
+        else:
+            zone_info = []
+            ts = obj.to_pydatetime().timestamp()
+        return (
+            [int(ts), obj.microsecond, obj.nanosecond],
+            zone_info,
+            bool(zone_info),
+        )
+
+    def deserial(self, serialized: List, context: Dict, subs: List):
+        if subs:
+            val = pd.Timestamp.utcfromtimestamp(serialized[0]).replace(
+                microsecond=serialized[1], nanosecond=serialized[2]
+            )
+            val = val.replace(tzinfo=datetime.timezone.utc).tz_convert(subs[0])
+        else:
+            val = pd.Timestamp.fromtimestamp(serialized[0]).replace(
+                microsecond=serialized[1], nanosecond=serialized[2]
+            )
+        return val
+
+
+class PdTimedeltaSerializer(Serializer):
+    def serial(self, obj: pd.Timedelta, context: Dict):
+        return [int(obj.seconds), obj.microseconds, obj.nanoseconds], [], True
+
+    def deserial(self, serialized: List, context: Dict, subs: List):
+        return pd.Timedelta(
+            seconds=serialized[0], microseconds=serialized[1], nanoseconds=serialized[2]
+        )
+
+
+class NoDefaultSerializer(Serializer):
+    def serial(self, obj: enum.Enum, context: Dict):
+        return [], [], True
+
+    def deserial(self, serialized: List, context: Dict, subs: List):
+        return no_default
+
+
+DataFrameSerializer.register(pd.DataFrame)
+SeriesSerializer.register(pd.Series)
+IndexSerializer.register(pd.Index)
+CategoricalSerializer.register(pd.Categorical)
+ArraySerializer.register(ExtensionArray)
+PdTimestampSerializer.register(pd.Timestamp)
+PdTimedeltaSerializer.register(pd.Timedelta)
+NoDefaultSerializer.register(type(no_default))

maxframe/serialization/scipy.py

@@ -0,0 +1,71 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, List
+
+import numpy as np
+
+try:
+    import scipy.sparse as sps
+except ImportError:  # pragma: no cover
+    sps = None
+
+from .core import Serializer, buffered, deserialize, serialize
+
+
+class CsrMatrixSerializer(Serializer):
+    @buffered
+    def serial(self, obj: Any, context: Dict):
+        data_header, data_buffers = serialize(obj.data)
+        idx_header, idx_buffers = serialize(obj.indices)
+        indptr_header, indptr_buffers = serialize(obj.indptr)
+        header = [
+            data_header,  # data_header
+            len(data_buffers),  # data_buf_num
+            idx_header,  # idx_header
+            len(idx_buffers),  # idx_buf_num
+            indptr_header,  # indptr_header
+            obj.shape,  # shape
+        ]
+        return header, data_buffers + idx_buffers + indptr_buffers, True
+
+    def deserial(self, serialized: List, context: Dict, subs: List):
+        (
+            data_header,
+            data_buf_num,
+            idx_header,
+            idx_buf_num,
+            indptr_header,
+            shape,
+        ) = serialized
+        data_buffers = subs[:data_buf_num]
+        idx_buffers = subs[data_buf_num : data_buf_num + idx_buf_num]
+        indptr_buffers = subs[data_buf_num + idx_buf_num :]
+
+        data = deserialize(data_header, data_buffers)
+        indices = deserialize(idx_header, idx_buffers)
+        indptr = deserialize(indptr_header, indptr_buffers)
+        shape = tuple(shape)
+
+        empty_arr = np.zeros(0, dtype=data.dtype)
+
+        target_csr = sps.coo_matrix(
+            (empty_arr, (empty_arr,) * 2), dtype=data.dtype, shape=shape
+        ).tocsr()
+        target_csr.data, target_csr.indices, target_csr.indptr = data, indices, indptr
+        return target_csr
+
+
+if sps:  # pragma: no branch
+    CsrMatrixSerializer.register(sps.csr_matrix)

maxframe/serialization/serializables/__init__.py

@@ -0,0 +1,55 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .core import Serializable, SerializableMeta
+from .field import (
+    AnyField,
+    BoolField,
+    BytesField,
+    Complex64Field,
+    Complex128Field,
+    DataFrameField,
+    DataTypeField,
+    Datetime64Field,
+    DictField,
+    EnumField,
+    Float16Field,
+    Float32Field,
+    Float64Field,
+    FunctionField,
+    IdentityField,
+    IndexField,
+    Int8Field,
+    Int16Field,
+    Int32Field,
+    Int64Field,
+    IntervalArrayField,
+    KeyField,
+    ListField,
+    NamedTupleField,
+    NDArrayField,
+    OneOfField,
+    ReferenceField,
+    SeriesField,
+    SliceField,
+    StringField,
+    Timedelta64Field,
+    TupleField,
+    TZInfoField,
+    UInt8Field,
+    UInt16Field,
+    UInt32Field,
+    UInt64Field,
+)
+from .field_type import FieldTypes