maxframe 0.1.0b5__cp39-cp39-win32.whl → 1.0.0__cp39-cp39-win32.whl

maxframe/serialization/serializables/tests/test_serializable.py

@@ -21,6 +21,7 @@ import pytest
 
 from ....core import EntityData
 from ....lib.wrapped_pickle import switch_unpickle
+from ....utils import no_default
 from ... import deserialize, serialize
 from .. import (
     AnyField,
@@ -94,6 +95,11 @@ class MySimpleSerializable(Serializable):
     _ref_val = ReferenceField("ref_val", "MySimpleSerializable")
 
 
+class MySubSerializable(MySimpleSerializable):
+    _m_int_val = Int64Field("m_int_val", default=250)
+    _m_str_val = StringField("m_str_val", default="SUB_STR")
+
+
 class MySerializable(Serializable):
     _id = IdentityField("id")
     _any_val = AnyField("any_val")
@@ -138,10 +144,11 @@ class MySerializable(Serializable):
         oneof1_val=f"{__name__}.MySerializable",
         oneof2_val=MySimpleSerializable,
     )
+    _no_default_val = Float64Field("no_default_val", default=no_default)
 
 
 @pytest.mark.parametrize("set_is_ci", [False, True], indirect=True)
-@switch_unpickle
+@switch_unpickle(forbidden=False)
 def test_serializable(set_is_ci):
     my_serializable = MySerializable(
         _id="1",
@@ -165,7 +172,9 @@ def test_serializable(set_is_ci):
         _key_val=MyHasKey("aaa"),
         _ndarray_val=np.random.rand(4, 3),
         _datetime64_val=pd.Timestamp(123),
-        _timedelta64_val=pd.Timedelta(days=1),
+        _timedelta64_val=pd.Timedelta(
+            days=1, seconds=123, microseconds=345, nanoseconds=132
+        ),
         _datatype_val=np.dtype(np.int32),
         _index_val=pd.Index([1, 2]),
         _series_val=pd.Series(["a", "b"]),
@@ -180,6 +189,7 @@ def test_serializable(set_is_ci):
         _dict_val={"a": b"bytes_value"},
         _ref_val=MySerializable(),
         _oneof_val=MySerializable(_id="2"),
+        _no_default_val=no_default,
     )
 
     header, buffers = serialize(my_serializable)
@@ -187,12 +197,51 @@ def test_serializable(set_is_ci):
     _assert_serializable_eq(my_serializable, my_serializable2)
 
 
+@pytest.mark.parametrize("set_is_ci", [False, True], indirect=True)
+@switch_unpickle
+def test_compatible_serializable(set_is_ci):
+    global MySimpleSerializable, MySubSerializable
+
+    old_base, old_sub = MySimpleSerializable, MySubSerializable
+
+    try:
+        my_sub_serializable = MySubSerializable(
+            _id="id_val",
+            _list_val=["abcd", "wxyz"],
+            _ref_val=MyHasKey(),
+            _m_int_val=3412,
+            _m_str_val="dfgghj",
+        )
+        header, buffers = serialize(my_sub_serializable)
+
+        class MySimpleSerializable(Serializable):
+            _id = IdentityField("id")
+            _int_val = Int64Field("int_val", default=1000)
+            _list_val = ListField("list_val", default_factory=list)
+            _ref_val = ReferenceField("ref_val", "MySimpleSerializable")
+            _dict_val = DictField("dict_val")
+
+        class MySubSerializable(MySimpleSerializable):
+            _m_int_val = Int64Field("m_int_val", default=250)
+            _m_str_val = StringField("m_str_val", default="SUB_STR")
+
+        my_sub_serializable2 = deserialize(header, buffers)
+        assert type(my_sub_serializable) is not type(my_sub_serializable2)
+        _assert_serializable_eq(my_sub_serializable, my_sub_serializable2)
+    finally:
+        MySimpleSerializable, MySubSerializable = old_base, old_sub
+
+
 def _assert_serializable_eq(my_serializable, my_serializable2):
     for field_name, field in my_serializable._FIELDS.items():
-        if not hasattr(my_serializable, field.tag):
+        if not hasattr(my_serializable, field.name):
             continue
         expect_value = getattr(my_serializable, field_name)
-        actual_value = getattr(my_serializable2, field_name)
+        if expect_value is no_default:
+            assert not hasattr(my_serializable2, field.name)
+            continue
+        else:
+            actual_value = getattr(my_serializable2, field_name)
         if isinstance(expect_value, np.ndarray):
             np.testing.assert_array_equal(expect_value, actual_value)
         elif isinstance(expect_value, pd.DataFrame):
@@ -208,7 +257,7 @@ def _assert_serializable_eq(my_serializable, my_serializable2):
         elif callable(expect_value):
             assert expect_value(1) == actual_value(1)
         else:
-            assert expect_value == actual_value
+            assert expect_value == actual_value, f"Field {field_name}"
 
 
 @pytest.mark.parametrize("set_is_ci", [True], indirect=True)

maxframe/serialization/tests/test_serial.py

@@ -42,7 +42,7 @@ except ImportError:
 from ...lib.sparse import SparseMatrix
 from ...lib.wrapped_pickle import switch_unpickle
 from ...tests.utils import require_cudf, require_cupy
-from ...utils import lazy_import
+from ...utils import lazy_import, no_default
 from .. import (
     PickleContainer,
     RemoteException,
@@ -90,6 +90,7 @@ class CustomNamedTuple(NamedTuple):
         pd.Timedelta(102.234154131),
         {"abc": 5.6, "def": [3.4], "gh": None, "ijk": {}},
         OrderedDict([("abcd", 5.6)]),
+        no_default,
     ],
 )
 @switch_unpickle

maxframe/session.py

@@ -150,6 +150,10 @@ class AbstractSession(ABC):
     def session_id(self):
         return self._session_id
 
+    @property
+    def closed(self) -> bool:
+        return self._closed
+
     def __eq__(self, other):
         return (
             isinstance(other, AbstractSession)
@@ -1283,9 +1287,12 @@ def get_default_or_create(**kwargs):
         if session is None:
             # no session attached, try to create one
             warnings.warn(warning_msg)
-            session = new_session(
-                ODPS.from_global() or ODPS.from_environments(), **kwargs
+            odps_entry = (
+                kwargs.pop("odps_entry", None)
+                or ODPS.from_global()
+                or ODPS.from_environments()
             )
+            session = new_session(odps_entry=odps_entry, **kwargs)
             session.as_default()
     if isinstance(session, IsolatedAsyncSession):
         session = SyncSession.from_isolated_session(session)

maxframe/tensor/__init__.py

@@ -114,7 +114,6 @@ from .arithmetic import (
 )
 from .arithmetic import truediv as true_divide
 from .arithmetic import trunc
-from .base import broadcast_to, transpose, unique, where
 from .core import Tensor
 from .datasource import (
     arange,
@@ -143,7 +142,16 @@ from .indexing import (
     take,
     unravel_index,
 )
-from .merge import stack
+from .merge import concatenate, stack, vstack
+from .misc import (
+    atleast_1d,
+    atleast_2d,
+    atleast_3d,
+    broadcast_to,
+    transpose,
+    unique,
+    where,
+)
 from .rechunk import rechunk
 from .reduction import (
     all,
@@ -180,4 +188,75 @@ from .reduction import std, sum, var
 from .reshape import reshape
 from .ufunc import ufunc
 
+# isort: off
+# noinspection PyUnresolvedReferences
+from numpy import (
+    e,
+    errstate,
+    geterr,
+    inf,
+    nan,
+    newaxis,
+    pi,
+    seterr,
+)
+
+try:
+    from numpy.exceptions import AxisError
+except ImportError:
+    from numpy import AxisError
+
+NAN = nan
+NINF = -inf
+Inf = inf
+NaN = nan
+
+# import numpy types
+# noinspection PyUnresolvedReferences
+from numpy import (
+    bool_ as bool,
+    bytes_,
+    character,
+    complex64,
+    complex128,
+    complexfloating,
+    datetime64,
+    double,
+    dtype,
+    flexible,
+    float16,
+    float32,
+    float64,
+    floating,
+    generic,
+    inexact,
+    int8,
+    int16,
+    int32,
+    int64,
+    intc,
+    intp,
+    number,
+    integer,
+    object_ as object,
+    signedinteger,
+    timedelta64,
+    uint,
+    uint8,
+    uint16,
+    uint32,
+    uint64,
+    unsignedinteger,
+    void,
+)
+
+try:
+    from numpy import cfloat
+except ImportError:
+    from numpy import cdouble as cfloat
+try:
+    from numpy import str_ as unicode_
+except ImportError:
+    from numpy import unicode_
+
 del fetch, ufunc

maxframe/tensor/arithmetic/isclose.py

@@ -23,6 +23,7 @@ from .core import TensorBinOp
 
 class TensorIsclose(TensorBinOp):
     _op_type_ = opcodes.ISCLOSE
+    _func_name = "isclose"
 
     rtol = Float64Field("rtol", default=None)
     atol = Float64Field("atol", default=None)

maxframe/tensor/arithmetic/tests/test_arithmetic.py

@@ -17,26 +17,13 @@
 import numpy as np
 import pytest
 
+from maxframe.tensor.arithmetic.core import TensorBinOp, TensorUnaryOp
+from maxframe.utils import collect_leaf_operators
+
 from ....core import enter_mode
 from ...core import SparseTensor, Tensor
 from ...datasource import array, empty, ones, tensor
-from .. import (
-    TensorAdd,
-    TensorGreaterThan,
-    TensorIsclose,
-    TensorLog,
-    TensorSubtract,
-    add,
-    around,
-    cos,
-    frexp,
-    isclose,
-    isfinite,
-    log,
-    negative,
-    subtract,
-    truediv,
-)
+from .. import *  # noqa: F401
 
 
 def test_add():
@@ -252,7 +239,7 @@ def test_compare():
 
 def test_frexp():
     t1 = ones((3, 4, 5), chunk_size=2)
-    t2 = empty((3, 4, 5), dtype=np.float_, chunk_size=2)
+    t2 = empty((3, 4, 5), dtype=np.dtype(float), chunk_size=2)
     op_type = type(t1.op)
 
     o1, o2 = frexp(t1)
@@ -412,3 +399,20 @@ def test_build_mode():
 
     with enter_mode(build=True):
         assert t1 != 2
+
+
+def test_unary_op_func_name():
+    # make sure all the unary op has defined the func name.
+
+    results = collect_leaf_operators(TensorUnaryOp)
+    for op_type in results:
+        assert hasattr(op_type, "_func_name")
+
+
+def test_binary_op_func_name():
+    # make sure all the binary op has defined the func name.
+
+    results = collect_leaf_operators(TensorBinOp)
+    for op_type in results:
+        if op_type not in (TensorSetImag, TensorSetReal):
+            assert hasattr(op_type, "_func_name")

maxframe/tensor/core.py

@@ -23,8 +23,6 @@ from typing import Any, Dict
 import numpy as np
 
 from ..core import (
-    Chunk,
-    ChunkData,
     HasShapeTileable,
     HasShapeTileableData,
     OutputType,
@@ -36,14 +34,9 @@ from ..core.entity.utils import refresh_tileable_shape
 from ..serialization.serializables import (
     AnyField,
     DataTypeField,
-    EnumField,
-    FieldTypes,
-    ListField,
     Serializable,
     StringField,
-    TupleField,
 )
-from ..utils import on_deserialize_shape, on_serialize_shape, skip_na_call
 from .utils import fetch_corner_data, get_chunk_slices
 
 logger = logging.getLogger(__name__)
@@ -56,134 +49,18 @@ class TensorOrder(Enum):
     F_ORDER = "F"
 
 
-class TensorChunkData(ChunkData):
-    __slots__ = ()
-    _no_copy_attrs_ = ChunkData._no_copy_attrs_ | {"dtype"}
-    type_name = "Tensor"
-
-    # required fields
-    _shape = TupleField(
-        "shape",
-        FieldTypes.int64,
-        on_serialize=on_serialize_shape,
-        on_deserialize=on_deserialize_shape,
-    )
-    _order = EnumField("order", TensorOrder, FieldTypes.string)
-    # optional fields
-    _dtype = DataTypeField("dtype")
-
-    def __init__(self, op=None, index=None, shape=None, dtype=None, order=None, **kw):
-        if isinstance(order, str):
-            order = getattr(TensorOrder, order)
-        super().__init__(
-            _op=op, _index=index, _shape=shape, _dtype=dtype, _order=order, **kw
-        )
-        if self.order is None and self.op is not None:
-            if len(self.inputs) == 0:
-                self._order = TensorOrder.C_ORDER
-            elif all(
-                hasattr(inp, "order") and inp.order == TensorOrder.F_ORDER
-                for inp in self.inputs
-            ):
-                self._order = TensorOrder.F_ORDER
-            else:
-                self._order = TensorOrder.C_ORDER
-
-    @property
-    def params(self) -> Dict[str, Any]:
-        # params return the properties which useful to rebuild a new chunk
-        return {
-            "shape": self.shape,
-            "dtype": self.dtype,
-            "order": self.order,
-            "index": self.index,
-        }
-
-    @params.setter
-    def params(self, new_params: Dict[str, Any]):
-        params = new_params.copy()
-        params.pop("index", None)  # index not needed to update
-        new_shape = params.pop("shape", None)
-        if new_shape is not None:
-            self._shape = new_shape
-        dtype = params.pop("dtype", None)
-        if dtype is not None:
-            self._dtype = dtype
-        order = params.pop("order", None)
-        if order is not None:
-            self._order = order
-        if params:  # pragma: no cover
-            raise TypeError(f"Unknown params: {list(params)}")
-
-    @classmethod
-    def get_params_from_data(cls, data: np.ndarray) -> Dict[str, Any]:
-        from .array_utils import is_cupy
-
-        if not is_cupy(data):
-            data = np.asarray(data)
-        order = (
-            TensorOrder.C_ORDER if data.flags["C_CONTIGUOUS"] else TensorOrder.F_ORDER
-        )
-        return {"shape": data.shape, "dtype": data.dtype, "order": order}
-
-    def __len__(self):
-        try:
-            return self.shape[0]
-        except IndexError:
-            if is_build_mode():
-                return 0
-            raise TypeError("len() of unsized object")
-
-    @property
-    def shape(self):
-        return getattr(self, "_shape", None)
-
-    @property
-    def ndim(self):
-        return len(self.shape)
-
-    @property
-    def size(self):
-        return np.prod(self.shape).item()
-
-    @property
-    def dtype(self):
-        return getattr(self, "_dtype", None) or self.op.dtype
-
-    @property
-    def order(self):
-        return getattr(self, "_order", None)
-
-    @property
-    def nbytes(self):
-        return np.prod(self.shape) * self.dtype.itemsize
-
-
-class TensorChunk(Chunk):
-    __slots__ = ()
-    _allow_data_type_ = (TensorChunkData,)
-    type_name = "Tensor"
-
-    def __len__(self):
-        return len(self._data)
-
-
 class TensorData(HasShapeTileableData, _ExecuteAndFetchMixin):
     __slots__ = ()
     type_name = "Tensor"
 
+    _legacy_deprecated_non_primitives = ["_chunks"]
+
     # required fields
     _order = StringField(
         "order", on_serialize=attrgetter("value"), on_deserialize=TensorOrder
     )
     # optional fields
     _dtype = DataTypeField("dtype")
-    _chunks = ListField(
-        "chunks",
-        FieldTypes.reference(TensorChunkData),
-        on_serialize=skip_na_call(lambda x: [it.data for it in x]),
-        on_deserialize=skip_na_call(lambda x: [TensorChunk(it) for it in x]),
-    )
 
     def __init__(
         self,
@@ -318,7 +195,7 @@ class TensorData(HasShapeTileableData, _ExecuteAndFetchMixin):
         return fromsparse(self, fill_value=fill_value)
 
     def transpose(self, *axes):
-        from .base import transpose
+        from .misc import transpose
 
         if len(axes) == 1 and isinstance(axes[0], Iterable):
             axes = axes[0]
@@ -346,11 +223,6 @@ class TensorData(HasShapeTileableData, _ExecuteAndFetchMixin):
 
         return reshape(self, shape, order=order)
 
-    def totiledb(self, uri, ctx=None, key=None, timestamp=None):
-        from .datastore import totiledb
-
-        return totiledb(uri, self, ctx=ctx, key=key, timestamp=timestamp)
-
     @staticmethod
     def from_dataframe(in_df):
         from .datasource import from_dataframe
@@ -526,9 +398,6 @@ class Tensor(HasShapeTileable):
         """
         return self._data.T
 
-    def totiledb(self, uri, ctx=None, key=None, timestamp=None):
-        return self._data.totiledb(uri, ctx=ctx, key=key, timestamp=timestamp)
-
     def copy(self, order="C"):
         return super().copy().astype(self.dtype, order=order, copy=False)
 
@@ -589,7 +458,7 @@ class Tensor(HasShapeTileable):
         array([('c', 1), ('a', 2)],
               dtype=[('x', '|S1'), ('y', '<i4')])
         """
-        from .base import sort
+        from .misc import sort
 
         self._data = sort(
             self,
@@ -651,7 +520,7 @@ class Tensor(HasShapeTileable):
         >>> a.execute()
         array([1, 2, 3, 4])
         """
-        from .base import partition
+        from .misc import partition
 
         self._data = partition(self, kth, axis=axis, kind=kind, order=order, **kw).data
 

maxframe/tensor/datasource/array.py

@@ -53,6 +53,9 @@ class ArrayDataSource(TensorNoInput):
 
         super().__init__(data=data, dtype=dtype, gpu=gpu, **kw)
 
+    def get_data(self):
+        return self.data
+
 
 class CSRMatrixDataSource(TensorNoInput):
     """

maxframe/tensor/datasource/full.py

@@ -89,7 +89,7 @@ def full(shape, fill_value, dtype=None, chunk_size=None, gpu=None, order="C"):
     """
     v = np.asarray(fill_value)
     if len(v.shape) > 0:
-        from ..base import broadcast_to
+        from ..misc import broadcast_to
 
         return broadcast_to(
             tensor(v, dtype=dtype, chunk_size=chunk_size, gpu=gpu, order=order), shape

maxframe/tensor/datasource/tests/test_datasource.py

@@ -141,7 +141,7 @@ def test_zeros():
 
 
 def test_data_source():
-    from ...base.broadcast_to import TensorBroadcastTo
+    from ...misc.broadcast_to import TensorBroadcastTo
 
     data = np.random.random((10, 3))
     t = tensor(data, chunk_size=2)

maxframe/tensor/indexing/flatnonzero.py

@@ -55,6 +55,6 @@ def flatnonzero(a):
     >>> x.ravel()[mt.flatnonzero(x)].execute()  # TODO(jisheng): accomplish this after fancy indexing is supported
 
     """
-    from ..base import ravel
+    from ..misc import ravel
 
     return nonzero(ravel(a))[0]

maxframe/tensor/indexing/getitem.py

@@ -130,6 +130,8 @@ def _calc_order(a, index):
             continue
         elif isinstance(ind, slice):
             shape = a.shape[in_axis]
+            if shape is np.nan:
+                return TensorOrder.C_ORDER
             slc = ind.indices(shape)
             if slc[0] == 0 and slc[1] == shape and slc[2] == 1:
                 continue

maxframe/tensor/merge/__init__.py

@@ -12,4 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from .concatenate import concatenate
 from .stack import stack
+from .vstack import vstack

maxframe/tensor/merge/concatenate.py

@@ -0,0 +1,101 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import numpy as np
+
+from ... import opcodes
+from ...serialization.serializables import Int32Field
+from ..datasource import tensor as astensor
+from ..operators import TensorOperator, TensorOperatorMixin
+from ..utils import validate_axis
+
+
+class TensorConcatenate(TensorOperator, TensorOperatorMixin):
+    _op_type_ = opcodes.CONCATENATE
+
+    axis = Int32Field("axis", default=0)
+
+    def __call__(self, tensors):
+        axis = self.axis
+        shape = _calc_concatenate_shape(tensors, axis)
+        shape[axis] = sum(t.shape[axis] for t in tensors)
+        return self.new_tensor(tensors, shape=tuple(shape))
+
+
+def concatenate(tensors, axis=0):
+    """
+    Join a sequence of arrays along an existing axis.
+
+    Parameters
+    ----------
+    a1, a2, ... : sequence of array_like
+        The tensors must have the same shape, except in the dimension
+        corresponding to `axis` (the first, by default).
+    axis : int, optional
+        The axis along which the tensors will be joined.  Default is 0.
+
+    Returns
+    -------
+    res : Tensor
+        The concatenated tensor.
+
+    See Also
+    --------
+    stack : Stack a sequence of tensors along a new axis.
+    vstack : Stack tensors in sequence vertically (row wise)
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+
+    >>> a = mt.array([[1, 2], [3, 4]])
+    >>> b = mt.array([[5, 6]])
+    >>> mt.concatenate((a, b), axis=0).execute()
+    array([[1, 2],
+           [3, 4],
+           [5, 6]])
+    >>> mt.concatenate((a, b.T), axis=1).execute()
+    array([[1, 2, 5],
+           [3, 4, 6]])
+
+    """
+    if axis is None:
+        axis = 0
+    tensors = [astensor(t) for t in tensors]
+    axis = validate_axis(tensors[0].ndim, axis)
+
+    if len(set(t.ndim for t in tensors)) != 1:
+        raise ValueError("all the input tensors must have same number of dimensions")
+
+    shapes = [t.shape[:axis] + t.shape[axis + 1 :] for t in tensors]
+    if len(set(shapes)) != 1:
+        raise ValueError(
+            "all the input tensor dimensions "
+            "except for the concatenation axis must match exactly"
+        )
+    shape = _calc_concatenate_shape(tensors, axis)
+    if any(np.isnan(s) for i, s in enumerate(shape) if i != axis):
+        raise ValueError("cannot concatenate tensor with unknown shape")
+
+    return _concatenate(tensors, axis)
+
+
+def _concatenate(tensors, axis=0):
+    dtype = np.result_type(*(t.dtype for t in tensors))
+
+    op = TensorConcatenate(axis=axis, dtype=dtype)
+    return op(tensors)
+
+
+def _calc_concatenate_shape(tensors, axis):
+    return [0 if i == axis else tensors[0].shape[i] for i in range(tensors[0].ndim)]