maxframe 0.1.0b4__cp311-cp311-win32.whl → 1.0.0__cp311-cp311-win32.whl

maxframe/tensor/__init__.py

@@ -114,7 +114,6 @@ from .arithmetic import (
 )
 from .arithmetic import truediv as true_divide
 from .arithmetic import trunc
-from .base import broadcast_to, transpose, where
 from .core import Tensor
 from .datasource import (
     arange,
@@ -143,7 +142,16 @@ from .indexing import (
     take,
     unravel_index,
 )
-from .merge import stack
+from .merge import concatenate, stack, vstack
+from .misc import (
+    atleast_1d,
+    atleast_2d,
+    atleast_3d,
+    broadcast_to,
+    transpose,
+    unique,
+    where,
+)
 from .rechunk import rechunk
 from .reduction import (
     all,
@@ -180,4 +188,75 @@ from .reduction import std, sum, var
 from .reshape import reshape
 from .ufunc import ufunc
 
+# isort: off
+# noinspection PyUnresolvedReferences
+from numpy import (
+    e,
+    errstate,
+    geterr,
+    inf,
+    nan,
+    newaxis,
+    pi,
+    seterr,
+)
+
+try:
+    from numpy.exceptions import AxisError
+except ImportError:
+    from numpy import AxisError
+
+NAN = nan
+NINF = -inf
+Inf = inf
+NaN = nan
+
+# import numpy types
+# noinspection PyUnresolvedReferences
+from numpy import (
+    bool_ as bool,
+    bytes_,
+    character,
+    complex64,
+    complex128,
+    complexfloating,
+    datetime64,
+    double,
+    dtype,
+    flexible,
+    float16,
+    float32,
+    float64,
+    floating,
+    generic,
+    inexact,
+    int8,
+    int16,
+    int32,
+    int64,
+    intc,
+    intp,
+    number,
+    integer,
+    object_ as object,
+    signedinteger,
+    timedelta64,
+    uint,
+    uint8,
+    uint16,
+    uint32,
+    uint64,
+    unsignedinteger,
+    void,
+)
+
+try:
+    from numpy import cfloat
+except ImportError:
+    from numpy import cdouble as cfloat
+try:
+    from numpy import str_ as unicode_
+except ImportError:
+    from numpy import unicode_
+
 del fetch, ufunc

maxframe/tensor/arithmetic/isclose.py

@@ -23,6 +23,7 @@ from .core import TensorBinOp
 
 class TensorIsclose(TensorBinOp):
     _op_type_ = opcodes.ISCLOSE
+    _func_name = "isclose"
 
     rtol = Float64Field("rtol", default=None)
     atol = Float64Field("atol", default=None)

maxframe/tensor/arithmetic/tests/test_arithmetic.py

@@ -17,26 +17,13 @@
 import numpy as np
 import pytest
 
+from maxframe.tensor.arithmetic.core import TensorBinOp, TensorUnaryOp
+from maxframe.utils import collect_leaf_operators
+
 from ....core import enter_mode
 from ...core import SparseTensor, Tensor
 from ...datasource import array, empty, ones, tensor
-from .. import (
-    TensorAdd,
-    TensorGreaterThan,
-    TensorIsclose,
-    TensorLog,
-    TensorSubtract,
-    add,
-    around,
-    cos,
-    frexp,
-    isclose,
-    isfinite,
-    log,
-    negative,
-    subtract,
-    truediv,
-)
+from .. import *  # noqa: F401
 
 
 def test_add():
@@ -252,7 +239,7 @@ def test_compare():
 
 def test_frexp():
     t1 = ones((3, 4, 5), chunk_size=2)
-    t2 = empty((3, 4, 5), dtype=np.float_, chunk_size=2)
+    t2 = empty((3, 4, 5), dtype=np.dtype(float), chunk_size=2)
     op_type = type(t1.op)
 
     o1, o2 = frexp(t1)
@@ -412,3 +399,20 @@ def test_build_mode():
 
     with enter_mode(build=True):
         assert t1 != 2
+
+
+def test_unary_op_func_name():
+    # make sure all the unary op has defined the func name.
+
+    results = collect_leaf_operators(TensorUnaryOp)
+    for op_type in results:
+        assert hasattr(op_type, "_func_name")
+
+
+def test_binary_op_func_name():
+    # make sure all the binary op has defined the func name.
+
+    results = collect_leaf_operators(TensorBinOp)
+    for op_type in results:
+        if op_type not in (TensorSetImag, TensorSetReal):
+            assert hasattr(op_type, "_func_name")

maxframe/tensor/core.py

@@ -23,8 +23,6 @@ from typing import Any, Dict
 import numpy as np
 
 from ..core import (
-    Chunk,
-    ChunkData,
     HasShapeTileable,
     HasShapeTileableData,
     OutputType,
@@ -36,14 +34,9 @@ from ..core.entity.utils import refresh_tileable_shape
 from ..serialization.serializables import (
     AnyField,
     DataTypeField,
-    EnumField,
-    FieldTypes,
-    ListField,
     Serializable,
     StringField,
-    TupleField,
 )
-from ..utils import on_deserialize_shape, on_serialize_shape, skip_na_call
 from .utils import fetch_corner_data, get_chunk_slices
 
 logger = logging.getLogger(__name__)
@@ -56,134 +49,18 @@ class TensorOrder(Enum):
     F_ORDER = "F"
 
 
-class TensorChunkData(ChunkData):
-    __slots__ = ()
-    _no_copy_attrs_ = ChunkData._no_copy_attrs_ | {"dtype"}
-    type_name = "Tensor"
-
-    # required fields
-    _shape = TupleField(
-        "shape",
-        FieldTypes.int64,
-        on_serialize=on_serialize_shape,
-        on_deserialize=on_deserialize_shape,
-    )
-    _order = EnumField("order", TensorOrder, FieldTypes.string)
-    # optional fields
-    _dtype = DataTypeField("dtype")
-
-    def __init__(self, op=None, index=None, shape=None, dtype=None, order=None, **kw):
-        if isinstance(order, str):
-            order = getattr(TensorOrder, order)
-        super().__init__(
-            _op=op, _index=index, _shape=shape, _dtype=dtype, _order=order, **kw
-        )
-        if self.order is None and self.op is not None:
-            if len(self.inputs) == 0:
-                self._order = TensorOrder.C_ORDER
-            elif all(
-                hasattr(inp, "order") and inp.order == TensorOrder.F_ORDER
-                for inp in self.inputs
-            ):
-                self._order = TensorOrder.F_ORDER
-            else:
-                self._order = TensorOrder.C_ORDER
-
-    @property
-    def params(self) -> Dict[str, Any]:
-        # params return the properties which useful to rebuild a new chunk
-        return {
-            "shape": self.shape,
-            "dtype": self.dtype,
-            "order": self.order,
-            "index": self.index,
-        }
-
-    @params.setter
-    def params(self, new_params: Dict[str, Any]):
-        params = new_params.copy()
-        params.pop("index", None)  # index not needed to update
-        new_shape = params.pop("shape", None)
-        if new_shape is not None:
-            self._shape = new_shape
-        dtype = params.pop("dtype", None)
-        if dtype is not None:
-            self._dtype = dtype
-        order = params.pop("order", None)
-        if order is not None:
-            self._order = order
-        if params:  # pragma: no cover
-            raise TypeError(f"Unknown params: {list(params)}")
-
-    @classmethod
-    def get_params_from_data(cls, data: np.ndarray) -> Dict[str, Any]:
-        from .array_utils import is_cupy
-
-        if not is_cupy(data):
-            data = np.asarray(data)
-        order = (
-            TensorOrder.C_ORDER if data.flags["C_CONTIGUOUS"] else TensorOrder.F_ORDER
-        )
-        return {"shape": data.shape, "dtype": data.dtype, "order": order}
-
-    def __len__(self):
-        try:
-            return self.shape[0]
-        except IndexError:
-            if is_build_mode():
-                return 0
-            raise TypeError("len() of unsized object")
-
-    @property
-    def shape(self):
-        return getattr(self, "_shape", None)
-
-    @property
-    def ndim(self):
-        return len(self.shape)
-
-    @property
-    def size(self):
-        return np.prod(self.shape).item()
-
-    @property
-    def dtype(self):
-        return getattr(self, "_dtype", None) or self.op.dtype
-
-    @property
-    def order(self):
-        return getattr(self, "_order", None)
-
-    @property
-    def nbytes(self):
-        return np.prod(self.shape) * self.dtype.itemsize
-
-
-class TensorChunk(Chunk):
-    __slots__ = ()
-    _allow_data_type_ = (TensorChunkData,)
-    type_name = "Tensor"
-
-    def __len__(self):
-        return len(self._data)
-
-
 class TensorData(HasShapeTileableData, _ExecuteAndFetchMixin):
     __slots__ = ()
     type_name = "Tensor"
 
+    _legacy_deprecated_non_primitives = ["_chunks"]
+
     # required fields
     _order = StringField(
         "order", on_serialize=attrgetter("value"), on_deserialize=TensorOrder
     )
     # optional fields
     _dtype = DataTypeField("dtype")
-    _chunks = ListField(
-        "chunks",
-        FieldTypes.reference(TensorChunkData),
-        on_serialize=skip_na_call(lambda x: [it.data for it in x]),
-        on_deserialize=skip_na_call(lambda x: [TensorChunk(it) for it in x]),
-    )
 
     def __init__(
         self,
@@ -318,7 +195,7 @@ class TensorData(HasShapeTileableData, _ExecuteAndFetchMixin):
         return fromsparse(self, fill_value=fill_value)
 
     def transpose(self, *axes):
-        from .base import transpose
+        from .misc import transpose
 
         if len(axes) == 1 and isinstance(axes[0], Iterable):
             axes = axes[0]
@@ -346,11 +223,6 @@ class TensorData(HasShapeTileableData, _ExecuteAndFetchMixin):
 
         return reshape(self, shape, order=order)
 
-    def totiledb(self, uri, ctx=None, key=None, timestamp=None):
-        from .datastore import totiledb
-
-        return totiledb(uri, self, ctx=ctx, key=key, timestamp=timestamp)
-
     @staticmethod
     def from_dataframe(in_df):
         from .datasource import from_dataframe
@@ -526,9 +398,6 @@ class Tensor(HasShapeTileable):
         """
         return self._data.T
 
-    def totiledb(self, uri, ctx=None, key=None, timestamp=None):
-        return self._data.totiledb(uri, ctx=ctx, key=key, timestamp=timestamp)
-
     def copy(self, order="C"):
         return super().copy().astype(self.dtype, order=order, copy=False)
 
@@ -589,7 +458,7 @@ class Tensor(HasShapeTileable):
         array([('c', 1), ('a', 2)],
               dtype=[('x', '|S1'), ('y', '<i4')])
         """
-        from .base import sort
+        from .misc import sort
 
         self._data = sort(
             self,
@@ -651,7 +520,7 @@ class Tensor(HasShapeTileable):
         >>> a.execute()
         array([1, 2, 3, 4])
         """
-        from .base import partition
+        from .misc import partition
 
         self._data = partition(self, kth, axis=axis, kind=kind, order=order, **kw).data
 

maxframe/tensor/datasource/array.py

@@ -20,6 +20,7 @@ from ...serialization.serializables import (
     AnyField,
     FieldTypes,
     NDArrayField,
+    StringField,
     TupleField,
 )
 from ...utils import on_deserialize_shape, on_serialize_shape
@@ -37,8 +38,9 @@ class ArrayDataSource(TensorNoInput):
 
     _op_type_ = opcodes.TENSOR_DATA_SOURCE
 
-    data = NDArrayField("data")
-    chunk_size = AnyField("chunk_size")
+    data = NDArrayField("data", default=None)
+    chunk_size = AnyField("chunk_size", default=None)
+    order = StringField("order", default=None)
 
     def __init__(self, data=None, dtype=None, gpu=None, **kw):
         if dtype is not None:
@@ -51,6 +53,9 @@ class ArrayDataSource(TensorNoInput):
 
         super().__init__(data=data, dtype=dtype, gpu=gpu, **kw)
 
+    def get_data(self):
+        return self.data
+
 
 class CSRMatrixDataSource(TensorNoInput):
     """

maxframe/tensor/datasource/full.py

@@ -89,7 +89,7 @@ def full(shape, fill_value, dtype=None, chunk_size=None, gpu=None, order="C"):
     """
     v = np.asarray(fill_value)
     if len(v.shape) > 0:
-        from ..base import broadcast_to
+        from ..misc import broadcast_to
 
         return broadcast_to(
             tensor(v, dtype=dtype, chunk_size=chunk_size, gpu=gpu, order=order), shape

maxframe/tensor/datasource/scalar.py

@@ -33,7 +33,7 @@ class Scalar(TensorNoInput):
 def scalar(data, dtype=None, gpu=None):
     try:
         arr = np.array(data, dtype=dtype)
-        op = Scalar(arr, dtype=arr.dtype, gpu=gpu)
+        op = Scalar(data=arr, dtype=arr.dtype, gpu=gpu)
         shape = ()
         return op(shape)
     except ValueError:

maxframe/tensor/datasource/tests/test_datasource.py

@@ -141,7 +141,7 @@ def test_zeros():
 
 
 def test_data_source():
-    from ...base.broadcast_to import TensorBroadcastTo
+    from ...misc.broadcast_to import TensorBroadcastTo
 
     data = np.random.random((10, 3))
     t = tensor(data, chunk_size=2)

maxframe/tensor/indexing/flatnonzero.py

@@ -55,6 +55,6 @@ def flatnonzero(a):
     >>> x.ravel()[mt.flatnonzero(x)].execute()  # TODO(jisheng): accomplish this after fancy indexing is supported
 
     """
-    from ..base import ravel
+    from ..misc import ravel
 
     return nonzero(ravel(a))[0]

maxframe/tensor/indexing/getitem.py

@@ -130,6 +130,8 @@ def _calc_order(a, index):
             continue
         elif isinstance(ind, slice):
             shape = a.shape[in_axis]
+            if shape is np.nan:
+                return TensorOrder.C_ORDER
             slc = ind.indices(shape)
             if slc[0] == 0 and slc[1] == shape and slc[2] == 1:
                 continue

maxframe/tensor/merge/__init__.py

@@ -12,4 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from .concatenate import concatenate
 from .stack import stack
+from .vstack import vstack

maxframe/tensor/merge/concatenate.py

@@ -0,0 +1,101 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import numpy as np
+
+from ... import opcodes
+from ...serialization.serializables import Int32Field
+from ..datasource import tensor as astensor
+from ..operators import TensorOperator, TensorOperatorMixin
+from ..utils import validate_axis
+
+
+class TensorConcatenate(TensorOperator, TensorOperatorMixin):
+    _op_type_ = opcodes.CONCATENATE
+
+    axis = Int32Field("axis", default=0)
+
+    def __call__(self, tensors):
+        axis = self.axis
+        shape = _calc_concatenate_shape(tensors, axis)
+        shape[axis] = sum(t.shape[axis] for t in tensors)
+        return self.new_tensor(tensors, shape=tuple(shape))
+
+
+def concatenate(tensors, axis=0):
+    """
+    Join a sequence of arrays along an existing axis.
+
+    Parameters
+    ----------
+    a1, a2, ... : sequence of array_like
+        The tensors must have the same shape, except in the dimension
+        corresponding to `axis` (the first, by default).
+    axis : int, optional
+        The axis along which the tensors will be joined.  Default is 0.
+
+    Returns
+    -------
+    res : Tensor
+        The concatenated tensor.
+
+    See Also
+    --------
+    stack : Stack a sequence of tensors along a new axis.
+    vstack : Stack tensors in sequence vertically (row wise)
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+
+    >>> a = mt.array([[1, 2], [3, 4]])
+    >>> b = mt.array([[5, 6]])
+    >>> mt.concatenate((a, b), axis=0).execute()
+    array([[1, 2],
+           [3, 4],
+           [5, 6]])
+    >>> mt.concatenate((a, b.T), axis=1).execute()
+    array([[1, 2, 5],
+           [3, 4, 6]])
+
+    """
+    if axis is None:
+        axis = 0
+    tensors = [astensor(t) for t in tensors]
+    axis = validate_axis(tensors[0].ndim, axis)
+
+    if len(set(t.ndim for t in tensors)) != 1:
+        raise ValueError("all the input tensors must have same number of dimensions")
+
+    shapes = [t.shape[:axis] + t.shape[axis + 1 :] for t in tensors]
+    if len(set(shapes)) != 1:
+        raise ValueError(
+            "all the input tensor dimensions "
+            "except for the concatenation axis must match exactly"
+        )
+    shape = _calc_concatenate_shape(tensors, axis)
+    if any(np.isnan(s) for i, s in enumerate(shape) if i != axis):
+        raise ValueError("cannot concatenate tensor with unknown shape")
+
+    return _concatenate(tensors, axis)
+
+
+def _concatenate(tensors, axis=0):
+    dtype = np.result_type(*(t.dtype for t in tensors))
+
+    op = TensorConcatenate(axis=axis, dtype=dtype)
+    return op(tensors)
+
+
+def _calc_concatenate_shape(tensors, axis):
+    return [0 if i == axis else tensors[0].shape[i] for i in range(tensors[0].ndim)]

maxframe/tensor/merge/tests/test_merge.py

@@ -18,7 +18,36 @@ import numpy as np
 import pytest
 
 from ...datasource import empty, ones
-from .. import stack
+from .. import concatenate, stack
+
+
+def test_concatenate():
+    a = ones((10, 20, 30), chunk_size=10)
+    b = ones((20, 20, 30), chunk_size=20)
+
+    c = concatenate([a, b])
+    assert c.shape == (30, 20, 30)
+
+    a = ones((10, 20, 30), chunk_size=10)
+    b = ones((10, 20, 40), chunk_size=20)
+
+    c = concatenate([a, b], axis=-1)
+    assert c.shape == (10, 20, 70)
+
+    with pytest.raises(ValueError):
+        a = ones((10, 20, 30), chunk_size=10)
+        b = ones((20, 30, 30), chunk_size=20)
+
+        concatenate([a, b])
+
+    with pytest.raises(ValueError):
+        a = ones((10, 20, 30), chunk_size=10)
+        b = ones((20, 20), chunk_size=20)
+
+        concatenate([a, b])
+
+    a = ones((10, 20, 30), chunk_size=5)
+    b = ones((20, 20, 30), chunk_size=10)
 
 
 def test_stack():

maxframe/tensor/merge/vstack.py

@@ -0,0 +1,74 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from ..misc import atleast_2d
+from .concatenate import _concatenate, concatenate
+
+
+def vstack(tup):
+    """
+    Stack tensors in sequence vertically (row wise).
+
+    This is equivalent to concatenation along the first axis after 1-D tensors
+    of shape `(N,)` have been reshaped to `(1,N)`. Rebuilds tensors divided by
+    `vsplit`.
+
+    This function makes most sense for tensors with up to 3 dimensions. For
+    instance, for pixel-data with a height (first axis), width (second axis),
+    and r/g/b channels (third axis). The functions `concatenate`, `stack` and
+    `block` provide more general stacking and concatenation operations.
+
+    Parameters
+    ----------
+    tup : sequence of tensors
+        The tensors must have the same shape along all but the first axis.
+        1-D tensors must have the same length.
+
+    Returns
+    -------
+    stacked : Tensor
+        The tensor formed by stacking the given tensors, will be at least 2-D.
+
+    See Also
+    --------
+    stack : Join a sequence of tensors along a new axis.
+    concatenate : Join a sequence of tensors along an existing axis.
+
+    Examples
+    --------
+    >>> import mars.tensor as mt
+
+    >>> a = mt.array([1, 2, 3])
+    >>> b = mt.array([2, 3, 4])
+    >>> mt.vstack((a,b)).execute()
+    array([[1, 2, 3],
+           [2, 3, 4]])
+
+    >>> a = mt.array([[1], [2], [3]])
+    >>> b = mt.array([[2], [3], [4]])
+    >>> mt.vstack((a,b)).execute()
+    array([[1],
+           [2],
+           [3],
+           [2],
+           [3],
+           [4]])
+
+    """
+    return concatenate([atleast_2d(t) for t in tup], axis=0)
+
+
+def _vstack(tup):
+    return _concatenate([atleast_2d(t) for t in tup], axis=0)

maxframe/tensor/{base → misc}/__init__.py

@@ -13,9 +13,13 @@
 # limitations under the License.
 
 from .astype import TensorAstype
+from .atleast_1d import atleast_1d
+from .atleast_2d import atleast_2d
+from .atleast_3d import atleast_3d
 from .broadcast_to import TensorBroadcastTo, broadcast_to
 from .ravel import ravel
 from .transpose import transpose
+from .unique import unique
 from .where import TensorWhere, where