maxframe 1.0.0rc2__cp39-cp39-macosx_10_9_universal2.whl → 1.0.0rc4__cp39-cp39-macosx_10_9_universal2.whl

maxframe/tensor/core.py

@@ -23,8 +23,6 @@ from typing import Any, Dict
 import numpy as np
 
 from ..core import (
-    Chunk,
-    ChunkData,
     HasShapeTileable,
     HasShapeTileableData,
     OutputType,
@@ -36,14 +34,9 @@ from ..core.entity.utils import refresh_tileable_shape
 from ..serialization.serializables import (
     AnyField,
     DataTypeField,
-    EnumField,
-    FieldTypes,
-    ListField,
     Serializable,
     StringField,
-    TupleField,
 )
-from ..utils import on_deserialize_shape, on_serialize_shape, skip_na_call
 from .utils import fetch_corner_data, get_chunk_slices
 
 logger = logging.getLogger(__name__)
@@ -56,134 +49,18 @@ class TensorOrder(Enum):
     F_ORDER = "F"
 
 
-class TensorChunkData(ChunkData):
-    __slots__ = ()
-    _no_copy_attrs_ = ChunkData._no_copy_attrs_ | {"dtype"}
-    type_name = "Tensor"
-
-    # required fields
-    _shape = TupleField(
-        "shape",
-        FieldTypes.int64,
-        on_serialize=on_serialize_shape,
-        on_deserialize=on_deserialize_shape,
-    )
-    _order = EnumField("order", TensorOrder, FieldTypes.string)
-    # optional fields
-    _dtype = DataTypeField("dtype")
-
-    def __init__(self, op=None, index=None, shape=None, dtype=None, order=None, **kw):
-        if isinstance(order, str):
-            order = getattr(TensorOrder, order)
-        super().__init__(
-            _op=op, _index=index, _shape=shape, _dtype=dtype, _order=order, **kw
-        )
-        if self.order is None and self.op is not None:
-            if len(self.inputs) == 0:
-                self._order = TensorOrder.C_ORDER
-            elif all(
-                hasattr(inp, "order") and inp.order == TensorOrder.F_ORDER
-                for inp in self.inputs
-            ):
-                self._order = TensorOrder.F_ORDER
-            else:
-                self._order = TensorOrder.C_ORDER
-
-    @property
-    def params(self) -> Dict[str, Any]:
-        # params return the properties which useful to rebuild a new chunk
-        return {
-            "shape": self.shape,
-            "dtype": self.dtype,
-            "order": self.order,
-            "index": self.index,
-        }
-
-    @params.setter
-    def params(self, new_params: Dict[str, Any]):
-        params = new_params.copy()
-        params.pop("index", None)  # index not needed to update
-        new_shape = params.pop("shape", None)
-        if new_shape is not None:
-            self._shape = new_shape
-        dtype = params.pop("dtype", None)
-        if dtype is not None:
-            self._dtype = dtype
-        order = params.pop("order", None)
-        if order is not None:
-            self._order = order
-        if params:  # pragma: no cover
-            raise TypeError(f"Unknown params: {list(params)}")
-
-    @classmethod
-    def get_params_from_data(cls, data: np.ndarray) -> Dict[str, Any]:
-        from .array_utils import is_cupy
-
-        if not is_cupy(data):
-            data = np.asarray(data)
-        order = (
-            TensorOrder.C_ORDER if data.flags["C_CONTIGUOUS"] else TensorOrder.F_ORDER
-        )
-        return {"shape": data.shape, "dtype": data.dtype, "order": order}
-
-    def __len__(self):
-        try:
-            return self.shape[0]
-        except IndexError:
-            if is_build_mode():
-                return 0
-            raise TypeError("len() of unsized object")
-
-    @property
-    def shape(self):
-        return getattr(self, "_shape", None)
-
-    @property
-    def ndim(self):
-        return len(self.shape)
-
-    @property
-    def size(self):
-        return np.prod(self.shape).item()
-
-    @property
-    def dtype(self):
-        return getattr(self, "_dtype", None) or self.op.dtype
-
-    @property
-    def order(self):
-        return getattr(self, "_order", None)
-
-    @property
-    def nbytes(self):
-        return np.prod(self.shape) * self.dtype.itemsize
-
-
-class TensorChunk(Chunk):
-    __slots__ = ()
-    _allow_data_type_ = (TensorChunkData,)
-    type_name = "Tensor"
-
-    def __len__(self):
-        return len(self._data)
-
-
 class TensorData(HasShapeTileableData, _ExecuteAndFetchMixin):
     __slots__ = ()
     type_name = "Tensor"
 
+    _legacy_deprecated_non_primitives = ["_chunks"]
+
     # required fields
     _order = StringField(
         "order", on_serialize=attrgetter("value"), on_deserialize=TensorOrder
     )
     # optional fields
     _dtype = DataTypeField("dtype")
-    _chunks = ListField(
-        "chunks",
-        FieldTypes.reference(TensorChunkData),
-        on_serialize=skip_na_call(lambda x: [it.data for it in x]),
-        on_deserialize=skip_na_call(lambda x: [TensorChunk(it) for it in x]),
-    )
 
     def __init__(
         self,
@@ -318,7 +195,7 @@ class TensorData(HasShapeTileableData, _ExecuteAndFetchMixin):
         return fromsparse(self, fill_value=fill_value)
 
     def transpose(self, *axes):
-        from .base import transpose
+        from .misc import transpose
 
         if len(axes) == 1 and isinstance(axes[0], Iterable):
             axes = axes[0]
@@ -346,11 +223,6 @@ class TensorData(HasShapeTileableData, _ExecuteAndFetchMixin):
 
         return reshape(self, shape, order=order)
 
-    def totiledb(self, uri, ctx=None, key=None, timestamp=None):
-        from .datastore import totiledb
-
-        return totiledb(uri, self, ctx=ctx, key=key, timestamp=timestamp)
-
     @staticmethod
     def from_dataframe(in_df):
         from .datasource import from_dataframe
@@ -526,9 +398,6 @@ class Tensor(HasShapeTileable):
         """
         return self._data.T
 
-    def totiledb(self, uri, ctx=None, key=None, timestamp=None):
-        return self._data.totiledb(uri, ctx=ctx, key=key, timestamp=timestamp)
-
     def copy(self, order="C"):
         return super().copy().astype(self.dtype, order=order, copy=False)
 
@@ -589,7 +458,7 @@ class Tensor(HasShapeTileable):
         array([('c', 1), ('a', 2)],
               dtype=[('x', '|S1'), ('y', '<i4')])
         """
-        from .base import sort
+        from .misc import sort
 
         self._data = sort(
             self,
@@ -651,7 +520,7 @@ class Tensor(HasShapeTileable):
         >>> a.execute()
         array([1, 2, 3, 4])
         """
-        from .base import partition
+        from .misc import partition
 
         self._data = partition(self, kth, axis=axis, kind=kind, order=order, **kw).data
 

maxframe/tensor/datasource/array.py

@@ -53,6 +53,9 @@ class ArrayDataSource(TensorNoInput):
 
         super().__init__(data=data, dtype=dtype, gpu=gpu, **kw)
 
+    def get_data(self):
+        return self.data
+
 
 class CSRMatrixDataSource(TensorNoInput):
     """

maxframe/tensor/datasource/full.py

@@ -89,7 +89,7 @@ def full(shape, fill_value, dtype=None, chunk_size=None, gpu=None, order="C"):
     """
     v = np.asarray(fill_value)
     if len(v.shape) > 0:
-        from ..base import broadcast_to
+        from ..misc import broadcast_to
 
         return broadcast_to(
             tensor(v, dtype=dtype, chunk_size=chunk_size, gpu=gpu, order=order), shape

maxframe/tensor/datasource/tests/test_datasource.py

@@ -141,7 +141,7 @@ def test_zeros():
 
 
 def test_data_source():
-    from ...base.broadcast_to import TensorBroadcastTo
+    from ...misc.broadcast_to import TensorBroadcastTo
 
     data = np.random.random((10, 3))
     t = tensor(data, chunk_size=2)

maxframe/tensor/indexing/flatnonzero.py

@@ -55,6 +55,6 @@ def flatnonzero(a):
     >>> x.ravel()[mt.flatnonzero(x)].execute()  # TODO(jisheng): accomplish this after fancy indexing is supported
 
     """
-    from ..base import ravel
+    from ..misc import ravel
 
     return nonzero(ravel(a))[0]

maxframe/tensor/indexing/getitem.py

@@ -130,6 +130,8 @@ def _calc_order(a, index):
             continue
         elif isinstance(ind, slice):
             shape = a.shape[in_axis]
+            if shape is np.nan:
+                return TensorOrder.C_ORDER
             slc = ind.indices(shape)
             if slc[0] == 0 and slc[1] == shape and slc[2] == 1:
                 continue

maxframe/tensor/merge/__init__.py

@@ -12,4 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from .concatenate import concatenate
 from .stack import stack
+from .vstack import vstack

maxframe/tensor/merge/concatenate.py

@@ -0,0 +1,101 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import numpy as np
+
+from ... import opcodes
+from ...serialization.serializables import Int32Field
+from ..datasource import tensor as astensor
+from ..operators import TensorOperator, TensorOperatorMixin
+from ..utils import validate_axis
+
+
+class TensorConcatenate(TensorOperator, TensorOperatorMixin):
+    _op_type_ = opcodes.CONCATENATE
+
+    axis = Int32Field("axis", default=0)
+
+    def __call__(self, tensors):
+        axis = self.axis
+        shape = _calc_concatenate_shape(tensors, axis)
+        shape[axis] = sum(t.shape[axis] for t in tensors)
+        return self.new_tensor(tensors, shape=tuple(shape))
+
+
+def concatenate(tensors, axis=0):
+    """
+    Join a sequence of arrays along an existing axis.
+
+    Parameters
+    ----------
+    a1, a2, ... : sequence of array_like
+        The tensors must have the same shape, except in the dimension
+        corresponding to `axis` (the first, by default).
+    axis : int, optional
+        The axis along which the tensors will be joined.  Default is 0.
+
+    Returns
+    -------
+    res : Tensor
+        The concatenated tensor.
+
+    See Also
+    --------
+    stack : Stack a sequence of tensors along a new axis.
+    vstack : Stack tensors in sequence vertically (row wise)
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+
+    >>> a = mt.array([[1, 2], [3, 4]])
+    >>> b = mt.array([[5, 6]])
+    >>> mt.concatenate((a, b), axis=0).execute()
+    array([[1, 2],
+           [3, 4],
+           [5, 6]])
+    >>> mt.concatenate((a, b.T), axis=1).execute()
+    array([[1, 2, 5],
+           [3, 4, 6]])
+
+    """
+    if axis is None:
+        axis = 0
+    tensors = [astensor(t) for t in tensors]
+    axis = validate_axis(tensors[0].ndim, axis)
+
+    if len(set(t.ndim for t in tensors)) != 1:
+        raise ValueError("all the input tensors must have same number of dimensions")
+
+    shapes = [t.shape[:axis] + t.shape[axis + 1 :] for t in tensors]
+    if len(set(shapes)) != 1:
+        raise ValueError(
+            "all the input tensor dimensions "
+            "except for the concatenation axis must match exactly"
+        )
+    shape = _calc_concatenate_shape(tensors, axis)
+    if any(np.isnan(s) for i, s in enumerate(shape) if i != axis):
+        raise ValueError("cannot concatenate tensor with unknown shape")
+
+    return _concatenate(tensors, axis)
+
+
+def _concatenate(tensors, axis=0):
+    dtype = np.result_type(*(t.dtype for t in tensors))
+
+    op = TensorConcatenate(axis=axis, dtype=dtype)
+    return op(tensors)
+
+
+def _calc_concatenate_shape(tensors, axis):
+    return [0 if i == axis else tensors[0].shape[i] for i in range(tensors[0].ndim)]

maxframe/tensor/merge/tests/test_merge.py

@@ -18,7 +18,36 @@ import numpy as np
 import pytest
 
 from ...datasource import empty, ones
-from .. import stack
+from .. import concatenate, stack
+
+
+def test_concatenate():
+    a = ones((10, 20, 30), chunk_size=10)
+    b = ones((20, 20, 30), chunk_size=20)
+
+    c = concatenate([a, b])
+    assert c.shape == (30, 20, 30)
+
+    a = ones((10, 20, 30), chunk_size=10)
+    b = ones((10, 20, 40), chunk_size=20)
+
+    c = concatenate([a, b], axis=-1)
+    assert c.shape == (10, 20, 70)
+
+    with pytest.raises(ValueError):
+        a = ones((10, 20, 30), chunk_size=10)
+        b = ones((20, 30, 30), chunk_size=20)
+
+        concatenate([a, b])
+
+    with pytest.raises(ValueError):
+        a = ones((10, 20, 30), chunk_size=10)
+        b = ones((20, 20), chunk_size=20)
+
+        concatenate([a, b])
+
+    a = ones((10, 20, 30), chunk_size=5)
+    b = ones((20, 20, 30), chunk_size=10)
 
 
 def test_stack():

maxframe/tensor/merge/vstack.py

@@ -0,0 +1,74 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from ..misc import atleast_2d
+from .concatenate import _concatenate, concatenate
+
+
+def vstack(tup):
+    """
+    Stack tensors in sequence vertically (row wise).
+
+    This is equivalent to concatenation along the first axis after 1-D tensors
+    of shape `(N,)` have been reshaped to `(1,N)`. Rebuilds tensors divided by
+    `vsplit`.
+
+    This function makes most sense for tensors with up to 3 dimensions. For
+    instance, for pixel-data with a height (first axis), width (second axis),
+    and r/g/b channels (third axis). The functions `concatenate`, `stack` and
+    `block` provide more general stacking and concatenation operations.
+
+    Parameters
+    ----------
+    tup : sequence of tensors
+        The tensors must have the same shape along all but the first axis.
+        1-D tensors must have the same length.
+
+    Returns
+    -------
+    stacked : Tensor
+        The tensor formed by stacking the given tensors, will be at least 2-D.
+
+    See Also
+    --------
+    stack : Join a sequence of tensors along a new axis.
+    concatenate : Join a sequence of tensors along an existing axis.
+
+    Examples
+    --------
+    >>> import mars.tensor as mt
+
+    >>> a = mt.array([1, 2, 3])
+    >>> b = mt.array([2, 3, 4])
+    >>> mt.vstack((a,b)).execute()
+    array([[1, 2, 3],
+           [2, 3, 4]])
+
+    >>> a = mt.array([[1], [2], [3]])
+    >>> b = mt.array([[2], [3], [4]])
+    >>> mt.vstack((a,b)).execute()
+    array([[1],
+           [2],
+           [3],
+           [2],
+           [3],
+           [4]])
+
+    """
+    return concatenate([atleast_2d(t) for t in tup], axis=0)
+
+
+def _vstack(tup):
+    return _concatenate([atleast_2d(t) for t in tup], axis=0)

maxframe/tensor/{base → misc}/__init__.py

@@ -14,6 +14,8 @@
 
 from .astype import TensorAstype
 from .atleast_1d import atleast_1d
+from .atleast_2d import atleast_2d
+from .atleast_3d import atleast_3d
 from .broadcast_to import TensorBroadcastTo, broadcast_to
 from .ravel import ravel
 from .transpose import transpose

maxframe/tensor/{base → misc}/atleast_1d.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2024 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

maxframe/tensor/misc/atleast_2d.py

@@ -0,0 +1,70 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ...core import ExecutableTuple
+from ..datasource import tensor as astensor
+
+
+def atleast_2d(*tensors):
+    """
+    View inputs as tensors with at least two dimensions.
+
+    Parameters
+    ----------
+    tensors1, tensors2, ... : array_like
+        One or more array-like sequences.  Non-tensor inputs are converted
+        to tensors.  Tensors that already have two or more dimensions are
+        preserved.
+
+    Returns
+    -------
+    res, res2, ... : Tensor
+        A tensor, or list of tensors, each with ``a.ndim >= 2``.
+        Copies are avoided where possible, and views with two or more
+        dimensions are returned.
+
+    See Also
+    --------
+    atleast_1d, atleast_3d
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+
+    >>> mt.atleast_2d(3.0).execute()
+    array([[ 3.]])
+
+    >>> x = mt.arange(3.0)
+    >>> mt.atleast_2d(x).execute()
+    array([[ 0.,  1.,  2.]])
+
+    >>> mt.atleast_2d(1, [1, 2], [[1, 2]]).execute()
+    [array([[1]]), array([[1, 2]]), array([[1, 2]])]
+
+    """
+    new_tensors = []
+    for x in tensors:
+        x = astensor(x)
+        if x.ndim == 0:
+            x = x[np.newaxis, np.newaxis]
+        elif x.ndim == 1:
+            x = x[np.newaxis, :]
+
+        new_tensors.append(x)
+
+    if len(new_tensors) == 1:
+        return new_tensors[0]
+    return ExecutableTuple(new_tensors)

maxframe/tensor/misc/atleast_3d.py

@@ -0,0 +1,85 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import numpy as np
+
+from ...core import ExecutableTuple
+from ..datasource import tensor as astensor
+
+
+def atleast_3d(*tensors):
+    """
+    View inputs as tensors with at least three dimensions.
+
+    Parameters
+    ----------
+    tensors1, tensors2, ... : array_like
+        One or more tensor-like sequences.  Non-tensor inputs are converted to
+        tensors.  Tensors that already have three or more dimensions are
+        preserved.
+
+    Returns
+    -------
+    res1, res2, ... : Tensor
+        A tensor, or list of tensors, each with ``a.ndim >= 3``.  Copies are
+        avoided where possible, and views with three or more dimensions are
+        returned.  For example, a 1-D tensor of shape ``(N,)`` becomes a view
+        of shape ``(1, N, 1)``, and a 2-D tensor of shape ``(M, N)`` becomes a
+        view of shape ``(M, N, 1)``.
+
+    See Also
+    --------
+    atleast_1d, atleast_2d
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+
+    >>> mt.atleast_3d(3.0).execute()
+    array([[[ 3.]]])
+
+    >>> x = mt.arange(3.0)
+    >>> mt.atleast_3d(x).shape
+    (1, 3, 1)
+
+    >>> x = mt.arange(12.0).reshape(4,3)
+    >>> mt.atleast_3d(x).shape
+    (4, 3, 1)
+
+    >>> for arr in mt.atleast_3d([1, 2], [[1, 2]], [[[1, 2]]]).execute():
+    ...     print(arr, arr.shape)
+    ...
+    [[[1]
+      [2]]] (1, 2, 1)
+    [[[1]
+      [2]]] (1, 2, 1)
+    [[[1 2]]] (1, 1, 2)
+
+    """
+    new_tensors = []
+    for x in tensors:
+        x = astensor(x)
+        if x.ndim == 0:
+            x = x[np.newaxis, np.newaxis, np.newaxis]
+        elif x.ndim == 1:
+            x = x[np.newaxis, :, np.newaxis]
+        elif x.ndim == 2:
+            x = x[:, :, None]
+
+        new_tensors.append(x)
+
+    if len(new_tensors) == 1:
+        return new_tensors[0]
+    return ExecutableTuple(new_tensors)

maxframe/tensor/misc/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.