maxframe 0.1.0b5__cp39-cp39-macosx_10_9_universal2.whl → 1.0.0__cp39-cp39-macosx_10_9_universal2.whl

maxframe/tensor/merge/tests/test_merge.py

@@ -18,7 +18,36 @@ import numpy as np
 import pytest
 
 from ...datasource import empty, ones
-from .. import stack
+from .. import concatenate, stack
+
+
+def test_concatenate():
+    a = ones((10, 20, 30), chunk_size=10)
+    b = ones((20, 20, 30), chunk_size=20)
+
+    c = concatenate([a, b])
+    assert c.shape == (30, 20, 30)
+
+    a = ones((10, 20, 30), chunk_size=10)
+    b = ones((10, 20, 40), chunk_size=20)
+
+    c = concatenate([a, b], axis=-1)
+    assert c.shape == (10, 20, 70)
+
+    with pytest.raises(ValueError):
+        a = ones((10, 20, 30), chunk_size=10)
+        b = ones((20, 30, 30), chunk_size=20)
+
+        concatenate([a, b])
+
+    with pytest.raises(ValueError):
+        a = ones((10, 20, 30), chunk_size=10)
+        b = ones((20, 20), chunk_size=20)
+
+        concatenate([a, b])
+
+    a = ones((10, 20, 30), chunk_size=5)
+    b = ones((20, 20, 30), chunk_size=10)
 
 
 def test_stack():

maxframe/tensor/merge/vstack.py

@@ -0,0 +1,74 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from ..misc import atleast_2d
+from .concatenate import _concatenate, concatenate
+
+
+def vstack(tup):
+    """
+    Stack tensors in sequence vertically (row wise).
+
+    This is equivalent to concatenation along the first axis after 1-D tensors
+    of shape `(N,)` have been reshaped to `(1,N)`. Rebuilds tensors divided by
+    `vsplit`.
+
+    This function makes most sense for tensors with up to 3 dimensions. For
+    instance, for pixel-data with a height (first axis), width (second axis),
+    and r/g/b channels (third axis). The functions `concatenate`, `stack` and
+    `block` provide more general stacking and concatenation operations.
+
+    Parameters
+    ----------
+    tup : sequence of tensors
+        The tensors must have the same shape along all but the first axis.
+        1-D tensors must have the same length.
+
+    Returns
+    -------
+    stacked : Tensor
+        The tensor formed by stacking the given tensors, will be at least 2-D.
+
+    See Also
+    --------
+    stack : Join a sequence of tensors along a new axis.
+    concatenate : Join a sequence of tensors along an existing axis.
+
+    Examples
+    --------
+    >>> import mars.tensor as mt
+
+    >>> a = mt.array([1, 2, 3])
+    >>> b = mt.array([2, 3, 4])
+    >>> mt.vstack((a,b)).execute()
+    array([[1, 2, 3],
+           [2, 3, 4]])
+
+    >>> a = mt.array([[1], [2], [3]])
+    >>> b = mt.array([[2], [3], [4]])
+    >>> mt.vstack((a,b)).execute()
+    array([[1],
+           [2],
+           [3],
+           [2],
+           [3],
+           [4]])
+
+    """
+    return concatenate([atleast_2d(t) for t in tup], axis=0)
+
+
+def _vstack(tup):
+    return _concatenate([atleast_2d(t) for t in tup], axis=0)

maxframe/tensor/{base → misc}/__init__.py

@@ -14,6 +14,8 @@
 
 from .astype import TensorAstype
 from .atleast_1d import atleast_1d
+from .atleast_2d import atleast_2d
+from .atleast_3d import atleast_3d
 from .broadcast_to import TensorBroadcastTo, broadcast_to
 from .ravel import ravel
 from .transpose import transpose

maxframe/tensor/{base → misc}/atleast_1d.py

@@ -1,6 +1,4 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright 1999-2021 Alibaba Group Holding Ltd.
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

maxframe/tensor/misc/atleast_2d.py

@@ -0,0 +1,70 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ...core import ExecutableTuple
+from ..datasource import tensor as astensor
+
+
+def atleast_2d(*tensors):
+    """
+    View inputs as tensors with at least two dimensions.
+
+    Parameters
+    ----------
+    tensors1, tensors2, ... : array_like
+        One or more array-like sequences.  Non-tensor inputs are converted
+        to tensors.  Tensors that already have two or more dimensions are
+        preserved.
+
+    Returns
+    -------
+    res, res2, ... : Tensor
+        A tensor, or list of tensors, each with ``a.ndim >= 2``.
+        Copies are avoided where possible, and views with two or more
+        dimensions are returned.
+
+    See Also
+    --------
+    atleast_1d, atleast_3d
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+
+    >>> mt.atleast_2d(3.0).execute()
+    array([[ 3.]])
+
+    >>> x = mt.arange(3.0)
+    >>> mt.atleast_2d(x).execute()
+    array([[ 0.,  1.,  2.]])
+
+    >>> mt.atleast_2d(1, [1, 2], [[1, 2]]).execute()
+    [array([[1]]), array([[1, 2]]), array([[1, 2]])]
+
+    """
+    new_tensors = []
+    for x in tensors:
+        x = astensor(x)
+        if x.ndim == 0:
+            x = x[np.newaxis, np.newaxis]
+        elif x.ndim == 1:
+            x = x[np.newaxis, :]
+
+        new_tensors.append(x)
+
+    if len(new_tensors) == 1:
+        return new_tensors[0]
+    return ExecutableTuple(new_tensors)

maxframe/tensor/misc/atleast_3d.py

@@ -0,0 +1,85 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import numpy as np
+
+from ...core import ExecutableTuple
+from ..datasource import tensor as astensor
+
+
+def atleast_3d(*tensors):
+    """
+    View inputs as tensors with at least three dimensions.
+
+    Parameters
+    ----------
+    tensors1, tensors2, ... : array_like
+        One or more tensor-like sequences.  Non-tensor inputs are converted to
+        tensors.  Tensors that already have three or more dimensions are
+        preserved.
+
+    Returns
+    -------
+    res1, res2, ... : Tensor
+        A tensor, or list of tensors, each with ``a.ndim >= 3``.  Copies are
+        avoided where possible, and views with three or more dimensions are
+        returned.  For example, a 1-D tensor of shape ``(N,)`` becomes a view
+        of shape ``(1, N, 1)``, and a 2-D tensor of shape ``(M, N)`` becomes a
+        view of shape ``(M, N, 1)``.
+
+    See Also
+    --------
+    atleast_1d, atleast_2d
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+
+    >>> mt.atleast_3d(3.0).execute()
+    array([[[ 3.]]])
+
+    >>> x = mt.arange(3.0)
+    >>> mt.atleast_3d(x).shape
+    (1, 3, 1)
+
+    >>> x = mt.arange(12.0).reshape(4,3)
+    >>> mt.atleast_3d(x).shape
+    (4, 3, 1)
+
+    >>> for arr in mt.atleast_3d([1, 2], [[1, 2]], [[[1, 2]]]).execute():
+    ...     print(arr, arr.shape)
+    ...
+    [[[1]
+      [2]]] (1, 2, 1)
+    [[[1]
+      [2]]] (1, 2, 1)
+    [[[1 2]]] (1, 1, 2)
+
+    """
+    new_tensors = []
+    for x in tensors:
+        x = astensor(x)
+        if x.ndim == 0:
+            x = x[np.newaxis, np.newaxis, np.newaxis]
+        elif x.ndim == 1:
+            x = x[np.newaxis, :, np.newaxis]
+        elif x.ndim == 2:
+            x = x[:, :, None]
+
+        new_tensors.append(x)
+
+    if len(new_tensors) == 1:
+        return new_tensors[0]
+    return ExecutableTuple(new_tensors)

maxframe/tensor/misc/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

maxframe/tensor/{base → misc}/transpose.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2024 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -66,33 +64,39 @@ class TensorTranspose(TensorHasInput, TensorOperatorMixin):
 
 def transpose(a, axes=None):
     """
-    Permute the dimensions of a tensor.
+    Returns an array with axes transposed.
+
+    For a 1-D array, this returns an unchanged view of the original array, as a
+    transposed vector is simply the same vector.
+    To convert a 1-D array into a 2-D column vector, an additional dimension
+    must be added, e.g., ``mt.atleast_2d(a).T`` achieves this, as does
+    ``a[:, mt.newaxis]``.
+    For a 2-D array, this is the standard matrix transpose.
+    For an n-D array, if axes are given, their order indicates how the
+    axes are permuted (see Examples). If axes are not provided, then
+    ``transpose(a).shape == a.shape[::-1]``.
 
     Parameters
     ----------
     a : array_like
-        Input tensor.
-    axes : list of ints, optional
-        By default, reverse the dimensions, otherwise permute the axes
-        according to the values given.
+        Input array.
+    axes : tuple or list of ints, optional
+        If specified, it must be a tuple or list which contains a permutation
+        of [0,1,...,N-1] where N is the number of axes of `a`. The `i`'th axis
+        of the returned array will correspond to the axis numbered ``axes[i]``
+        of the input. If not specified, defaults to ``range(a.ndim)[::-1]``,
+        which reverses the order of the axes.
 
     Returns
     -------
-    p : Tensor
-        `a` with its axes permuted.  A view is returned whenever
-        possible.
-
-    See Also
-    --------
-    moveaxis
-    argsort
+    p : ndarray
+        `a` with its axes permuted. A view is returned whenever possible.
 
     Notes
     -----
-    Use `transpose(a, argsort(axes))` to invert the transposition of tensors
+    Use ``transpose(a, argsort(axes))`` to invert the transposition of tensors
     when using the `axes` keyword argument.
 
-    Transposing a 1-D array returns an unchanged view of the original tensor.
 
     Examples
     --------
@@ -121,5 +125,5 @@ def transpose(a, axes=None):
         axes = list(range(a.ndim))[::-1]
     else:
         axes = list(axes)
-    op = TensorTranspose(axes, dtype=a.dtype, sparse=a.issparse())
+    op = TensorTranspose(axes, dtype=a.dtype)
     return op(a)

maxframe/tensor/{base → misc}/unique.py

@@ -15,7 +15,7 @@
 
 import numpy as np
 
-from ... import opcodes as OperandDef
+from ... import opcodes
 from ...serialization.serializables import BoolField, Int32Field
 from ..core import TensorOrder
 from ..operators import TensorHasInput, TensorOperatorMixin
@@ -23,7 +23,7 @@ from ..utils import validate_axis
 
 
 class TensorUnique(TensorHasInput, TensorOperatorMixin):
-    _op_type_ = OperandDef.UNIQUE
+    _op_type_ = opcodes.UNIQUE
 
     return_index = BoolField("return_index", default=False)
     return_inverse = BoolField("return_inverse", default=False)
@@ -75,7 +75,7 @@ class TensorUnique(TensorHasInput, TensorOperatorMixin):
         if self.return_counts:
             kw = {
                 "shape": (np.nan,),
-                "dtype": np.dtype(np.int_),
+                "dtype": np.dtype(int),
                 "gpu": input_obj.op.gpu,
                 "type": "counts",
             }

maxframe/tensor/operators.py

@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 from ..core import OutputType
 from ..core.operator import (
-    Fuse,
     HasInput,
     MapReduceOperator,
     Operator,
@@ -115,9 +115,3 @@ class TensorMapReduceOperator(MapReduceOperator):
     _output_type_ = OutputType.tensor
 
     dtype = DataTypeField("dtype", default=None)
-
-
-class TensorFuse(Fuse):
-    _output_type_ = OutputType.tensor
-
-    dtype = DataTypeField("dtype", default=None)

maxframe/tensor/random/core.py

@@ -19,9 +19,9 @@ from contextlib import contextmanager
 import numpy as np
 
 from ...serialization.serializables import FieldTypes, Int32Field, TupleField
-from ..base import broadcast_to
 from ..core import TENSOR_TYPE
 from ..datasource import tensor as astensor
+from ..misc import broadcast_to
 from ..operators import TensorMapReduceOperator, TensorOperator, TensorOperatorMixin
 from ..utils import broadcast_shape
 

maxframe/tensor/reduction/count_nonzero.py

@@ -22,6 +22,7 @@ from .core import TensorReduction, TensorReductionMixin
 
 class TensorCountNonzero(TensorReduction, TensorReductionMixin):
     _op_type_ = opcodes.COUNT_NONZERO
+    _func_name = "count_nonzero"
 
     def __init__(self, dtype=None, **kw):
         if dtype is None:
@@ -77,5 +78,5 @@ def count_nonzero(a, axis=None):
     array([2, 3])
 
     """
-    op = TensorCountNonzero(axis=axis, dtype=np.dtype(np.int_), keepdims=None)
+    op = TensorCountNonzero(axis=axis, dtype=np.dtype(int), keepdims=None)
     return op(a)

maxframe/tensor/reduction/mean.py

@@ -23,6 +23,7 @@ from .core import TensorReduction, TensorReductionMixin
 
 class TensorMean(TensorReduction, TensorReductionMixin):
     _op_type_ = opcodes.MEAN
+    _func_name = "mean"
 
 
 def mean(a, axis=None, dtype=None, out=None, keepdims=None):

maxframe/tensor/reduction/nanmean.py

@@ -23,6 +23,7 @@ from .core import TensorReduction, TensorReductionMixin
 
 class TensorNanMean(TensorReduction, TensorReductionMixin):
     _op_type_ = opcodes.NANMEAN
+    _func_name = "nanmean"
 
 
 def nanmean(a, axis=None, dtype=None, out=None, keepdims=None):

maxframe/tensor/reduction/nanvar.py

@@ -24,6 +24,7 @@ from .core import TensorReduction, TensorReductionMixin
 
 class TensorNanMoment(TensorReduction, TensorReductionMixin):
     _op_type_ = opcodes.NANMOMENT
+    _func_name = "nanvar"
 
     moment = Int32Field("moment", default=2)
     ddof = Int32Field("ddof", default=None)
@@ -36,6 +37,7 @@ class TensorNanMoment(TensorReduction, TensorReductionMixin):
 
 class TensorNanVar(TensorReduction, TensorReductionMixin):
     _op_type_ = opcodes.NANVAR
+    _func_name = "nanvar"
 
     ddof = Int32Field("ddof", default=0)
 

maxframe/tensor/reduction/tests/test_reduction.py

@@ -17,8 +17,11 @@
 import numpy as np
 import pytest
 
+from maxframe.tensor.reduction.core import TensorReduction
+
+from ....utils import collect_leaf_operators
 from ...datasource import ones, tensor
-from .. import all
+from .. import *  # noqa: F401
 
 
 def test_base_reduction():
@@ -179,3 +182,11 @@ def test_var_reduction():
 
     res1 = var(ones((10, 8, 8), chunk_size=3), axis=1)
     assert res1.shape == (10, 8)
+
+
+def test_reduction_op_func_name():
+    # make sure all the binary op has defined the func name.
+
+    results = collect_leaf_operators(TensorReduction)
+    for op_type in results:
+        assert hasattr(op_type, "_func_name")

maxframe/tensor/reduction/var.py

@@ -42,6 +42,7 @@ def reduce_var_square(var_square, avg_diff, count, op, axis, sum_func):
 
 class TensorMoment(TensorReduction, TensorReductionMixin):
     _op_type_ = opcodes.MOMENT
+    _func_name = "var"
 
     moment = Int32Field("moment", default=2)
     ddof = Int32Field("ddof", default=None)
@@ -54,6 +55,7 @@ class TensorMoment(TensorReduction, TensorReductionMixin):
 
 class TensorVar(TensorReduction, TensorReductionMixin):
     _op_type_ = opcodes.VAR
+    _func_name = "var"
 
     ddof = Int32Field("ddof", default=0)
 

maxframe/tensor/statistics/quantile.py

@@ -16,7 +16,7 @@ from collections.abc import Iterable
 
 import numpy as np
 
-from ... import opcodes as OperandDef
+from ... import opcodes
 from ...core import ENTITY_TYPE
 from ...serialization.serializables import AnyField, BoolField, KeyField, StringField
 from ..core import TENSOR_TYPE, TensorOrder
@@ -43,7 +43,7 @@ q_error_msg = "Quantiles must be in the range [0, 1]"
 
 class TensorQuantile(TensorOperator, TensorOperatorMixin):
     __slots__ = ("q_error_msg",)
-    _op_type_ = OperandDef.QUANTILE
+    _op_type_ = opcodes.QUANTILE
 
     a = KeyField("a")
     q = AnyField("q")

maxframe/tensor/utils.py

@@ -19,18 +19,13 @@ import itertools
 import operator
 from collections import OrderedDict
 from collections.abc import Iterable
-from functools import lru_cache, wraps
+from functools import wraps
 from math import ceil
 from numbers import Integral
 from typing import Dict, List, Union
 
 import numpy as np
 
-try:
-    import tiledb
-except (ImportError, OSError):  # pragma: no cover
-    tildb = None
-
 from ..core import ExecutableTuple
 from ..lib.mmh3 import hash_from_buffer
 from ..utils import lazy_import
@@ -508,7 +503,7 @@ def decide_unify_split(*splits):
 
 
 def check_out_param(out, t, casting):
-    from .base import broadcast_to
+    from .misc import broadcast_to
 
     if not hasattr(out, "shape"):
         raise TypeError("return arrays must be a tensor")
@@ -563,21 +558,6 @@ def filter_inputs(inputs):
     return [inp for inp in inputs if isinstance(inp, ENTITY_TYPE)]
 
 
-# As TileDB Ctx's creation is a bit time-consuming,
-# we just cache the Ctx
-# also remember the arguments should be hashable
-@lru_cache(10)
-def _create_tiledb_ctx(conf_tuple):
-    if conf_tuple is not None:
-        return tiledb.Ctx(dict(conf_tuple))
-    return tiledb.Ctx()
-
-
-def get_tiledb_ctx(conf):
-    key = tuple(conf.items()) if conf is not None else None
-    return _create_tiledb_ctx(key)
-
-
 # this function is only used for pandas' compatibility
 def to_numpy(pdf):
     try:

maxframe/tests/test_protocol.py

@@ -85,6 +85,40 @@ def test_error_info_json_serialize():
         deserial_err_info.reraise()
 
 
+class CannotPickleException(Exception):
+    def __reduce__(self):
+        raise ValueError
+
+
+class CannotUnpickleException(Exception):
+    @classmethod
+    def load_from_pk(cls, _):
+        raise ValueError
+
+    def __reduce__(self):
+        return type(self).load_from_pk, (0,)
+
+
+def test_error_info_fallback_json_serialize():
+    try:
+        raise CannotPickleException
+    except CannotPickleException as ex:
+        err_info1 = ErrorInfo.from_exception(ex)
+
+    try:
+        raise CannotUnpickleException
+    except CannotUnpickleException as ex:
+        err_info2 = ErrorInfo.from_exception(ex)
+
+    for err_info in (err_info1, err_info2):
+        deserial_err_info = ErrorInfo.from_json(err_info.to_json())
+        assert deserial_err_info.raw_error_source is None
+        assert deserial_err_info.raw_error_data is None
+
+        with pytest.raises(RemoteException):
+            deserial_err_info.reraise()
+
+
 def test_dag_info_json_serialize():
     try:
         raise ValueError("ERR_DATA")

maxframe/tests/test_utils.py

@@ -288,15 +288,6 @@ def test_estimate_pandas_size():
     df2 = pd.DataFrame(np.random.rand(1000, 10))
     assert utils.estimate_pandas_size(df2) == sys.getsizeof(df2)
 
-    df3 = pd.DataFrame(
-        {
-            "A": np.random.choice(["abcd", "def", "gh"], size=(1000,)),
-            "B": np.random.rand(1000),
-            "C": np.random.rand(1000),
-        }
-    )
-    assert utils.estimate_pandas_size(df3) != sys.getsizeof(df3)
-
     s1 = pd.Series(np.random.rand(1000))
     assert utils.estimate_pandas_size(s1) == sys.getsizeof(s1)
 
@@ -307,7 +298,6 @@ def test_estimate_pandas_size():
     assert utils.estimate_pandas_size(s2) == sys.getsizeof(s2)
 
     s3 = pd.Series(np.random.choice(["abcd", "def", "gh"], size=(1000,)))
-    assert utils.estimate_pandas_size(s3) != sys.getsizeof(s3)
     assert (
         pytest.approx(utils.estimate_pandas_size(s3) / sys.getsizeof(s3), abs=0.5) == 1
     )
@@ -318,7 +308,6 @@ def test_estimate_pandas_size():
     assert utils.estimate_pandas_size(idx1) == sys.getsizeof(idx1)
 
     string_idx = pd.Index(np.random.choice(["a", "bb", "cc"], size=(1000,)))
-    assert utils.estimate_pandas_size(string_idx) != sys.getsizeof(string_idx)
     assert (
         pytest.approx(
             utils.estimate_pandas_size(string_idx) / sys.getsizeof(string_idx), abs=0.5
@@ -338,7 +327,6 @@ def test_estimate_pandas_size():
         },
         index=idx2,
     )
-    assert utils.estimate_pandas_size(df4) != sys.getsizeof(df4)
     assert (
         pytest.approx(utils.estimate_pandas_size(df4) / sys.getsizeof(df4), abs=0.5)
         == 1