maxframe 0.1.0b5__cp310-cp310-macosx_11_0_arm64.whl

maxframe/tensor/datasource/scalar.py

@@ -0,0 +1,40 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import numpy as np
+
+from ... import opcodes
+from ...serialization.serializables import AnyField
+from .core import TensorNoInput
+
+
+class Scalar(TensorNoInput):
+    """
+    Operator represents scalar type.
+    """
+
+    _op_type_ = opcodes.SCALAR
+
+    data = AnyField("data", default=None)
+
+
+def scalar(data, dtype=None, gpu=None):
+    try:
+        arr = np.array(data, dtype=dtype)
+        op = Scalar(data=arr, dtype=arr.dtype, gpu=gpu)
+        shape = ()
+        return op(shape)
+    except ValueError:
+        raise TypeError(f"Expect scalar, got: {data}")

maxframe/tensor/datasource/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

maxframe/tensor/datasource/tests/test_datasource.py

@@ -0,0 +1,278 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from copy import copy
+
+import numpy as np
+import pytest
+import scipy.sparse as sps
+
+from .... import dataframe as md
+from ....core import enter_mode
+from ... import arange, full, ones, ones_like, tensor, zeros
+from ...core import SparseTensor, Tensor
+from .. import array, asarray, ascontiguousarray, asfortranarray, fromdense
+from ..array import CSRMatrixDataSource
+from ..from_dataframe import from_dataframe
+from ..from_dense import DenseToSparse
+from ..ones import TensorOnesLike
+
+
+def test_array():
+    a = tensor([0, 1, 2], chunk_size=2)
+
+    b = array(a)
+    assert a is not b
+
+    c = asarray(a)
+    assert a is c
+
+
+def test_ascontiguousarray():
+    # dtype different
+    raw_a = np.asfortranarray(np.random.rand(2, 4))
+    raw_b = np.ascontiguousarray(raw_a, dtype="f4")
+
+    a = tensor(raw_a, chunk_size=2)
+    b = ascontiguousarray(a, dtype="f4")
+
+    assert a.dtype == raw_a.dtype
+    assert a.flags["C_CONTIGUOUS"] == raw_a.flags["C_CONTIGUOUS"]
+    assert a.flags["F_CONTIGUOUS"] == raw_a.flags["F_CONTIGUOUS"]
+
+    assert b.dtype == raw_b.dtype
+    assert b.flags["C_CONTIGUOUS"] == raw_b.flags["C_CONTIGUOUS"]
+    assert b.flags["F_CONTIGUOUS"] == raw_b.flags["F_CONTIGUOUS"]
+
+    # no copy
+    raw_a = np.random.rand(2, 4)
+    raw_b = np.ascontiguousarray(raw_a)
+
+    a = tensor(raw_a, chunk_size=2)
+    b = ascontiguousarray(a)
+
+    assert a.dtype == raw_a.dtype
+    assert a.flags["C_CONTIGUOUS"] == raw_a.flags["C_CONTIGUOUS"]
+    assert a.flags["F_CONTIGUOUS"] == raw_a.flags["F_CONTIGUOUS"]
+
+    assert b.dtype == raw_b.dtype
+    assert b.flags["C_CONTIGUOUS"] == raw_b.flags["C_CONTIGUOUS"]
+    assert b.flags["F_CONTIGUOUS"] == raw_b.flags["F_CONTIGUOUS"]
+
+
+def test_asfortranarray():
+    # dtype different
+    raw_a = np.random.rand(2, 4)
+    raw_b = np.asfortranarray(raw_a, dtype="f4")
+
+    a = tensor(raw_a, chunk_size=2)
+    b = asfortranarray(a, dtype="f4")
+
+    assert a.dtype == raw_a.dtype
+    assert a.flags["C_CONTIGUOUS"] == raw_a.flags["C_CONTIGUOUS"]
+    assert a.flags["F_CONTIGUOUS"] == raw_a.flags["F_CONTIGUOUS"]
+
+    assert b.dtype == raw_b.dtype
+    assert b.flags["C_CONTIGUOUS"] == raw_b.flags["C_CONTIGUOUS"]
+    assert b.flags["F_CONTIGUOUS"] == raw_b.flags["F_CONTIGUOUS"]
+
+    # no copy
+    raw_a = np.asfortranarray(np.random.rand(2, 4))
+    raw_b = np.asfortranarray(raw_a)
+
+    a = tensor(raw_a, chunk_size=2)
+    b = asfortranarray(a)
+
+    assert a.dtype == raw_a.dtype
+    assert a.flags["C_CONTIGUOUS"] == raw_a.flags["C_CONTIGUOUS"]
+    assert a.flags["F_CONTIGUOUS"] == raw_a.flags["F_CONTIGUOUS"]
+
+    assert b.dtype == raw_b.dtype
+    assert b.flags["C_CONTIGUOUS"] == raw_b.flags["C_CONTIGUOUS"]
+    assert b.flags["F_CONTIGUOUS"] == raw_b.flags["F_CONTIGUOUS"]
+
+
+def test_ones():
+    tensor = ones((10, 10, 8), chunk_size=(3, 3, 5))
+    assert tensor.shape == (10, 10, 8)
+
+    tensor = ones((10, 3), chunk_size=(4, 2))
+    assert tensor.shape == (10, 3)
+
+    tensor = ones((10, 5), chunk_size=(2, 3), gpu=True)
+    assert tensor.op.gpu is True
+
+    tensor = ones((2, 3, 4))
+    assert len(list(tensor)) == 2
+
+    tensor2 = ones((2, 3, 4), chunk_size=1)
+    assert tensor.op.key != tensor2.op.key
+    assert tensor.key != tensor2.key
+
+    tensor3 = ones((2, 3, 3))
+    assert tensor.op.key != tensor3.op.key
+    assert tensor.key != tensor3.key
+
+
+def test_zeros():
+    tensor = zeros((2, 3, 4))
+    assert len(list(tensor)) == 2
+    assert tensor.op.gpu is None
+
+    tensor2 = zeros((2, 3, 4), chunk_size=1)
+    # tensor's op key must be equal to tensor2
+    assert tensor.op.key != tensor2.op.key
+    assert tensor.key != tensor2.key
+
+    tensor3 = zeros((2, 3, 3))
+    assert tensor.op.key != tensor3.op.key
+    assert tensor.key != tensor3.key
+
+
+def test_data_source():
+    from ...base.broadcast_to import TensorBroadcastTo
+
+    data = np.random.random((10, 3))
+    t = tensor(data, chunk_size=2)
+    assert t.op.gpu is None
+
+    t = tensor(data, chunk_size=2, gpu=True)
+    assert t.op.gpu is True
+
+    t = full((2, 2), 2, dtype="f4")
+    assert t.op.gpu is None
+    assert t.shape == (2, 2)
+    assert t.dtype == np.float32
+
+    t = full((2, 2), [1.0, 2.0], dtype="f4")
+    assert t.shape == (2, 2)
+    assert t.dtype == np.float32
+    assert isinstance(t.op, TensorBroadcastTo)
+
+    with pytest.raises(ValueError):
+        full((2, 2), [1.0, 2.0, 3.0], dtype="f4")
+
+
+def test_ufunc():
+    t = ones((3, 10), chunk_size=2)
+
+    x = np.add(t, [[1], [2], [3]])
+    assert isinstance(x, Tensor)
+
+    y = np.sum(t, axis=1)
+    assert isinstance(y, Tensor)
+
+
+def test_arange():
+    t = arange(10, chunk_size=3)
+
+    assert t.op.gpu is False
+    assert t.shape == (10,)
+
+    t = arange(0, 10, 3, chunk_size=2)
+    assert t.shape == (4,)
+
+    pytest.raises(TypeError, lambda: arange(10, start=0))
+    pytest.raises(TypeError, lambda: arange(0, 10, stop=0))
+    pytest.raises(TypeError, lambda: arange())
+    pytest.raises(
+        ValueError, lambda: arange("1066-10-13", dtype=np.datetime64, chunks=3)
+    )
+
+
+def test_set_tensor_inputs():
+    t1 = tensor([1, 2], chunk_size=2)
+    t2 = tensor([2, 3], chunk_size=2)
+    t3 = t1 + t2
+
+    t1c = copy(t1)
+    t2c = copy(t2)
+
+    assert t1c is not t1
+    assert t2c is not t2
+
+    assert t3.op.lhs is t1.data
+    assert t3.op.rhs is t2.data
+    assert t3.op.inputs == [t1.data, t2.data]
+    assert t3.inputs == [t1.data, t2.data]
+
+    with pytest.raises(StopIteration):
+        t3.inputs = []
+
+    t1 = tensor([1, 2], chunk_size=2)
+    t2 = tensor([True, False], chunk_size=2)
+    t3 = t1[t2]
+
+    t1c = copy(t1)
+    t2c = copy(t2)
+    t3c = copy(t3)
+    t3c.inputs = [t1c, t2c]
+
+    with enter_mode(build=True):
+        assert t3c.op.input is t1c.data
+        assert t3c.op.indexes[0] is t2c.data
+
+
+def test_from_spmatrix():
+    t = tensor(sps.csr_matrix([[0, 0, 1], [1, 0, 0]], dtype="f8"), chunk_size=2)
+
+    assert isinstance(t, SparseTensor)
+    assert isinstance(t.op, CSRMatrixDataSource)
+    assert t.issparse() is True
+    assert not t.op.gpu
+    assert isinstance(t.op, CSRMatrixDataSource)
+    assert not t.op.gpu
+
+
+def test_from_dense():
+    t = fromdense(tensor([[0, 0, 1], [1, 0, 0]], chunk_size=2))
+
+    assert isinstance(t, SparseTensor)
+    assert isinstance(t.op, DenseToSparse)
+    assert t.issparse() is True
+    assert isinstance(t.op, DenseToSparse)
+
+
+def test_ones_like():
+    t1 = tensor([[0, 0, 1], [1, 0, 0]], chunk_size=2).tosparse()
+    t = ones_like(t1, dtype="f8")
+
+    assert isinstance(t, SparseTensor)
+    assert isinstance(t.op, TensorOnesLike)
+    assert t.issparse() is True
+    assert t.op.gpu is None
+
+    assert isinstance(t.op, TensorOnesLike)
+
+
+def test_from_array():
+    x = array([1, 2, 3])
+    assert x.shape == (3,)
+
+    y = array([x, x])
+    assert y.shape == (2, 3)
+
+    z = array((x, x, x))
+    assert z.shape == (3, 3)
+
+
+def test_from_dataframe():
+    mdf = md.DataFrame(
+        {"a": [0, 1, 2], "b": [3, 4, 5], "c": [0.1, 0.2, 0.3]},
+        index=["c", "d", "e"],
+        chunk_size=2,
+    )
+    tensor = from_dataframe(mdf)
+    assert tensor.shape == (3, 3)
+    assert np.float64 == tensor.dtype

maxframe/tensor/datasource/zeros.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import numpy as np
+
+from ... import opcodes
+from ...serialization.serializables import (
+    AnyField,
+    FieldTypes,
+    KeyField,
+    StringField,
+    TupleField,
+)
+from ..utils import get_order
+from .array import tensor
+from .core import TensorLike, TensorNoInput
+
+
+class TensorZeros(TensorNoInput):
+    _op_type_ = opcodes.TENSOR_ZEROS
+
+    order = StringField("order")
+    shape = TupleField("shape", FieldTypes.int64)
+    chunk_size = AnyField("chunk_size")
+
+    def __init__(self, shape=None, **kwargs):
+        if type(shape) is int:
+            shape = (shape,)
+        super().__init__(shape=shape, **kwargs)
+
+
+def zeros(shape, dtype=None, chunk_size=None, gpu=None, sparse=False, order="C"):
+    """
+    Return a new tensor of given shape and type, filled with zeros.
+
+    Parameters
+    ----------
+    shape : int or sequence of ints
+        Shape of the new tensor, e.g., ``(2, 3)`` or ``2``.
+    dtype : data-type, optional
+        The desired data-type for the array, e.g., `mt.int8`.  Default is
+        `mt.float64`.
+    chunk_size : int or tuple of int or tuple of ints, optional
+        Desired chunk size on each dimension
+    gpu : bool, optional
+        Allocate the tensor on GPU if True, False as default
+    sparse: bool, optional
+        Create sparse tensor if True, False as default
+    order : {'C', 'F'}, optional, default: 'C'
+        Whether to store multi-dimensional data in row-major
+        (C-style) or column-major (Fortran-style) order in
+        memory.
+
+    Returns
+    -------
+    out : Tensor
+        Tensor of zeros with the given shape, dtype, and order.
+
+    See Also
+    --------
+    zeros_like : Return a tensor of zeros with shape and type of input.
+    ones_like : Return a tensor of ones with shape and type of input.
+    empty_like : Return a empty tensor with shape and type of input.
+    ones : Return a new tensor setting values to one.
+    empty : Return a new uninitialized tensor.
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+    >>> mt.zeros(5).execute()
+    array([ 0.,  0.,  0.,  0.,  0.])
+
+    >>> mt.zeros((5,), dtype=int).execute()
+    array([0, 0, 0, 0, 0])
+
+    >>> mt.zeros((2, 1)).execute()
+    array([[ 0.],
+           [ 0.]])
+
+    >>> s = (2,2)
+    >>> mt.zeros(s).execute()
+    array([[ 0.,  0.],
+           [ 0.,  0.]])
+
+    >>> mt.zeros((2,), dtype=[('x', 'i4'), ('y', 'i4')]).execute() # custom dtype
+    array([(0, 0), (0, 0)],
+          dtype=[('x', '<i4'), ('y', '<i4')])
+    """
+    tensor_order = get_order(
+        order,
+        None,
+        available_options="CF",
+        err_msg="only 'C' or 'F' order is permitted",
+    )
+    dtype = np.dtype(dtype or "f8")
+    op = TensorZeros(
+        dtype=dtype,
+        shape=shape,
+        chunk_size=chunk_size,
+        gpu=gpu,
+        sparse=sparse,
+        order=order,
+    )
+    return op(shape, chunk_size=chunk_size, order=tensor_order)
+
+
+class TensorZerosLike(TensorLike):
+    _op_type_ = opcodes.TENSOR_ZEROS_LIKE
+
+    _input = KeyField("input")
+    order = StringField("order")
+
+    def __init__(self, dtype=None, gpu=None, sparse=False, **kw):
+        dtype = np.dtype(dtype) if dtype is not None else None
+        super().__init__(dtype=dtype, gpu=gpu, sparse=sparse, **kw)
+
+
+def zeros_like(a, dtype=None, gpu=None, order="K"):
+    """
+    Return a tensor of zeros with the same shape and type as a given tensor.
+
+    Parameters
+    ----------
+    a : array_like
+        The shape and data-type of `a` define these same attributes of
+        the returned array.
+    dtype : data-type, optional
+        Overrides the data type of the result.
+    gpu : bool, optional
+        Allocate the tensor on GPU if True, None as default
+    order : {'C', 'F', 'A', or 'K'}, optional
+        Overrides the memory layout of the result. 'C' means C-order,
+        'F' means F-order, 'A' means 'F' if `a` is Fortran contiguous,
+        'C' otherwise. 'K' means match the layout of `a` as closely
+        as possible.
+
+    Returns
+    -------
+    out : Tensor
+        tensor of zeros with the same shape and type as `a`.
+
+    See Also
+    --------
+    ones_like : Return an array of ones with shape and type of input.
+    empty_like : Return an empty array with shape and type of input.
+    zeros : Return a new array setting values to zero.
+    ones : Return a new array setting values to one.
+    empty : Return a new uninitialized array.
+
+    Examples
+    --------
+    >>> import maxframe.tensr as mt
+    >>> x = mt.arange(6)
+    >>> x = x.reshape((2, 3))
+    >>> x.execute()
+    array([[0, 1, 2],
+           [3, 4, 5]])
+
+    >>> mt.zeros_like(x).execute()
+    array([[0, 0, 0],
+           [0, 0, 0]])
+
+    >>> y = mt.arange(3, dtype=float)
+    >>> y.execute()
+    array([ 0.,  1.,  2.])
+
+    >>> mt.zeros_like(y).execute()
+    array([ 0.,  0.,  0.])
+    """
+    a = tensor(a)
+    tensor_order = get_order(order, a.order)
+    gpu = a.op.gpu if gpu is None else gpu
+    op = TensorZerosLike(dtype=dtype, gpu=gpu, sparse=a.issparse(), order=order)
+    return op(a, order=tensor_order)

maxframe/tensor/fetch/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .core import TensorFetch, TensorFetchShuffle

maxframe/tensor/fetch/core.py

@@ -0,0 +1,54 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...core import OutputType, register_fetch_class
+from ...core.operator import Fetch, FetchMixin, FetchShuffle
+from ...serialization.serializables import DataTypeField
+from ..operators import TensorOperatorMixin
+
+
+class TensorFetchMixin(TensorOperatorMixin, FetchMixin):
+    __slots__ = ()
+    _output_type_ = OutputType.tensor
+
+
+class TensorFetch(TensorFetchMixin, Fetch):
+    dtype = DataTypeField("dtype")
+
+    def __init__(self, **kw):
+        kw.pop("output_types", None)
+        kw.pop("_output_types", None)
+        super().__init__(**kw)
+
+    def _new_tileables(self, inputs, kws=None, **kw):
+        if "_key" in kw and self.source_key is None:
+            self.source_key = kw["_key"]
+        return super()._new_tileables(inputs, kws=kws, **kw)
+
+
+class TensorFetchShuffle(TensorFetchMixin, FetchShuffle):
+    _dtype = DataTypeField("dtype")
+
+    def __init__(self, **kw):
+        kw.pop("output_types", None)
+        kw.pop("_output_types", None)
+        super().__init__(**kw)
+
+    @property
+    def dtype(self):
+        return getattr(self, "_dtype", None)
+
+
+register_fetch_class(OutputType.tensor, TensorFetch, TensorFetchShuffle)
+register_fetch_class(OutputType.scalar, TensorFetch, TensorFetchShuffle)

maxframe/tensor/indexing/__init__.py

@@ -0,0 +1,47 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .choose import TensorChoose, choose
+from .compress import compress
+from .extract import extract
+from .fill_diagonal import TensorFillDiagonal, fill_diagonal
+from .flatnonzero import flatnonzero
+from .getitem import FancyIndexingConcat, FancyIndexingDistribute, TensorIndex
+from .nonzero import TensorNonzero, nonzero
+from .setitem import TensorIndexSetValue
+from .slice import TensorSlice
+from .take import take
+from .unravel_index import TensorUnravelIndex, unravel_index
+
+
+def _install():
+    from ..core import Tensor, TensorData
+    from .getitem import _getitem
+    from .setitem import _setitem
+
+    setattr(Tensor, "__getitem__", _getitem)
+    setattr(TensorData, "__getitem__", _getitem)
+    setattr(Tensor, "__setitem__", _setitem)
+    setattr(Tensor, "take", take)
+    setattr(
+        Tensor,
+        "compress",
+        lambda a, condition, axis=None: compress(condition, a, axis=axis),
+    )
+    setattr(Tensor, "choose", choose)
+    setattr(Tensor, "nonzero", nonzero)
+
+
+_install()
+del _install