maxframe 0.1.0b5__cp310-cp310-macosx_11_0_arm64.whl

maxframe/tensor/datasource/__init__.py

@@ -0,0 +1,32 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .arange import arange
+from .array import (
+    ArrayDataSource,
+    CSRMatrixDataSource,
+    array,
+    asarray,
+    ascontiguousarray,
+    asfortranarray,
+    tensor,
+)
+from .empty import TensorEmpty, TensorEmptyLike, empty, empty_like
+from .from_dataframe import from_dataframe, from_index, from_series
+from .from_dense import fromdense
+from .from_sparse import fromsparse
+from .full import full, full_like
+from .ones import TensorOnes, TensorOnesLike, ones, ones_like
+from .scalar import Scalar, scalar
+from .zeros import TensorZeros, zeros

maxframe/tensor/datasource/arange.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ... import opcodes
+from ...serialization.serializables import AnyField
+from .core import TensorNoInput
+
+
+class TensorArange(TensorNoInput):
+    _op_type_ = opcodes.TENSOR_ARANGE
+
+    start = AnyField("start")
+    stop = AnyField("stop")
+    step = AnyField("step")
+
+    def __init__(self, start=None, stop=None, step=None, dtype=None, **kw):
+        if dtype is not None:
+            dtype = np.dtype(dtype)
+        elif stop is not None and step is not None:
+            dtype = (
+                np.dtype(dtype)
+                if dtype is not None
+                else np.arange(0, type(stop)(1), step).dtype
+            )
+        super().__init__(start=start, stop=stop, step=step, dtype=dtype, **kw)
+
+
+def arange(*args, **kwargs):
+    """
+    Return evenly spaced values within a given interval.
+
+    Values are generated within the half-open interval ``[start, stop)``
+    (in other words, the interval including `start` but excluding `stop`).
+    For integer arguments the function is equivalent to the Python built-in
+    `range <http://docs.python.org/lib/built-in-funcs.html>`_ function,
+    but returns a tensor rather than a list.
+
+    When using a non-integer step, such as 0.1, the results will often not
+    be consistent.  It is better to use ``linspace`` for these cases.
+
+    Parameters
+    ----------
+    start : number, optional
+        Start of interval.  The interval includes this value.  The default
+        start value is 0.
+    stop : number
+        End of interval.  The interval does not include this value, except
+        in some cases where `step` is not an integer and floating point
+        round-off affects the length of `out`.
+    step : number, optional
+        Spacing between values.  For any output `out`, this is the distance
+        between two adjacent values, ``out[i+1] - out[i]``.  The default
+        step size is 1.  If `step` is specified as a position argument,
+        `start` must also be given.
+    dtype : dtype
+        The type of the output tensor.  If `dtype` is not given, infer the data
+        type from the other input arguments.
+    gpu : bool, optional
+        Allocate the tensor on GPU if True, False as default
+
+    Returns
+    -------
+    arange : Tensor
+        Tensor of evenly spaced values.
+
+        For floating point arguments, the length of the result is
+        ``ceil((stop - start)/step)``.  Because of floating point overflow,
+        this rule may result in the last element of `out` being greater
+        than `stop`.
+
+    See Also
+    --------
+    linspace : Evenly spaced numbers with careful handling of endpoints.
+    ogrid: Tensors of evenly spaced numbers in N-dimensions.
+    mgrid: Grid-shaped tensors of evenly spaced numbers in N-dimensions.
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+
+    >>> mt.arange(3).execute()
+    array([0, 1, 2])
+    >>> mt.arange(3.0).execute()
+    array([ 0.,  1.,  2.])
+    >>> mt.arange(3,7).execute()
+    array([3, 4, 5, 6])
+    >>> mt.arange(3,7,2).execute()
+    array([3, 5])
+    """
+    kw_args = [kwargs.get("start"), kwargs.get("stop"), kwargs.get("step")]
+    kw_def = any(arg is not None for arg in kw_args)
+    dtype = None
+    if not kw_def:
+        if len(args) == 1:
+            start = 0
+            stop = args[0]
+            step = 1
+        elif len(args) == 2:
+            start = args[0]
+            stop = args[1]
+            step = 1
+        elif len(args) == 3:
+            start, stop, step = args
+        elif len(args) == 4:
+            start, stop, step, dtype = args
+            dtype = np.dtype(dtype)
+        else:
+            raise TypeError("Required argument 'start' (pos 1) not found")
+    else:
+        names = "start", "stop", "step"
+        for i, arg in enumerate(args):
+            if kw_args[i] is not None:
+                raise TypeError(
+                    f"Argument given by name ('{names[i]}') and position ({i})"
+                )
+            kw_args[i] = arg
+        start, stop, step = kw_args
+
+    if dtype is None:
+        if "dtype" in kwargs:
+            dtype = np.dtype(kwargs["dtype"])
+        else:
+            dtype = np.arange(0, type(stop)(1), step).dtype
+
+    start, stop = dtype.type(start), dtype.type(stop)
+    if dtype == np.datetime64 and not start:
+        raise ValueError(
+            "arange requires both a start and a stop for MaxFrame datetime64 ranges"
+        )
+    if dtype == np.datetime64:
+        span = np.array([stop - start])
+        span[0] = step
+        step = span[0]
+        dtype = np.dtype(stop.dtype)
+    else:
+        step = dtype.type(step)
+    size = max(int(np.ceil(np.true_divide(stop - start, step))), 0)
+
+    op = TensorArange(start, stop, step, dtype=dtype, gpu=kwargs.get("gpu", False))
+    shape = (size,)
+    return op(shape, chunk_size=kwargs.pop("chunk_size", None))

maxframe/tensor/datasource/array.py

@@ -0,0 +1,415 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ... import opcodes
+from ...lib.sparse.core import cp, cps, get_array_module, issparse, sps
+from ...serialization.serializables import (
+    AnyField,
+    FieldTypes,
+    NDArrayField,
+    StringField,
+    TupleField,
+)
+from ...utils import on_deserialize_shape, on_serialize_shape
+from ..array_utils import is_array, is_cupy
+from ..core import TENSOR_TYPE, Tensor, TensorData, TensorOrder
+from ..utils import get_chunk_slices
+from .core import TensorNoInput
+from .scalar import scalar
+
+
+class ArrayDataSource(TensorNoInput):
+    """
+    Represents data from numpy or cupy array
+    """
+
+    _op_type_ = opcodes.TENSOR_DATA_SOURCE
+
+    data = NDArrayField("data", default=None)
+    chunk_size = AnyField("chunk_size", default=None)
+    order = StringField("order", default=None)
+
+    def __init__(self, data=None, dtype=None, gpu=None, **kw):
+        if dtype is not None:
+            dtype = np.dtype(dtype)
+        elif data is not None:
+            dtype = np.dtype(data.dtype)
+
+        if gpu is None and is_cupy(data):  # pragma: no cover
+            gpu = True
+
+        super().__init__(data=data, dtype=dtype, gpu=gpu, **kw)
+
+
+class CSRMatrixDataSource(TensorNoInput):
+    """
+    Represents data from sparse array include scipy sparse or cupy sparse matrix.
+    """
+
+    _op_type_ = opcodes.SPARSE_MATRIX_DATA_SOURCE
+
+    indices = NDArrayField("indices")
+    indptr = NDArrayField("indptr")
+    data = NDArrayField("data")
+    shape = TupleField(
+        "shape",
+        FieldTypes.int64,
+        on_serialize=on_serialize_shape,
+        on_deserialize=on_deserialize_shape,
+    )
+    chunk_size = AnyField("chunk_size")
+
+    def __init__(self, data=None, **kw):
+        kw["sparse"] = True
+        if is_cupy(data):  # pragma: no cover
+            kw["gpu"] = True
+        super().__init__(data=data, **kw)
+
+    def to_chunk_op(self, *args):
+        _, idx, chunk_size = args
+
+        xps = cps if self.gpu else sps
+        if len(self.shape) == 1:
+            shape = (1, self.shape[0])
+        else:
+            shape = self.shape
+        data = xps.csr_matrix((self.data, self.indices, self.indptr), shape)
+        chunk_data = data[get_chunk_slices(chunk_size, idx)]
+
+        chunk_op = self.copy().reset_key()
+        chunk_op.data = chunk_data.data
+        chunk_op.indices = chunk_data.indices
+        chunk_op.indptr = chunk_data.indptr
+        chunk_shape = chunk_data.shape[1:] if len(self.shape) == 1 else chunk_data.shape
+        chunk_op.shape = chunk_shape
+
+        return chunk_op
+
+
+def _from_spmatrix(spmatrix, dtype=None, chunk_size=None, gpu=None):
+    if gpu is None:
+        m = get_array_module(spmatrix)
+        if cp is not None and m is cp:
+            gpu = True
+        elif cp is np:
+            gpu = False
+    if dtype and spmatrix.dtype != dtype:
+        spmatrix = spmatrix.astype(dtype)
+    spmatrix = spmatrix.tocsr()
+    op = CSRMatrixDataSource(
+        indices=spmatrix.indices,
+        indptr=spmatrix.indptr,
+        data=spmatrix.data,
+        shape=spmatrix.shape,
+        dtype=spmatrix.dtype,
+        gpu=gpu,
+        chunk_size=chunk_size,
+    )
+    return op(spmatrix.shape, chunk_size=chunk_size)
+
+
+def tensor(
+    data=None, dtype=None, order="K", chunk_size=None, gpu=None, sparse=False
+) -> Tensor:
+    order = order or "K"
+    if isinstance(data, TENSOR_TYPE):
+        if isinstance(data, TensorData):
+            data = Tensor(data)
+        return data.astype(dtype or data.dtype, order=order, copy=False)
+    elif (
+        isinstance(data, (tuple, list))
+        and len(data) > 0
+        and all(isinstance(d, TENSOR_TYPE) for d in data)
+    ):
+        from ..merge import stack
+
+        data = stack(data)
+        return data.astype(dtype or data.dtype, order=order, copy=False)
+    elif np.isscalar(data):
+        return scalar(data, dtype=dtype)
+    elif issparse(data):
+        return _from_spmatrix(data, dtype=dtype, chunk_size=chunk_size, gpu=gpu)
+    elif hasattr(data, "__maxframe_tensor__"):
+        return data.__maxframe_tensor__(dtype=dtype, order=order)
+    else:
+        m = get_array_module(data)
+        try:
+            data = m.asarray(data, dtype=dtype, order=order)
+        except ValueError:
+            arr = data.__array__(dtype=dtype)
+            if isinstance(arr, TENSOR_TYPE):
+                return arr.astype(arr.dtype, order=order, copy=False)
+            raise
+        if gpu is None:
+            if cp is not None and m is cp:
+                gpu = True
+
+    if is_array(data):
+        if data.ndim == 0:
+            return scalar(data.item(), dtype=dtype)
+        tensor_order = (
+            TensorOrder.C_ORDER if data.flags["C_CONTIGUOUS"] else TensorOrder.F_ORDER
+        )
+        op = ArrayDataSource(data, dtype=dtype, gpu=gpu, chunk_size=chunk_size)
+        t = op(data.shape, chunk_size=chunk_size, order=tensor_order)
+        if sparse and not t.issparse():
+            return t.tosparse()
+        return t
+    else:
+        raise ValueError(f"Cannot create tensor by given data: {data}")
+
+
+def array(x, dtype=None, copy=True, order="K", ndmin=None, chunk_size=None):
+    """
+    Create a tensor.
+
+    Parameters
+    ----------
+    object : array_like
+        An array, any object exposing the array interface, an object whose
+        __array__ method returns an array, or any (nested) sequence.
+    dtype : data-type, optional
+        The desired data-type for the array.  If not given, then the type will
+        be determined as the minimum type required to hold the objects in the
+        sequence.  This argument can only be used to 'upcast' the array.  For
+        downcasting, use the .astype(t) method.
+    copy : bool, optional
+        If true (default), then the object is copied.  Otherwise, a copy will
+        only be made if __array__ returns a copy, if obj is a nested sequence,
+        or if a copy is needed to satisfy any of the other requirements
+        (`dtype`, `order`, etc.).
+    order : {'K', 'A', 'C', 'F'}, optional
+        Specify the memory layout of the array. If object is not an array, the
+        newly created array will be in C order (row major) unless 'F' is
+        specified, in which case it will be in Fortran order (column major).
+        If object is an array the following holds.
+
+        ===== ========= ===================================================
+        order  no copy                     copy=True
+        ===== ========= ===================================================
+        'K'   unchanged F & C order preserved, otherwise most similar order
+        'A'   unchanged F order if input is F and not C, otherwise C order
+        'C'   C order   C order
+        'F'   F order   F order
+        ===== ========= ===================================================
+
+        When ``copy=False`` and a copy is made for other reasons, the result is
+        the same as if ``copy=True``, with some exceptions for `A`, see the
+        Notes section. The default order is 'K'.
+    ndmin : int, optional
+        Specifies the minimum number of dimensions that the resulting
+        array should have.  Ones will be prepended to the shape as
+        needed to meet this requirement.
+    chunk_size: int, tuple, optional
+        Specifies chunk size for each dimension.
+
+    Returns
+    -------
+    out : Tensor
+        An tensor object satisfying the specified requirements.
+
+    See Also
+    --------
+    empty, empty_like, zeros, zeros_like, ones, ones_like, full, full_like
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+
+    >>> mt.array([1, 2, 3]).execute()
+    array([1, 2, 3])
+
+    Upcasting:
+
+    >>> mt.array([1, 2, 3.0]).execute()
+    array([ 1.,  2.,  3.])
+
+    More than one dimension:
+
+    >>> mt.array([[1, 2], [3, 4]]).execute()
+    array([[1, 2],
+           [3, 4]])
+
+    Minimum dimensions 2:
+
+    >>> mt.array([1, 2, 3], ndmin=2).execute()
+    array([[1, 2, 3]])
+
+    Type provided:
+
+    >>> mt.array([1, 2, 3], dtype=complex).execute()
+    array([ 1.+0.j,  2.+0.j,  3.+0.j])
+
+    """
+    raw_x = x
+    order = order or "K"
+    x = tensor(x, dtype=dtype, order=order, chunk_size=chunk_size)
+    while ndmin is not None and x.ndim < ndmin:
+        x = x[np.newaxis]
+
+    if copy and x is raw_x:
+        x = x.copy(order=order)
+    elif (
+        not copy
+        and isinstance(raw_x, TENSOR_TYPE)
+        and raw_x.dtype == x.dtype
+        and raw_x.order == x.order
+        and raw_x.shape == x.shape
+        and raw_x is not x
+        and hasattr(raw_x, "data")
+    ):
+        raw_x.data = x.data
+
+    return x
+
+
+def asarray(x, dtype=None, order=None, chunk_size=None):
+    """Convert the input to an array.
+
+    Parameters
+    ----------
+    a : array_like
+        Input data, in any form that can be converted to a tensor.  This
+        includes lists, lists of tuples, tuples, tuples of tuples, tuples
+        of lists and tensors.
+    dtype : data-type, optional
+        By default, the data-type is inferred from the input data.
+    order : {'C', 'F'}, optional
+        Whether to use row-major (C-style) or
+        column-major (Fortran-style) memory representation.
+    chunk_size: int, tuple, optional
+        Specifies chunk size for each dimension.
+
+    Returns
+    -------
+    out : Tensor
+        Tensor interpretation of `a`.  No copy is performed if the input
+        is already an ndarray with matching dtype and order.  If `a` is a
+        subclass of ndarray, a base class ndarray is returned.
+
+    See Also
+    --------
+    ascontiguousarray : Convert input to a contiguous tensor.
+    asfortranarray : Convert input to a tensor with column-major
+                     memory order.
+
+    Examples
+    --------
+    Convert a list into a tensor:
+
+    >>> import maxframe.tensor as mt
+
+    >>> a = [1, 2]
+    >>> mt.asarray(a).execute()
+    array([1, 2])
+
+    Existing arrays are not copied:
+
+    >>> a = mt.array([1, 2])
+    >>> mt.asarray(a) is a
+    True
+
+    If `dtype` is set, array is copied only if dtype does not match:
+
+    >>> a = mt.array([1, 2], dtype=mt.float32)
+    >>> mt.asarray(a, dtype=mt.float32) is a
+    True
+    >>> mt.asarray(a, dtype=mt.float64) is a
+    False
+    """
+    return array(x, dtype=dtype, copy=False, order=order, chunk_size=chunk_size)
+
+
+def ascontiguousarray(a, dtype=None, chunk_size=None):
+    """
+    Return a contiguous tensor (ndim >= 1) in memory (C order).
+
+    Parameters
+    ----------
+    a : array_like
+        Input tensor.
+    dtype : str or dtype object, optional
+        Data-type of returned tensor.
+    chunk_size: int, tuple, optional
+        Specifies chunk size for each dimension.
+
+    Returns
+    -------
+    out : Tensor
+        Contiguous tensor of same shape and content as `a`, with type `dtype`
+        if specified.
+
+    See Also
+    --------
+    asfortranarray : Convert input to a tensor with column-major
+                     memory order.
+    Tensor.flags : Information about the memory layout of the tensor.
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+    >>> x = mt.arange(6).reshape(2,3)
+    >>> mt.ascontiguousarray(x, dtype=mt.float32)
+    array([[ 0.,  1.,  2.],
+           [ 3.,  4.,  5.]], dtype=float32)
+    >>> x.flags['C_CONTIGUOUS']
+    True
+
+    Note: This function returns a tensor with at least one-dimension (1-d)
+    so it will not preserve 0-d tensors.
+
+    """
+
+    return array(a, dtype, copy=False, order="C", ndmin=1, chunk_size=chunk_size)
+
+
+def asfortranarray(a, dtype=None, chunk_size=None):
+    """
+    Return a tensor (ndim >= 1) laid out in Fortran order in memory.
+
+    Parameters
+    ----------
+    a : array_like
+        Input tensor.
+    dtype : str or dtype object, optional
+        By default, the data-type is inferred from the input data.
+    chunk_size: int, tuple, optional
+        Specifies chunk size for each dimension.
+
+    Returns
+    -------
+    out : Tensor
+        The input `a` in Fortran, or column-major, order.
+
+    See Also
+    --------
+    ascontiguousarray : Convert input to a contiguous (C order) tensor.
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+    >>> x = mt.arange(6).reshape(2,3)
+    >>> y = mt.asfortranarray(x)
+    >>> x.flags['F_CONTIGUOUS']
+    False
+    >>> y.flags['F_CONTIGUOUS']
+    True
+
+    Note: This function returns a tensor with at least one-dimension (1-d)
+    so it will not preserve 0-d tensors.
+
+    """
+    return array(a, dtype, copy=False, order="F", ndmin=1, chunk_size=chunk_size)

maxframe/tensor/datasource/core.py

@@ -0,0 +1,109 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ...serialization.serializables import FieldTypes, StringField, TupleField
+from ..core import TensorOrder
+from ..operators import TensorOperator, TensorOperatorMixin
+from ..utils import normalize_shape
+
+
+class TensorDataSource(TensorOperator, TensorOperatorMixin):
+    """
+    Tensor data source base class, provide universal tile logic,
+    subclass can overwrite tile method.
+    """
+
+    __slots__ = ()
+
+
+class TensorNoInput(TensorDataSource):
+    """
+    Tensor operator with no inputs.
+    """
+
+    def check_inputs(self, inputs):
+        # no inputs
+        if inputs and len(inputs) > 0:
+            raise ValueError("Tensor data source has no inputs")
+
+    def _new_tileables(self, inputs, kws=None, **kw):
+        shape = kw.get("shape", None)
+        self.extra_params[
+            "shape"
+        ] = shape  # set shape to make the operator key different
+        return super()._new_tileables(inputs, kws=kws, **kw)
+
+    def __call__(self, shape, chunk_size=None, order=None):
+        shape = normalize_shape(shape)
+        order = TensorOrder.C_ORDER if order is None else order
+        return self.new_tensor(None, shape, raw_chunk_size=chunk_size, order=order)
+
+
+class TensorHasInput(TensorDataSource):
+    """
+    Tensor operator with a single input.
+    """
+
+    @property
+    def input(self):
+        return self._input
+
+    def check_inputs(self, inputs):
+        # no inputs
+        if len(inputs) != 1:
+            raise ValueError("Tensor can only have 1 input")
+
+    def _set_inputs(self, inputs):
+        super()._set_inputs(inputs)
+        self._input = self._inputs[0]
+
+    def __call__(self, a, order=None):
+        order = a.order if order is None else order
+        return self.new_tensor([a], a.shape, order=order)
+
+
+class TensorLike(TensorHasInput):
+    def _set_inputs(self, inputs):
+        super()._set_inputs(inputs)
+        if self.dtype is None:
+            self.dtype = self.input.dtype
+        if self.gpu is None:
+            self.gpu = self.input.op.gpu
+
+        # FIXME: remove when cupy supports other dtypes
+        if self.gpu and self.dtype not in (np.float32, np.float64):
+            raise NotImplementedError(
+                "Sparse tensor on GPU only supports float32 and float64"
+            )
+
+
+class TensorFromHDF5Like(TensorNoInput):
+    filename = StringField("filename", default=None)
+    group = StringField("group", default=None)
+    dataset = StringField("dataset", default=None)
+    axis_offsets = TupleField("axis_offsets", FieldTypes.int64, default=None)
+
+    @property
+    def path(self):
+        return self.get_path(self.group, self.dataset)
+
+    @staticmethod
+    def get_path(group, dataset):
+        paths = []
+        if group:
+            paths.append(group)
+        paths.append(dataset)
+        return "/".join(paths)