maxframe 0.1.0b4__cp311-cp311-win32.whl → 0.1.0b5__cp311-cp311-win32.whl

maxframe/learn/utils/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .core import convert_to_tensor_or_dataframe

maxframe/learn/utils/core.py

@@ -0,0 +1,29 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pandas as pd
+
+from ...dataframe import DataFrame, Series
+from ...dataframe.core import DATAFRAME_TYPE, SERIES_TYPE
+from ...tensor import tensor as astensor
+
+
+def convert_to_tensor_or_dataframe(item):
+    if isinstance(item, (DATAFRAME_TYPE, pd.DataFrame)):
+        item = DataFrame(item)
+    elif isinstance(item, (SERIES_TYPE, pd.Series)):
+        item = Series(item)
+    else:
+        item = astensor(item)
+    return item

maxframe/odpsio/arrow.py

@@ -17,10 +17,9 @@ from typing import Any, Tuple, Union
 import pandas as pd
 import pyarrow as pa
 
-import maxframe.tensor as mt
-
 from ..core import OutputType
 from ..protocol import DataFrameTableMeta
+from ..tensor.core import TENSOR_TYPE
 from ..typing_ import ArrowTableType, PandasObjectTypes
 from .schema import build_dataframe_table_meta
 
@@ -83,7 +82,7 @@ def pandas_to_arrow(
         df = df.to_frame(name=names[0] if len(names) == 1 else names)
     elif table_meta.type == OutputType.scalar:
         names = ["_idx_0"]
-        if isinstance(df, mt.Tensor):
+        if isinstance(df, TENSOR_TYPE):
             df = pd.DataFrame([], columns=names).astype({names[0]: df.dtype})
         else:
             df = pd.DataFrame([[df]], columns=names)

maxframe/odpsio/tableio.py

@@ -183,6 +183,28 @@ class HaloTableIO(MCTableIO):
             for pt in partitions
         ]
 
+    def get_table_record_count(
+        self, full_table_name: str, partitions: PartitionsType = None
+    ):
+        from odps.apis.storage_api import SplitOptions, TableBatchScanRequest
+
+        table = self._odps.get_table(full_table_name)
+        client = StorageApiArrowClient(
+            self._odps, table, rest_endpoint=self._storage_api_endpoint
+        )
+
+        split_option = SplitOptions.SplitMode.SIZE
+
+        scan_kw = {
+            "required_partitions": self._convert_partitions(partitions),
+            "split_options": SplitOptions.get_default_options(split_option),
+        }
+
+        # todo add more options for partition column handling
+        req = TableBatchScanRequest(**scan_kw)
+        resp = client.create_read_session(req)
+        return resp.record_count
+
     @contextmanager
     def open_reader(
         self,

maxframe/odpsio/tests/test_schema.py

@@ -30,20 +30,23 @@ from ..schema import (
 )
 
 
-def _wrap_maxframe_obj(obj, wrap=True):
-    if not wrap:
+def _wrap_maxframe_obj(obj, wrap="no"):
+    if wrap == "no":
         return obj
     if isinstance(obj, pd.DataFrame):
-        return md.DataFrame(obj)
+        obj = md.DataFrame(obj)
     elif isinstance(obj, pd.Series):
-        return md.Series(obj)
+        obj = md.Series(obj)
     elif isinstance(obj, pd.Index):
-        return md.Index(obj)
+        obj = md.Index(obj)
     else:
-        return mt.scalar(obj)
+        obj = mt.scalar(obj)
+    if wrap == "data":
+        return obj.data
+    return obj
 
 
-@pytest.mark.parametrize("wrap_obj", [False, True])
+@pytest.mark.parametrize("wrap_obj", ["no", "yes", "data"])
 def test_pandas_to_odps_schema_dataframe(wrap_obj):
     data = pd.DataFrame(np.random.rand(100, 5), columns=list("ABCDE"))
 
@@ -94,7 +97,7 @@ def test_pandas_to_odps_schema_dataframe(wrap_obj):
     assert meta.pd_index_level_names == [None, None]
 
 
-@pytest.mark.parametrize("wrap_obj", [False, True])
+@pytest.mark.parametrize("wrap_obj", ["no", "yes", "data"])
 def test_pandas_to_odps_schema_series(wrap_obj):
     data = pd.Series(np.random.rand(100))
 
@@ -135,7 +138,7 @@ def test_pandas_to_odps_schema_series(wrap_obj):
     assert meta.pd_index_level_names == ["c1", "c2"]
 
 
-@pytest.mark.parametrize("wrap_obj", [False, True])
+@pytest.mark.parametrize("wrap_obj", ["no", "yes", "data"])
 def test_pandas_to_odps_schema_index(wrap_obj):
     data = pd.Index(np.random.randint(0, 100, 100))
 
@@ -167,11 +170,13 @@ def test_pandas_to_odps_schema_index(wrap_obj):
     assert meta.pd_index_level_names == ["c1", "c2"]
 
 
-@pytest.mark.parametrize("wrap_obj", [False, True])
+@pytest.mark.parametrize("wrap_obj", ["no", "yes", "data"])
 def test_pandas_to_odps_schema_scalar(wrap_obj):
     data = 1234.56
 
     test_scalar = _wrap_maxframe_obj(data, wrap=wrap_obj)
+    if wrap_obj != "no":
+        test_scalar.op.data = None
     schema, meta = pandas_to_odps_schema(test_scalar, unknown_as_string=True)
     assert schema.columns[0].name == "_idx_0"
     assert schema.columns[0].type.name == "double"
@@ -279,7 +284,7 @@ def test_build_column_name():
     assert build_table_column_name(4, ("A", 1), records) == "a_1"
 
 
-@pytest.mark.parametrize("wrap_obj", [False, True])
+@pytest.mark.parametrize("wrap_obj", ["no", "yes", "data"])
 def test_build_table_meta(wrap_obj):
     data = pd.DataFrame(
         np.random.rand(100, 7),

maxframe/opcodes.py

@@ -386,6 +386,9 @@ DATAFRAME_EVAL = 738
 DUPLICATED = 739
 DELETE = 740
 ALIGN = 741
+CASE_WHEN = 742
+PIVOT = 743
+PIVOT_TABLE = 744
 
 FUSE = 801
 

maxframe/serialization/core.pyi

@@ -0,0 +1,61 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from concurrent.futures import Executor
+from typing import Any, Callable, Dict, List, TypeVar
+
+def buffered(func: Callable) -> Callable: ...
+def fast_id(obj: Any) -> int: ...
+
+LoadType = TypeVar("LoadType")
+
+def load_type(class_name: str, parent_class: LoadType) -> LoadType: ...
+
+class PickleContainer:
+    def __init__(self, buffers: List[bytes]): ...
+    def get(self) -> Any: ...
+    def get_buffers(self) -> List[bytes]: ...
+
+class Serializer:
+    serializer_id: int
+    def serial(self, obj: Any, context: Dict): ...
+    def deserial(self, serialized: List, context: Dict, subs: List[Any]): ...
+    def on_deserial_error(
+        self,
+        serialized: List,
+        context: Dict,
+        subs_serialized: List,
+        error_index: int,
+        exc: BaseException,
+    ): ...
+    @classmethod
+    def register(cls, obj_type): ...
+    @classmethod
+    def unregister(cls, obj_type): ...
+
+class Placeholder:
+    id: int
+    callbacks: List[Callable]
+    def __init__(self, id_: int): ...
+    def __hash__(self): ...
+    def __eq__(self, other): ...
+
+def serialize(obj: Any, context: Dict = None): ...
+async def serialize_with_spawn(
+    obj: Any,
+    context: Dict = None,
+    spawn_threshold: int = 100,
+    executor: Executor = None,
+): ...
+def deserialize(headers: List, buffers: List, context: Dict = None): ...

maxframe/session.py

@@ -365,6 +365,15 @@ class AbstractAsyncSession(AbstractSession, metaclass=ABCMeta):
         Stop server.
         """
 
+    @abstractmethod
+    async def get_logview_address(self, hours=None) -> Optional[str]:
+        """
+        Get Logview address
+        Returns
+        -------
+            Logview address
+        """
+
     def close(self):
         asyncio.run(self.destroy())
 
@@ -549,6 +558,15 @@ class AbstractSyncSession(AbstractSession, metaclass=ABCMeta):
 
         return fetch(tileables, self, offsets=offsets, sizes=sizes)
 
+    @abstractmethod
+    def get_logview_address(self, hours=None) -> Optional[str]:
+        """
+        Get logview address
+        Returns
+        -------
+            logview address
+        """
+
 
 def _delegate_to_isolated_session(func: Union[Callable, Coroutine]):
     if asyncio.iscoroutinefunction(func):
@@ -728,6 +746,11 @@ class AsyncSession(AbstractAsyncSession):
         await asyncio.wrap_future(asyncio.run_coroutine_threadsafe(coro, self._loop))
         stop_isolation()
 
+    @implements(AbstractAsyncSession.get_logview_address)
+    @_delegate_to_isolated_session
+    async def get_logview_address(self, hours=None) -> Optional[str]:
+        pass  # pragma: no cover
+
 
 class ProgressBar:
     def __init__(self, show_progress):
@@ -949,6 +972,11 @@ class SyncSession(AbstractSyncSession):
     def get_cluster_versions(self) -> List[str]:
         pass  # pragma: no cover
 
+    @implements(AbstractSyncSession.get_logview_address)
+    @_delegate_to_isolated_session
+    def get_logview_address(self, hours=None) -> Optional[str]:
+        pass  # pragma: no cover
+
     def destroy(self):
         coro = self._isolated_session.destroy()
         asyncio.run_coroutine_threadsafe(coro, self._loop).result()

maxframe/tensor/__init__.py

@@ -114,7 +114,7 @@ from .arithmetic import (
 )
 from .arithmetic import truediv as true_divide
 from .arithmetic import trunc
-from .base import broadcast_to, transpose, where
+from .base import broadcast_to, transpose, unique, where
 from .core import Tensor
 from .datasource import (
     arange,

maxframe/tensor/base/__init__.py

@@ -13,9 +13,11 @@
 # limitations under the License.
 
 from .astype import TensorAstype
+from .atleast_1d import atleast_1d
 from .broadcast_to import TensorBroadcastTo, broadcast_to
 from .ravel import ravel
 from .transpose import transpose
+from .unique import unique
 from .where import TensorWhere, where
 
 

maxframe/tensor/base/atleast_1d.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright 1999-2021 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ...core import ExecutableTuple
+from ..datasource import tensor as astensor
+
+
+def atleast_1d(*tensors):
+    """
+    Convert inputs to tensors with at least one dimension.
+
+    Scalar inputs are converted to 1-dimensional tensors, whilst
+    higher-dimensional inputs are preserved.
+
+    Parameters
+    ----------
+    tensors1, tensors2, ... : array_like
+        One or more input tensors.
+
+    Returns
+    -------
+    ret : Tensor
+        An tensor, or list of tensors, each with ``a.ndim >= 1``.
+        Copies are made only if necessary.
+
+    See Also
+    --------
+    atleast_2d, atleast_3d
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+
+    >>> mt.atleast_1d(1.0).execute()
+    array([ 1.])
+
+    >>> x = mt.arange(9.0).reshape(3,3)
+    >>> mt.atleast_1d(x).execute()
+    array([[ 0.,  1.,  2.],
+           [ 3.,  4.,  5.],
+           [ 6.,  7.,  8.]])
+    >>> mt.atleast_1d(x) is x
+    True
+
+    >>> mt.atleast_1d(1, [3, 4]).execute()
+    [array([1]), array([3, 4])]
+
+    """
+    new_tensors = []
+    for x in tensors:
+        x = astensor(x)
+        if x.ndim == 0:
+            x = x[np.newaxis]
+
+        new_tensors.append(x)
+
+    if len(new_tensors) == 1:
+        return new_tensors[0]
+    return ExecutableTuple(new_tensors)

maxframe/tensor/base/unique.py

@@ -0,0 +1,205 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import numpy as np
+
+from ... import opcodes as OperandDef
+from ...serialization.serializables import BoolField, Int32Field
+from ..core import TensorOrder
+from ..operators import TensorHasInput, TensorOperatorMixin
+from ..utils import validate_axis
+
+
+class TensorUnique(TensorHasInput, TensorOperatorMixin):
+    _op_type_ = OperandDef.UNIQUE
+
+    return_index = BoolField("return_index", default=False)
+    return_inverse = BoolField("return_inverse", default=False)
+    return_counts = BoolField("return_counts", default=False)
+    axis = Int32Field("axis", default=None)
+
+    @property
+    def output_limit(self):
+        return 1
+
+    def _gen_kws(self, input_obj, chunk=False, chunk_index=None):
+        kws = []
+
+        # unique tensor
+        shape = list(input_obj.shape)
+        shape[self.axis] = np.nan
+        kw = {"shape": tuple(shape), "dtype": input_obj.dtype, "gpu": input_obj.op.gpu}
+        if chunk:
+            idx = [0] * len(shape)
+            idx[self.axis] = chunk_index or 0
+            kw["index"] = tuple(idx)
+        kws.append(kw)
+
+        # unique indices tensor
+        if self.return_index:
+            kw = {
+                "shape": (np.nan,),
+                "dtype": np.dtype(np.intp),
+                "gpu": input_obj.op.gpu,
+                "type": "indices",
+            }
+            if chunk:
+                kw["index"] = (chunk_index or 0,)
+            kws.append(kw)
+
+        # unique inverse tensor
+        if self.return_inverse:
+            kw = {
+                "shape": (input_obj.shape[self.axis],),
+                "dtype": np.dtype(np.intp),
+                "gpu": input_obj.op.gpu,
+                "type": "inverse",
+            }
+            if chunk:
+                kw["index"] = (chunk_index or 0,)
+            kws.append(kw)
+
+        # unique counts tensor
+        if self.return_counts:
+            kw = {
+                "shape": (np.nan,),
+                "dtype": np.dtype(np.int_),
+                "gpu": input_obj.op.gpu,
+                "type": "counts",
+            }
+            if chunk:
+                kw["index"] = (chunk_index or 0,)
+            kws.append(kw)
+
+        return kws
+
+    def __call__(self, ar):
+        from .atleast_1d import atleast_1d
+
+        ar = atleast_1d(ar)
+        if self.axis is None:
+            if ar.ndim > 1:
+                ar = ar.flatten()
+            self._axis = 0
+        else:
+            self._axis = validate_axis(ar.ndim, self._axis)
+
+        kws = self._gen_kws(self, ar)
+        tensors = self.new_tensors([ar], kws=kws, order=TensorOrder.C_ORDER)
+        if len(tensors) == 1:
+            return tensors[0]
+        return tensors
+
+
+def unique(
+    ar,
+    return_index=False,
+    return_inverse=False,
+    return_counts=False,
+    axis=None,
+):
+    """
+    Find the unique elements of a tensor.
+
+    Returns the sorted unique elements of a tensor. There are three optional
+    outputs in addition to the unique elements:
+
+    * the indices of the input tensor that give the unique values
+    * the indices of the unique tensor that reconstruct the input tensor
+    * the number of times each unique value comes up in the input tensor
+
+    Parameters
+    ----------
+    ar : array_like
+        Input tensor. Unless `axis` is specified, this will be flattened if it
+        is not already 1-D.
+    return_index : bool, optional
+        If True, also return the indices of `ar` (along the specified axis,
+        if provided, or in the flattened tensor) that result in the unique tensor.
+    return_inverse : bool, optional
+        If True, also return the indices of the unique tensor (for the specified
+        axis, if provided) that can be used to reconstruct `ar`.
+    return_counts : bool, optional
+        If True, also return the number of times each unique item appears
+        in `ar`.
+    axis : int or None, optional
+        The axis to operate on. If None, `ar` will be flattened. If an integer,
+        the subarrays indexed by the given axis will be flattened and treated
+        as the elements of a 1-D tensor with the dimension of the given axis,
+        see the notes for more details.  Object tensors or structured tensors
+        that contain objects are not supported if the `axis` kwarg is used. The
+        default is None.
+
+    Returns
+    -------
+    unique : Tensor
+        The sorted unique values.
+    unique_indices : Tensor, optional
+        The indices of the first occurrences of the unique values in the
+        original tensor. Only provided if `return_index` is True.
+    unique_inverse : Tensor, optional
+        The indices to reconstruct the original tensor from the
+        unique tensor. Only provided if `return_inverse` is True.
+    unique_counts : Tensor, optional
+        The number of times each of the unique values comes up in the
+        original tensor. Only provided if `return_counts` is True.
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+
+    >>> mt.unique([1, 1, 2, 2, 3, 3]).execute()
+    array([1, 2, 3])
+    >>> a = mt.array([[1, 1], [2, 3]])
+    >>> mt.unique(a).execute()
+    array([1, 2, 3])
+
+    Return the unique rows of a 2D tensor
+
+    >>> a = mt.array([[1, 0, 0], [1, 0, 0], [2, 3, 4]])
+    >>> mt.unique(a, axis=0).execute()
+    array([[1, 0, 0], [2, 3, 4]])
+
+    Return the indices of the original tensor that give the unique values:
+
+    >>> a = mt.array(['a', 'b', 'b', 'c', 'a'])
+    >>> u, indices = mt.unique(a, return_index=True)
+    >>> u.execute()
+    array(['a', 'b', 'c'],
+           dtype='|S1')
+    >>> indices.execute()
+    array([0, 1, 3])
+    >>> a[indices].execute()
+    array(['a', 'b', 'c'],
+           dtype='|S1')
+
+    Reconstruct the input array from the unique values:
+
+    >>> a = mt.array([1, 2, 6, 4, 2, 3, 2])
+    >>> u, indices = mt.unique(a, return_inverse=True)
+    >>> u.execute()
+    array([1, 2, 3, 4, 6])
+    >>> indices.execute()
+    array([0, 1, 4, 3, 1, 2, 1])
+    >>> u[indices].execute()
+    array([1, 2, 6, 4, 2, 3, 2])
+    """
+    op = TensorUnique(
+        return_index=return_index,
+        return_inverse=return_inverse,
+        return_counts=return_counts,
+        axis=axis,
+    )
+    return op(ar)

maxframe/tensor/datasource/array.py

@@ -20,6 +20,7 @@ from ...serialization.serializables import (
     AnyField,
     FieldTypes,
     NDArrayField,
+    StringField,
     TupleField,
 )
 from ...utils import on_deserialize_shape, on_serialize_shape
@@ -37,8 +38,9 @@ class ArrayDataSource(TensorNoInput):
 
     _op_type_ = opcodes.TENSOR_DATA_SOURCE
 
-    data = NDArrayField("data")
-    chunk_size = AnyField("chunk_size")
+    data = NDArrayField("data", default=None)
+    chunk_size = AnyField("chunk_size", default=None)
+    order = StringField("order", default=None)
 
     def __init__(self, data=None, dtype=None, gpu=None, **kw):
         if dtype is not None:

maxframe/tensor/datasource/scalar.py

@@ -33,7 +33,7 @@ class Scalar(TensorNoInput):
 def scalar(data, dtype=None, gpu=None):
     try:
         arr = np.array(data, dtype=dtype)
-        op = Scalar(arr, dtype=arr.dtype, gpu=gpu)
+        op = Scalar(data=arr, dtype=arr.dtype, gpu=gpu)
         shape = ()
         return op(shape)
     except ValueError: