maxframe 0.1.0b3__cp38-cp38-win_amd64.whl → 0.1.0b5__cp38-cp38-win_amd64.whl

maxframe/learn/utils/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .core import convert_to_tensor_or_dataframe

maxframe/learn/utils/core.py

@@ -0,0 +1,29 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pandas as pd
+
+from ...dataframe import DataFrame, Series
+from ...dataframe.core import DATAFRAME_TYPE, SERIES_TYPE
+from ...tensor import tensor as astensor
+
+
+def convert_to_tensor_or_dataframe(item):
+    if isinstance(item, (DATAFRAME_TYPE, pd.DataFrame)):
+        item = DataFrame(item)
+    elif isinstance(item, (SERIES_TYPE, pd.Series)):
+        item = Series(item)
+    else:
+        item = astensor(item)
+    return item

maxframe/odpsio/arrow.py

@@ -17,10 +17,9 @@ from typing import Any, Tuple, Union
 import pandas as pd
 import pyarrow as pa
 
-import maxframe.tensor as mt
-
 from ..core import OutputType
 from ..protocol import DataFrameTableMeta
+from ..tensor.core import TENSOR_TYPE
 from ..typing_ import ArrowTableType, PandasObjectTypes
 from .schema import build_dataframe_table_meta
 
@@ -65,20 +64,25 @@ def arrow_to_pandas(
         raise ValueError(f"Does not support meta type {table_meta.type!r}")
 
 
-def pandas_to_arrow(df: Any, nthreads=1) -> Tuple[ArrowTableType, DataFrameTableMeta]:
-    table_meta = build_dataframe_table_meta(df)
+def pandas_to_arrow(
+    df: Any, nthreads=1, ignore_index=False
+) -> Tuple[ArrowTableType, DataFrameTableMeta]:
+    table_meta = build_dataframe_table_meta(df, ignore_index)
     df = df.copy() if callable(getattr(df, "copy", None)) else df
     if table_meta.type in (OutputType.dataframe, OutputType.series):
         if table_meta.type == OutputType.series:
             df = df.to_frame("_data" if df.name is None else df.name)
         df.columns = pd.Index(table_meta.table_column_names)
-        df = df.rename_axis(table_meta.table_index_column_names).reset_index()
+        if not ignore_index:
+            df = df.rename_axis(table_meta.table_index_column_names).reset_index()
+    elif ignore_index:
+        df = pd.DataFrame([], columns=[])
     elif table_meta.type == OutputType.index:
         names = [f"_idx_{idx}" for idx in range(len(df.names))]
         df = df.to_frame(name=names[0] if len(names) == 1 else names)
     elif table_meta.type == OutputType.scalar:
         names = ["_idx_0"]
-        if isinstance(df, mt.Tensor):
+        if isinstance(df, TENSOR_TYPE):
             df = pd.DataFrame([], columns=names).astype({names[0]: df.dtype})
         else:
             df = pd.DataFrame([[df]], columns=names)

maxframe/odpsio/schema.py

@@ -175,7 +175,9 @@ def _scalar_as_index(df_obj: Any) -> pd.Index:
 
 
 def pandas_to_odps_schema(
-    df_obj: Any, unknown_as_string: bool = False
+    df_obj: Any,
+    unknown_as_string: bool = False,
+    ignore_index=False,
 ) -> Tuple[odps_types.OdpsSchema, DataFrameTableMeta]:
     from .. import dataframe as md
     from .arrow import pandas_to_arrow
@@ -209,7 +211,7 @@ def pandas_to_odps_schema(
     else:
         empty_df_obj = df_obj
 
-    arrow_data, table_meta = pandas_to_arrow(empty_df_obj)
+    arrow_data, table_meta = pandas_to_arrow(empty_df_obj, ignore_index=ignore_index)
     return (
         arrow_schema_to_odps_schema(
             arrow_data.schema, unknown_as_string=unknown_as_string
@@ -268,7 +270,9 @@ def build_table_column_name(
     return col_name
 
 
-def build_dataframe_table_meta(df_obj: Any) -> DataFrameTableMeta:
+def build_dataframe_table_meta(
+    df_obj: Any, ignore_index: bool = False
+) -> DataFrameTableMeta:
     from .. import dataframe as md
 
     col_to_count = defaultdict(lambda: 0)
@@ -285,6 +289,8 @@ def build_dataframe_table_meta(df_obj: Any) -> DataFrameTableMeta:
     else:  # pragma: no cover
         raise TypeError(f"Cannot accept type {type(df_obj)}")
 
+    assert not ignore_index or obj_type in (OutputType.dataframe, OutputType.series)
+
     if obj_type == OutputType.scalar:
         pd_dtypes = pd.Series([])
         column_index_names = []
@@ -340,12 +346,19 @@ def build_dataframe_table_meta(df_obj: Any) -> DataFrameTableMeta:
     else:
         index_dtypes = pd.Series([pd_index_val.dtype], index=pd_index_val.names)
 
+    if ignore_index:
+        table_index_column_names = []
+        pd_index_dtypes = pd.Series([], index=[])
+    else:
+        table_index_column_names = [f"_idx_{i}" for i in range(len(index_obj.names))]
+        pd_index_dtypes = index_dtypes
+
     return DataFrameTableMeta(
         table_name=table_name,
         type=obj_type,
         table_column_names=final_sql_columns,
-        table_index_column_names=[f"_idx_{i}" for i in range(len(index_obj.names))],
+        table_index_column_names=table_index_column_names,
         pd_column_dtypes=pd_dtypes,
         pd_column_level_names=column_index_names,
-        pd_index_dtypes=index_dtypes,
+        pd_index_dtypes=pd_index_dtypes,
     )

maxframe/odpsio/tableio.py

@@ -183,6 +183,28 @@ class HaloTableIO(MCTableIO):
             for pt in partitions
         ]
 
+    def get_table_record_count(
+        self, full_table_name: str, partitions: PartitionsType = None
+    ):
+        from odps.apis.storage_api import SplitOptions, TableBatchScanRequest
+
+        table = self._odps.get_table(full_table_name)
+        client = StorageApiArrowClient(
+            self._odps, table, rest_endpoint=self._storage_api_endpoint
+        )
+
+        split_option = SplitOptions.SplitMode.SIZE
+
+        scan_kw = {
+            "required_partitions": self._convert_partitions(partitions),
+            "split_options": SplitOptions.get_default_options(split_option),
+        }
+
+        # todo add more options for partition column handling
+        req = TableBatchScanRequest(**scan_kw)
+        resp = client.create_read_session(req)
+        return resp.record_count
+
     @contextmanager
     def open_reader(
         self,

maxframe/odpsio/tests/test_schema.py

@@ -30,20 +30,23 @@ from ..schema import (
 )
 
 
-def _wrap_maxframe_obj(obj, wrap=True):
-    if not wrap:
+def _wrap_maxframe_obj(obj, wrap="no"):
+    if wrap == "no":
         return obj
     if isinstance(obj, pd.DataFrame):
-        return md.DataFrame(obj)
+        obj = md.DataFrame(obj)
     elif isinstance(obj, pd.Series):
-        return md.Series(obj)
+        obj = md.Series(obj)
     elif isinstance(obj, pd.Index):
-        return md.Index(obj)
+        obj = md.Index(obj)
     else:
-        return mt.scalar(obj)
+        obj = mt.scalar(obj)
+    if wrap == "data":
+        return obj.data
+    return obj
 
 
-@pytest.mark.parametrize("wrap_obj", [False, True])
+@pytest.mark.parametrize("wrap_obj", ["no", "yes", "data"])
 def test_pandas_to_odps_schema_dataframe(wrap_obj):
     data = pd.DataFrame(np.random.rand(100, 5), columns=list("ABCDE"))
 
@@ -61,6 +64,16 @@ def test_pandas_to_odps_schema_dataframe(wrap_obj):
     assert meta.pd_column_level_names == [None]
     assert meta.pd_index_level_names == [None]
 
+    test_df = _wrap_maxframe_obj(data, wrap=wrap_obj)
+    schema, meta = pandas_to_odps_schema(test_df, ignore_index=True)
+    assert [c.name for c in schema.columns] == list(test_df.dtypes.index.str.lower())
+    assert [c.type.name for c in schema.columns] == ["double"] * len(test_df.columns)
+    assert meta.type == OutputType.dataframe
+    assert meta.table_column_names == list(test_df.dtypes.index.str.lower())
+    assert meta.table_index_column_names == []
+    assert meta.pd_column_level_names == [None]
+    assert meta.pd_index_level_names == []
+
     data.columns = pd.MultiIndex.from_tuples(
         [("A", "A"), ("A", "B"), ("A", "C"), ("B", "A"), ("B", "B")], names=["c1", "c2"]
     )
@@ -84,7 +97,7 @@ def test_pandas_to_odps_schema_dataframe(wrap_obj):
     assert meta.pd_index_level_names == [None, None]
 
 
-@pytest.mark.parametrize("wrap_obj", [False, True])
+@pytest.mark.parametrize("wrap_obj", ["no", "yes", "data"])
 def test_pandas_to_odps_schema_series(wrap_obj):
     data = pd.Series(np.random.rand(100))
 
@@ -99,6 +112,15 @@ def test_pandas_to_odps_schema_series(wrap_obj):
     assert meta.pd_column_level_names == [None]
     assert meta.pd_index_level_names == [None]
 
+    schema, meta = pandas_to_odps_schema(test_s, ignore_index=True)
+    assert [c.name for c in schema.columns] == ["_data"]
+    assert [c.type.name for c in schema.columns] == ["double"]
+    assert meta.type == OutputType.series
+    assert meta.table_column_names == ["_data"]
+    assert meta.table_index_column_names == []
+    assert meta.pd_column_level_names == [None]
+    assert meta.pd_index_level_names == []
+
     data.index = pd.MultiIndex.from_arrays(
         [np.random.choice(list("ABC"), 100), np.random.randint(0, 10, 100)],
         names=["c1", "c2"],
@@ -116,7 +138,7 @@ def test_pandas_to_odps_schema_series(wrap_obj):
     assert meta.pd_index_level_names == ["c1", "c2"]
 
 
-@pytest.mark.parametrize("wrap_obj", [False, True])
+@pytest.mark.parametrize("wrap_obj", ["no", "yes", "data"])
 def test_pandas_to_odps_schema_index(wrap_obj):
     data = pd.Index(np.random.randint(0, 100, 100))
 
@@ -130,6 +152,9 @@ def test_pandas_to_odps_schema_index(wrap_obj):
     assert meta.pd_column_level_names == []
     assert meta.pd_index_level_names == [None]
 
+    with pytest.raises(AssertionError):
+        pandas_to_odps_schema(test_idx, unknown_as_string=True, ignore_index=True)
+
     data = pd.MultiIndex.from_arrays(
         [np.random.choice(list("ABC"), 100), np.random.randint(0, 10, 100)],
         names=["c1", "c2"],
@@ -145,11 +170,13 @@ def test_pandas_to_odps_schema_index(wrap_obj):
     assert meta.pd_index_level_names == ["c1", "c2"]
 
 
-@pytest.mark.parametrize("wrap_obj", [False, True])
+@pytest.mark.parametrize("wrap_obj", ["no", "yes", "data"])
 def test_pandas_to_odps_schema_scalar(wrap_obj):
     data = 1234.56
 
     test_scalar = _wrap_maxframe_obj(data, wrap=wrap_obj)
+    if wrap_obj != "no":
+        test_scalar.op.data = None
     schema, meta = pandas_to_odps_schema(test_scalar, unknown_as_string=True)
     assert schema.columns[0].name == "_idx_0"
     assert schema.columns[0].type.name == "double"
@@ -159,6 +186,9 @@ def test_pandas_to_odps_schema_scalar(wrap_obj):
     assert meta.pd_column_level_names == []
     assert meta.pd_index_level_names == [None]
 
+    with pytest.raises(AssertionError):
+        pandas_to_odps_schema(test_scalar, unknown_as_string=True, ignore_index=True)
+
 
 def test_odps_arrow_schema_conversion():
     odps_schema = odps_types.OdpsSchema(
@@ -254,7 +284,7 @@ def test_build_column_name():
     assert build_table_column_name(4, ("A", 1), records) == "a_1"
 
 
-@pytest.mark.parametrize("wrap_obj", [False, True])
+@pytest.mark.parametrize("wrap_obj", ["no", "yes", "data"])
 def test_build_table_meta(wrap_obj):
     data = pd.DataFrame(
         np.random.rand(100, 7),

maxframe/opcodes.py

@@ -386,6 +386,9 @@ DATAFRAME_EVAL = 738
 DUPLICATED = 739
 DELETE = 740
 ALIGN = 741
+CASE_WHEN = 742
+PIVOT = 743
+PIVOT_TABLE = 744
 
 FUSE = 801
 
@@ -564,6 +567,11 @@ CHOLESKY_FUSE = 999988
 # MaxFrame-dedicated functions
 DATAFRAME_RESHUFFLE = 10001
 
+# MaxFrame internal operators
+DATAFRAME_PROJECTION_SAME_INDEX_MERGE = 100001
+GROUPBY_AGGR_SAME_INDEX_MERGE = 100002
+DATAFRAME_ILOC_GET_AND_RENAME_ITEM = 100003
+
 # fetches
 FETCH_SHUFFLE = 999998
 FETCH = 999999

maxframe/serialization/core.pyi

@@ -0,0 +1,61 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from concurrent.futures import Executor
+from typing import Any, Callable, Dict, List, TypeVar
+
+def buffered(func: Callable) -> Callable: ...
+def fast_id(obj: Any) -> int: ...
+
+LoadType = TypeVar("LoadType")
+
+def load_type(class_name: str, parent_class: LoadType) -> LoadType: ...
+
+class PickleContainer:
+    def __init__(self, buffers: List[bytes]): ...
+    def get(self) -> Any: ...
+    def get_buffers(self) -> List[bytes]: ...
+
+class Serializer:
+    serializer_id: int
+    def serial(self, obj: Any, context: Dict): ...
+    def deserial(self, serialized: List, context: Dict, subs: List[Any]): ...
+    def on_deserial_error(
+        self,
+        serialized: List,
+        context: Dict,
+        subs_serialized: List,
+        error_index: int,
+        exc: BaseException,
+    ): ...
+    @classmethod
+    def register(cls, obj_type): ...
+    @classmethod
+    def unregister(cls, obj_type): ...
+
+class Placeholder:
+    id: int
+    callbacks: List[Callable]
+    def __init__(self, id_: int): ...
+    def __hash__(self): ...
+    def __eq__(self, other): ...
+
+def serialize(obj: Any, context: Dict = None): ...
+async def serialize_with_spawn(
+    obj: Any,
+    context: Dict = None,
+    spawn_threshold: int = 100,
+    executor: Executor = None,
+): ...
+def deserialize(headers: List, buffers: List, context: Dict = None): ...

maxframe/session.py

@@ -365,6 +365,15 @@ class AbstractAsyncSession(AbstractSession, metaclass=ABCMeta):
         Stop server.
         """
 
+    @abstractmethod
+    async def get_logview_address(self, hours=None) -> Optional[str]:
+        """
+        Get Logview address
+        Returns
+        -------
+            Logview address
+        """
+
     def close(self):
         asyncio.run(self.destroy())
 
@@ -549,6 +558,15 @@ class AbstractSyncSession(AbstractSession, metaclass=ABCMeta):
 
         return fetch(tileables, self, offsets=offsets, sizes=sizes)
 
+    @abstractmethod
+    def get_logview_address(self, hours=None) -> Optional[str]:
+        """
+        Get logview address
+        Returns
+        -------
+            logview address
+        """
+
 
 def _delegate_to_isolated_session(func: Union[Callable, Coroutine]):
     if asyncio.iscoroutinefunction(func):
@@ -728,6 +746,11 @@ class AsyncSession(AbstractAsyncSession):
         await asyncio.wrap_future(asyncio.run_coroutine_threadsafe(coro, self._loop))
         stop_isolation()
 
+    @implements(AbstractAsyncSession.get_logview_address)
+    @_delegate_to_isolated_session
+    async def get_logview_address(self, hours=None) -> Optional[str]:
+        pass  # pragma: no cover
+
 
 class ProgressBar:
     def __init__(self, show_progress):
@@ -949,6 +972,11 @@ class SyncSession(AbstractSyncSession):
     def get_cluster_versions(self) -> List[str]:
         pass  # pragma: no cover
 
+    @implements(AbstractSyncSession.get_logview_address)
+    @_delegate_to_isolated_session
+    def get_logview_address(self, hours=None) -> Optional[str]:
+        pass  # pragma: no cover
+
     def destroy(self):
         coro = self._isolated_session.destroy()
         asyncio.run_coroutine_threadsafe(coro, self._loop).result()
@@ -1211,7 +1239,7 @@ def new_session(
     # load third party extensions.
     ensure_isolation_created(kwargs)
 
-    odps_entry = odps_entry or ODPS.from_environments()
+    odps_entry = odps_entry or ODPS.from_global() or ODPS.from_environments()
     if address is None:
         from maxframe_client.session.consts import ODPS_SESSION_INSECURE_SCHEME
 
@@ -1255,7 +1283,9 @@ def get_default_or_create(**kwargs):
         if session is None:
             # no session attached, try to create one
             warnings.warn(warning_msg)
-            session = new_session(ODPS.from_environments(), **kwargs)
+            session = new_session(
+                ODPS.from_global() or ODPS.from_environments(), **kwargs
+            )
             session.as_default()
     if isinstance(session, IsolatedAsyncSession):
         session = SyncSession.from_isolated_session(session)

maxframe/tensor/__init__.py

@@ -114,7 +114,7 @@ from .arithmetic import (
 )
 from .arithmetic import truediv as true_divide
 from .arithmetic import trunc
-from .base import broadcast_to, transpose, where
+from .base import broadcast_to, transpose, unique, where
 from .core import Tensor
 from .datasource import (
     arange,

maxframe/tensor/base/__init__.py

@@ -13,9 +13,11 @@
 # limitations under the License.
 
 from .astype import TensorAstype
+from .atleast_1d import atleast_1d
 from .broadcast_to import TensorBroadcastTo, broadcast_to
 from .ravel import ravel
 from .transpose import transpose
+from .unique import unique
 from .where import TensorWhere, where
 
 

maxframe/tensor/base/atleast_1d.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright 1999-2021 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ...core import ExecutableTuple
+from ..datasource import tensor as astensor
+
+
+def atleast_1d(*tensors):
+    """
+    Convert inputs to tensors with at least one dimension.
+
+    Scalar inputs are converted to 1-dimensional tensors, whilst
+    higher-dimensional inputs are preserved.
+
+    Parameters
+    ----------
+    tensors1, tensors2, ... : array_like
+        One or more input tensors.
+
+    Returns
+    -------
+    ret : Tensor
+        An tensor, or list of tensors, each with ``a.ndim >= 1``.
+        Copies are made only if necessary.
+
+    See Also
+    --------
+    atleast_2d, atleast_3d
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+
+    >>> mt.atleast_1d(1.0).execute()
+    array([ 1.])
+
+    >>> x = mt.arange(9.0).reshape(3,3)
+    >>> mt.atleast_1d(x).execute()
+    array([[ 0.,  1.,  2.],
+           [ 3.,  4.,  5.],
+           [ 6.,  7.,  8.]])
+    >>> mt.atleast_1d(x) is x
+    True
+
+    >>> mt.atleast_1d(1, [3, 4]).execute()
+    [array([1]), array([3, 4])]
+
+    """
+    new_tensors = []
+    for x in tensors:
+        x = astensor(x)
+        if x.ndim == 0:
+            x = x[np.newaxis]
+
+        new_tensors.append(x)
+
+    if len(new_tensors) == 1:
+        return new_tensors[0]
+    return ExecutableTuple(new_tensors)