maxframe 2.0.0b2__cp37-cp37m-win32.whl → 2.3.0rc1__cp37-cp37m-win32.whl

maxframe/dataframe/misc/map.py

@@ -21,8 +21,8 @@ import pandas as pd
 from ... import opcodes
 from ...core import EntityData, OutputType
 from ...serialization.serializables import AnyField, KeyField, StringField
-from ...udf import BuiltinFunction, MarkedFunction
-from ...utils import quiet_stdio
+from ...udf import BuiltinFunction, MarkedFunction, ODPSFunction
+from ...utils import make_dtype, quiet_stdio
 from ..core import SERIES_TYPE
 from ..operators import DataFrameOperator, DataFrameOperatorMixin
 from ..utils import build_series, copy_func_scheduling_hints
@@ -40,6 +40,7 @@ class DataFrameMap(DataFrameOperator, DataFrameOperatorMixin):
         if not self.output_types:
             self.output_types = [OutputType.series]
         if hasattr(self, "arg"):
+            self.arg = ODPSFunction.wrap(self.arg)
             copy_func_scheduling_hints(self.arg, self)
 
     @classmethod
@@ -55,25 +56,34 @@ class DataFrameMap(DataFrameOperator, DataFrameOperatorMixin):
         ) and not isinstance(self.arg, BuiltinFunction)
 
     def __call__(self, series, dtype, skip_infer=False):
-        if dtype is None and not skip_infer:
-            inferred_dtype = None
-            if callable(self.arg):
+        if dtype is not None:
+            dtype = make_dtype(dtype)
+        else:
+            # obtain dtype from existing hints
+            if isinstance(self.arg, ODPSFunction):
+                if self.arg.result_dtype is not None:
+                    dtype = self.arg.result_dtype
+            elif callable(self.arg):
                 # arg is a function, try to inspect the signature
                 sig = inspect.signature(self.arg)
                 return_type = sig.return_annotation
                 if return_type is not inspect._empty:
-                    inferred_dtype = np.dtype(return_type)
-                else:
-                    try:
-                        with quiet_stdio():
-                            # try to infer dtype by calling the function
-                            inferred_dtype = (
-                                build_series(series)
-                                .map(self.arg, na_action=self.na_action)
-                                .dtype
-                            )
-                    except:  # noqa: E722  # nosec
-                        pass
+                    dtype = np.dtype(return_type)
+
+        err_prefix = None
+        if dtype is None and not skip_infer:
+            inferred_dtype = None
+            if callable(self.arg):
+                try:
+                    with quiet_stdio():
+                        # try to infer dtype by calling the function
+                        inferred_dtype = (
+                            build_series(series)
+                            .map(self.arg, na_action=self.na_action)
+                            .dtype
+                        )
+                except:  # noqa: E722  # nosec
+                    pass
             else:
                 if isinstance(self.arg, MutableMapping):
                     inferred_dtype = pd.Series(self.arg).dtype
@@ -86,13 +96,16 @@ class DataFrameMap(DataFrameOperator, DataFrameOperatorMixin):
                     # but for int, due to the nan which may occur,
                     # we cannot infer the dtype
                     dtype = inferred_dtype
+                else:
+                    err_prefix = "int type may not be exact"
             else:
                 dtype = inferred_dtype
 
         if dtype is None:
             if not skip_infer:
+                err_prefix = err_prefix or "cannot infer dtype"
                 raise ValueError(
-                    "cannot infer dtype, it needs to be specified manually for `map`"
+                    f"{err_prefix}, it needs to be specified manually for `map`"
                 )
         else:
             dtype = np.int64 if dtype is int else dtype
@@ -251,3 +264,97 @@ def index_map(
     """
     op = DataFrameMap(arg=mapper, na_action=na_action, memory_scale=memory_scale)
     return op(idx, dtype=dtype, skip_infer=skip_infer)
+
+
+def df_map(
+    df, func, na_action=None, dtypes=None, dtype=None, skip_infer=False, **kwargs
+):
+    """
+    Apply a function to a Dataframe elementwise.
+
+    This method applies a function that accepts and returns a scalar
+    to every element of a DataFrame.
+
+    Parameters
+    ----------
+    func : callable
+        Python function, returns a single value from a single value.
+    na_action : {None, 'ignore'}, default None
+        If 'ignore', propagate NaN values, without passing them to func.
+    dtypes : Series, default None
+        Specify dtypes of returned DataFrames.
+    dtype : np.dtype, default None
+        Specify dtypes of all columns of returned DataFrames, only
+        effective when dtypes is not specified.
+    skip_infer: bool, default False
+        Whether infer dtypes when dtypes or dtype is not specified.
+    **kwargs
+        Additional keyword arguments to pass as keywords arguments to
+        `func`.
+
+    Returns
+    -------
+    DataFrame
+        Transformed DataFrame.
+
+    See Also
+    --------
+    DataFrame.apply : Apply a function along input axis of DataFrame.
+    DataFrame.replace: Replace values given in `to_replace` with `value`.
+    Series.map : Apply a function elementwise on a Series.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame([[1, 2.12], [3.356, 4.567]])
+    >>> df.execute()
+           0      1
+    0  1.000  2.120
+    1  3.356  4.567
+
+    >>> df.map(lambda x: len(str(x))).execute()
+       0  1
+    0  3  4
+    1  5  5
+
+    Like Series.map, NA values can be ignored:
+
+    >>> df_copy = df.copy()
+    >>> df_copy.iloc[0, 0] = md.NA
+    >>> df_copy.map(lambda x: len(str(x)), na_action='ignore').execute()
+         0  1
+    0  NaN  4
+    1  5.0  5
+
+    It is also possible to use `map` with functions that are not
+    `lambda` functions:
+
+    >>> df.map(round, ndigits=1).execute()
+         0    1
+    0  1.0  2.1
+    1  3.4  4.6
+
+    Note that a vectorized version of `func` often exists, which will
+    be much faster. You could square each number elementwise.
+
+    >>> df.map(lambda x: x**2).execute()
+               0          1
+    0   1.000000   4.494400
+    1  11.262736  20.857489
+
+    But it's better to avoid map in that case.
+
+    >>> (df ** 2).execute()
+               0          1
+    0   1.000000   4.494400
+    1  11.262736  20.857489
+    """
+    if dtypes is None and dtype is not None:
+        dtypes = pd.Series([dtype] * df.shape[1], index=df.dtypes.index)
+
+    def _wrapper(row):
+        return row.map(func, na_action=na_action, **kwargs)
+
+    return df.apply(
+        _wrapper, axis=1, dtypes=dtypes, skip_infer=skip_infer, elementwise=True
+    )

maxframe/dataframe/misc/repeat.py

@@ -0,0 +1,159 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+import numpy as np
+from pandas.api.types import is_list_like
+
+from ... import opcodes
+from ...core import ENTITY_TYPE, EntityData, get_output_types
+from ...serialization.serializables import AnyField
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import parse_index, validate_axis
+
+
+class DataFrameRepeat(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.REPEAT
+
+    repeats = AnyField("repeats", default=None)
+
+    def __init__(self, output_types=None, **kw):
+        super().__init__(_output_types=output_types, **kw)
+
+    @classmethod
+    def _set_inputs(cls, op: "DataFrameRepeat", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        if isinstance(op.repeats, ENTITY_TYPE):
+            op.repeats = inputs[1]
+
+    def __call__(self, obj, repeats):
+        self._output_types = get_output_types(obj)
+        test_index = obj.index_value.to_pandas()[:0]
+
+        params = obj.params
+        params["index_value"] = parse_index(test_index, obj, type(self), self.repeats)
+        params["shape"] = (np.nan,)
+
+        inputs = [obj]
+        if isinstance(repeats, ENTITY_TYPE):
+            inputs.append(repeats)
+        return self.new_tileable(inputs, **params)
+
+
+def _repeat(obj, repeats, axis=None):
+    from ...tensor.datasource import tensor
+
+    axis = validate_axis(axis or 0, obj)
+    if is_list_like(repeats):
+        repeats = tensor(repeats)
+    op = DataFrameRepeat(repeats=repeats, axis=axis)
+    return op(obj, repeats)
+
+
+def series_repeat(obj, repeats, axis=None):
+    """
+    Repeat elements of a Series.
+
+    Returns a new Series where each element of the current Series
+    is repeated consecutively a given number of times.
+
+    Parameters
+    ----------
+    repeats : int or array of ints
+        The number of repetitions for each element. This should be a
+        non-negative integer. Repeating 0 times will return an empty
+        Series.
+    axis : None
+        Must be ``None``. Has no effect but is accepted for compatibility
+        with numpy.
+
+    Returns
+    -------
+    Series
+        Newly created Series with repeated elements.
+
+    See Also
+    --------
+    Index.repeat : Equivalent function for Index.
+    numpy.repeat : Similar method for :class:`numpy.ndarray`.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> s = md.Series(['a', 'b', 'c'])
+    >>> s.execute()
+    0    a
+    1    b
+    2    c
+    dtype: object
+    >>> s.repeat(2).execute()
+    0    a
+    0    a
+    1    b
+    1    b
+    2    c
+    2    c
+    dtype: object
+    >>> s.repeat([1, 2, 3]).execute()
+    0    a
+    1    b
+    1    b
+    2    c
+    2    c
+    2    c
+    dtype: object
+    """
+    return _repeat(obj, repeats, axis=axis)
+
+
+def index_repeat(obj, repeats, axis=None):
+    """
+    Repeat elements of an Index.
+
+    Returns a new Index where each element of the current Index
+    is repeated consecutively a given number of times.
+
+    Parameters
+    ----------
+    repeats : int or array of ints
+        The number of repetitions for each element. This should be a
+        non-negative integer. Repeating 0 times will return an empty
+        Index.
+    axis : None
+        Must be ``None``. Has no effect but is accepted for compatibility
+        with numpy.
+
+    Returns
+    -------
+    repeated_index : Index
+        Newly created Index with repeated elements.
+
+    See Also
+    --------
+    Series.repeat : Equivalent function for Series.
+    numpy.repeat : Similar method for :class:`numpy.ndarray`.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> idx = md.Index(['a', 'b', 'c'])
+    >>> idx.execute()
+    Index(['a', 'b', 'c'], dtype='object')
+    >>> idx.repeat(2).execute()
+    Index(['a', 'a', 'b', 'b', 'c', 'c'], dtype='object')
+    >>> idx.repeat([1, 2, 3]).execute()
+    Index(['a', 'b', 'b', 'c', 'c', 'c'], dtype='object')
+    """
+    return _repeat(obj, repeats, axis=axis)

maxframe/dataframe/misc/tests/test_misc.py

@@ -16,11 +16,13 @@ import numpy as np
 import pandas as pd
 import pytest
 
+from maxframe import options
+
 from .... import opcodes
 from ....core import OutputType
 from ....dataframe import DataFrame
 from ....tensor.core import TENSOR_TYPE
-from ....udf import with_running_options
+from ....udf import ODPSFunction, with_running_options
 from ... import eval as maxframe_eval
 from ... import get_dummies, to_numeric
 from ...arithmetic import DataFrameGreater, DataFrameLess
@@ -124,6 +126,7 @@ def test_dataframe_apply():
         dtypes=pd.Series([np.dtype(float)] * 3),
     )
     assert df2.ndim == 2
+    assert df2.op.expect_resources == options.function.default_running_options
 
 
 def test_series_apply():
@@ -180,6 +183,8 @@ def test_series_apply():
         pd.Series, output_type="dataframe", dtypes=dtypes, index=pd.RangeIndex(2)
     )
     assert r.ndim == 2
+    assert r.op.expect_resources == options.function.default_running_options
+
     pd.testing.assert_series_equal(r.dtypes, dtypes)
     assert r.shape == (2, 3)
 
@@ -305,6 +310,7 @@ def test_transform():
     assert r.shape == series.shape
     assert r.op._op_type_ == opcodes.TRANSFORM
     assert r.op.output_types[0] == OutputType.series
+    assert r.op.expect_resources == options.function.default_running_options
 
 
 def test_series_isin():
@@ -563,12 +569,17 @@ def test_apply():
     )
     assert apply_df.shape == (3, 2)
     assert apply_df.op.expect_engine == "SPE"
-    assert apply_df.op.expect_resources == {"cpu": 1, "memory": "40GB", "gpu": 0}
+    assert apply_df.op.expect_resources == {
+        "cpu": 4,
+        "memory": "40GB",
+        "gpu": 0,
+        "gu_quota": None,
+    }
 
 
 def test_pivot_table():
     from ...groupby.aggregation import DataFrameGroupByAgg
-    from ...misc.pivot_table import DataFramePivotTable
+    from ...reshape.pivot_table import DataFramePivotTable
 
     raw = pd.DataFrame(
         {
@@ -602,3 +613,37 @@ def test_pivot_table():
     t = df.pivot_table(index=["A", "B"], columns="C", aggfunc="sum")
     assert isinstance(t.op, DataFramePivotTable)
     assert t.shape == (np.nan, np.nan)
+
+
+def test_map_with_functions():
+    raw = pd.Series([1, 2, 3], name="s_name")
+    series = from_pandas_series(raw, chunk_size=2)
+
+    # inferred type may not be exact
+    def fn1(val):
+        return val
+
+    with pytest.raises(ValueError, match="int type"):
+        series.map(fn1)
+    mapped = series.map(fn1, dtype="float64", skip_infer=True)
+    assert mapped.dtype == np.dtype("float64")
+
+    # test when type infer is valid
+    def fn2(val):
+        return val * 1.0
+
+    mapped = series.map(fn2)
+    assert mapped.dtype == np.dtype("float64")
+
+    # test function with type annotations
+    def fn3(val) -> int:
+        return val
+
+    mapped = series.map(fn3)
+    assert mapped.dtype == np.dtype("int64")
+
+    # test odps function
+    odps_func = ODPSFunction("test_odps_udf", dtype=np.float64)
+    mapped = series.map(odps_func)
+    assert isinstance(mapped.op.arg, ODPSFunction)
+    assert mapped.dtype == np.dtype("float64")

maxframe/dataframe/misc/to_numeric.py

@@ -15,6 +15,7 @@
 import numpy as np
 import pandas as pd
 
+from ... import opcodes
 from ...core import ENTITY_TYPE, OutputType
 from ...serialization.serializables import StringField
 from ...tensor import tensor as astensor
@@ -23,6 +24,8 @@ from ..operators import DataFrameOperator, DataFrameOperatorMixin
 
 
 class DataFrameToNumeric(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.TO_NUMERIC
+
     errors = StringField("errors")
     downcast = StringField("downcast")
 

maxframe/dataframe/misc/transform.py

@@ -38,8 +38,9 @@ from ..utils import (
 _with_convert_dtype = pd_release_version < (1, 2, 0)
 
 
-class TransformOperator(DataFrameOperator, DataFrameOperatorMixin):
+class DataFrameTransform(DataFrameOperator, DataFrameOperatorMixin):
     _op_type_ = opcodes.TRANSFORM
+    _legacy_name = "TransformOperator"
 
     func = AnyField("func", default=None)
     axis = AnyField("axis", default=None)
@@ -141,13 +142,17 @@ class TransformOperator(DataFrameOperator, DataFrameOperatorMixin):
 
     @classmethod
     def estimate_size(
-        cls, ctx: MutableMapping[str, Union[int, float]], op: "TransformOperator"
+        cls, ctx: MutableMapping[str, Union[int, float]], op: "DataFrameTransform"
     ) -> None:
         if isinstance(op.func, MarkedFunction):
             ctx[op.outputs[0].key] = float("inf")
         super().estimate_size(ctx, op)
 
 
+# keep for import compatibility
+TransformOperator = DataFrameTransform
+
+
 def get_packed_funcs(df, output_type, func, *args, **kwds) -> Any:
     stub_df = _build_stub_pandas_obj(df, output_type)
     n_args = copy_if_possible(args)
@@ -235,7 +240,7 @@ def df_transform(df, func, axis=0, *args, dtypes=None, skip_infer=False, **kwarg
     """
     call_agg = kwargs.pop("_call_agg", False)
     func = get_packed_funcs(df, OutputType.dataframe, func, *args, **kwargs)
-    op = TransformOperator(
+    op = DataFrameTransform(
         func=func,
         axis=axis,
         args=args,
@@ -327,13 +332,15 @@ def series_transform(
     """
     call_agg = kwargs.pop("_call_agg", False)
     func = get_packed_funcs(series, OutputType.series, func, *args, **kwargs)
-    op = TransformOperator(
+    op = DataFrameTransform(
         func=func,
         axis=axis,
         convert_dtype=convert_dtype,
         args=args,
         kwds=kwargs,
-        output_types=[OutputType.series],
+        output_types=[OutputType.series]
+        if not call_agg and not isinstance(func, list)
+        else None,
         call_agg=call_agg,
     )
     return op(series, dtype=dtype, name=series.name, skip_infer=skip_infer)

maxframe/dataframe/misc/transpose.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import pandas as pd
+
 from ... import opcodes
 from ...core import OutputType
 from ..operators import DataFrameOperator, DataFrameOperatorMixin
@@ -30,10 +32,20 @@ class DataFrameTranspose(DataFrameOperator, DataFrameOperatorMixin):
         new_shape = arg.shape[::-1]
         columns_value = arg.index_value
         index_value = parse_index(arg.dtypes.index)
+
+        if not arg.index_value.has_value:
+            dtypes = None
+        else:
+            from pandas.core.dtypes.cast import find_common_type
+
+            dtype = find_common_type(list(arg.dtypes))
+            pd_index = arg.index_value.to_pandas()
+            dtypes = pd.Series([dtype] * len(pd_index), index=pd_index)
+
         return self.new_dataframe(
             [arg],
             shape=new_shape,
-            dtypes=None,
+            dtypes=dtypes,
             columns_value=columns_value,
             index_value=index_value,
         )

maxframe/dataframe/misc/valid_index.py

@@ -0,0 +1,115 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...udf import builtin_function
+
+
+@builtin_function
+def _item_or_none(item):
+    if len(item) > 0:
+        return item[0]
+    return None
+
+
+def _valid_index(df_or_series, slc: slice):
+    from ... import tensor as mt
+
+    idx = df_or_series.dropna(how="all").index[slc]
+    return mt.array(idx).mf.apply_chunk(_item_or_none, dtype=idx.dtype)
+
+
+_doc = """
+Return index for %(pos)s non-NA value or None, if no non-NA value is found.
+
+Returns
+-------
+type of index
+
+Examples
+--------
+For Series:
+
+>>> import maxframe.dataframe as md
+>>> s = md.Series([None, 3, 4])
+>>> s.first_valid_index().execute()
+1
+>>> s.last_valid_index().execute()
+2
+
+>>> s = md.Series([None, None])
+>>> print(s.first_valid_index()).execute()
+None
+>>> print(s.last_valid_index()).execute()
+None
+
+If all elements in Series are NA/null, returns None.
+
+>>> s = md.Series()
+>>> print(s.first_valid_index()).execute()
+None
+>>> print(s.last_valid_index()).execute()
+None
+
+If Series is empty, returns None.
+
+For DataFrame:
+
+>>> df = md.DataFrame({'A': [None, None, 2], 'B': [None, 3, 4]})
+>>> df.execute()
+     A      B
+0  NaN    NaN
+1  NaN    3.0
+2  2.0    4.0
+>>> df.first_valid_index().execute()
+1
+>>> df.last_valid_index().execute()
+2
+
+>>> df = md.DataFrame({'A': [None, None, None], 'B': [None, None, None]})
+>>> df.execute()
+     A      B
+0  None   None
+1  None   None
+2  None   None
+>>> print(df.first_valid_index()).execute()
+None
+>>> print(df.last_valid_index()).execute()
+None
+
+If all elements in DataFrame are NA/null, returns None.
+
+>>> df = md.DataFrame()
+>>> df.execute()
+Empty DataFrame
+Columns: []
+Index: []
+>>> print(df.first_valid_index()).execute()
+None
+>>> print(df.last_valid_index()).execute()
+None
+
+If DataFrame is empty, returns None.
+"""
+
+
+def first_valid_index(df_or_series):
+    return _valid_index(df_or_series, slice(None, 1))
+
+
+def last_valid_index(df_or_series):
+    return _valid_index(df_or_series, slice(-1, None))
+
+
+first_valid_index.__doc__ = _doc % dict(pos="first")
+last_valid_index.__doc__ = _doc % dict(pos="last")