maxframe 2.4.0rc1__cp312-cp312-win32.whl

maxframe/dataframe/misc/valid_index.py

@@ -0,0 +1,115 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ...udf import builtin_function
+
+
+@builtin_function
+def _item_or_none(item):
+    if len(item) > 0:
+        return item[0]
+    return None
+
+
+def _valid_index(df_or_series, slc: slice):
+    from ... import tensor as mt
+
+    idx = df_or_series.dropna(how="all").index[slc]
+    return mt.array(idx).mf.apply_chunk(_item_or_none, dtype=idx.dtype)
+
+
+_doc = """
+Return index for %(pos)s non-NA value or None, if no non-NA value is found.
+
+Returns
+-------
+type of index
+
+Examples
+--------
+For Series:
+
+>>> import maxframe.dataframe as md
+>>> s = md.Series([None, 3, 4])
+>>> s.first_valid_index().execute()
+1
+>>> s.last_valid_index().execute()
+2
+
+>>> s = md.Series([None, None])
+>>> print(s.first_valid_index()).execute()
+None
+>>> print(s.last_valid_index()).execute()
+None
+
+If all elements in Series are NA/null, returns None.
+
+>>> s = md.Series()
+>>> print(s.first_valid_index()).execute()
+None
+>>> print(s.last_valid_index()).execute()
+None
+
+If Series is empty, returns None.
+
+For DataFrame:
+
+>>> df = md.DataFrame({'A': [None, None, 2], 'B': [None, 3, 4]})
+>>> df.execute()
+     A      B
+0  NaN    NaN
+1  NaN    3.0
+2  2.0    4.0
+>>> df.first_valid_index().execute()
+1
+>>> df.last_valid_index().execute()
+2
+
+>>> df = md.DataFrame({'A': [None, None, None], 'B': [None, None, None]})
+>>> df.execute()
+     A      B
+0  None   None
+1  None   None
+2  None   None
+>>> print(df.first_valid_index()).execute()
+None
+>>> print(df.last_valid_index()).execute()
+None
+
+If all elements in DataFrame are NA/null, returns None.
+
+>>> df = md.DataFrame()
+>>> df.execute()
+Empty DataFrame
+Columns: []
+Index: []
+>>> print(df.first_valid_index()).execute()
+None
+>>> print(df.last_valid_index()).execute()
+None
+
+If DataFrame is empty, returns None.
+"""
+
+
+def first_valid_index(df_or_series):
+    return _valid_index(df_or_series, slice(None, 1))
+
+
+def last_valid_index(df_or_series):
+    return _valid_index(df_or_series, slice(-1, None))
+
+
+first_valid_index.__doc__ = _doc % dict(pos="first")
+last_valid_index.__doc__ = _doc % dict(pos="last")

maxframe/dataframe/misc/value_counts.py

@@ -0,0 +1,206 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+import numpy as np
+import pandas as pd
+
+from ... import opcodes
+from ...core import EntityData, OutputType
+from ...serialization.serializables import BoolField, Int64Field, KeyField, StringField
+from ...utils import pd_release_version
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import build_df, build_series, parse_index
+
+_keep_original_order = pd_release_version >= (1, 3, 0)
+_name_count_or_proportion = pd_release_version >= (2, 0, 0)
+
+
+class DataFrameValueCounts(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.VALUE_COUNTS
+
+    input = KeyField("input")
+    normalize = BoolField("normalize", default=False)
+    sort = BoolField("sort", default=True)
+    ascending = BoolField("ascending", default=False)
+    bins = Int64Field("bins", default=None)
+    dropna = BoolField("dropna", default=True)
+    method = StringField("method", default=None)
+    convert_index_to_interval = BoolField("convert_index_to_interval", default=None)
+    nrows = Int64Field("nrows", default=None)
+
+    def __init__(self, **kw):
+        super().__init__(**kw)
+        self.output_types = [OutputType.series]
+
+    @classmethod
+    def _set_inputs(cls, op: "DataFrameValueCounts", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        op.input = op._inputs[0]
+
+    def __call__(self, inp):
+        if inp.ndim == 2:
+            idx = pd.MultiIndex.from_frame(build_df(inp))
+            test_series = build_series(index=idx, dtype=int)
+            if _name_count_or_proportion:
+                out_name = "proportion" if self.normalize else "count"
+            else:
+                out_name = None
+        else:
+            test_series = build_series(inp).value_counts(normalize=self.normalize)
+            out_name = test_series.name
+        if self.bins is not None:
+            from .cut import cut
+
+            # cut
+            try:
+                inp = cut(inp, self.bins, include_lowest=True)
+            except TypeError:  # pragma: no cover
+                raise TypeError("bins argument only works with numeric data.")
+
+            self.bins = None
+            self.convert_index_to_interval = True
+            return self.new_series(
+                [inp],
+                shape=(np.nan,),
+                index_value=parse_index(pd.CategoricalIndex([]), inp, store_data=False),
+                name=out_name,
+                dtype=test_series.dtype,
+            )
+        else:
+            return self.new_series(
+                [inp],
+                shape=(np.nan,),
+                index_value=parse_index(test_series.index, store_data=False),
+                name=out_name,
+                dtype=test_series.dtype,
+            )
+
+
+def value_counts(
+    series,
+    normalize=False,
+    sort=True,
+    ascending=False,
+    bins=None,
+    dropna=True,
+    method="auto",
+):
+    # FIXME: https://github.com/aliyun/alibabacloud-odps-maxframe-client/issues/33
+    """
+    Return a Series containing counts of unique values.
+
+    The resulting object will be in descending order so that the
+    first element is the most frequently-occurring element.
+    Excludes NA values by default.
+
+    Parameters
+    ----------
+    normalize : bool, default False
+        If True then the object returned will contain the relative
+        frequencies of the unique values.
+    sort : bool, default True
+        Sort by frequencies.
+    ascending : bool, default False
+        Sort in ascending order.
+    bins : int, optional
+        Rather than count values, group them into half-open bins,
+        a convenience for ``pd.cut``, only works with numeric data.
+    dropna : bool, default True
+        Don't include counts of NaN.
+    method : str, default 'auto'
+        'auto', 'shuffle', or 'tree', 'tree' method provide
+        a better performance, while 'shuffle' is recommended
+        if aggregated result is very large, 'auto' will use
+        'shuffle' method in distributed mode and use 'tree'
+        in local mode.
+
+    Returns
+    -------
+    Series
+
+    See Also
+    --------
+    Series.count: Number of non-NA elements in a Series.
+    DataFrame.count: Number of non-NA elements in a DataFrame.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> import numpy as np
+    >>> s = md.Series([3, 1, 2, 3, 4, np.nan])
+    >>> s.value_counts().execute()
+    3.0    2
+    4.0    1
+    2.0    1
+    1.0    1
+    dtype: int64
+
+    With `normalize` set to `True`, returns the relative frequency by
+    dividing all values by the sum of values.
+
+    >>> s = md.Series([3, 1, 2, 3, 4, np.nan])
+    >>> s.value_counts(normalize=True).execute()
+    3.0    0.4
+    4.0    0.2
+    2.0    0.2
+    1.0    0.2
+    dtype: float64
+
+    **dropna**
+
+    With `dropna` set to `False` we can also see NaN index values.
+
+    >>> s.value_counts(dropna=False).execute()
+    3.0    2
+    NaN    1
+    4.0    1
+    2.0    1
+    1.0    1
+    dtype: int64
+    """
+    op = DataFrameValueCounts(
+        normalize=normalize,
+        sort=sort,
+        ascending=ascending,
+        bins=bins,
+        dropna=dropna,
+        method=method,
+    )
+    return op(series)
+
+
+def df_value_counts(
+    df,
+    subset=None,
+    normalize=False,
+    sort=True,
+    ascending=False,
+    dropna=True,
+    method="auto",
+):
+    if not subset:
+        df_to_count = df
+    else:
+        df_to_count = df[subset]
+
+    op = DataFrameValueCounts(
+        normalize=normalize,
+        sort=sort,
+        ascending=ascending,
+        dropna=dropna,
+        method=method,
+    )
+    return op(df_to_count)

maxframe/dataframe/missing/__init__.py

@@ -0,0 +1,53 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .checkna import isna, isnull, notna, notnull
+from .dropna import df_dropna, index_dropna, series_dropna
+from .fillna import bfill, ffill, fillna, index_fillna
+from .replace import df_replace, series_replace
+
+
+def _install():
+    from ..core import DATAFRAME_TYPE, INDEX_TYPE, SERIES_TYPE
+
+    for cls in DATAFRAME_TYPE + SERIES_TYPE:
+        setattr(cls, "backfill", bfill)
+        setattr(cls, "bfill", bfill)
+        setattr(cls, "fillna", fillna)
+        setattr(cls, "ffill", ffill)
+        setattr(cls, "isna", isna)
+        setattr(cls, "isnull", isnull)
+        setattr(cls, "notna", notna)
+        setattr(cls, "notnull", notnull)
+        setattr(cls, "pad", ffill)
+
+    for cls in DATAFRAME_TYPE:
+        setattr(cls, "dropna", df_dropna)
+        setattr(cls, "replace", df_replace)
+
+    for cls in SERIES_TYPE:
+        setattr(cls, "dropna", series_dropna)
+        setattr(cls, "replace", series_replace)
+
+    for cls in INDEX_TYPE:
+        setattr(cls, "dropna", index_dropna)
+        setattr(cls, "fillna", index_fillna)
+        setattr(cls, "isna", isna)
+        setattr(cls, "isnull", isnull)
+        setattr(cls, "notna", notna)
+        setattr(cls, "notnull", notnull)
+
+
+_install()
+del _install

maxframe/dataframe/missing/checkna.py

@@ -0,0 +1,231 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+from ... import opcodes
+from ... import tensor as mt
+from ...core import ENTITY_TYPE, OutputType
+from ...serialization.serializables import BoolField
+from ...tensor.core import TENSOR_TYPE
+from ...utils import get_pd_option
+from ..core import DATAFRAME_TYPE, INDEX_TYPE, SERIES_TYPE, MultiIndex
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+
+
+class DataFrameCheckNA(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.CHECK_NA
+
+    positive = BoolField("positive", default=None)
+    use_inf_as_na = BoolField("use_inf_as_na", default=None)
+
+    def __call__(self, df):
+        if isinstance(df, DATAFRAME_TYPE):
+            self.output_types = [OutputType.dataframe]
+        elif isinstance(df, SERIES_TYPE):
+            self.output_types = [OutputType.series]
+        elif isinstance(df, TENSOR_TYPE):
+            self.output_types = [OutputType.tensor]
+        elif isinstance(df, INDEX_TYPE):
+            self.output_types = [OutputType.index]
+        else:
+            raise TypeError(
+                f"Expecting maxframe dataframe, series, index, or tensor, got {type(df)}"
+            )
+
+        params = df.params
+        if self.output_types[0] == OutputType.dataframe:
+            if df.dtypes is None:
+                params["dtypes"] = None
+            else:
+                params["dtypes"] = pd.Series(
+                    [np.dtype("bool")] * len(df.dtypes),
+                    index=df.columns_value.to_pandas(),
+                )
+        else:
+            params["dtype"] = np.dtype("bool")
+        return self.new_tileable([df], **params)
+
+
+def _from_pandas(obj: Any):
+    if isinstance(obj, pd.DataFrame):
+        from ..datasource.dataframe import from_pandas
+
+        return from_pandas(obj)
+    elif isinstance(obj, pd.Series):
+        from ..datasource.series import from_pandas
+
+        return from_pandas(obj)
+    elif isinstance(obj, np.ndarray):
+        return mt.tensor(obj)
+    else:
+        return obj
+
+
+def isna(obj):
+    """
+    Detect missing values.
+
+    Return a boolean same-sized object indicating if the values are NA.
+    NA values, such as None or :attr:`numpy.NaN`, gets mapped to True
+    values.
+
+    Everything else gets mapped to False values. Characters such as empty
+    strings ``''`` or :attr:`numpy.inf` are not considered NA values
+    (unless you set ``pandas.options.mode.use_inf_as_na = True``).
+
+    Returns
+    -------
+    DataFrame
+        Mask of bool values for each element in DataFrame that
+        indicates whether an element is not an NA value.
+
+    See Also
+    --------
+    DataFrame.isnull : Alias of isna.
+    DataFrame.notna : Boolean inverse of isna.
+    DataFrame.dropna : Omit axes labels with missing values.
+    isna : Top-level isna.
+
+    Examples
+    --------
+    Show which entries in a DataFrame are NA.
+
+    >>> import numpy as np
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame({'age': [5, 6, np.NaN],
+    ...                    'born': [md.NaT, md.Timestamp('1939-05-27'),
+    ...                             md.Timestamp('1940-04-25')],
+    ...                    'name': ['Alfred', 'Batman', ''],
+    ...                    'toy': [None, 'Batmobile', 'Joker']})
+    >>> df.execute()
+       age       born    name        toy
+    0  5.0        NaT  Alfred       None
+    1  6.0 1939-05-27  Batman  Batmobile
+    2  NaN 1940-04-25              Joker
+
+    >>> df.isna().execute()
+         age   born   name    toy
+    0  False   True  False   True
+    1  False  False  False  False
+    2   True  False  False  False
+
+    Show which entries in a Series are NA.
+
+    >>> ser = md.Series([5, 6, np.NaN])
+    >>> ser.execute()
+    0    5.0
+    1    6.0
+    2    NaN
+    dtype: float64
+
+    >>> ser.isna().execute()
+    0    False
+    1    False
+    2     True
+    dtype: bool
+    """
+    use_inf_as_na = get_pd_option("mode.use_inf_as_na", False)
+    if isinstance(obj, MultiIndex):
+        raise NotImplementedError("isna is not defined for MultiIndex")
+    elif isinstance(obj, ENTITY_TYPE):
+        if isinstance(obj, TENSOR_TYPE):
+            return mt.isnan(obj)
+        else:
+            op = DataFrameCheckNA(positive=True, use_inf_as_na=use_inf_as_na)
+            return op(obj)
+    else:
+        return _from_pandas(pd.isna(obj))
+
+
+def notna(obj):
+    """
+    Detect existing (non-missing) values.
+
+    Return a boolean same-sized object indicating if the values are not NA.
+    Non-missing values get mapped to True. Characters such as empty
+    strings ``''`` or :attr:`numpy.inf` are not considered NA values
+    (unless you set ``pandas.options.mode.use_inf_as_na = True``).
+    NA values, such as None or :attr:`numpy.NaN`, get mapped to False
+    values.
+
+    Returns
+    -------
+    DataFrame
+        Mask of bool values for each element in DataFrame that
+        indicates whether an element is not an NA value.
+
+    See Also
+    --------
+    DataFrame.notnull : Alias of notna.
+    DataFrame.isna : Boolean inverse of notna.
+    DataFrame.dropna : Omit axes labels with missing values.
+    notna : Top-level notna.
+
+    Examples
+    --------
+    Show which entries in a DataFrame are not NA.
+
+    >>> import numpy as np
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame({'age': [5, 6, np.NaN],
+    ...                    'born': [md.NaT, md.Timestamp('1939-05-27'),
+    ...                             md.Timestamp('1940-04-25')],
+    ...                    'name': ['Alfred', 'Batman', ''],
+    ...                    'toy': [None, 'Batmobile', 'Joker']})
+    >>> df.execute()
+       age       born    name        toy
+    0  5.0        NaT  Alfred       None
+    1  6.0 1939-05-27  Batman  Batmobile
+    2  NaN 1940-04-25              Joker
+
+    >>> df.notna().execute()
+         age   born  name    toy
+    0   True  False  True  False
+    1   True   True  True   True
+    2  False   True  True   True
+
+    Show which entries in a Series are not NA.
+
+    >>> ser = md.Series([5, 6, np.NaN])
+    >>> ser.execute()
+    0    5.0
+    1    6.0
+    2    NaN
+    dtype: float64
+
+    >>> ser.notna().execute()
+    0     True
+    1     True
+    2    False
+    dtype: bool
+    """
+    use_inf_as_na = get_pd_option("mode.use_inf_as_na", False)
+    if isinstance(obj, MultiIndex):
+        raise NotImplementedError("isna is not defined for MultiIndex")
+    elif isinstance(obj, ENTITY_TYPE):
+        if isinstance(obj, TENSOR_TYPE):
+            return ~mt.isnan(obj)
+        else:
+            op = DataFrameCheckNA(positive=False, use_inf_as_na=use_inf_as_na)
+            return op(obj)
+    else:
+        return _from_pandas(pd.notna(obj))
+
+
+isnull = isna
+notnull = notna