maxframe 0.1.0b5__cp310-cp310-macosx_11_0_arm64.whl

maxframe/dataframe/misc/value_counts.py

@@ -0,0 +1,182 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pandas as pd
+
+from ... import opcodes
+from ...core import OutputType
+from ...serialization.serializables import BoolField, Int64Field, KeyField, StringField
+from ...utils import pd_release_version
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import build_series, parse_index
+
+_keep_original_order = pd_release_version >= (1, 3, 0)
+
+
+class DataFrameValueCounts(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.VALUE_COUNTS
+
+    input = KeyField("input")
+    normalize = BoolField("normalize", default=False)
+    sort = BoolField("sort", default=True)
+    ascending = BoolField("ascending", default=False)
+    bins = Int64Field("bins", default=None)
+    dropna = BoolField("dropna", default=True)
+    method = StringField("method", default=None)
+    convert_index_to_interval = BoolField("convert_index_to_interval", default=None)
+    nrows = Int64Field("nrows", default=None)
+
+    def __init__(self, **kw):
+        super().__init__(**kw)
+        self.output_types = [OutputType.series]
+
+    def _set_inputs(self, inputs):
+        super()._set_inputs(inputs)
+        self.input = self._inputs[0]
+
+    def __call__(self, inp):
+        test_series = build_series(inp).value_counts(normalize=self.normalize)
+        if self.bins is not None:
+            from .cut import cut
+
+            # cut
+            try:
+                inp = cut(inp, self.bins, include_lowest=True)
+            except TypeError:  # pragma: no cover
+                raise TypeError("bins argument only works with numeric data.")
+
+            self.bins = None
+            self.convert_index_to_interval = True
+            return self.new_series(
+                [inp],
+                shape=(np.nan,),
+                index_value=parse_index(pd.CategoricalIndex([]), inp, store_data=False),
+                name=inp.name,
+                dtype=test_series.dtype,
+            )
+        else:
+            return self.new_series(
+                [inp],
+                shape=(np.nan,),
+                index_value=parse_index(test_series.index, store_data=False),
+                name=inp.name,
+                dtype=test_series.dtype,
+            )
+
+
+def value_counts(
+    series,
+    normalize=False,
+    sort=True,
+    ascending=False,
+    bins=None,
+    dropna=True,
+    method="auto",
+):
+    """
+    Return a Series containing counts of unique values.
+
+    The resulting object will be in descending order so that the
+    first element is the most frequently-occurring element.
+    Excludes NA values by default.
+
+    Parameters
+    ----------
+    normalize : bool, default False
+        If True then the object returned will contain the relative
+        frequencies of the unique values.
+    sort : bool, default True
+        Sort by frequencies.
+    ascending : bool, default False
+        Sort in ascending order.
+    bins : int, optional
+        Rather than count values, group them into half-open bins,
+        a convenience for ``pd.cut``, only works with numeric data.
+    dropna : bool, default True
+        Don't include counts of NaN.
+    method : str, default 'auto'
+        'auto', 'shuffle', or 'tree', 'tree' method provide
+        a better performance, while 'shuffle' is recommended
+        if aggregated result is very large, 'auto' will use
+        'shuffle' method in distributed mode and use 'tree'
+        in local mode.
+
+    Returns
+    -------
+    Series
+
+    See Also
+    --------
+    Series.count: Number of non-NA elements in a Series.
+    DataFrame.count: Number of non-NA elements in a DataFrame.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> import maxframe.tensor as mt
+
+    >>> s = md.Series([3, 1, 2, 3, 4, mt.nan])
+    >>> s.value_counts().execute()
+    3.0    2
+    4.0    1
+    2.0    1
+    1.0    1
+    dtype: int64
+
+    With `normalize` set to `True`, returns the relative frequency by
+    dividing all values by the sum of values.
+
+    >>> s = md.Series([3, 1, 2, 3, 4, mt.nan])
+    >>> s.value_counts(normalize=True).execute()
+    3.0    0.4
+    4.0    0.2
+    2.0    0.2
+    1.0    0.2
+    dtype: float64
+
+    **bins**
+
+    Bins can be useful for going from a continuous variable to a
+    categorical variable; instead of counting unique
+    apparitions of values, divide the index in the specified
+    number of half-open bins.
+
+    >>> s.value_counts(bins=3).execute()
+    (2.0, 3.0]      2
+    (0.996, 2.0]    2
+    (3.0, 4.0]      1
+    dtype: int64
+
+    **dropna**
+
+    With `dropna` set to `False` we can also see NaN index values.
+
+    >>> s.value_counts(dropna=False).execute()
+    3.0    2
+    NaN    1
+    4.0    1
+    2.0    1
+    1.0    1
+    dtype: int64
+    """
+    op = DataFrameValueCounts(
+        normalize=normalize,
+        sort=sort,
+        ascending=ascending,
+        bins=bins,
+        dropna=dropna,
+        method=method,
+    )
+    return op(series)

maxframe/dataframe/missing/__init__.py

@@ -0,0 +1,53 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .checkna import isna, isnull, notna, notnull
+from .dropna import df_dropna, index_dropna, series_dropna
+from .fillna import bfill, ffill, fillna, index_fillna
+from .replace import df_replace, series_replace
+
+
+def _install():
+    from ..core import DATAFRAME_TYPE, INDEX_TYPE, SERIES_TYPE
+
+    for cls in DATAFRAME_TYPE + SERIES_TYPE:
+        setattr(cls, "backfill", bfill)
+        setattr(cls, "bfill", bfill)
+        setattr(cls, "fillna", fillna)
+        setattr(cls, "ffill", ffill)
+        setattr(cls, "isna", isna)
+        setattr(cls, "isnull", isnull)
+        setattr(cls, "notna", notna)
+        setattr(cls, "notnull", notnull)
+        setattr(cls, "pad", ffill)
+
+    for cls in DATAFRAME_TYPE:
+        setattr(cls, "dropna", df_dropna)
+        setattr(cls, "replace", df_replace)
+
+    for cls in SERIES_TYPE:
+        setattr(cls, "dropna", series_dropna)
+        setattr(cls, "replace", series_replace)
+
+    for cls in INDEX_TYPE:
+        setattr(cls, "dropna", index_dropna)
+        setattr(cls, "fillna", index_fillna)
+        setattr(cls, "isna", isna)
+        setattr(cls, "isnull", isnull)
+        setattr(cls, "notna", notna)
+        setattr(cls, "notnull", notnull)
+
+
+_install()
+del _install

maxframe/dataframe/missing/checkna.py

@@ -0,0 +1,223 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+from ... import opcodes
+from ... import tensor as mt
+from ...core import ENTITY_TYPE, OutputType
+from ...serialization.serializables import BoolField
+from ...tensor.core import TENSOR_TYPE
+from ..core import DATAFRAME_TYPE, INDEX_TYPE, SERIES_TYPE, MultiIndex
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+
+
+class DataFrameCheckNA(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.CHECK_NA
+
+    positive = BoolField("positive", default=None)
+
+    def __call__(self, df):
+        if isinstance(df, DATAFRAME_TYPE):
+            self.output_types = [OutputType.dataframe]
+        elif isinstance(df, SERIES_TYPE):
+            self.output_types = [OutputType.series]
+        elif isinstance(df, TENSOR_TYPE):
+            self.output_types = [OutputType.tensor]
+        elif isinstance(df, INDEX_TYPE):
+            self.output_types = [OutputType.index]
+        else:
+            raise TypeError(
+                f"Expecting maxframe dataframe, series, index, or tensor, got {type(df)}"
+            )
+
+        params = df.params.copy()
+        if self.output_types[0] == OutputType.dataframe:
+            params["dtypes"] = pd.Series(
+                [np.dtype("bool")] * len(df.dtypes), index=df.columns_value.to_pandas()
+            )
+        else:
+            params["dtype"] = np.dtype("bool")
+        return self.new_tileable([df], **params)
+
+
+def _from_pandas(obj: Any):
+    if isinstance(obj, pd.DataFrame):
+        from ..datasource.dataframe import from_pandas
+
+        return from_pandas(obj)
+    elif isinstance(obj, pd.Series):
+        from ..datasource.series import from_pandas
+
+        return from_pandas(obj)
+    elif isinstance(obj, np.ndarray):
+        return mt.tensor(obj)
+    else:
+        return obj
+
+
+def isna(obj):
+    """
+    Detect missing values.
+
+    Return a boolean same-sized object indicating if the values are NA.
+    NA values, such as None or :attr:`numpy.NaN`, gets mapped to True
+    values.
+
+    Everything else gets mapped to False values. Characters such as empty
+    strings ``''`` or :attr:`numpy.inf` are not considered NA values
+    (unless you set ``pandas.options.mode.use_inf_as_na = True``).
+
+    Returns
+    -------
+    DataFrame
+        Mask of bool values for each element in DataFrame that
+        indicates whether an element is not an NA value.
+
+    See Also
+    --------
+    DataFrame.isnull : Alias of isna.
+    DataFrame.notna : Boolean inverse of isna.
+    DataFrame.dropna : Omit axes labels with missing values.
+    isna : Top-level isna.
+
+    Examples
+    --------
+    Show which entries in a DataFrame are NA.
+
+    >>> import numpy as np
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame({'age': [5, 6, np.NaN],
+    ...                    'born': [md.NaT, md.Timestamp('1939-05-27'),
+    ...                             md.Timestamp('1940-04-25')],
+    ...                    'name': ['Alfred', 'Batman', ''],
+    ...                    'toy': [None, 'Batmobile', 'Joker']})
+    >>> df.execute()
+       age       born    name        toy
+    0  5.0        NaT  Alfred       None
+    1  6.0 1939-05-27  Batman  Batmobile
+    2  NaN 1940-04-25              Joker
+
+    >>> df.isna().execute()
+         age   born   name    toy
+    0  False   True  False   True
+    1  False  False  False  False
+    2   True  False  False  False
+
+    Show which entries in a Series are NA.
+
+    >>> ser = md.Series([5, 6, np.NaN])
+    >>> ser.execute()
+    0    5.0
+    1    6.0
+    2    NaN
+    dtype: float64
+
+    >>> ser.isna().execute()
+    0    False
+    1    False
+    2     True
+    dtype: bool
+    """
+    if isinstance(obj, MultiIndex):
+        raise NotImplementedError("isna is not defined for MultiIndex")
+    elif isinstance(obj, ENTITY_TYPE):
+        if isinstance(obj, TENSOR_TYPE):
+            return mt.isnan(obj)
+        else:
+            op = DataFrameCheckNA(positive=True)
+            return op(obj)
+    else:
+        return _from_pandas(pd.isna(obj))
+
+
+def notna(obj):
+    """
+    Detect existing (non-missing) values.
+
+    Return a boolean same-sized object indicating if the values are not NA.
+    Non-missing values get mapped to True. Characters such as empty
+    strings ``''`` or :attr:`numpy.inf` are not considered NA values
+    (unless you set ``pandas.options.mode.use_inf_as_na = True``).
+    NA values, such as None or :attr:`numpy.NaN`, get mapped to False
+    values.
+
+    Returns
+    -------
+    DataFrame
+        Mask of bool values for each element in DataFrame that
+        indicates whether an element is not an NA value.
+
+    See Also
+    --------
+    DataFrame.notnull : Alias of notna.
+    DataFrame.isna : Boolean inverse of notna.
+    DataFrame.dropna : Omit axes labels with missing values.
+    notna : Top-level notna.
+
+    Examples
+    --------
+    Show which entries in a DataFrame are not NA.
+
+    >>> import numpy as np
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame({'age': [5, 6, np.NaN],
+    ...                    'born': [md.NaT, md.Timestamp('1939-05-27'),
+    ...                             md.Timestamp('1940-04-25')],
+    ...                    'name': ['Alfred', 'Batman', ''],
+    ...                    'toy': [None, 'Batmobile', 'Joker']})
+    >>> df.execute()
+       age       born    name        toy
+    0  5.0        NaT  Alfred       None
+    1  6.0 1939-05-27  Batman  Batmobile
+    2  NaN 1940-04-25              Joker
+
+    >>> df.notna().execute()
+         age   born  name    toy
+    0   True  False  True  False
+    1   True   True  True   True
+    2  False   True  True   True
+
+    Show which entries in a Series are not NA.
+
+    >>> ser = md.Series([5, 6, np.NaN])
+    >>> ser.execute()
+    0    5.0
+    1    6.0
+    2    NaN
+    dtype: float64
+
+    >>> ser.notna().execute()
+    0     True
+    1     True
+    2    False
+    dtype: bool
+    """
+    if isinstance(obj, MultiIndex):
+        raise NotImplementedError("isna is not defined for MultiIndex")
+    elif isinstance(obj, ENTITY_TYPE):
+        if isinstance(obj, TENSOR_TYPE):
+            return ~mt.isnan(obj)
+        else:
+            op = DataFrameCheckNA(positive=False)
+            return op(obj)
+    else:
+        return _from_pandas(pd.notna(obj))
+
+
+isnull = isna
+notnull = notna