maxframe 0.1.0b4__cp311-cp311-win32.whl → 0.1.0b5__cp311-cp311-win32.whl

maxframe/dataframe/misc/pivot_table.py

@@ -0,0 +1,262 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pandas as pd
+from pandas.api.types import is_list_like
+
+from ... import opcodes
+from ...core import OutputType
+from ...serialization.serializables import AnyField, BoolField, StringField
+from ...utils import no_default
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import build_df, parse_index
+
+
+class DataFramePivotTable(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.PIVOT_TABLE
+
+    values = AnyField("values", default=None)
+    index = AnyField("index", default=None)
+    columns = AnyField("columns", default=None)
+    aggfunc = AnyField("aggfunc", default="mean")
+    fill_value = AnyField("fill_value", default=None)
+    margins = BoolField("margins", default=False)
+    dropna = BoolField("dropna", default=True)
+    margins_name = StringField("margins_name", default=None)
+    sort = BoolField("sort", default=False)
+
+    def __init__(self, **kw):
+        super().__init__(**kw)
+        self.output_types = [OutputType.dataframe]
+
+    def __call__(self, df):
+        index_value = columns_value = dtypes = None
+        if self.index is not None:
+            # index is now a required field
+            if len(self.index) == 1:
+                index_data = pd.Index(
+                    [], dtype=df.dtypes[self.index[0]], name=self.index[0]
+                )
+            else:
+                index_data = pd.MultiIndex.from_frame(build_df(df[self.index]))
+            index_value = parse_index(index_data)
+
+            if self.columns is None:  # output columns can be determined
+                sel_df = df
+                groupby_obj = sel_df.groupby(self.index)
+                if self.values:
+                    groupby_obj = groupby_obj[self.values]
+                aggregated_df = groupby_obj.agg(self.aggfunc)
+                index_value = aggregated_df.index_value
+                columns_value = aggregated_df.columns_value
+                dtypes = aggregated_df.dtypes
+            else:
+                columns_value = dtypes = None
+        return self.new_dataframe(
+            [df],
+            shape=(np.nan, np.nan),
+            dtypes=dtypes,
+            columns_value=columns_value,
+            index_value=index_value,
+        )
+
+
+def pivot_table(
+    data,
+    values=None,
+    index=None,
+    columns=None,
+    aggfunc="mean",
+    fill_value=None,
+    margins=False,
+    dropna=True,
+    margins_name="All",
+    sort=True,
+):
+    """
+    Create a spreadsheet-style pivot table as a DataFrame.
+
+    The levels in the pivot table will be stored in MultiIndex objects
+    (hierarchical indexes) on the index and columns of the result DataFrame.
+
+    Parameters
+    ----------
+    values : column to aggregate, optional
+    index : column, Grouper, array, or list of the previous
+        If an array is passed, it must be the same length as the data. The
+        list can contain any of the other types (except list).
+        Keys to group by on the pivot table index.  If an array is passed,
+        it is being used as the same manner as column values.
+    columns : column, Grouper, array, or list of the previous
+        If an array is passed, it must be the same length as the data. The
+        list can contain any of the other types (except list).
+        Keys to group by on the pivot table column.  If an array is passed,
+        it is being used as the same manner as column values.
+    aggfunc : function, list of functions, dict, default numpy.mean
+        If list of functions passed, the resulting pivot table will have
+        hierarchical columns whose top level are the function names
+        (inferred from the function objects themselves)
+        If dict is passed, the key is column to aggregate and value
+        is function or list of functions.
+    fill_value : scalar, default None
+        Value to replace missing values with (in the resulting pivot table,
+        after aggregation).
+    margins : bool, default False
+        Add all row / columns (e.g. for subtotal / grand totals).
+    dropna : bool, default True
+        Do not include columns whose entries are all NaN.
+    margins_name : str, default 'All'
+        Name of the row / column that will contain the totals
+        when margins is True.
+    sort : bool, default True
+        Specifies if the result should be sorted.
+
+    Returns
+    -------
+    DataFrame
+        An Excel style pivot table.
+
+    See Also
+    --------
+    DataFrame.pivot : Pivot without aggregation that can handle
+        non-numeric data.
+    DataFrame.melt: Unpivot a DataFrame from wide to long format,
+        optionally leaving identifiers set.
+    wide_to_long : Wide panel to long format. Less flexible but more
+        user-friendly than melt.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame({"A": ["foo", "foo", "foo", "foo", "foo",
+    ...                          "bar", "bar", "bar", "bar"],
+    ...                    "B": ["one", "one", "one", "two", "two",
+    ...                          "one", "one", "two", "two"],
+    ...                    "C": ["small", "large", "large", "small",
+    ...                          "small", "large", "small", "small",
+    ...                          "large"],
+    ...                    "D": [1, 2, 2, 3, 3, 4, 5, 6, 7],
+    ...                    "E": [2, 4, 5, 5, 6, 6, 8, 9, 9]})
+    >>> df.execute()
+         A    B      C  D  E
+    0  foo  one  small  1  2
+    1  foo  one  large  2  4
+    2  foo  one  large  2  5
+    3  foo  two  small  3  5
+    4  foo  two  small  3  6
+    5  bar  one  large  4  6
+    6  bar  one  small  5  8
+    7  bar  two  small  6  9
+    8  bar  two  large  7  9
+
+    This first example aggregates values by taking the sum.
+
+    >>> table = md.pivot_table(df, values='D', index=['A', 'B'],
+    ...                        columns=['C'], aggfunc=np.sum)
+    >>> table.execute()
+    C        large  small
+    A   B
+    bar one    4.0    5.0
+        two    7.0    6.0
+    foo one    4.0    1.0
+        two    NaN    6.0
+
+    We can also fill missing values using the `fill_value` parameter.
+
+    >>> table = md.pivot_table(df, values='D', index=['A', 'B'],
+    ...                        columns=['C'], aggfunc=np.sum, fill_value=0)
+    >>> table.execute()
+    C        large  small
+    A   B
+    bar one      4      5
+        two      7      6
+    foo one      4      1
+        two      0      6
+
+    The next example aggregates by taking the mean across multiple columns.
+
+    >>> table = md.pivot_table(df, values=['D', 'E'], index=['A', 'C'],
+    ...                        aggfunc={'D': np.mean,
+    ...                                 'E': np.mean})
+    >>> table.execute()
+                    D         E
+    A   C
+    bar large  5.500000  7.500000
+        small  5.500000  8.500000
+    foo large  2.000000  4.500000
+        small  2.333333  4.333333
+
+    We can also calculate multiple types of aggregations for any given
+    value column.
+
+    >>> table = md.pivot_table(df, values=['D', 'E'], index=['A', 'C'],
+    ...                        aggfunc={'D': np.mean,
+    ...                                 'E': [min, max, np.mean]})
+    >>> table.execute()
+                    D    E
+                mean  max      mean  min
+    A   C
+    bar large  5.500000  9.0  7.500000  6.0
+        small  5.500000  9.0  8.500000  8.0
+    foo large  2.000000  5.0  4.500000  4.0
+        small  2.333333  6.0  4.333333  2.0
+    """
+    if index is None and columns is None:
+        raise ValueError(
+            "No group keys passed, need to specify at least one of index or columns"
+        )
+
+    def make_col_list(col):
+        try:
+            if col in data.dtypes.index:
+                return [col]
+        except TypeError:
+            return col
+        return col
+
+    values_list = make_col_list(values)
+    index_list = make_col_list(index)
+    columns_list = make_col_list(columns)
+
+    name_to_attr = {"values": values_list, "index": index_list, "columns": columns_list}
+    for key, val in name_to_attr.items():
+        if val is None:
+            continue
+        if not is_list_like(val):
+            raise ValueError(f"Need to specify {key} as a list-like object.")
+        non_exist_key = next((c for c in val if c not in data.dtypes.index), no_default)
+        if non_exist_key is not no_default:
+            raise ValueError(
+                f"Column {non_exist_key} specified in {key} is not a valid column."
+            )
+
+    if columns is None and not margins:
+        if values_list:
+            data = data[index_list + values_list]
+        return data.groupby(index, sort=sort).agg(aggfunc)
+
+    op = DataFramePivotTable(
+        values=values,
+        index=index,
+        columns=columns,
+        aggfunc=aggfunc,
+        fill_value=fill_value,
+        margins=margins,
+        dropna=dropna,
+        margins_name=margins_name,
+        sort=sort,
+    )
+    return op(data)

maxframe/dataframe/misc/tests/test_misc.py

@@ -21,6 +21,7 @@ from ....core import OutputType
 from ....tensor.core import TENSOR_TYPE
 from ... import eval as maxframe_eval
 from ... import get_dummies, to_numeric
+from ...arithmetic import DataFrameGreater, DataFrameLess
 from ...core import CATEGORICAL_TYPE, DATAFRAME_TYPE, INDEX_TYPE, SERIES_TYPE
 from ...datasource.dataframe import from_pandas as from_pandas_df
 from ...datasource.index import from_pandas as from_pandas_index
@@ -405,3 +406,63 @@ def test_to_numeric():
 
     with pytest.raises(ValueError):
         _ = to_numeric([])
+
+
+def test_case_when():
+    rs = np.random.RandomState(0)
+    raw = pd.DataFrame(
+        rs.randint(1000, size=(20, 8)), columns=["c" + str(i + 1) for i in range(8)]
+    )
+    df = from_pandas_df(raw, chunk_size=8)
+
+    with pytest.raises(TypeError):
+        df.c1.case_when(df.c2)
+    with pytest.raises(ValueError):
+        df.c1.case_when([])
+    with pytest.raises(TypeError):
+        df.c1.case_when([[]])
+    with pytest.raises(ValueError):
+        df.c1.case_when([()])
+
+    col = df.c1.case_when([(df.c2 < 10, 10), (df.c2 > 20, df.c3)])
+    assert len(col.inputs) == 4
+    assert isinstance(col.inputs[1].op, DataFrameLess)
+    assert isinstance(col.inputs[2].op, DataFrameGreater)
+
+
+def test_pivot_table():
+    from ...groupby.aggregation import DataFrameGroupByAgg
+    from ...misc.pivot_table import DataFramePivotTable
+
+    raw = pd.DataFrame(
+        {
+            "A": "foo foo foo foo foo bar bar bar bar".split(),
+            "B": "one one one two two one one two two".split(),
+            "C": "small large large small small large small small large".split(),
+            "D": [1, 2, 2, 3, 3, 4, 5, 6, 7],
+            "E": [2, 4, 5, 5, 6, 6, 8, 9, 9],
+        }
+    )
+    df = from_pandas_df(raw, chunk_size=8)
+    with pytest.raises(ValueError):
+        df.pivot_table(index=123)
+    with pytest.raises(ValueError):
+        df.pivot_table(index=["F"])
+    with pytest.raises(ValueError):
+        df.pivot_table(values=["D", "E"], aggfunc="sum")
+
+    t = df.pivot_table(index="A")
+    assert isinstance(t.op, DataFrameGroupByAgg)
+    t = df.pivot_table(index="A", values=["D", "E"], aggfunc="sum")
+    assert isinstance(t.op, DataFrameGroupByAgg)
+
+    t = df.pivot_table(index=["A", "B"], values=["D", "E"], aggfunc="sum", margins=True)
+    assert isinstance(t.op, DataFramePivotTable)
+
+    t = df.pivot_table(index="A", columns=["B", "C"], aggfunc="sum")
+    assert isinstance(t.op, DataFramePivotTable)
+    assert t.shape == (np.nan, np.nan)
+
+    t = df.pivot_table(index=["A", "B"], columns="C", aggfunc="sum")
+    assert isinstance(t.op, DataFramePivotTable)
+    assert t.shape == (np.nan, np.nan)

maxframe/dataframe/plotting/core.py

@@ -17,7 +17,7 @@ from collections import OrderedDict
 import pandas as pd
 
 from ...core import ENTITY_TYPE, ExecutableTuple
-from ...utils import adapt_docstring
+from ...utils import adapt_docstring, get_item_if_scalar
 
 
 class PlotAccessor:
@@ -34,7 +34,7 @@ class PlotAccessor:
             .fetch(session=session)
         )
         for p, v in zip(to_executes, executed):
-            result[p] = v
+            result[p] = get_item_if_scalar(v)
 
         data = result.pop("__object__")
         pd_kwargs = kwargs.copy()

maxframe/dataframe/reduction/core.py

@@ -30,7 +30,7 @@ from ...serialization.serializables import (
     StringField,
 )
 from ...typing_ import TileableType
-from ...utils import pd_release_version, tokenize
+from ...utils import get_item_if_scalar, pd_release_version, tokenize
 from ..operators import DATAFRAME_TYPE, DataFrameOperator, DataFrameOperatorMixin
 from ..utils import (
     build_df,
@@ -715,6 +715,7 @@ class ReductionCompiler:
                 keys_to_vars = {inp.key: local_key_to_var[inp.key] for inp in t.inputs}
 
                 def _interpret_var(v):
+                    v = get_item_if_scalar(v)
                     # get representation for variables
                     if hasattr(v, "key"):
                         return keys_to_vars[v.key]

maxframe/dataframe/utils.py

@@ -1136,6 +1136,13 @@ def concat_on_columns(objs: List) -> Any:
     return result
 
 
+def apply_if_callable(maybe_callable, obj, **kwargs):
+    if callable(maybe_callable):
+        return maybe_callable(obj, **kwargs)
+
+    return maybe_callable
+
+
 def patch_sa_engine_execute():
     """
     pandas did not resolve compatibility issue of sqlalchemy 2.0, the issue

maxframe/learn/contrib/utils.py

@@ -0,0 +1,52 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import sys
+
+
+def make_import_error_func(package_name):
+    def _func(*_, **__):  # pragma: no cover
+        raise ImportError(
+            f"Cannot import {package_name}, please reinstall that package."
+        )
+
+    return _func
+
+
+def config_mod_getattr(mod_dict, globals_):
+    def __getattr__(name):
+        import importlib
+
+        if name in mod_dict:
+            mod_name, cls_name = mod_dict[name].rsplit(".", 1)
+            mod = importlib.import_module(mod_name, globals_["__name__"])
+            cls = globals_[name] = getattr(mod, cls_name)
+            return cls
+        else:  # pragma: no cover
+            raise AttributeError(name)
+
+    if sys.version_info[:2] < (3, 7):
+        for _mod in mod_dict.keys():
+            __getattr__(_mod)
+
+    def __dir__():
+        return sorted([n for n in globals_ if not n.startswith("_")] + list(mod_dict))
+
+    globals_.update(
+        {
+            "__getattr__": __getattr__,
+            "__dir__": __dir__,
+            "__all__": list(__dir__()),
+            "__warningregistry__": dict(),
+        }
+    )

maxframe/learn/contrib/xgboost/__init__.py

@@ -0,0 +1,26 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ..utils import config_mod_getattr as _config_mod_getattr
+from .dmatrix import DMatrix
+from .predict import predict
+from .train import train
+
+_config_mod_getattr(
+    {
+        "XGBClassifier": ".classifier.XGBClassifier",
+        "XGBRegressor": ".regressor.XGBRegressor",
+    },
+    globals(),
+)

maxframe/learn/contrib/xgboost/classifier.py

@@ -0,0 +1,86 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ....tensor import argmax
+from ..utils import make_import_error_func
+from .core import XGBScikitLearnBase, xgboost
+
+if not xgboost:
+    XGBClassifier = make_import_error_func("xgboost")
+else:
+    from xgboost.sklearn import XGBClassifierBase
+
+    from .core import wrap_evaluation_matrices
+    from .predict import predict
+    from .train import train
+
+    class XGBClassifier(XGBScikitLearnBase, XGBClassifierBase):
+        """
+        Implementation of the scikit-learn API for XGBoost classification.
+        """
+
+        def fit(
+            self,
+            X,
+            y,
+            sample_weight=None,
+            base_margin=None,
+            eval_set=None,
+            sample_weight_eval_set=None,
+            base_margin_eval_set=None,
+            num_class=None,
+        ):
+            dtrain, evals = wrap_evaluation_matrices(
+                None,
+                X,
+                y,
+                sample_weight,
+                base_margin,
+                eval_set,
+                sample_weight_eval_set,
+                base_margin_eval_set,
+            )
+            params = self.get_xgb_params()
+            self.n_classes_ = num_class or 1
+            if self.n_classes_ > 2:
+                params["objective"] = "multi:softprob"
+                params["num_class"] = self.n_classes_
+            else:
+                params["objective"] = "binary:logistic"
+            self.evals_result_ = dict()
+            result = train(
+                params,
+                dtrain,
+                num_boost_round=self.get_num_boosting_rounds(),
+                evals=evals,
+                evals_result=self.evals_result_,
+                num_class=num_class,
+            )
+            self._Booster = result
+            return self
+
+        def predict(self, data, **kw):
+            prob = self.predict_proba(data, flag=True, **kw)
+            if prob.ndim > 1:
+                prediction = argmax(prob, axis=1)
+            else:
+                prediction = (prob > 0.5).astype(np.int64)
+            return prediction
+
+        def predict_proba(self, data, ntree_limit=None, flag=False, **kw):
+            if ntree_limit is not None:
+                raise NotImplementedError("ntree_limit is not currently supported")
+            return predict(self.get_booster(), data, flag=flag, **kw)