maxframe 2.3.0__cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

maxframe/dataframe/misc/explode.py

@@ -0,0 +1,171 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pandas as pd
+
+from ... import opcodes
+from ...core import OutputType
+from ...serialization.serializables import AnyField, BoolField
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import parse_index
+
+
+class DataFrameExplode(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.EXPLODE
+
+    column = AnyField("column", default=None)
+    ignore_index = BoolField("ignore_field", default=None)
+
+    def __init__(self, output_types=None, **kw):
+        super().__init__(_output_types=output_types, **kw)
+
+    def _rewrite_params(self, in_obj):
+        params = in_obj.params.copy()
+        new_shape = list(in_obj.shape)
+        new_shape[0] = np.nan
+        params["shape"] = tuple(new_shape)
+
+        if self.ignore_index:
+            params["index_value"] = parse_index(
+                pd.RangeIndex(-1), (in_obj.key, in_obj.index_value.key)
+            )
+        else:
+            params["index_value"] = parse_index(
+                None, (in_obj.key, in_obj.index_value.key)
+            )
+        return params
+
+    def __call__(self, df_or_series):
+        return self.new_tileable([df_or_series], **self._rewrite_params(df_or_series))
+
+
+def df_explode(df, column, ignore_index=False):
+    """
+    Transform each element of a list-like to a row, replicating index values.
+
+    Parameters
+    ----------
+    column : str or tuple
+        Column to explode.
+    ignore_index : bool, default False
+        If True, the resulting index will be labeled 0, 1, …, n - 1.
+
+    Returns
+    -------
+    DataFrame
+        Exploded lists to rows of the subset columns;
+        index will be duplicated for these rows.
+
+    Raises
+    ------
+    ValueError :
+        if columns of the frame are not unique.
+
+    See Also
+    --------
+    DataFrame.unstack : Pivot a level of the (necessarily hierarchical)
+        index labels.
+    DataFrame.melt : Unpivot a DataFrame from wide format to long format.
+    Series.explode : Explode a DataFrame from list-like columns to long format.
+
+    Notes
+    -----
+    This routine will explode list-likes including lists, tuples,
+    Series, and np.ndarray. The result dtype of the subset rows will
+    be object. Scalars will be returned unchanged. Empty list-likes will
+    result in a np.nan for that row.
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame({'A': [[1, 2, 3], 'foo', [], [3, 4]], 'B': 1})
+    >>> df.execute()
+               A  B
+    0  [1, 2, 3]  1
+    1        foo  1
+    2         []  1
+    3     [3, 4]  1
+
+    >>> df.explode('A').execute()
+         A  B
+    0    1  1
+    0    2  1
+    0    3  1
+    1  foo  1
+    2  NaN  1
+    3    3  1
+    3    4  1
+    """
+    op = DataFrameExplode(
+        column=column, ignore_index=ignore_index, output_types=[OutputType.dataframe]
+    )
+    return op(df)
+
+
+def series_explode(series, ignore_index=False):
+    """
+    Transform each element of a list-like to a row.
+
+    Parameters
+    ----------
+    ignore_index : bool, default False
+        If True, the resulting index will be labeled 0, 1, …, n - 1.
+
+    Returns
+    -------
+    Series
+        Exploded lists to rows; index will be duplicated for these rows.
+
+    See Also
+    --------
+    Series.str.split : Split string values on specified separator.
+    Series.unstack : Unstack, a.k.a. pivot, Series with MultiIndex
+        to produce DataFrame.
+    DataFrame.melt : Unpivot a DataFrame from wide format to long format.
+    DataFrame.explode : Explode a DataFrame from list-like
+        columns to long format.
+
+    Notes
+    -----
+    This routine will explode list-likes including lists, tuples,
+    Series, and np.ndarray. The result dtype of the subset rows will
+    be object. Scalars will be returned unchanged. Empty list-likes will
+    result in a np.nan for that row.
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+    >>> import maxframe.dataframe as md
+    >>> s = md.Series([[1, 2, 3], 'foo', [], [3, 4]])
+    >>> s.execute()
+    0    [1, 2, 3]
+    1          foo
+    2           []
+    3       [3, 4]
+    dtype: object
+
+    >>> s.explode().execute()
+    0      1
+    0      2
+    0      3
+    1    foo
+    2    NaN
+    3      3
+    3      4
+    dtype: object
+    """
+    op = DataFrameExplode(ignore_index=ignore_index, output_types=[OutputType.series])
+    return op(series)

maxframe/dataframe/misc/get_dummies.py

@@ -0,0 +1,241 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+import numpy as np
+import pandas as pd
+
+from ...core import EntityData, OutputType
+from ...serialization.serializables import (
+    AnyField,
+    BoolField,
+    KeyField,
+    ListField,
+    StringField,
+)
+from ...utils import make_dtype, pd_release_version
+from ..datasource.dataframe import from_pandas as from_pandas_df
+from ..datasource.series import from_pandas as from_pandas_series
+from ..initializer import Series as asseries
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+
+_encoding_dtype_kind = ["O", "S", "U"]
+_ret_uint8 = pd_release_version < (2, 0, 0)
+
+
+class DataFrameGetDummies(DataFrameOperator, DataFrameOperatorMixin):
+    prefix = AnyField("prefix", default=None)
+    prefix_sep = StringField("prefix_sep", default=None)
+    dummy_na = BoolField("dummy_na", default=None)
+    columns = ListField("columns", default=None)
+    sparse = BoolField("sparse", default=None)
+    drop_first = BoolField("drop_first", default=None)
+    dtype = AnyField("dtype", default=None)
+
+    agg_results = KeyField("agg_results", default=None)
+
+    def __init__(self, **kw):
+        super().__init__(**kw)
+        self.output_types = [OutputType.dataframe]
+
+    @classmethod
+    def _set_inputs(cls, op: "DataFrameGetDummies", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        if op.agg_results is not None:  # pragma: no branch
+            op.agg_results = inputs[-1]
+
+    def __call__(self, data):
+        if not self.columns:
+            self.agg_results = data.agg(["unique"])
+        else:
+            self.agg_results = data[self.columns].agg(["unique"])
+
+        return self.new_tileable(
+            [data, self.agg_results],
+            shape=(np.nan, np.nan),
+            dtypes=None,
+            index_value=data.index_value,
+            columns_value=None,
+        )
+
+
+def get_dummies(
+    data,
+    prefix=None,
+    prefix_sep="_",
+    dummy_na=False,
+    columns=None,
+    sparse=False,
+    drop_first=False,
+    dtype=None,
+):
+    """
+    Convert categorical variable into dummy/indicator variables.
+
+    Parameters
+    ----------
+    data : array-like, Series, or DataFrame
+        Data of which to get dummy indicators.
+    prefix : str, list of str, or dict of str, default None
+        String to append DataFrame column names.
+        Pass a list with length equal to the number of columns
+        when calling get_dummies on a DataFrame. Alternatively, `prefix`
+        can be a dictionary mapping column names to prefixes.
+    prefix_sep : str, default '_'
+        If appending prefix, separator/delimiter to use. Or pass a
+        list or dictionary as with `prefix`.
+    dummy_na : bool, default False
+        Add a column to indicate NaNs, if False NaNs are ignored.
+    columns : list-like, default None
+        Column names in the DataFrame to be encoded.
+        If `columns` is None then all the columns with
+        `object` or `category` dtype will be converted.
+    sparse : bool, default False
+        Whether the dummy-encoded columns should be backed by
+        a :class:`SparseArray` (True) or a regular NumPy array (False).
+    drop_first : bool, default False
+        Whether to get k-1 dummies out of k categorical levels by removing the
+        first level.
+    dtype : dtype, default bool
+        Data type for new columns. Only a single dtype is allowed.
+
+    Returns
+    -------
+    DataFrame
+        Dummy-coded data.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> s = md.Series(list('abca'))
+
+    >>> md.get_dummies(s).execute()
+       a  b  c
+    0  1  0  0
+    1  0  1  0
+    2  0  0  1
+    3  1  0  0
+
+    >>> s1 = ['a', 'b', np.nan]
+
+    >>> md.get_dummies(s1).execute()
+       a  b
+    0  1  0
+    1  0  1
+    2  0  0
+
+    >>> md.get_dummies(s1, dummy_na=True).execute()
+       a  b  NaN
+    0  1  0    0
+    1  0  1    0
+    2  0  0    1
+
+    >>> df = md.DataFrame({'A': ['a', 'b', 'a'], 'B': ['b', 'a', 'c'],
+    ...                    'C': [1, 2, 3]})
+
+    >>> md.get_dummies(df, prefix=['col1', 'col2']).execute()
+       C  col1_a  col1_b  col2_a  col2_b  col2_c
+    0  1       1       0       0       1       0
+    1  2       0       1       1       0       0
+    2  3       1       0       0       0       1
+
+    >>> md.get_dummies(pd.Series(list('abcaa'))).execute()
+       a  b  c
+    0  1  0  0
+    1  0  1  0
+    2  0  0  1
+    3  1  0  0
+    4  1  0  0
+
+    >>> md.get_dummies(pd.Series(list('abcaa')), drop_first=True).execute()
+       b  c
+    0  0  0
+    1  1  0
+    2  0  1
+    3  0  0
+    4  0  0
+
+    >>> md.get_dummies(pd.Series(list('abc')), dtype=float).execute()
+         a    b    c
+    0  1.0  0.0  0.0
+    1  0.0  1.0  0.0
+    2  0.0  0.0  1.0
+    """
+    if columns is not None and not isinstance(columns, list):
+        raise TypeError("Input must be a list-like for parameter `columns`")
+
+    if isinstance(data, (list, tuple)):
+        data = asseries(data)
+    elif isinstance(data, pd.Series):
+        data = from_pandas_series(data)
+    elif isinstance(data, pd.DataFrame):
+        data = from_pandas_df(data)
+
+    dtype = make_dtype(
+        dtype if dtype is not None else np.dtype(np.uint8 if _ret_uint8 else bool)
+    )
+
+    if prefix is not None:
+        if isinstance(prefix, list):
+            if columns is not None:
+                encoding_col_num = len(columns)
+            else:
+                encoding_col_num = 0
+                for dt in data.dtypes.values:
+                    if dt.kind in _encoding_dtype_kind:
+                        encoding_col_num += 1
+            prefix_num = len(prefix)
+            if prefix_num != encoding_col_num:
+                raise ValueError(
+                    f"Length of 'prefix' ({prefix_num}) did not match "
+                    + f"the length of the columns being encoded ({encoding_col_num})"
+                )
+        elif isinstance(prefix, dict):
+            if columns is not None:
+                encoding_col_num = len(columns)
+                prefix_num = len(prefix)
+                if prefix_num != encoding_col_num:
+                    raise ValueError(
+                        f"Length of 'prefix' ({prefix_num}) did not match "
+                        + f"the length of the columns being encoded ({encoding_col_num})"
+                    )
+                prefix_cols = prefix.keys()
+                for columns_columnname, prefix_columnname in zip(
+                    prefix_cols, list(columns)
+                ):
+                    if columns_columnname != prefix_columnname:
+                        raise KeyError(f"{columns_columnname}")
+            else:
+                columns = list(prefix.keys())
+            # Convert prefix from dict to list, to simplify tile work
+            prefix = list(prefix.values())
+
+    if not columns and data.ndim == 2:
+        columns = []
+        for col_name, dt in data.dtypes.items():
+            if dt.kind in _encoding_dtype_kind:
+                columns.append(col_name)
+
+    op = DataFrameGetDummies(
+        prefix=prefix,
+        prefix_sep=prefix_sep,
+        dummy_na=dummy_na,
+        columns=columns,
+        sparse=sparse,
+        drop_first=drop_first,
+        dtype=dtype,
+    )
+
+    return op(data)

maxframe/dataframe/misc/infer_dtypes.py

@@ -0,0 +1,251 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ... import opcodes
+from ...serialization.serializables import AnyField, StringField
+from ..core import DATAFRAME_TYPE, SERIES_TYPE
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+
+
+class DataFrameInferDtypes(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.DATAFRAME_INFER_DTYPES
+
+    infer_method = StringField("infer_method")
+    infer_kwargs = AnyField("infer_kwargs")
+
+    infer_stage = StringField("infer_stage", default=None)
+
+    def __init__(self, output_types=None, **kw):
+        super().__init__(_output_types=output_types, **kw)
+
+    def __call__(self, df):
+        if isinstance(df, DATAFRAME_TYPE):
+            return self.new_dataframe(
+                [df],
+                shape=df.shape,
+                dtypes=None,
+                index_value=df.index_value,
+                columns_value=df.columns_value,
+            )
+        else:
+            assert isinstance(df, SERIES_TYPE)
+            return self.new_series(
+                [df],
+                shape=df.shape,
+                dtype=None,
+                name=df.name,
+                index_value=df.index_value,
+            )
+
+
+def convert_dtypes(
+    df_or_series,
+    infer_objects=True,
+    convert_string=True,
+    convert_integer=True,
+    convert_boolean=True,
+    convert_floating=True,
+    dtype_backend="numpy",
+):
+    """
+    Convert columns to best possible dtypes using dtypes supporting ``pd.NA``.
+
+    Parameters
+    ----------
+    infer_objects : bool, default True
+        Whether object dtypes should be converted to the best possible types.
+    convert_string : bool, default True
+        Whether object dtypes should be converted to ``StringDtype()``.
+    convert_integer : bool, default True
+        Whether, if possible, conversion can be done to integer extension types.
+    convert_boolean : bool, defaults True
+        Whether object dtypes should be converted to ``BooleanDtypes()``.
+    convert_floating : bool, defaults True
+        Whether, if possible, conversion can be done to floating extension types.
+        If `convert_integer` is also True, preference will be give to integer
+        dtypes if the floats can be faithfully casted to integers.
+
+    Returns
+    -------
+    Series or DataFrame
+        Copy of input object with new dtype.
+
+    See Also
+    --------
+    infer_objects : Infer dtypes of objects.
+    to_datetime : Convert argument to datetime.
+    to_timedelta : Convert argument to timedelta.
+    to_numeric : Convert argument to a numeric type.
+
+    Notes
+    -----
+    By default, ``convert_dtypes`` will attempt to convert a Series (or each
+    Series in a DataFrame) to dtypes that support ``pd.NA``. By using the options
+    ``convert_string``, ``convert_integer``, ``convert_boolean`` and
+    ``convert_boolean``, it is possible to turn off individual conversions
+    to ``StringDtype``, the integer extension types, ``BooleanDtype``
+    or floating extension types, respectively.
+
+    For object-dtyped columns, if ``infer_objects`` is ``True``, use the inference
+    rules as during normal Series/DataFrame construction.  Then, if possible,
+    convert to ``StringDtype``, ``BooleanDtype`` or an appropriate integer
+    or floating extension type, otherwise leave as ``object``.
+
+    If the dtype is integer, convert to an appropriate integer extension type.
+
+    If the dtype is numeric, and consists of all integers, convert to an
+    appropriate integer extension type. Otherwise, convert to an
+    appropriate floating extension type.
+
+    .. versionchanged:: 1.2
+        Starting with pandas 1.2, this method also converts float columns
+        to the nullable floating extension type.
+
+    In the future, as new dtypes are added that support ``pd.NA``, the results
+    of this method will change to support those new dtypes.
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame(
+    ...     {
+    ...         "a": md.Series([1, 2, 3], dtype=mt.dtype("int32")),
+    ...         "b": md.Series(["x", "y", "z"], dtype=mt.dtype("O")),
+    ...         "c": md.Series([True, False, mt.nan], dtype=mt.dtype("O")),
+    ...         "d": md.Series(["h", "i", mt.nan], dtype=mt.dtype("O")),
+    ...         "e": md.Series([10, mt.nan, 20], dtype=mt.dtype("float")),
+    ...         "f": md.Series([mt.nan, 100.5, 200], dtype=mt.dtype("float")),
+    ...     }
+    ... )
+
+    Start with a DataFrame with default dtypes.
+
+    >>> df.execute()
+       a  b      c    d     e      f
+    0  1  x   True    h  10.0    NaN
+    1  2  y  False    i   NaN  100.5
+    2  3  z    NaN  NaN  20.0  200.0
+
+    >>> df.dtypes.execute()
+    a      int32
+    b     object
+    c     object
+    d     object
+    e    float64
+    f    float64
+    dtype: object
+
+    Convert the DataFrame to use best possible dtypes.
+
+    >>> dfn = df.convert_dtypes()
+    >>> dfn.execute()
+       a  b      c     d     e      f
+    0  1  x   True     h    10   <NA>
+    1  2  y  False     i  <NA>  100.5
+    2  3  z   <NA>  <NA>    20  200.0
+
+    >>> dfn.dtypes.execute()
+    a      Int32
+    b     string
+    c    boolean
+    d     string
+    e      Int64
+    f    Float64
+    dtype: object
+
+    Start with a Series of strings and missing data represented by ``np.nan``.
+
+    >>> s = md.Series(["a", "b", mt.nan])
+    >>> s.execute()
+    0      a
+    1      b
+    2    NaN
+    dtype: object
+
+    Obtain a Series with dtype ``StringDtype``.
+
+    >>> s.convert_dtypes().execute()
+    0       a
+    1       b
+    2    <NA>
+    dtype: string
+    """
+    dtype_backend = "numpy" if dtype_backend == "numpy_nullable" else dtype_backend
+    op = DataFrameInferDtypes(
+        infer_method="convert_dtypes",
+        infer_kwargs=dict(
+            infer_objects=infer_objects,
+            convert_string=convert_string,
+            convert_integer=convert_integer,
+            convert_boolean=convert_boolean,
+            convert_floating=convert_floating,
+            dtype_backend=dtype_backend,
+        ),
+    )
+    return op(df_or_series)
+
+
+def infer_objects(df_or_series, copy=True):
+    """
+    Attempt to infer better dtypes for object columns.
+
+    Attempts soft conversion of object-dtyped
+    columns, leaving non-object and unconvertible
+    columns unchanged. The inference rules are the
+    same as during normal Series/DataFrame construction.
+
+    Returns
+    -------
+    converted : same type as input object
+
+    See Also
+    --------
+    to_datetime : Convert argument to datetime.
+    to_timedelta : Convert argument to timedelta.
+    to_numeric : Convert argument to numeric type.
+    convert_dtypes : Convert argument to best possible dtype.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame({"A": ["a", 1, 2, 3]})
+    >>> df = df.iloc[1:]
+    >>> df.execute()
+       A
+    1  1
+    2  2
+    3  3
+
+    >>> df.dtypes.execute()
+    A    object
+    dtype: object
+
+    >>> df.infer_objects().dtypes.execute()
+    A    int64
+    dtype: object
+    """
+    if (isinstance(df_or_series, SERIES_TYPE) and df_or_series.dtype != "O") or (
+        isinstance(df_or_series, DATAFRAME_TYPE)
+        and all(dt != "O" for dt in df_or_series.dtypes)
+    ):
+        # no objects to cast
+        return df_or_series
+
+    _ = copy  # in MaxFrame data are immutable, thus ignore the parameter
+    op = DataFrameInferDtypes(
+        infer_method="infer_objects",
+        infer_kwargs={},
+    )
+    return op(df_or_series)