maxframe 2.0.0b2__cp37-cp37m-win32.whl → 2.3.0rc1__cp37-cp37m-win32.whl

maxframe/dataframe/misc/describe.py

@@ -22,7 +22,7 @@ from ...core import EntityData
 from ...serialization.serializables import AnyField, FieldTypes, KeyField, ListField
 from ..core import SERIES_TYPE
 from ..operators import DataFrameOperator, DataFrameOperatorMixin
-from ..utils import build_empty_df, parse_index
+from ..utils import build_df, parse_index
 
 
 class DataFrameDescribe(DataFrameOperator, DataFrameOperatorMixin):
@@ -43,8 +43,6 @@ class DataFrameDescribe(DataFrameOperator, DataFrameOperatorMixin):
 
     def __call__(self, df_or_series):
         if isinstance(df_or_series, SERIES_TYPE):
-            if not np.issubdtype(df_or_series.dtype, np.number):
-                raise NotImplementedError("non-numeric type is not supported for now")
             test_series = pd.Series([], dtype=df_or_series.dtype).describe(
                 percentiles=self.percentiles,
                 include=self.include,
@@ -57,7 +55,7 @@ class DataFrameDescribe(DataFrameOperator, DataFrameOperatorMixin):
                 index_value=parse_index(test_series.index, store_data=True),
             )
         else:
-            test_inp_df = build_empty_df(df_or_series.dtypes)
+            test_inp_df = build_df(df_or_series)
             test_df = test_inp_df.describe(
                 percentiles=self.percentiles,
                 include=self.include,
@@ -69,11 +67,6 @@ class DataFrameDescribe(DataFrameOperator, DataFrameOperatorMixin):
                 # MaxFrame DataFrame allows user to specify percentiles=False
                 # to skip computation about percentiles
                 test_df.drop(["50%"], axis=0, inplace=True)
-            for dtype in test_df.dtypes:
-                if not np.issubdtype(dtype, np.number):
-                    raise NotImplementedError(
-                        "non-numeric type is not supported for now"
-                    )
             return self.new_dataframe(
                 [df_or_series],
                 shape=test_df.shape,
@@ -84,6 +77,179 @@ class DataFrameDescribe(DataFrameOperator, DataFrameOperatorMixin):
 
 
 def describe(df_or_series, percentiles=None, include=None, exclude=None):
+    """
+    Generate descriptive statistics.
+
+    Descriptive statistics include those that summarize the central
+    tendency, dispersion and shape of a
+    dataset's distribution, excluding ``NaN`` values.
+
+    Analyzes both numeric and object series, as well
+    as ``DataFrame`` column sets of mixed data types. The output
+    will vary depending on what is provided. Refer to the notes
+    below for more detail.
+
+    Parameters
+    ----------
+    percentiles : list-like of numbers, optional
+        The percentiles to include in the output. All should
+        fall between 0 and 1. The default is
+        ``[.25, .5, .75]``, which returns the 25th, 50th, and
+        75th percentiles.
+    include : 'all', list-like of dtypes or None (default), optional
+        A white list of data types to include in the result. Ignored
+        for ``Series``. Here are the options:
+
+        - 'all' : All columns of the input will be included in the output.
+        - A list-like of dtypes : Limits the results to the
+          provided data types.
+          To limit the result to numeric types submit
+          ``numpy.number``. To limit it instead to object columns submit
+          the ``numpy.object`` data type. Strings
+          can also be used in the style of
+          ``select_dtypes`` (e.g. ``df.describe(include=['O'])``).
+        - None (default) : The result will include all numeric columns.
+    exclude : list-like of dtypes or None (default), optional,
+        A black list of data types to omit from the result. Ignored
+        for ``Series``. Here are the options:
+
+        - A list-like of dtypes : Excludes the provided data types
+          from the result. To exclude numeric types submit
+          ``numpy.number``. To exclude object columns submit the data
+          type ``numpy.object``. Strings can also be used in the style of
+          ``select_dtypes`` (e.g. ``df.describe(exclude=['O'])``).
+        - None (default) : The result will exclude nothing.
+
+    Returns
+    -------
+    Series or DataFrame
+        Summary statistics of the Series or Dataframe provided.
+
+    See Also
+    --------
+    DataFrame.count: Count number of non-NA/null observations.
+    DataFrame.max: Maximum of the values in the object.
+    DataFrame.min: Minimum of the values in the object.
+    DataFrame.mean: Mean of the values.
+    DataFrame.std: Standard deviation of the observations.
+    DataFrame.select_dtypes: Subset of a DataFrame including/excluding
+        columns based on their dtype.
+
+    Notes
+    -----
+    For numeric data, the result's index will include ``count``,
+    ``mean``, ``std``, ``min``, ``max`` as well as lower, ``50`` and
+    upper percentiles. By default the lower percentile is ``25`` and the
+    upper percentile is ``75``. The ``50`` percentile is the
+    same as the median.
+
+    For object data (e.g. strings or timestamps), the result's index
+    will include ``count``, ``unique``, ``top``, and ``freq``. The ``top``
+    is the most common value. The ``freq`` is the most common value's
+    frequency. Timestamps also include the ``first`` and ``last`` items.
+
+    If multiple object values have the highest count, then the
+    ``count`` and ``top`` results will be arbitrarily chosen from
+    among those with the highest count.
+
+    For mixed data types provided via a ``DataFrame``, the default is to
+    return only an analysis of numeric columns. If the dataframe consists
+    only of object data without any numeric columns, the default is to
+    return an analysis of object columns. If ``include='all'`` is provided
+    as an option, the result will include a union of attributes of each type.
+
+    The `include` and `exclude` parameters can be used to limit
+    which columns in a ``DataFrame`` are analyzed for the output.
+    The parameters are ignored when analyzing a ``Series``.
+
+    Examples
+    --------
+    Describing a numeric ``Series``.
+
+    >>> import maxframe.tensor as mt
+    >>> import maxframe.dataframe as md
+    >>> s = md.Series([1, 2, 3])
+    >>> s.describe().execute()
+    count    3.0
+    mean     2.0
+    std      1.0
+    min      1.0
+    25%      1.5
+    50%      2.0
+    75%      2.5
+    max      3.0
+    dtype: float64
+
+    Describing a ``DataFrame``. By default only numeric fields
+    are returned.
+
+    >>> df = md.DataFrame({'numeric': [1, 2, 3],
+    ...                    'object': ['a', 'b', 'c']
+    ...                    })
+    >>> df.describe().execute()
+           numeric
+    count      3.0
+    mean       2.0
+    std        1.0
+    min        1.0
+    25%        1.5
+    50%        2.0
+    75%        2.5
+    max        3.0
+
+    Describing all columns of a ``DataFrame`` regardless of data type.
+
+    >>> df.describe(include='all').execute()  # doctest: +SKIP.execute()
+           numeric object
+    count      3.0      3
+    unique     NaN      3
+    top        NaN      a
+    freq       NaN      1
+    mean       2.0    NaN
+    std        1.0    NaN
+    min        1.0    NaN
+    25%        1.5    NaN
+    50%        2.0    NaN
+    75%        2.5    NaN
+    max        3.0    NaN
+
+    Describing a column from a ``DataFrame`` by accessing it as
+    an attribute.
+
+    >>> df.numeric.describe().execute()
+    count    3.0
+    mean     2.0
+    std      1.0
+    min      1.0
+    25%      1.5
+    50%      2.0
+    75%      2.5
+    max      3.0
+    Name: numeric, dtype: float64
+
+    Including only numeric columns in a ``DataFrame`` description.
+
+    >>> df.describe(include=[mt.number]).execute()
+           numeric
+    count      3.0
+    mean       2.0
+    std        1.0
+    min        1.0
+    25%        1.5
+    50%        2.0
+    75%        2.5
+    max        3.0
+
+    Including only string columns in a ``DataFrame`` description.
+
+    >>> df.describe(include=[object]).execute()  # doctest: +SKIP.execute()
+           object
+    count       3
+    unique      3
+    top         a
+    freq        1
+    """
+    # fixme add support for categorical columns once implemented
     if percentiles is False:
         percentiles = []
     elif percentiles is None:

maxframe/dataframe/misc/drop.py

@@ -419,6 +419,37 @@ def series_drop(
     )
 
 
+def series_pop(series, item):
+    """
+    Return item and drops from series. Raise KeyError if not found.
+
+    Parameters
+    ----------
+    item : label
+        Index of the element that needs to be removed.
+
+    Returns
+    -------
+    Value that is popped from series.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> ser = md.Series([1,2,3])
+
+    >>> ser.pop(0).execute()
+    1
+
+    >>> ser.execute()
+    1    2
+    2    3
+    dtype: int64
+    """
+    scalar = series.data[item]
+    series_drop(series, item, inplace=True)
+    return scalar
+
+
 def index_drop(index, labels, errors="raise"):
     """
     Make new Index with passed list of labels deleted.

maxframe/dataframe/misc/drop_duplicates.py

@@ -19,10 +19,10 @@ from ... import opcodes
 from ...serialization.serializables import BoolField
 from ..operators import OutputType
 from ..utils import gen_unknown_index_value, parse_index
-from ._duplicate import DuplicateOperand, validate_subset
+from ._duplicate import BaseDuplicateOp, validate_subset
 
 
-class DataFrameDropDuplicates(DuplicateOperand):
+class DataFrameDropDuplicates(BaseDuplicateOp):
     _op_type_ = opcodes.DROP_DUPLICATES
 
     ignore_index = BoolField("ignore_index", default=True)

maxframe/dataframe/misc/duplicated.py

@@ -16,10 +16,10 @@ import numpy as np
 
 from ... import opcodes
 from ...core import OutputType
-from ._duplicate import DuplicateOperand, validate_subset
+from ._duplicate import BaseDuplicateOp, validate_subset
 
 
-class DataFrameDuplicated(DuplicateOperand):
+class DataFrameDuplicated(BaseDuplicateOp):
     _op_type_ = opcodes.DUPLICATED
 
     def __init__(self, output_types=None, **kw):

maxframe/dataframe/misc/get_dummies.py

@@ -25,12 +25,14 @@ from ...serialization.serializables import (
     ListField,
     StringField,
 )
+from ...utils import make_dtype, pd_release_version
 from ..datasource.dataframe import from_pandas as from_pandas_df
 from ..datasource.series import from_pandas as from_pandas_series
 from ..initializer import Series as asseries
 from ..operators import DataFrameOperator, DataFrameOperatorMixin
 
 _encoding_dtype_kind = ["O", "S", "U"]
+_ret_uint8 = pd_release_version < (2, 0, 0)
 
 
 class DataFrameGetDummies(DataFrameOperator, DataFrameOperatorMixin):
@@ -181,7 +183,9 @@ def get_dummies(
     elif isinstance(data, pd.DataFrame):
         data = from_pandas_df(data)
 
-    dtype = dtype if dtype is not None else np.dtype(bool)
+    dtype = make_dtype(
+        dtype if dtype is not None else np.dtype(np.uint8 if _ret_uint8 else bool)
+    )
 
     if prefix is not None:
         if isinstance(prefix, list):

maxframe/dataframe/misc/infer_dtypes.py

@@ -0,0 +1,251 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ... import opcodes
+from ...serialization.serializables import AnyField, StringField
+from ..core import DATAFRAME_TYPE, SERIES_TYPE
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+
+
+class DataFrameInferDtypes(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.DATAFRAME_INFER_DTYPES
+
+    infer_method = StringField("infer_method")
+    infer_kwargs = AnyField("infer_kwargs")
+
+    infer_stage = StringField("infer_stage", default=None)
+
+    def __init__(self, output_types=None, **kw):
+        super().__init__(_output_types=output_types, **kw)
+
+    def __call__(self, df):
+        if isinstance(df, DATAFRAME_TYPE):
+            return self.new_dataframe(
+                [df],
+                shape=df.shape,
+                dtypes=None,
+                index_value=df.index_value,
+                columns_value=df.columns_value,
+            )
+        else:
+            assert isinstance(df, SERIES_TYPE)
+            return self.new_series(
+                [df],
+                shape=df.shape,
+                dtype=None,
+                name=df.name,
+                index_value=df.index_value,
+            )
+
+
+def convert_dtypes(
+    df_or_series,
+    infer_objects=True,
+    convert_string=True,
+    convert_integer=True,
+    convert_boolean=True,
+    convert_floating=True,
+    dtype_backend="numpy",
+):
+    """
+    Convert columns to best possible dtypes using dtypes supporting ``pd.NA``.
+
+    Parameters
+    ----------
+    infer_objects : bool, default True
+        Whether object dtypes should be converted to the best possible types.
+    convert_string : bool, default True
+        Whether object dtypes should be converted to ``StringDtype()``.
+    convert_integer : bool, default True
+        Whether, if possible, conversion can be done to integer extension types.
+    convert_boolean : bool, defaults True
+        Whether object dtypes should be converted to ``BooleanDtypes()``.
+    convert_floating : bool, defaults True
+        Whether, if possible, conversion can be done to floating extension types.
+        If `convert_integer` is also True, preference will be give to integer
+        dtypes if the floats can be faithfully casted to integers.
+
+    Returns
+    -------
+    Series or DataFrame
+        Copy of input object with new dtype.
+
+    See Also
+    --------
+    infer_objects : Infer dtypes of objects.
+    to_datetime : Convert argument to datetime.
+    to_timedelta : Convert argument to timedelta.
+    to_numeric : Convert argument to a numeric type.
+
+    Notes
+    -----
+    By default, ``convert_dtypes`` will attempt to convert a Series (or each
+    Series in a DataFrame) to dtypes that support ``pd.NA``. By using the options
+    ``convert_string``, ``convert_integer``, ``convert_boolean`` and
+    ``convert_boolean``, it is possible to turn off individual conversions
+    to ``StringDtype``, the integer extension types, ``BooleanDtype``
+    or floating extension types, respectively.
+
+    For object-dtyped columns, if ``infer_objects`` is ``True``, use the inference
+    rules as during normal Series/DataFrame construction.  Then, if possible,
+    convert to ``StringDtype``, ``BooleanDtype`` or an appropriate integer
+    or floating extension type, otherwise leave as ``object``.
+
+    If the dtype is integer, convert to an appropriate integer extension type.
+
+    If the dtype is numeric, and consists of all integers, convert to an
+    appropriate integer extension type. Otherwise, convert to an
+    appropriate floating extension type.
+
+    .. versionchanged:: 1.2
+        Starting with pandas 1.2, this method also converts float columns
+        to the nullable floating extension type.
+
+    In the future, as new dtypes are added that support ``pd.NA``, the results
+    of this method will change to support those new dtypes.
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame(
+    ...     {
+    ...         "a": md.Series([1, 2, 3], dtype=mt.dtype("int32")),
+    ...         "b": md.Series(["x", "y", "z"], dtype=mt.dtype("O")),
+    ...         "c": md.Series([True, False, mt.nan], dtype=mt.dtype("O")),
+    ...         "d": md.Series(["h", "i", mt.nan], dtype=mt.dtype("O")),
+    ...         "e": md.Series([10, mt.nan, 20], dtype=mt.dtype("float")),
+    ...         "f": md.Series([mt.nan, 100.5, 200], dtype=mt.dtype("float")),
+    ...     }
+    ... )
+
+    Start with a DataFrame with default dtypes.
+
+    >>> df.execute()
+       a  b      c    d     e      f
+    0  1  x   True    h  10.0    NaN
+    1  2  y  False    i   NaN  100.5
+    2  3  z    NaN  NaN  20.0  200.0
+
+    >>> df.dtypes.execute()
+    a      int32
+    b     object
+    c     object
+    d     object
+    e    float64
+    f    float64
+    dtype: object
+
+    Convert the DataFrame to use best possible dtypes.
+
+    >>> dfn = df.convert_dtypes()
+    >>> dfn.execute()
+       a  b      c     d     e      f
+    0  1  x   True     h    10   <NA>
+    1  2  y  False     i  <NA>  100.5
+    2  3  z   <NA>  <NA>    20  200.0
+
+    >>> dfn.dtypes.execute()
+    a      Int32
+    b     string
+    c    boolean
+    d     string
+    e      Int64
+    f    Float64
+    dtype: object
+
+    Start with a Series of strings and missing data represented by ``np.nan``.
+
+    >>> s = md.Series(["a", "b", mt.nan])
+    >>> s.execute()
+    0      a
+    1      b
+    2    NaN
+    dtype: object
+
+    Obtain a Series with dtype ``StringDtype``.
+
+    >>> s.convert_dtypes().execute()
+    0       a
+    1       b
+    2    <NA>
+    dtype: string
+    """
+    dtype_backend = "numpy" if dtype_backend == "numpy_nullable" else dtype_backend
+    op = DataFrameInferDtypes(
+        infer_method="convert_dtypes",
+        infer_kwargs=dict(
+            infer_objects=infer_objects,
+            convert_string=convert_string,
+            convert_integer=convert_integer,
+            convert_boolean=convert_boolean,
+            convert_floating=convert_floating,
+            dtype_backend=dtype_backend,
+        ),
+    )
+    return op(df_or_series)
+
+
+def infer_objects(df_or_series, copy=True):
+    """
+    Attempt to infer better dtypes for object columns.
+
+    Attempts soft conversion of object-dtyped
+    columns, leaving non-object and unconvertible
+    columns unchanged. The inference rules are the
+    same as during normal Series/DataFrame construction.
+
+    Returns
+    -------
+    converted : same type as input object
+
+    See Also
+    --------
+    to_datetime : Convert argument to datetime.
+    to_timedelta : Convert argument to timedelta.
+    to_numeric : Convert argument to numeric type.
+    convert_dtypes : Convert argument to best possible dtype.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame({"A": ["a", 1, 2, 3]})
+    >>> df = df.iloc[1:]
+    >>> df.execute()
+       A
+    1  1
+    2  2
+    3  3
+
+    >>> df.dtypes.execute()
+    A    object
+    dtype: object
+
+    >>> df.infer_objects().dtypes.execute()
+    A    int64
+    dtype: object
+    """
+    if (isinstance(df_or_series, SERIES_TYPE) and df_or_series.dtype != "O") or (
+        isinstance(df_or_series, DATAFRAME_TYPE)
+        and all(dt != "O" for dt in df_or_series.dtypes)
+    ):
+        # no objects to cast
+        return df_or_series
+
+    _ = copy  # in MaxFrame data are immutable, thus ignore the parameter
+    op = DataFrameInferDtypes(
+        infer_method="infer_objects",
+        infer_kwargs={},
+    )
+    return op(df_or_series)

maxframe/dataframe/misc/isin.py

@@ -133,7 +133,7 @@ def series_isin(elements, values):
     5    False
     Name: animal, dtype: bool
     """
-    if is_list_like(values):
+    if is_list_like(values) and not isinstance(values, ENTITY_TYPE):
         values = list(values)
     elif not isinstance(values, (SERIES_TYPE, TENSOR_TYPE, INDEX_TYPE)):
         raise TypeError(
@@ -207,7 +207,7 @@ def df_isin(df, values):
     falcon      True       True
     dog        False      False
     """
-    if is_list_like(values) and not isinstance(values, dict):
+    if is_list_like(values) and not isinstance(values, (dict, ENTITY_TYPE)):
         values = list(values)
     elif not isinstance(
         values, (SERIES_TYPE, DATAFRAME_TYPE, TENSOR_TYPE, INDEX_TYPE, dict)