maxframe 0.1.0b5__cp310-cp310-macosx_11_0_arm64.whl

maxframe/dataframe/statistics/corr.py

@@ -0,0 +1,280 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pandas as pd
+
+from ... import opcodes
+from ...core import ENTITY_TYPE
+from ...serialization.serializables import AnyField, BoolField, Int32Field, KeyField
+from ...tensor.utils import filter_inputs
+from ..core import DATAFRAME_TYPE, SERIES_TYPE
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import build_empty_df, parse_index, validate_axis
+
+
+class DataFrameCorr(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.CORR
+
+    other = KeyField("other", default=None)
+    method = AnyField("method", default=None)
+    min_periods = Int32Field("min_periods", default=None)
+    axis = Int32Field("axis", default=None)
+    drop = BoolField("drop", default=None)
+
+    def _set_inputs(self, inputs):
+        super()._set_inputs(inputs)
+        inputs_iter = iter(self._inputs)
+        next(inputs_iter)
+        if isinstance(self.other, ENTITY_TYPE):
+            self.other = next(inputs_iter)
+
+    def __call__(self, df_or_series):
+        if isinstance(df_or_series, SERIES_TYPE):
+            inputs = filter_inputs([df_or_series, self.other])
+            return self.new_scalar(inputs, dtype=np.dtype(np.float_))
+        else:
+
+            def _filter_numeric(obj):
+                if not isinstance(obj, DATAFRAME_TYPE):
+                    return obj
+                num_dtypes = build_empty_df(obj.dtypes)._get_numeric_data().dtypes
+                if len(num_dtypes) != len(obj.dtypes):
+                    return obj[list(num_dtypes.index)]
+                return obj
+
+            df_or_series = _filter_numeric(df_or_series)
+            self.other = _filter_numeric(self.other)
+
+            inputs = filter_inputs([df_or_series, self.other])
+            if self.axis is None:
+                dtypes = pd.Series(
+                    [np.dtype(np.float_)] * len(df_or_series.dtypes),
+                    index=df_or_series.dtypes.index,
+                )
+                return self.new_dataframe(
+                    inputs,
+                    shape=(df_or_series.shape[1],) * 2,
+                    dtypes=dtypes,
+                    index_value=df_or_series.columns_value,
+                    columns_value=df_or_series.columns_value,
+                )
+            else:
+                new_index_value = df_or_series.axes[1 - self.axis].index_value
+                if isinstance(self.other, DATAFRAME_TYPE):
+                    align_dtypes = pd.concat(
+                        [self.other.dtypes, df_or_series.dtypes], axis=1
+                    )
+                    align_shape = (np.nan, align_dtypes.shape[0])
+                    new_index_value = parse_index(align_dtypes.index)
+                else:
+                    align_shape = df_or_series.shape
+
+                shape = (np.nan,) if self.drop else (align_shape[1 - self.axis],)
+                return self.new_series(
+                    inputs,
+                    shape=shape,
+                    dtype=np.dtype(np.float_),
+                    index_value=new_index_value,
+                )
+
+
+def df_corr(df, method="pearson", min_periods=1):
+    """
+    Compute pairwise correlation of columns, excluding NA/null values.
+
+    Parameters
+    ----------
+    method : {'pearson', 'kendall', 'spearman'} or callable
+        Method of correlation:
+
+        * pearson : standard correlation coefficient
+        * kendall : Kendall Tau correlation coefficient
+        * spearman : Spearman rank correlation
+        * callable: callable with input two 1d ndarrays
+            and returning a float. Note that the returned matrix from corr
+            will have 1 along the diagonals and will be symmetric
+            regardless of the callable's behavior.
+
+        .. note::
+            kendall, spearman and callables not supported on multiple chunks yet.
+
+    min_periods : int, optional
+        Minimum number of observations required per pair of columns
+        to have a valid result. Currently only available for Pearson
+        and Spearman correlation.
+
+    Returns
+    -------
+    DataFrame
+        Correlation matrix.
+
+    See Also
+    --------
+    DataFrame.corrwith : Compute pairwise correlation with another
+        DataFrame or Series.
+    Series.corr : Compute the correlation between two Series.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame([(.2, .3), (.0, .6), (.6, .0), (.2, .1)],
+    ...                   columns=['dogs', 'cats'])
+    >>> df.corr(method='pearson').execute()
+              dogs      cats
+    dogs  1.000000 -0.851064
+    cats -0.851064  1.000000
+    """
+    op = DataFrameCorr(method=method, min_periods=min_periods)
+    return op(df)
+
+
+def df_corrwith(df, other, axis=0, drop=False, method="pearson"):
+    """
+    Compute pairwise correlation.
+
+    Pairwise correlation is computed between rows or columns of
+    DataFrame with rows or columns of Series or DataFrame. DataFrames
+    are first aligned along both axes before computing the
+    correlations.
+
+    Parameters
+    ----------
+    other : DataFrame, Series
+        Object with which to compute correlations.
+    axis : {0 or 'index', 1 or 'columns'}, default 0
+        The axis to use. 0 or 'index' to compute column-wise, 1 or 'columns' for
+        row-wise.
+    drop : bool, default False
+        Drop missing indices from result.
+    method : {'pearson', 'kendall', 'spearman'} or callable
+        Method of correlation:
+
+        * pearson : standard correlation coefficient
+        * kendall : Kendall Tau correlation coefficient
+        * spearman : Spearman rank correlation
+        * callable: callable with input two 1d ndarrays
+            and returning a float.
+
+        .. note::
+            kendall, spearman and callables not supported on multiple chunks yet.
+
+    Returns
+    -------
+    Series
+        Pairwise correlations.
+
+    See Also
+    --------
+    DataFrame.corr : Compute pairwise correlation of columns.
+    """
+    axis = validate_axis(axis, df)
+    if drop:
+        # TODO implement with df.align(method='inner')
+        raise NotImplementedError("drop=True not implemented")
+    op = DataFrameCorr(other=other, method=method, axis=axis, drop=drop)
+    return op(df)
+
+
+def series_corr(series, other, method="pearson", min_periods=None):
+    """
+    Compute correlation with `other` Series, excluding missing values.
+
+    Parameters
+    ----------
+    other : Series
+        Series with which to compute the correlation.
+    method : {'pearson', 'kendall', 'spearman'} or callable
+        Method used to compute correlation:
+
+        - pearson : Standard correlation coefficient
+        - kendall : Kendall Tau correlation coefficient
+        - spearman : Spearman rank correlation
+        - callable: Callable with input two 1d ndarrays and returning a float.
+
+        .. note::
+            kendall, spearman and callables not supported on multiple chunks yet.
+
+    min_periods : int, optional
+        Minimum number of observations needed to have a valid result.
+
+    Returns
+    -------
+    float
+        Correlation with other.
+
+    See Also
+    --------
+    DataFrame.corr : Compute pairwise correlation between columns.
+    DataFrame.corrwith : Compute pairwise correlation with another
+        DataFrame or Series.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> s1 = md.Series([.2, .0, .6, .2])
+    >>> s2 = md.Series([.3, .6, .0, .1])
+    >>> s1.corr(s2, method='pearson').execute()
+    -0.8510644963469898
+    """
+    op = DataFrameCorr(other=other, method=method, min_periods=min_periods)
+    return op(series)
+
+
+def series_autocorr(series, lag=1):
+    """
+    Compute the lag-N autocorrelation.
+
+    This method computes the Pearson correlation between
+    the Series and its shifted self.
+
+    Parameters
+    ----------
+    lag : int, default 1
+        Number of lags to apply before performing autocorrelation.
+
+    Returns
+    -------
+    float
+        The Pearson correlation between self and self.shift(lag).
+
+    See Also
+    --------
+    Series.corr : Compute the correlation between two Series.
+    Series.shift : Shift index by desired number of periods.
+    DataFrame.corr : Compute pairwise correlation of columns.
+    DataFrame.corrwith : Compute pairwise correlation between rows or
+        columns of two DataFrame objects.
+
+    Notes
+    -----
+    If the Pearson correlation is not well defined return 'NaN'.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> s = md.Series([0.25, 0.5, 0.2, -0.05])
+    >>> s.autocorr().execute()  # doctest: +ELLIPSIS.execute()
+    0.10355...
+    >>> s.autocorr(lag=2).execute()  # doctest: +ELLIPSIS.execute()
+    -0.99999...
+
+    If the Pearson correlation is not well defined, then 'NaN' is returned.
+
+    >>> s = md.Series([1, 0, 0, 0])
+    >>> s.autocorr().execute()
+    nan
+    """
+    op = DataFrameCorr(other=series.shift(lag), method="pearson")
+    return op(series)

maxframe/dataframe/statistics/quantile.py

@@ -0,0 +1,341 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pandas as pd
+
+from ... import opcodes as OperandDef
+from ...core import ENTITY_TYPE
+from ...serialization.serializables import (
+    AnyField,
+    BoolField,
+    DataTypeField,
+    Int32Field,
+    KeyField,
+    StringField,
+)
+from ...tensor.core import TENSOR_TYPE
+from ...tensor.datasource import empty
+from ...tensor.datasource import from_series as tensor_from_series
+from ...tensor.datasource import tensor as astensor
+from ...tensor.statistics.quantile import quantile as tensor_quantile
+from ..core import DATAFRAME_TYPE
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import build_empty_df, find_common_type, parse_index, validate_axis
+
+
+class DataFrameQuantile(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = OperandDef.QUANTILE
+
+    input = KeyField("input", default=None)
+    q = AnyField("q", default=None)
+    axis = Int32Field("axis", default=None)
+    numeric_only = BoolField("numeric_only", default=None)
+    interpolation = StringField("interpolation", default=None)
+
+    dtype = DataTypeField("dtype", default=None)
+
+    def __init__(self, output_types=None, **kw):
+        super().__init__(_output_types=output_types, **kw)
+
+    def _set_inputs(self, inputs):
+        super()._set_inputs(inputs)
+        self.input = self._inputs[0]
+        if isinstance(self.q, TENSOR_TYPE):
+            self.q = self._inputs[-1]
+
+    def _calc_dtype_on_axis_1(self, a, dtypes):
+        quantile_dtypes = []
+        for name in dtypes.index:
+            dt = tensor_quantile(
+                tensor_from_series(a[name]),
+                self.q,
+                interpolation=self.interpolation,
+                handle_non_numeric=not self.numeric_only,
+            ).dtype
+            quantile_dtypes.append(dt)
+        return find_common_type(quantile_dtypes)
+
+    def _call_dataframe(self, a, inputs):
+        if self.numeric_only:
+            empty_df = build_empty_df(a.dtypes)
+            dtypes = empty_df._get_numeric_data().dtypes
+        else:
+            dtypes = a.dtypes
+        if isinstance(self.q, TENSOR_TYPE):
+            q_val = self.q
+            pd_index = pd.Index([], dtype=q_val.dtype)
+            name = None
+            store_index_value = False
+        else:
+            q_val = np.asanyarray(self.q)
+            pd_index = pd.Index(q_val)
+            name = self.q if q_val.size == 1 else None
+            store_index_value = True
+        tokenize_objects = (a, q_val, self.interpolation, type(self).__name__)
+
+        if q_val.ndim == 0 and self.axis == 0:
+            index_value = parse_index(dtypes.index, store_data=store_index_value)
+            shape = (len(dtypes),)
+            # calc dtype
+            dtype = self._calc_dtype_on_axis_1(a, dtypes)
+            return self.new_series(
+                inputs,
+                shape=shape,
+                dtype=dtype,
+                index_value=index_value,
+                name=name or dtypes.index.name,
+            )
+        elif q_val.ndim == 0 and self.axis == 1:
+            index_value = a.index_value
+            shape = (len(a),)
+            # calc dtype
+            dt = tensor_quantile(
+                empty(a.shape[1], dtype=find_common_type(list(dtypes))),
+                self.q,
+                interpolation=self.interpolation,
+                handle_non_numeric=not self.numeric_only,
+            ).dtype
+            return self.new_series(
+                inputs,
+                shape=shape,
+                dtype=dt,
+                index_value=index_value,
+                name=name or index_value.name,
+            )
+        elif q_val.ndim == 1 and self.axis == 0:
+            shape = (len(q_val), len(dtypes))
+            index_value = parse_index(
+                pd_index, *tokenize_objects, store_data=store_index_value
+            )
+            dtype_list = []
+            for name in dtypes.index:
+                dtype_list.append(
+                    tensor_quantile(
+                        tensor_from_series(a[name]),
+                        self.q,
+                        interpolation=self.interpolation,
+                        handle_non_numeric=not self.numeric_only,
+                    ).dtype
+                )
+            dtypes = pd.Series(dtype_list, index=dtypes.index)
+            return self.new_dataframe(
+                inputs,
+                shape=shape,
+                dtypes=dtypes,
+                index_value=index_value,
+                columns_value=parse_index(dtypes.index, store_data=True),
+            )
+        else:
+            assert q_val.ndim == 1 and self.axis == 1
+            shape = (len(q_val), a.shape[0])
+            index_value = parse_index(
+                pd_index, *tokenize_objects, store_data=store_index_value
+            )
+            pd_columns = a.index_value.to_pandas()
+            dtype_list = np.full(len(pd_columns), self._calc_dtype_on_axis_1(a, dtypes))
+            dtypes = pd.Series(dtype_list, index=pd_columns)
+            return self.new_dataframe(
+                inputs,
+                shape=shape,
+                dtypes=dtypes,
+                index_value=index_value,
+                columns_value=parse_index(
+                    dtypes.index, store_data=True, key=a.index_value.key
+                ),
+            )
+
+    def _call_series(self, a, inputs):
+        if isinstance(self.q, TENSOR_TYPE):
+            q_val = self.q
+            index_val = pd.Index([], dtype=q_val.dtype)
+            store_index_value = False
+        else:
+            q_val = np.asanyarray(self.q)
+            index_val = pd.Index(q_val)
+            store_index_value = True
+
+        # get dtype by tensor
+        a_t = astensor(a)
+        self._dtype = dtype = tensor_quantile(
+            a_t,
+            self.q,
+            interpolation=self.interpolation,
+            handle_non_numeric=not self.numeric_only,
+        ).dtype
+
+        if q_val.ndim == 0:
+            return self.new_scalar(inputs, dtype=dtype)
+        else:
+            return self.new_series(
+                inputs,
+                shape=q_val.shape,
+                dtype=dtype,
+                index_value=parse_index(
+                    index_val,
+                    a,
+                    q_val,
+                    self.interpolation,
+                    type(self).__name__,
+                    store_data=store_index_value,
+                ),
+                name=a.name,
+            )
+
+    def __call__(self, a, q_input=None):
+        inputs = [a]
+        if q_input is not None:
+            inputs.append(q_input)
+        if isinstance(a, DATAFRAME_TYPE):
+            return self._call_dataframe(a, inputs)
+        else:
+            return self._call_series(a, inputs)
+
+
+def quantile_series(series, q=0.5, interpolation="linear"):
+    """
+    Return value at the given quantile.
+
+    Parameters
+    ----------
+    q : float or array-like, default 0.5 (50% quantile)
+        0 <= q <= 1, the quantile(s) to compute.
+    interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'}
+
+        This optional parameter specifies the interpolation method to use,
+        when the desired quantile lies between two data points `i` and `j`:
+
+            * linear: `i + (j - i) * fraction`, where `fraction` is the
+              fractional part of the index surrounded by `i` and `j`.
+            * lower: `i`.
+            * higher: `j`.
+            * nearest: `i` or `j` whichever is nearest.
+            * midpoint: (`i` + `j`) / 2.
+
+    Returns
+    -------
+    float or Series
+        If ``q`` is an array or a tensor, a Series will be returned where the
+        index is ``q`` and the values are the quantiles, otherwise
+        a float will be returned.
+
+    See Also
+    --------
+    core.window.Rolling.quantile
+    numpy.percentile
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> s = md.Series([1, 2, 3, 4])
+    >>> s.quantile(.5).execute()
+    2.5
+    >>> s.quantile([.25, .5, .75]).execute()
+    0.25    1.75
+    0.50    2.50
+    0.75    3.25
+    dtype: float64
+    """
+
+    if isinstance(q, ENTITY_TYPE):
+        q = astensor(q)
+        q_input = q
+    else:
+        q_input = None
+
+    op = DataFrameQuantile(q=q, interpolation=interpolation, gpu=series.op.gpu)
+    return op(series, q_input=q_input)
+
+
+def quantile_dataframe(df, q=0.5, axis=0, numeric_only=True, interpolation="linear"):
+    """
+    Return values at the given quantile over requested axis.
+
+    Parameters
+    ----------
+    q : float or array-like, default 0.5 (50% quantile)
+        Value between 0 <= q <= 1, the quantile(s) to compute.
+    axis : {0, 1, 'index', 'columns'} (default 0)
+        Equals 0 or 'index' for row-wise, 1 or 'columns' for column-wise.
+    numeric_only : bool, default True
+        If False, the quantile of datetime and timedelta data will be
+        computed as well.
+    interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'}
+        This optional parameter specifies the interpolation method to use,
+        when the desired quantile lies between two data points `i` and `j`:
+        * linear: `i + (j - i) * fraction`, where `fraction` is the
+          fractional part of the index surrounded by `i` and `j`.
+        * lower: `i`.
+        * higher: `j`.
+        * nearest: `i` or `j` whichever is nearest.
+        * midpoint: (`i` + `j`) / 2.
+
+    Returns
+    -------
+    Series or DataFrame
+        If ``q`` is an array or a tensor, a DataFrame will be returned where the
+          index is ``q``, the columns are the columns of self, and the
+          values are the quantiles.
+        If ``q`` is a float, a Series will be returned where the
+          index is the columns of self and the values are the quantiles.
+
+    See Also
+    --------
+    core.window.Rolling.quantile: Rolling quantile.
+    numpy.percentile: Numpy function to compute the percentile.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame(np.array([[1, 1], [2, 10], [3, 100], [4, 100]]),
+    ...                   columns=['a', 'b'])
+    >>> df.quantile(.1).execute()
+    a    1.3
+    b    3.7
+    Name: 0.1, dtype: float64
+
+    >>> df.quantile([.1, .5]).execute()
+           a     b
+    0.1  1.3   3.7
+    0.5  2.5  55.0
+
+    Specifying `numeric_only=False` will also compute the quantile of
+    datetime and timedelta data.
+
+    >>> df = md.DataFrame({'A': [1, 2],
+    ...                    'B': [md.Timestamp('2010'),
+    ...                          md.Timestamp('2011')],
+    ...                    'C': [md.Timedelta('1 days'),
+    ...                          md.Timedelta('2 days')]})
+    >>> df.quantile(0.5, numeric_only=False).execute()
+    A                    1.5
+    B    2010-07-02 12:00:00
+    C        1 days 12:00:00
+    Name: 0.5, dtype: object
+    """
+    if isinstance(q, ENTITY_TYPE):
+        q = astensor(q)
+        q_input = q
+    else:
+        q_input = None
+    axis = validate_axis(axis, df)
+
+    op = DataFrameQuantile(
+        q=q,
+        interpolation=interpolation,
+        axis=axis,
+        numeric_only=numeric_only,
+        gpu=df.op.gpu,
+    )
+    return op(df, q_input=q_input)

maxframe/dataframe/statistics/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.