maxframe 2.0.0b2__cp37-cp37m-win32.whl → 2.3.0rc1__cp37-cp37m-win32.whl

maxframe/dataframe/merge/append.py

@@ -14,108 +14,107 @@
 
 import pandas as pd
 
-from ... import opcodes
-from ...core import OutputType
-from ...serialization.serializables import BoolField
 from ..datasource.dataframe import from_pandas
-from ..operators import (
-    DATAFRAME_TYPE,
-    SERIES_TYPE,
-    DataFrameOperator,
-    DataFrameOperatorMixin,
-)
-from ..utils import parse_index
-
-
-class DataFrameAppend(DataFrameOperator, DataFrameOperatorMixin):
-    _op_type_ = opcodes.APPEND
-
-    ignore_index = BoolField("ignore_index", default=False)
-    verify_integrity = BoolField("verify_integrity", default=False)
-    sort = BoolField("sort", default=False)
-
-    def __init__(self, output_types=None, **kw):
-        super().__init__(_output_types=output_types, **kw)
-
-    def _call_dataframe(self, df, other):
-        if isinstance(other, DATAFRAME_TYPE):
-            shape = (df.shape[0] + other.shape[0], df.shape[1])
-            inputs = [df, other]
-            if self.ignore_index:
-                index_value = parse_index(pd.RangeIndex(shape[0]))
-            else:
-                index_value = parse_index(
-                    df.index_value.to_pandas().append(other.index_value.to_pandas())
-                )
-        elif isinstance(other, list):
-            row_length = df.shape[0]
-            index = df.index_value.to_pandas()
-            for item in other:
-                if not isinstance(item, DATAFRAME_TYPE):  # pragma: no cover
-                    raise ValueError(f"Invalid type {type(item)} to append")
-                row_length += item.shape[0]
-                index = index.append(item.index_value.to_pandas())
-            shape = (row_length, df.shape[1])
-            if self.ignore_index:  # pragma: no cover
-                index_value = parse_index(pd.RangeIndex(shape[0]))
-            else:
-                index_value = parse_index(index)
-            inputs = [df] + other
-        else:  # pragma: no cover
-            raise ValueError(f"Invalid type {type(other)} to append")
-        return self.new_dataframe(
-            inputs,
-            shape=shape,
-            dtypes=df.dtypes,
-            index_value=index_value,
-            columns_value=df.columns_value,
-        )
-
-    def _call_series(self, df, other):
-        if isinstance(other, SERIES_TYPE):
-            shape = (df.shape[0] + other.shape[0],)
-            inputs = [df, other]
-            if self.ignore_index:
-                index_value = parse_index(pd.RangeIndex(shape[0]))
-            else:
-                index_value = parse_index(
-                    df.index_value.to_pandas().append(other.index_value.to_pandas())
-                )
-        elif isinstance(other, list):
-            row_length = df.shape[0]
-            index = df.index_value.to_pandas()
-            for item in other:
-                if not isinstance(item, SERIES_TYPE):  # pragma: no cover
-                    raise ValueError(f"Invalid type {type(item)} to append")
-                row_length += item.shape[0]
-                index = index.append(item.index_value.to_pandas())
-            shape = (row_length,)
-            if self.ignore_index:  # pragma: no cover
-                index_value = parse_index(pd.RangeIndex(shape[0]))
-            else:
-                index_value = parse_index(index)
-            inputs = [df] + other
-        else:  # pragma: no cover
-            raise ValueError(f"Invalid type {type(other)} to append")
-        return self.new_series(
-            inputs, shape=shape, dtype=df.dtype, index_value=index_value, name=df.name
-        )
-
-    def __call__(self, df, other):
-        if isinstance(df, DATAFRAME_TYPE):
-            self.output_types = [OutputType.dataframe]
-            return self._call_dataframe(df, other)
-        else:
-            self.output_types = [OutputType.series]
-            return self._call_series(df, other)
 
 
 def append(df, other, ignore_index=False, verify_integrity=False, sort=False):
-    if verify_integrity or sort:  # pragma: no cover
-        raise NotImplementedError("verify_integrity and sort are not supported now")
+    """
+    Append rows of `other` to the end of caller, returning a new object.
+
+    Columns in `other` that are not in the caller are added as new columns.
+
+    Parameters
+    ----------
+    other : DataFrame or Series/dict-like object, or list of these
+        The data to append.
+    ignore_index : bool, default False
+        If True, the resulting axis will be labeled 0, 1, …, n - 1.
+    verify_integrity : bool, default False
+        If True, raise ValueError on creating index with duplicates.
+    sort : bool, default False
+        Sort columns if the columns of `self` and `other` are not aligned.
+
+    Returns
+    -------
+    DataFrame
+        A new DataFrame consisting of the rows of caller and the rows of `other`.
+
+    See Also
+    --------
+    concat : General function to concatenate DataFrame or Series objects.
+
+    Notes
+    -----
+    If a list of dict/series is passed and the keys are all contained in
+    the DataFrame's index, the order of the columns in the resulting
+    DataFrame will be unchanged.
+
+    Iteratively appending rows to a DataFrame can be more computationally
+    intensive than a single concatenate. A better solution is to append
+    those rows to a list and then concatenate the list with the original
+    DataFrame all at once.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame([[1, 2], [3, 4]], columns=list('AB'), index=['x', 'y'])
+    >>> df.execute()
+       A  B
+    x  1  2
+    y  3  4
+    >>> df2 = md.DataFrame([[5, 6], [7, 8]], columns=list('AB'), index=['x', 'y'])
+    >>> df.append(df2).execute()
+       A  B
+    x  1  2
+    y  3  4
+    x  5  6
+    y  7  8
+
+    With `ignore_index` set to True:
+
+    >>> df.append(df2, ignore_index=True).execute()
+       A  B
+    0  1  2
+    1  3  4
+    2  5  6
+    3  7  8
+
+    The following, while not recommended methods for generating DataFrames,
+    show two ways to generate a DataFrame from multiple data sources.
+
+    Less efficient:
+
+    >>> df = md.DataFrame(columns=['A'])
+    >>> for i in range(5):
+    ...     df = df.append({'A': i}, ignore_index=True)
+    >>> df.execute()
+       A
+    0  0
+    1  1
+    2  2
+    3  3
+    4  4
+
+    More efficient:
+
+    >>> md.concat([md.DataFrame([i], columns=['A']) for i in range(5)],
+    ...           ignore_index=True).execute()
+       A
+    0  0
+    1  1
+    2  2
+    3  3
+    4  4
+    """
+    from .concat import concat
+
     if isinstance(other, dict):
         other = from_pandas(pd.DataFrame(dict((k, [v]) for k, v in other.items())))
-    op = DataFrameAppend(
-        ignore_index=ignore_index, verify_integrity=verify_integrity, sort=sort
+    if not isinstance(other, list):
+        other = [other]
+    return concat(
+        [df] + other,
+        ignore_index=ignore_index,
+        verify_integrity=verify_integrity,
+        sort=sort,
     )
-    return op(df, other)

maxframe/dataframe/merge/combine.py

@@ -0,0 +1,244 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ... import opcodes
+from ...serialization.serializables import AnyField, BoolField, FunctionField
+from ...udf import BuiltinFunction
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+
+
+class DataFrameCombine(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.DATAFRAME_COMBINE
+
+    func = FunctionField("func")
+    fill_value = AnyField("fill_value")
+    overwrite = BoolField("overwrite")
+
+    def has_custom_code(self) -> bool:
+        return not isinstance(self.func, BuiltinFunction)
+
+    def __call__(self, obj1, obj2):
+        from ..indexing.align import align
+
+        assert obj1.ndim == 1 and obj2.ndim == 1
+        obj1, obj2 = align(obj1, obj2)
+        # Create the output series based on the result series
+        return self.new_series(
+            [obj1, obj2],
+            shape=obj1.shape,
+            dtype=obj1.dtype,
+            index_value=obj1.index_value,
+            name=obj1.name,
+        )
+
+
+def df_combine(df, other, func, fill_value=None, overwrite=True):
+    """
+    Perform column-wise combine with another DataFrame.
+
+    Combines a DataFrame with `other` DataFrame using `func`
+    to element-wise combine columns. The row and column indexes of the
+    resulting DataFrame will be the union of the two.
+
+    Parameters
+    ----------
+    other : DataFrame
+        The DataFrame to merge column-wise.
+    func : function
+        Function that takes two series as inputs and return a Series or a
+        scalar. Used to merge the two dataframes column by columns.
+    fill_value : scalar value, default None
+        The value to fill NaNs with prior to passing any column to the
+        merge func.
+    overwrite : bool, default True
+        If True, columns in `self` that do not exist in `other` will be
+        overwritten with NaNs.
+
+    Returns
+    -------
+    DataFrame
+        Combination of the provided DataFrames.
+
+    See Also
+    --------
+    DataFrame.combine_first : Combine two DataFrame objects and default to
+        non-null values in frame calling the method.
+
+    Examples
+    --------
+    Combine using a simple function that chooses the smaller column.
+
+    >>> import maxframe.tensor as mt
+    >>> import maxframe.dataframe as md
+    >>> df1 = md.DataFrame({'A': [0, 0], 'B': [4, 4]})
+    >>> df2 = md.DataFrame({'A': [1, 1], 'B': [3, 3]})
+    >>> take_smaller = lambda s1, s2: s1 if s1.sum() < s2.sum() else s2
+    >>> df1.combine(df2, take_smaller).execute()
+       A  B
+    0  0  3
+    1  0  3
+
+    Example using a true element-wise combine function.
+
+    >>> df1 = md.DataFrame({'A': [5, 0], 'B': [2, 4]})
+    >>> df2 = md.DataFrame({'A': [1, 1], 'B': [3, 3]})
+    >>> df1.combine(df2, mt.minimum).execute()
+       A  B
+    0  1  2
+    1  0  3
+
+    Using `fill_value` fills Nones prior to passing the column to the
+    merge function.
+
+    >>> df1 = md.DataFrame({'A': [0, 0], 'B': [None, 4]})
+    >>> df2 = md.DataFrame({'A': [1, 1], 'B': [3, 3]})
+    >>> df1.combine(df2, take_smaller, fill_value=-5).execute()
+       A    B
+    0  0 -5.0
+    1  0  4.0
+
+    However, if the same element in both dataframes is None, that None
+    is preserved
+
+    >>> df1 = md.DataFrame({'A': [0, 0], 'B': [None, 4]})
+    >>> df2 = md.DataFrame({'A': [1, 1], 'B': [None, 3]})
+    >>> df1.combine(df2, take_smaller, fill_value=-5).execute()
+        A    B
+    0  0 -5.0
+    1  0  3.0
+
+    Example that demonstrates the use of `overwrite` and behavior when
+    the axis differ between the dataframes.
+
+    >>> df1 = md.DataFrame({'A': [0, 0], 'B': [4, 4]})
+    >>> df2 = md.DataFrame({'B': [3, 3], 'C': [-10, 1], }, index=[1, 2])
+    >>> df1.combine(df2, take_smaller).execute()
+         A    B     C
+    0  NaN  NaN   NaN
+    1  NaN  3.0 -10.0
+    2  NaN  3.0   1.0
+
+    >>> df1.combine(df2, take_smaller, overwrite=False).execute()
+         A    B     C
+    0  0.0  NaN   NaN
+    1  0.0  3.0 -10.0
+    2  NaN  3.0   1.0
+
+    Demonstrating the preference of the passed in dataframe.
+
+    >>> df2 = md.DataFrame({'B': [3, 3], 'C': [1, 1], }, index=[1, 2])
+    >>> df2.combine(df1, take_smaller).execute()
+       A    B   C
+    0  0.0  NaN NaN
+    1  0.0  3.0 NaN
+    2  NaN  3.0 NaN
+
+    >>> df2.combine(df1, take_smaller, overwrite=False).execute()
+         A    B   C
+    0  0.0  NaN NaN
+    1  0.0  3.0 1.0
+    2  NaN  3.0 1.0
+    """
+    # todo merge series logic into whole dataframe to reduce latency
+    from ..indexing.align import align
+    from .concat import concat
+
+    src_df_cols = set(df.dtypes.index)
+    src_other_cols = set(other.dtypes.index)
+
+    df, other = align(df, other)
+    col_data = []
+    for c in df.dtypes.index:
+        if c in src_df_cols and c in src_other_cols:
+            col_data.append(func(df[c], other[c]))
+        elif c in src_other_cols and not overwrite:
+            col_data.append(df[c])
+        else:
+            col_data.append(other[c])
+    res = concat(col_data, axis=1)
+    if fill_value is not None:
+        res = res.fillna(fill_value)
+    return res
+
+
+def series_combine(series, other, func, fill_value=None):
+    """
+    Combine the Series with a Series or scalar according to `func`.
+
+    Combine the Series and `other` using `func` to perform elementwise
+    selection for combined Series.
+    `fill_value` is assumed when value is missing at some index
+    from one of the two objects being combined.
+
+    Parameters
+    ----------
+    other : Series or scalar
+        The value(s) to be combined with the `Series`.
+    func : function
+        Function that takes two scalars as inputs and returns an element.
+    fill_value : scalar, optional
+        The value to assume when an index is missing from
+        one Series or the other. The default specifies to use the
+        appropriate NaN value for the underlying dtype of the Series.
+
+    Returns
+    -------
+    Series
+        The result of combining the Series with the other object.
+
+    See Also
+    --------
+    Series.combine_first : Combine Series values, choosing the calling
+        Series' values first.
+
+    Examples
+    --------
+    Consider 2 Datasets ``s1`` and ``s2`` containing
+    highest clocked speeds of different birds.
+
+    >>> import maxframe.dataframe as md
+    >>> s1 = md.Series({'falcon': 330.0, 'eagle': 160.0})
+    >>> s1.execute()
+    falcon    330.0
+    eagle     160.0
+    dtype: float64
+    >>> s2 = md.Series({'falcon': 345.0, 'eagle': 200.0, 'duck': 30.0})
+    >>> s2.execute()
+    falcon    345.0
+    eagle     200.0
+    duck       30.0
+    dtype: float64
+
+    Now, to combine the two datasets and view the highest speeds
+    of the birds across the two datasets
+
+    >>> s1.combine(s2, max).execute()
+    duck        NaN
+    eagle     200.0
+    falcon    345.0
+    dtype: float64
+
+    In the previous example, the resulting value for duck is missing,
+    because the maximum of a NaN and a float is a NaN.
+    So, in the example, we set ``fill_value=0``,
+    so the maximum value returned will be the value from some dataset.
+
+    >>> s1.combine(s2, max, fill_value=0).execute()
+    duck       30.0
+    eagle     200.0
+    falcon    345.0
+    dtype: float64
+    """
+    op = DataFrameCombine(func=func, fill_value=fill_value, overwrite=True)
+    return op(series, other)

maxframe/dataframe/merge/combine_first.py

@@ -0,0 +1,120 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def df_combine_first(df, other):
+    """
+    Update null elements with value in the same location in `other`.
+
+    Combine two DataFrame objects by filling null values in one DataFrame
+    with non-null values from other DataFrame. The row and column indexes
+    of the resulting DataFrame will be the union of the two. The resulting
+    dataframe contains the 'first' dataframe values and overrides the
+    second one values where both first.loc[index, col] and
+    second.loc[index, col] are not missing values, upon calling
+    first.combine_first(second).
+
+    Parameters
+    ----------
+    other : DataFrame
+        Provided DataFrame to use to fill null values.
+
+    Returns
+    -------
+    DataFrame
+        The result of combining the provided DataFrame with the other object.
+
+    See Also
+    --------
+    DataFrame.combine : Perform series-wise operation on two DataFrames
+        using a given function.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> df1 = md.DataFrame({'A': [None, 0], 'B': [None, 4]})
+    >>> df2 = md.DataFrame({'A': [1, 1], 'B': [3, 3]})
+    >>> df1.combine_first(df2).execute()
+         A    B
+    0  1.0  3.0
+    1  0.0  4.0
+
+    Null values still persist if the location of that null value
+    does not exist in `other`
+
+    >>> df1 = md.DataFrame({'A': [None, 0], 'B': [4, None]})
+    >>> df2 = md.DataFrame({'B': [3, 3], 'C': [1, 1]}, index=[1, 2])
+    >>> df1.combine_first(df2).execute()
+         A    B    C
+    0  NaN  4.0  NaN
+    1  0.0  3.0  1.0
+    2  NaN  3.0  1.0
+    """
+    ret = df.copy()
+    ret.update(other, join="outer", overwrite=False)
+    return ret
+
+
+def series_combine_first(series, other):
+    """
+    Update null elements with value in the same location in 'other'.
+
+    Combine two Series objects by filling null values in one Series with
+    non-null values from the other Series. Result index will be the union
+    of the two indexes.
+
+    Parameters
+    ----------
+    other : Series
+        The value(s) to be used for filling null values.
+
+    Returns
+    -------
+    Series
+        The result of combining the provided Series with the other object.
+
+    See Also
+    --------
+    Series.combine : Perform element-wise operation on two Series
+        using a given function.
+
+    Examples
+    --------
+    >>> import maxframe.tensor as mt
+    >>> import maxframe.dataframe as md
+    >>> s1 = md.Series([1, mt.nan])
+    >>> s2 = md.Series([3, 4, 5])
+    >>> s1.combine_first(s2).execute()
+    0    1.0
+    1    4.0
+    2    5.0
+    dtype: float64
+
+    Null values still persist if the location of that null value
+    does not exist in `other`
+
+    >>> s1 = md.Series({'falcon': mt.nan, 'eagle': 160.0})
+    >>> s2 = md.Series({'eagle': 200.0, 'duck': 30.0})
+    >>> s1.combine_first(s2).execute()
+    duck       30.0
+    eagle     160.0
+    falcon      NaN
+    dtype: float64
+    """
+    ret = series.copy()
+    # as Series.update does not have other args, we add them manually
+    #  to the operator object
+    ret.update(other)
+    ret.op.join, ret.op.overwrite = "outer", False
+    return ret