maxframe 2.2.0__cp310-cp310-macosx_10_9_universal2.whl → 2.3.0rc1__cp310-cp310-macosx_10_9_universal2.whl

maxframe/dataframe/indexing/insert.py

@@ -17,10 +17,10 @@ from typing import List
 import pandas as pd
 
 from ... import opcodes
-from ...core import EntityData
+from ...core import EntityData, get_output_types
 from ...serialization.serializables import AnyField, BoolField, Int64Field
 from ...tensor.core import TENSOR_TYPE
-from ..core import SERIES_TYPE
+from ..core import INDEX_TYPE, SERIES_TYPE
 from ..operators import DataFrameOperator, DataFrameOperatorMixin
 from ..utils import build_empty_df, parse_index
 
@@ -29,9 +29,9 @@ class DataFrameInsert(DataFrameOperator, DataFrameOperatorMixin):
     _op_type_ = opcodes.INSERT
 
     loc = Int64Field("loc")
-    column = AnyField("column")
-    value = AnyField("value")
-    allow_duplicates = BoolField("allow_duplicates")
+    column = AnyField("column", default=None)
+    value = AnyField("value", default=None)
+    allow_duplicates = BoolField("allow_duplicates", default=False)
 
     @classmethod
     def _set_inputs(cls, op: "DataFrameInsert", inputs: List[EntityData]):
@@ -40,6 +40,7 @@ class DataFrameInsert(DataFrameOperator, DataFrameOperatorMixin):
             op.value = op._inputs[-1]
 
     def __call__(self, df):
+        self._output_types = get_output_types(df)
         inputs = [df]
         if isinstance(self.value, (SERIES_TYPE, TENSOR_TYPE)):
             value_dtype = self.value.dtype
@@ -47,19 +48,27 @@ class DataFrameInsert(DataFrameOperator, DataFrameOperatorMixin):
         else:
             value_dtype = pd.Series(self.value).dtype
 
-        empty_df = build_empty_df(df.dtypes)
-        empty_df.insert(
-            loc=self.loc,
-            column=self.column,
-            allow_duplicates=self.allow_duplicates,
-            value=pd.Series([], dtype=value_dtype),
-        )
-
         params = df.params
-        params["columns_value"] = parse_index(empty_df.columns, store_data=True)
-        params["dtypes"] = empty_df.dtypes
-        params["shape"] = (df.shape[0], df.shape[1] + 1)
-        return self.new_dataframe(inputs, **params)
+
+        if df.ndim == 2:
+            empty_obj = build_empty_df(df.dtypes)
+            empty_obj.insert(
+                loc=self.loc,
+                column=self.column,
+                allow_duplicates=self.allow_duplicates,
+                value=pd.Series([], dtype=value_dtype),
+            )
+
+            params["columns_value"] = parse_index(empty_obj.columns, store_data=True)
+            params["dtypes"] = empty_obj.dtypes
+            params["shape"] = (df.shape[0], df.shape[1] + 1)
+        else:
+            assert isinstance(df, INDEX_TYPE)
+            params["index_value"] = parse_index(
+                df.index_value, type(self), df, self.loc, self.value
+            )
+            params["shape"] = (df.shape[0] + 1,)
+        return self.new_tileable(inputs, **params)
 
 
 def df_insert(df, loc, column, value, allow_duplicates=False):
@@ -88,3 +97,22 @@ def df_insert(df, loc, column, value, allow_duplicates=False):
     )
     out_df = op(df)
     df.data = out_df.data
+
+
+def index_insert(idx, loc, value):
+    """
+    Make new Index inserting new item at location.
+
+    Follows Python list.append semantics for negative values.
+
+    Parameters
+    ----------
+    loc : int
+    item : object
+
+    Returns
+    -------
+    new_index : Index
+    """
+    op = DataFrameInsert(loc=loc, value=value)
+    return op(idx)

maxframe/dataframe/merge/__init__.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from .append import append
+from .combine import DataFrameCombine, df_combine, series_combine
 from .combine_first import df_combine_first, series_combine_first
 from .compare import DataFrameCompare, df_compare, series_compare
 from .concat import DataFrameConcat, concat
@@ -36,11 +37,13 @@ def _install():
         setattr(cls, "join", join)
         setattr(cls, "merge", merge)
         setattr(cls, "update", df_update)
+        setattr(cls, "combine", df_combine)
 
     for cls in SERIES_TYPE:
         setattr(cls, "combine_first", series_combine_first)
         setattr(cls, "compare", series_compare)
         setattr(cls, "update", series_update)
+        setattr(cls, "combine", series_combine)
 
     for cls in DATAFRAME_TYPE + SERIES_TYPE:
         setattr(cls, "append", append)

maxframe/dataframe/merge/combine.py

@@ -0,0 +1,244 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ... import opcodes
+from ...serialization.serializables import AnyField, BoolField, FunctionField
+from ...udf import BuiltinFunction
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+
+
+class DataFrameCombine(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.DATAFRAME_COMBINE
+
+    func = FunctionField("func")
+    fill_value = AnyField("fill_value")
+    overwrite = BoolField("overwrite")
+
+    def has_custom_code(self) -> bool:
+        return not isinstance(self.func, BuiltinFunction)
+
+    def __call__(self, obj1, obj2):
+        from ..indexing.align import align
+
+        assert obj1.ndim == 1 and obj2.ndim == 1
+        obj1, obj2 = align(obj1, obj2)
+        # Create the output series based on the result series
+        return self.new_series(
+            [obj1, obj2],
+            shape=obj1.shape,
+            dtype=obj1.dtype,
+            index_value=obj1.index_value,
+            name=obj1.name,
+        )
+
+
+def df_combine(df, other, func, fill_value=None, overwrite=True):
+    """
+    Perform column-wise combine with another DataFrame.
+
+    Combines a DataFrame with `other` DataFrame using `func`
+    to element-wise combine columns. The row and column indexes of the
+    resulting DataFrame will be the union of the two.
+
+    Parameters
+    ----------
+    other : DataFrame
+        The DataFrame to merge column-wise.
+    func : function
+        Function that takes two series as inputs and return a Series or a
+        scalar. Used to merge the two dataframes column by columns.
+    fill_value : scalar value, default None
+        The value to fill NaNs with prior to passing any column to the
+        merge func.
+    overwrite : bool, default True
+        If True, columns in `self` that do not exist in `other` will be
+        overwritten with NaNs.
+
+    Returns
+    -------
+    DataFrame
+        Combination of the provided DataFrames.
+
+    See Also
+    --------
+    DataFrame.combine_first : Combine two DataFrame objects and default to
+        non-null values in frame calling the method.
+
+    Examples
+    --------
+    Combine using a simple function that chooses the smaller column.
+
+    >>> import maxframe.tensor as mt
+    >>> import maxframe.dataframe as md
+    >>> df1 = md.DataFrame({'A': [0, 0], 'B': [4, 4]})
+    >>> df2 = md.DataFrame({'A': [1, 1], 'B': [3, 3]})
+    >>> take_smaller = lambda s1, s2: s1 if s1.sum() < s2.sum() else s2
+    >>> df1.combine(df2, take_smaller).execute()
+       A  B
+    0  0  3
+    1  0  3
+
+    Example using a true element-wise combine function.
+
+    >>> df1 = md.DataFrame({'A': [5, 0], 'B': [2, 4]})
+    >>> df2 = md.DataFrame({'A': [1, 1], 'B': [3, 3]})
+    >>> df1.combine(df2, mt.minimum).execute()
+       A  B
+    0  1  2
+    1  0  3
+
+    Using `fill_value` fills Nones prior to passing the column to the
+    merge function.
+
+    >>> df1 = md.DataFrame({'A': [0, 0], 'B': [None, 4]})
+    >>> df2 = md.DataFrame({'A': [1, 1], 'B': [3, 3]})
+    >>> df1.combine(df2, take_smaller, fill_value=-5).execute()
+       A    B
+    0  0 -5.0
+    1  0  4.0
+
+    However, if the same element in both dataframes is None, that None
+    is preserved
+
+    >>> df1 = md.DataFrame({'A': [0, 0], 'B': [None, 4]})
+    >>> df2 = md.DataFrame({'A': [1, 1], 'B': [None, 3]})
+    >>> df1.combine(df2, take_smaller, fill_value=-5).execute()
+        A    B
+    0  0 -5.0
+    1  0  3.0
+
+    Example that demonstrates the use of `overwrite` and behavior when
+    the axis differ between the dataframes.
+
+    >>> df1 = md.DataFrame({'A': [0, 0], 'B': [4, 4]})
+    >>> df2 = md.DataFrame({'B': [3, 3], 'C': [-10, 1], }, index=[1, 2])
+    >>> df1.combine(df2, take_smaller).execute()
+         A    B     C
+    0  NaN  NaN   NaN
+    1  NaN  3.0 -10.0
+    2  NaN  3.0   1.0
+
+    >>> df1.combine(df2, take_smaller, overwrite=False).execute()
+         A    B     C
+    0  0.0  NaN   NaN
+    1  0.0  3.0 -10.0
+    2  NaN  3.0   1.0
+
+    Demonstrating the preference of the passed in dataframe.
+
+    >>> df2 = md.DataFrame({'B': [3, 3], 'C': [1, 1], }, index=[1, 2])
+    >>> df2.combine(df1, take_smaller).execute()
+       A    B   C
+    0  0.0  NaN NaN
+    1  0.0  3.0 NaN
+    2  NaN  3.0 NaN
+
+    >>> df2.combine(df1, take_smaller, overwrite=False).execute()
+         A    B   C
+    0  0.0  NaN NaN
+    1  0.0  3.0 1.0
+    2  NaN  3.0 1.0
+    """
+    # todo merge series logic into whole dataframe to reduce latency
+    from ..indexing.align import align
+    from .concat import concat
+
+    src_df_cols = set(df.dtypes.index)
+    src_other_cols = set(other.dtypes.index)
+
+    df, other = align(df, other)
+    col_data = []
+    for c in df.dtypes.index:
+        if c in src_df_cols and c in src_other_cols:
+            col_data.append(func(df[c], other[c]))
+        elif c in src_other_cols and not overwrite:
+            col_data.append(df[c])
+        else:
+            col_data.append(other[c])
+    res = concat(col_data, axis=1)
+    if fill_value is not None:
+        res = res.fillna(fill_value)
+    return res
+
+
+def series_combine(series, other, func, fill_value=None):
+    """
+    Combine the Series with a Series or scalar according to `func`.
+
+    Combine the Series and `other` using `func` to perform elementwise
+    selection for combined Series.
+    `fill_value` is assumed when value is missing at some index
+    from one of the two objects being combined.
+
+    Parameters
+    ----------
+    other : Series or scalar
+        The value(s) to be combined with the `Series`.
+    func : function
+        Function that takes two scalars as inputs and returns an element.
+    fill_value : scalar, optional
+        The value to assume when an index is missing from
+        one Series or the other. The default specifies to use the
+        appropriate NaN value for the underlying dtype of the Series.
+
+    Returns
+    -------
+    Series
+        The result of combining the Series with the other object.
+
+    See Also
+    --------
+    Series.combine_first : Combine Series values, choosing the calling
+        Series' values first.
+
+    Examples
+    --------
+    Consider 2 Datasets ``s1`` and ``s2`` containing
+    highest clocked speeds of different birds.
+
+    >>> import maxframe.dataframe as md
+    >>> s1 = md.Series({'falcon': 330.0, 'eagle': 160.0})
+    >>> s1.execute()
+    falcon    330.0
+    eagle     160.0
+    dtype: float64
+    >>> s2 = md.Series({'falcon': 345.0, 'eagle': 200.0, 'duck': 30.0})
+    >>> s2.execute()
+    falcon    345.0
+    eagle     200.0
+    duck       30.0
+    dtype: float64
+
+    Now, to combine the two datasets and view the highest speeds
+    of the birds across the two datasets
+
+    >>> s1.combine(s2, max).execute()
+    duck        NaN
+    eagle     200.0
+    falcon    345.0
+    dtype: float64
+
+    In the previous example, the resulting value for duck is missing,
+    because the maximum of a NaN and a float is a NaN.
+    So, in the example, we set ``fill_value=0``,
+    so the maximum value returned will be the value from some dataset.
+
+    >>> s1.combine(s2, max, fill_value=0).execute()
+    duck       30.0
+    eagle     200.0
+    falcon    345.0
+    dtype: float64
+    """
+    op = DataFrameCombine(func=func, fill_value=fill_value, overwrite=True)
+    return op(series, other)

maxframe/dataframe/misc/__init__.py

@@ -21,12 +21,12 @@ from .check_monotonic import (
     is_monotonic_decreasing,
     is_monotonic_increasing,
 )
-from .check_unique import is_unique
+from .check_unique import index_is_unique, series_is_unique
 from .clip import clip
 from .cut import cut
 from .describe import describe
 from .diff import df_diff, series_diff
-from .drop import df_drop, df_pop, index_drop, series_drop
+from .drop import df_drop, df_pop, index_drop, series_drop, series_pop
 from .drop_duplicates import (
     df_drop_duplicates,
     index_drop_duplicates,
@@ -35,12 +35,14 @@ from .drop_duplicates import (
 from .duplicated import df_duplicated, index_duplicated, series_duplicated
 from .eval import df_eval, df_query
 from .explode import df_explode, series_explode
+from .infer_dtypes import convert_dtypes, infer_objects
 from .isin import df_isin, series_isin
 from .map import df_map, index_map, series_map
 from .memory_usage import df_memory_usage, index_memory_usage, series_memory_usage
 from .pct_change import pct_change
 from .qcut import qcut
 from .rechunk import rechunk
+from .repeat import index_repeat, series_repeat
 from .select_dtypes import select_dtypes
 from .shift import shift, tshift
 from .transform import df_transform, series_transform
@@ -57,6 +59,7 @@ def _install():
         setattr(t, "applymap", df_map)
         setattr(t, "astype", astype)
         setattr(t, "clip", clip)
+        setattr(t, "convert_dtypes", convert_dtypes)
         setattr(t, "describe", describe)
         setattr(
             t, "__delitem__", lambda df, items: df_drop(df, items, axis=1, inplace=True)
@@ -68,6 +71,7 @@ def _install():
         setattr(t, "eval", df_eval)
         setattr(t, "explode", df_explode)
         setattr(t, "first_valid_index", first_valid_index)
+        setattr(t, "infer_objects", infer_objects)
         setattr(t, "isin", df_isin)
         setattr(t, "last_valid_index", last_valid_index)
         setattr(t, "map", df_map)
@@ -89,6 +93,7 @@ def _install():
         setattr(t, "case_when", case_when)
         setattr(t, "check_monotonic", check_monotonic)
         setattr(t, "clip", clip)
+        setattr(t, "convert_dtypes", convert_dtypes)
         setattr(t, "describe", describe)
         setattr(t, "diff", series_diff)
         setattr(t, "drop", series_drop)
@@ -96,16 +101,19 @@ def _install():
         setattr(t, "duplicated", series_duplicated)
         setattr(t, "explode", series_explode)
         setattr(t, "first_valid_index", first_valid_index)
+        setattr(t, "infer_objects", infer_objects)
         setattr(t, "is_monotonic", property(fget=is_monotonic))
         setattr(t, "is_monotonic_decreasing", property(fget=is_monotonic_decreasing))
         setattr(t, "is_monotonic_increasing", property(fget=is_monotonic_increasing))
         setattr(t, "isin", series_isin)
-        setattr(t, "is_unique", property(fget=is_unique))
+        setattr(t, "is_unique", property(fget=series_is_unique))
         setattr(t, "last_valid_index", last_valid_index)
         setattr(t, "map", series_map)
         setattr(t, "memory_usage", series_memory_usage)
         setattr(t, "pct_change", pct_change)
+        setattr(t, "pop", series_pop)
         setattr(t, "rechunk", rechunk)
+        setattr(t, "repeat", series_repeat)
         setattr(t, "shift", shift)
         setattr(t, "transform", series_transform)
         setattr(t, "tshift", tshift)
@@ -118,12 +126,15 @@ def _install():
         setattr(t, "drop", index_drop)
         setattr(t, "drop_duplicates", index_drop_duplicates)
         setattr(t, "duplicated", index_duplicated)
+        setattr(t, "has_duplicates", property(fget=lambda x: not index_is_unique(x)))
         setattr(t, "is_monotonic", property(fget=is_monotonic))
         setattr(t, "is_monotonic_increasing", property(fget=is_monotonic_increasing))
         setattr(t, "is_monotonic_decreasing", property(fget=is_monotonic_decreasing))
+        setattr(t, "is_unique", property(fget=index_is_unique))
         setattr(t, "map", index_map)
         setattr(t, "memory_usage", index_memory_usage)
         setattr(t, "rechunk", rechunk)
+        setattr(t, "repeat", index_repeat)
         setattr(t, "value_counts", value_counts)
 
 

maxframe/dataframe/misc/check_unique.py

@@ -18,13 +18,27 @@ from ...udf import builtin_function
 
 
 @builtin_function
-def _tailor_unique(series):
-    if not series.is_unique:
-        return pd.Series([], name=series.name, dtype=series.dtype)
-    return series
+def _tailor_unique(series_or_idx):
+    if not series_or_idx.is_unique:
+        if isinstance(series_or_idx, pd.Series):
+            return series_or_idx.iloc[:0]
+        else:
+            return series_or_idx[:0]
+    return series_or_idx
 
 
-def is_unique(series):
+def _is_unique(series_or_index):
+    from ... import tensor as mt
+
+    return mt.equal(
+        series_or_index.mf.apply_chunk(
+            _tailor_unique, dtype=series_or_index.dtype
+        ).nunique(),
+        mt.shape(series_or_index)[0],
+    )
+
+
+def series_is_unique(series):
     """
     Return boolean if values in the object are unique.
 
@@ -43,9 +57,26 @@ def is_unique(series):
     >>> s.is_unique.execute()
     False
     """
-    from ... import tensor as mt
+    return _is_unique(series)
 
-    return mt.equal(
-        series.mf.apply_chunk(_tailor_unique, dtype=series.dtype).nunique(),
-        mt.shape(series)[0],
-    )
+
+def index_is_unique(index):
+    """
+    Return boolean if values in the index are unique.
+
+    Returns
+    -------
+    bool
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> index = md.Index([1, 2, 3])
+    >>> index.is_unique.execute()
+    True
+
+    >>> index = md.Index([1, 2, 3, 1])
+    >>> index.is_unique.execute()
+    False
+    """
+    return index.to_series().is_unique

maxframe/dataframe/misc/drop.py

@@ -419,6 +419,37 @@ def series_drop(
     )
 
 
+def series_pop(series, item):
+    """
+    Return item and drops from series. Raise KeyError if not found.
+
+    Parameters
+    ----------
+    item : label
+        Index of the element that needs to be removed.
+
+    Returns
+    -------
+    Value that is popped from series.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> ser = md.Series([1,2,3])
+
+    >>> ser.pop(0).execute()
+    1
+
+    >>> ser.execute()
+    1    2
+    2    3
+    dtype: int64
+    """
+    scalar = series.data[item]
+    series_drop(series, item, inplace=True)
+    return scalar
+
+
 def index_drop(index, labels, errors="raise"):
     """
     Make new Index with passed list of labels deleted.