maxframe 0.1.0b5__cp39-cp39-macosx_10_9_universal2.whl → 1.0.0__cp39-cp39-macosx_10_9_universal2.whl

maxframe/dataframe/indexing/align.py

@@ -138,7 +138,7 @@ class DataFrameAlign(DataFrameOperator, DataFrameOperatorMixin):
                 series_index = rhs.index_value.to_pandas()
                 dtypes = lhs.dtypes.reindex(
                     lhs.dtypes.index.join(series_index, how=self.join)
-                ).fillna(np.dtype(np.float_))
+                ).fillna(np.dtype(float))
                 l_shape[1] = r_size = len(dtypes)
                 col_val = r_idx_val = parse_index(dtypes.index, store_data=True)
 

maxframe/dataframe/indexing/loc.py

@@ -25,13 +25,14 @@ from ...core import ENTITY_TYPE, OutputType
 from ...serialization.serializables import AnyField, KeyField, ListField
 from ...tensor.datasource import asarray
 from ...tensor.utils import calc_sliced_size, filter_inputs
-from ...utils import is_full_slice, lazy_import
+from ...utils import is_full_slice, lazy_import, pd_release_version
 from ..core import DATAFRAME_TYPE, IndexValue
 from ..operators import DataFrameOperator, DataFrameOperatorMixin
 from ..utils import parse_index
 from .iloc import DataFrameIlocSetItem
 
 cudf = lazy_import("cudf")
+with_slice_locs_kind = pd_release_version < (1, 4, 0)
 
 
 def process_loc_indexes(inp, indexes, fetch_index: bool = True):
@@ -210,9 +211,10 @@ class DataFrameLocGetItem(DataFrameOperator, DataFrameOperatorMixin):
             if axis == 1:
                 param["dtypes"] = inp.dtypes
         elif input_index_value.has_value():
-            start, end = pd_index.slice_locs(
-                index.start, index.stop, index.step, kind="loc"
-            )
+            kw = {}
+            if with_slice_locs_kind:
+                kw["kind"] = "loc"
+            start, end = pd_index.slice_locs(index.start, index.stop, index.step, **kw)
             slc = slice(start, end, index.step)
             size = calc_sliced_size(inp.shape[axis], slc)
             param["shape"] = size

maxframe/dataframe/indexing/rename.py

@@ -17,7 +17,7 @@ import warnings
 from ... import opcodes
 from ...core import get_output_types
 from ...serialization.serializables import AnyField, StringField
-from ..core import SERIES_TYPE
+from ..core import INDEX_TYPE, SERIES_TYPE
 from ..operators import DataFrameOperator, DataFrameOperatorMixin
 from ..utils import build_df, build_series, parse_index, validate_axis
 
@@ -73,6 +73,8 @@ class DataFrameRename(DataFrameOperator, DataFrameOperatorMixin):
             params["index_value"] = parse_index(new_index)
         if df.ndim == 1:
             params["name"] = new_df.name
+            if isinstance(df, INDEX_TYPE):
+                params["names"] = new_df.names
         return self.new_tileable([df], **params)
 
 
@@ -246,6 +248,7 @@ def df_rename(
     )
 
 
+# fixme https://github.com/aliyun/alibabacloud-odps-maxframe-client/issues/58
 def series_rename(
     series,
     index=None,
@@ -303,11 +306,6 @@ def series_rename(
     1    2
     2    3
     Name: my_name, dtype: int64
-    >>> s.rename(lambda x: x ** 2).execute()  # function, changes labels.execute()
-    0    1
-    1    2
-    4    3
-    dtype: int64
     >>> s.rename({1: 3, 2: 5}).execute()  # mapping, changes labels.execute()
     0    1
     3    2
@@ -385,6 +383,7 @@ def index_rename(index, name, inplace=False):
         return ret
 
 
+# fixme https://github.com/aliyun/alibabacloud-odps-maxframe-client/issues/59
 def index_set_names(index, names, level=None, inplace=False):
     """
     Set Index or MultiIndex name.
@@ -419,28 +418,6 @@ def index_set_names(index, names, level=None, inplace=False):
     Int64Index([1, 2, 3, 4], dtype='int64')
     >>> idx.set_names('quarter').execute()
     Int64Index([1, 2, 3, 4], dtype='int64', name='quarter')
-
-    >>> idx = md.MultiIndex.from_product([['python', 'cobra'],
-    ...                                   [2018, 2019]])
-    >>> idx.execute()
-    MultiIndex([('python', 2018),
-                ('python', 2019),
-                ( 'cobra', 2018),
-                ( 'cobra', 2019)],
-               )
-    >>> idx.set_names(['kind', 'year'], inplace=True)
-    >>> idx.execute()
-    MultiIndex([('python', 2018),
-                ('python', 2019),
-                ( 'cobra', 2018),
-                ( 'cobra', 2019)],
-               names=['kind', 'year'])
-    >>> idx.set_names('species', level=0).execute()
-    MultiIndex([('python', 2018),
-                ('python', 2019),
-                ( 'cobra', 2018),
-                ( 'cobra', 2019)],
-               names=['species', 'year'])
     """
     op = DataFrameRename(
         index_mapper=names, level=level, output_types=get_output_types(index)

maxframe/dataframe/indexing/sample.py

@@ -195,7 +195,6 @@ def sample(
             num_legs  num_wings  num_specimen_seen
     falcon         2          2                 10
     fish           0          0                  8
-
     """
     axis = validate_axis(axis or 0, df_or_series)
     if axis == 1:

maxframe/dataframe/indexing/set_index.py

@@ -31,7 +31,7 @@ class DataFrameSetIndex(DataFrameOperator, DataFrameOperatorMixin):
         super().__init__(_output_types=output_types, **kw)
 
     def __call__(self, df):
-        new_df = build_empty_df(df.dtypes).set_index(
+        new_df = build_empty_df(df.dtypes, index=df.index_value.to_pandas()).set_index(
             keys=self.keys,
             drop=self.drop,
             append=self.append,
@@ -47,6 +47,73 @@ class DataFrameSetIndex(DataFrameOperator, DataFrameOperatorMixin):
 
 
 def set_index(df, keys, drop=True, append=False, inplace=False, verify_integrity=False):
+    # TODO add support for set index by series, index, mt.ndarray, etc.
+    """
+    Set the DataFrame index using existing columns.
+
+    Set the DataFrame index (row labels) using one or more existing
+    columns. The index can replace the existing index or expand on it.
+
+    Parameters
+    ----------
+    keys : label or array-like or list of labels
+        This parameter can be either a single column key, or a list containing column keys.
+    drop : bool, default True
+        Delete columns to be used as the new index.
+    append : bool, default False
+        Whether to append columns to existing index.
+    inplace : bool, default False
+        If True, modifies the DataFrame in place (do not create a new object).
+    verify_integrity : bool, default False
+        Check the new index for duplicates. Otherwise defer the check until
+        necessary. Setting to False will improve the performance of this
+        method.
+
+    Returns
+    -------
+    DataFrame or None
+        Changed row labels or None if ``inplace=True``.
+
+    See Also
+    --------
+    DataFrame.reset_index : Opposite of set_index.
+    DataFrame.reindex : Change to new indices or expand indices.
+    DataFrame.reindex_like : Change to same indices as other DataFrame.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+
+    >>> df = md.DataFrame({'month': [1, 4, 7, 10],
+    ...                    'year': [2012, 2014, 2013, 2014],
+    ...                    'sale': [55, 40, 84, 31]})
+    >>> df
+       month  year  sale
+    0      1  2012    55
+    1      4  2014    40
+    2      7  2013    84
+    3     10  2014    31
+
+    Set the index to become the 'month' column:
+
+    >>> df.set_index('month')
+           year  sale
+    month
+    1      2012    55
+    4      2014    40
+    7      2013    84
+    10     2014    31
+
+    Create a MultiIndex using columns 'year' and 'month':
+
+    >>> df.set_index(['year', 'month'])
+                sale
+    year  month
+    2012  1     55
+    2014  4     40
+    2013  7     84
+    2014  10    31
+    """
     op = DataFrameSetIndex(
         keys=keys,
         drop=drop,

maxframe/dataframe/initializer.py

@@ -15,6 +15,7 @@
 from typing import Union
 
 import pandas as pd
+from pandas.api.types import is_list_like
 from pandas.core.dtypes.common import pandas_dtype
 
 from ..core import ENTITY_TYPE
@@ -61,6 +62,8 @@ class DataFrame(_Frame, metaclass=InitializerMeta):
         num_partitions=None,
     ):
         need_repart = False
+        if columns is not None and not is_list_like(columns):
+            raise ValueError("columns must be a list-like object")
         if isinstance(data, TENSOR_TYPE):
             if chunk_size is not None:
                 data = data.rechunk(chunk_size)
@@ -69,7 +72,10 @@ class DataFrame(_Frame, metaclass=InitializerMeta):
             )
             need_repart = num_partitions is not None
         elif isinstance(data, SERIES_TYPE):
-            df = data.to_frame()
+            if columns is not None and len(columns) != 1:
+                raise ValueError("columns' length must be 1 when data is Series")
+            col_name = columns[0] if columns else None
+            df = data.to_frame(name=col_name)
             need_repart = num_partitions is not None
         elif isinstance(data, DATAFRAME_TYPE):
             if not hasattr(data, "data"):
@@ -77,6 +83,10 @@ class DataFrame(_Frame, metaclass=InitializerMeta):
                 df = _Frame(data)
             else:
                 df = data
+            if columns is not None:
+                if len(df.columns) != len(columns):
+                    raise ValueError("columns' length must be equal to the data's")
+                df.columns = columns
             need_repart = num_partitions is not None
         elif isinstance(data, dict) and self._can_process_by_1d_tileables(data):
             # data is a dict and some value is tensor

maxframe/dataframe/merge/__init__.py

@@ -14,7 +14,15 @@
 
 from .append import DataFrameAppend, append
 from .concat import DataFrameConcat, concat
-from .merge import DataFrameMerge, DataFrameMergeAlign, join, merge
+from .merge import (
+    DataFrameMerge,
+    DataFrameMergeAlign,
+    DistributedMapJoinHint,
+    MapJoinHint,
+    SkewJoinHint,
+    join,
+    merge,
+)
 
 
 def _install():

maxframe/dataframe/merge/concat.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from typing import List, Union
 
 import pandas as pd
 
@@ -24,6 +25,7 @@ from ...serialization.serializables import (
     StringField,
 )
 from ...utils import lazy_import
+from ..core import DataFrame, Series
 from ..operators import SERIES_TYPE, DataFrameOperator, DataFrameOperatorMixin
 from ..utils import build_empty_df, build_empty_series, parse_index, validate_axis
 
@@ -55,41 +57,53 @@ class DataFrameConcat(DataFrameOperator, DataFrameOperatorMixin):
         return self.names
 
     @classmethod
-    def _concat_index(cls, prev_index: pd.Index, cur_index: pd.Index):
-        if isinstance(prev_index, pd.RangeIndex) and isinstance(
-            cur_index, pd.RangeIndex
-        ):
-            # handle RangeIndex that append may generate huge amount of data
-            # e.g. pd.RangeIndex(10_000) and pd.RangeIndex(10_000)
-            # will generate a Int64Index full of data
-            # for details see GH#1647
-            prev_stop = prev_index.start + prev_index.size * prev_index.step
-            cur_start = cur_index.start
-            if prev_stop == cur_start and prev_index.step == cur_index.step:
-                # continuous RangeIndex, still return RangeIndex
-                return prev_index.append(cur_index)
-            else:
-                # otherwise, return an empty index
-                return pd.Index([], dtype=prev_index.dtype)
-        elif isinstance(prev_index, pd.RangeIndex):
-            return pd.Index([], prev_index.dtype).append(cur_index)
-        elif isinstance(cur_index, pd.RangeIndex):
-            return prev_index.append(pd.Index([], cur_index.dtype))
-        return prev_index.append(cur_index)
+    def _concat_index(cls, df_or_series_list: Union[List[DataFrame], List[Series]]):
+        concat_index = None
+        all_indexes_have_value = all(
+            input.index_value.has_value() for input in df_or_series_list
+        )
+
+        def _concat(prev_index: pd.Index, cur_index: pd.Index):
+            if prev_index is None:
+                return cur_index
+
+            if (
+                all_indexes_have_value
+                and isinstance(prev_index, pd.RangeIndex)
+                and isinstance(cur_index, pd.RangeIndex)
+            ):
+                # handle RangeIndex that append may generate huge amount of data
+                # e.g. pd.RangeIndex(10_000) and pd.RangeIndex(10_000)
+                # will generate a Int64Index full of data
+                # for details see GH#1647
+                prev_stop = prev_index.start + prev_index.size * prev_index.step
+                cur_start = cur_index.start
+                if prev_stop == cur_start and prev_index.step == cur_index.step:
+                    # continuous RangeIndex, still return RangeIndex
+                    return prev_index.append(cur_index)
+                else:
+                    # otherwise, return an empty index
+                    return pd.Index([], dtype=prev_index.dtype)
+            elif isinstance(prev_index, pd.RangeIndex):
+                return pd.Index([], prev_index.dtype).append(cur_index)
+            elif isinstance(cur_index, pd.RangeIndex):
+                return prev_index.append(pd.Index([], cur_index.dtype))
+            return prev_index.append(cur_index)
+
+        for input in df_or_series_list:
+            concat_index = _concat(concat_index, input.index_value.to_pandas())
+
+        return concat_index
 
     def _call_series(self, objs):
         if self.axis == 0:
             row_length = 0
-            index = None
             for series in objs:
-                if index is None:
-                    index = series.index_value.to_pandas()
-                else:
-                    index = self._concat_index(index, series.index_value.to_pandas())
                 row_length += series.shape[0]
             if self.ignore_index:  # pragma: no cover
                 index_value = parse_index(pd.RangeIndex(row_length))
             else:
+                index = self._concat_index(objs)
                 index_value = parse_index(index, objs)
             obj_names = {obj.name for obj in objs}
             return self.new_series(
@@ -130,13 +144,8 @@ class DataFrameConcat(DataFrameOperator, DataFrameOperatorMixin):
     def _call_dataframes(self, objs):
         if self.axis == 0:
             row_length = 0
-            index = None
             empty_dfs = []
             for df in objs:
-                if index is None:
-                    index = df.index_value.to_pandas()
-                else:
-                    index = self._concat_index(index, df.index_value.to_pandas())
                 row_length += df.shape[0]
                 if df.ndim == 2:
                     empty_dfs.append(build_empty_df(df.dtypes))
@@ -153,6 +162,7 @@ class DataFrameConcat(DataFrameOperator, DataFrameOperatorMixin):
             if self.ignore_index:  # pragma: no cover
                 index_value = parse_index(pd.RangeIndex(row_length))
             else:
+                index = self._concat_index(objs)
                 index_value = parse_index(index, objs)
 
             new_objs = []