maxframe 2.0.0b2__cp37-cp37m-win32.whl → 2.3.0rc1__cp37-cp37m-win32.whl

maxframe/dataframe/core.py

@@ -56,8 +56,11 @@ from ..utils import (
     ceildiv,
     estimate_pandas_size,
     on_serialize_numpy_type,
+    pd_release_version,
+    prevent_called_from_pandas,
     tokenize,
 )
+from .typing_ import DataFrameType, IndexType, SeriesType
 from .utils import (
     ReprSeries,
     apply_if_callable,
@@ -66,6 +69,8 @@ from .utils import (
     parse_index,
 )
 
+_df_with_iteritems = pd_release_version[:2] < (2, 0)
+
 
 class IndexValue(Serializable):
     """
@@ -478,9 +483,17 @@ _lazy_chunk_meta_properties = (
 )
 
 
+def _calc_cum_nsplit(nsplit: Tuple[int]) -> List[int]:
+    return [0] + np.cumsum(nsplit).tolist()
+
+
+def calc_cum_nsplits(nsplits: Tuple[Tuple[int]]) -> List[List[int]]:
+    return tuple(_calc_cum_nsplit(nsplit) for nsplit in nsplits)
+
+
 @functools.lru_cache(maxsize=128)
 def _get_cum_nsplit(nsplit: Tuple[int]) -> List[int]:
-    return [0] + np.cumsum(nsplit).tolist()
+    return _calc_cum_nsplit(nsplit)
 
 
 def _calc_axis_slice(nsplit: Tuple[int], index: int) -> slice:
@@ -684,6 +697,10 @@ class IndexData(HasShapeTileableData, _ToPandasMixin):
     def names(self):
         return getattr(self, "_names", None) or [self.name]
 
+    @property
+    def nlevels(self) -> int:
+        return len(self.names)
+
     @property
     def index_value(self) -> IndexValue:
         return self._index_value
@@ -698,55 +715,6 @@ class IndexData(HasShapeTileableData, _ToPandasMixin):
         return from_index(self, dtype=dtype, extract_multi_index=extract_multi_index)
 
     def to_frame(self, index: bool = True, name=None):
-        """
-        Create a DataFrame with a column containing the Index.
-
-        Parameters
-        ----------
-        index : bool, default True
-            Set the index of the returned DataFrame as the original Index.
-
-        name : object, default None
-            The passed name should substitute for the index name (if it has
-            one).
-
-        Returns
-        -------
-        DataFrame
-            DataFrame containing the original Index data.
-
-        See Also
-        --------
-        Index.to_series : Convert an Index to a Series.
-        Series.to_frame : Convert Series to DataFrame.
-
-        Examples
-        --------
-        >>> import maxframe.dataframe as md
-        >>> idx = md.Index(['Ant', 'Bear', 'Cow'], name='animal')
-        >>> idx.to_frame().execute()
-               animal
-        animal
-        Ant       Ant
-        Bear     Bear
-        Cow       Cow
-
-        By default, the original Index is reused. To enforce a new Index:
-
-        >>> idx.to_frame(index=False).execute()
-          animal
-        0    Ant
-        1   Bear
-        2    Cow
-
-        To override the name of the resulting column, specify `name`:
-
-        >>> idx.to_frame(index=False, name='zoo').execute()
-            zoo
-        0   Ant
-        1  Bear
-        2   Cow
-        """
         from . import dataframe_from_tensor
 
         if isinstance(self.index_value.value, IndexValue.MultiIndex):
@@ -772,34 +740,20 @@ class IndexData(HasShapeTileableData, _ToPandasMixin):
             columns = [name or self.name or 0]
         index_ = self if index else None
         return dataframe_from_tensor(
-            self._to_maxframe_tensor(self, extract_multi_index=True),
+            self._to_maxframe_tensor(extract_multi_index=True),
             index=index_,
             columns=columns,
         )
 
     def to_series(self, index=None, name=None):
-        """
-        Create a Series with both index and values equal to the index keys.
-
-        Useful with map for returning an indexer based on an index.
-
-        Parameters
-        ----------
-        index : Index, optional
-            Index of resulting Series. If None, defaults to original index.
-        name : str, optional
-            Dame of resulting Series. If None, defaults to name of original
-            index.
-
-        Returns
-        -------
-        Series
-            The dtype will be based on the type of the Index values.
-        """
         from . import series_from_index
 
         return series_from_index(self, index=index, name=name)
 
+    @property
+    def hasnans(self):
+        return self.isna().any()
+
 
 class Index(HasShapeTileable, _ToPandasMixin):
     __slots__ = "_df_or_series", "_parent_key", "_axis"
@@ -818,6 +772,9 @@ class Index(HasShapeTileable, _ToPandasMixin):
     def __len__(self):
         return len(self._data)
 
+    def __class_getitem__(cls, item):
+        return IndexType.from_getitem_args(item)
+
     def __maxframe_tensor__(self, dtype=None, order="K"):
         return self._data.__maxframe_tensor__(dtype=dtype, order=order)
 
@@ -867,6 +824,99 @@ class Index(HasShapeTileable, _ToPandasMixin):
     def values(self):
         return self.to_tensor()
 
+    def to_frame(self, index: bool = True, name=None):
+        """
+        Create a DataFrame with a column containing the Index.
+
+        Parameters
+        ----------
+        index : bool, default True
+            Set the index of the returned DataFrame as the original Index.
+
+        name : object, default None
+            The passed name should substitute for the index name (if it has
+            one).
+
+        Returns
+        -------
+        DataFrame
+            DataFrame containing the original Index data.
+
+        See Also
+        --------
+        Index.to_series : Convert an Index to a Series.
+        Series.to_frame : Convert Series to DataFrame.
+
+        Examples
+        --------
+        >>> import maxframe.dataframe as md
+        >>> idx = md.Index(['Ant', 'Bear', 'Cow'], name='animal')
+        >>> idx.to_frame().execute()
+               animal
+        animal
+        Ant       Ant
+        Bear     Bear
+        Cow       Cow
+
+        By default, the original Index is reused. To enforce a new Index:
+
+        >>> idx.to_frame(index=False).execute()
+          animal
+        0    Ant
+        1   Bear
+        2    Cow
+
+        To override the name of the resulting column, specify `name`:
+
+        >>> idx.to_frame(index=False, name='zoo').execute()
+            zoo
+        0   Ant
+        1  Bear
+        2   Cow
+        """
+        return self._data.to_frame(index=index, name=name)
+
+    def to_series(self, index=None, name=None):
+        """
+        Create a Series with both index and values equal to the index keys.
+
+        Useful with map for returning an indexer based on an index.
+
+        Parameters
+        ----------
+        index : Index, optional
+            Index of resulting Series. If None, defaults to original index.
+        name : str, optional
+            Dame of resulting Series. If None, defaults to name of original
+            index.
+
+        Returns
+        -------
+        Series
+            The dtype will be based on the type of the Index values.
+        """
+        return self._data.to_series(index=index, name=name)
+
+    @property
+    def hasnans(self):
+        """
+        Return True if there are any NaNs.
+
+        Returns
+        -------
+        bool
+
+        Examples
+        --------
+        >>> import maxframe.dataframe as md
+        >>> idx = md.Index([1, 2, 3, None])
+        >>> idx.execute()
+        Index([1.0, 2.0, 3.0, nan], dtype='float64')
+        >>> idx.hasnans.execute()
+        True
+        """
+        return self._data.hasnans
+
 
 class RangeIndex(Index):
     __slots__ = ()
@@ -1049,12 +1099,6 @@ class BaseSeriesData(HasShapeTileableData, _ToPandasMixin):
 
         return from_series(self, dtype=dtype)
 
-    @staticmethod
-    def from_tensor(in_tensor, index=None, name=None):
-        from .datasource.from_tensor import series_from_tensor
-
-        return series_from_tensor(in_tensor, index=index, name=name)
-
 
 class SeriesData(_BatchedFetcher, BaseSeriesData):
     type_name = "Series"
@@ -1065,29 +1109,51 @@ class SeriesData(_BatchedFetcher, BaseSeriesData):
         return tensor.astype(dtype=dtype, order=order, copy=False)
 
     def iteritems(self, batch_size=10000, session=None):
+        method_name = "iteritems" if _df_with_iteritems else "items"
         for batch_data in self.iterbatch(batch_size=batch_size, session=session):
-            yield from getattr(batch_data, "iteritems")()
+            yield from getattr(batch_data, method_name)()
 
     items = iteritems
 
-    def to_dict(self, into=dict, batch_size=10000, session=None):
-        fetch_kwargs = dict(batch_size=batch_size)
-        return self.to_pandas(session=session, fetch_kwargs=fetch_kwargs).to_dict(
-            into=into
-        )
-
     def to_frame(self, name=None):
         from . import dataframe_from_tensor
 
         name = name or self.name or 0
         return dataframe_from_tensor(self, columns=[name])
 
+    @property
+    def hasnans(self):
+        """
+        Return True if there are any NaNs.
+
+        Returns
+        -------
+        bool
+
+        Examples
+        --------
+        >>> import maxframe.dataframe as md
+        >>> s = md.Series([1, 2, 3, None])
+        >>> s.execute()
+        0    1.0
+        1    2.0
+        2    3.0
+        3    NaN
+        dtype: float64
+        >>> s.hasnans.execute()
+        True
+        """
+        return self.isna().any()
+
 
 class Series(HasShapeTileable, _ToPandasMixin):
     __slots__ = ("_cache",)
     _allow_data_type_ = (SeriesData,)
     type_name = "Series"
 
+    def __class_getitem__(cls, item):
+        return SeriesType.from_getitem_args(item)
+
     def to_tensor(self, dtype=None):
         return self._data.to_tensor(dtype=dtype)
 
@@ -1185,6 +1251,11 @@ class Series(HasShapeTileable, _ToPandasMixin):
         else:
             return super()._view()
 
+    def __iter__(self):
+        # prevent being called by pandas to make sure `__eq__` works
+        prevent_called_from_pandas()
+        return (tp[1] for tp in self.items())
+
     def __len__(self):
         return len(self._data)
 
@@ -1238,38 +1309,6 @@ class Series(HasShapeTileable, _ToPandasMixin):
 
     items = iteritems
 
-    def to_dict(self, into=dict, batch_size=10000, session=None):
-        """
-        Convert Series to {label -> value} dict or dict-like object.
-
-        Parameters
-        ----------
-        into : class, default dict
-            The collections.abc.Mapping subclass to use as the return
-            object. Can be the actual class or an empty
-            instance of the mapping type you want.  If you want a
-            collections.defaultdict, you must pass it initialized.
-
-        Returns
-        -------
-        collections.abc.Mapping
-            Key-value representation of Series.
-
-        Examples
-        --------
-        >>> import maxframe.dataframe as md
-        >>> s = md.Series([1, 2, 3, 4])
-        >>> s.to_dict()
-        {0: 1, 1: 2, 2: 3, 3: 4}
-        >>> from collections import OrderedDict, defaultdict
-        >>> s.to_dict(OrderedDict)
-        OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)])
-        >>> dd = defaultdict(list)
-        >>> s.to_dict(dd)
-        defaultdict(<class 'list'>, {0: 1, 1: 2, 2: 3, 3: 4})
-        """
-        return self._data.to_dict(into=into, batch_size=batch_size, session=session)
-
     def to_frame(self, name=None):
         """
         Convert Series to DataFrame.
@@ -1297,98 +1336,6 @@ class Series(HasShapeTileable, _ToPandasMixin):
         """
         return self._data.to_frame(name=name)
 
-    def between(self, left, right, inclusive="both"):
-        """
-        Return boolean Series equivalent to left <= series <= right.
-        This function returns a boolean vector containing `True` wherever the
-        corresponding Series element is between the boundary values `left` and
-        `right`. NA values are treated as `False`.
-
-        Parameters
-        ----------
-        left : scalar or list-like
-            Left boundary.
-        right : scalar or list-like
-            Right boundary.
-        inclusive : {"both", "neither", "left", "right"}
-            Include boundaries. Whether to set each bound as closed or open.
-
-        Returns
-        -------
-        Series
-            Series representing whether each element is between left and
-            right (inclusive).
-
-        See Also
-        --------
-        Series.gt : Greater than of series and other.
-        Series.lt : Less than of series and other.
-
-        Notes
-        -----
-        This function is equivalent to ``(left <= ser) & (ser <= right)``
-
-        Examples
-        --------
-        >>> import maxframe.dataframe as md
-        >>> s = md.Series([2, 0, 4, 8, np.nan])
-
-        Boundary values are included by default:
-
-        >>> s.between(1, 4).execute()
-        0     True
-        1    False
-        2     True
-        3    False
-        4    False
-        dtype: bool
-
-        With `inclusive` set to ``"neither"`` boundary values are excluded:
-
-        >>> s.between(1, 4, inclusive="neither").execute()
-        0     True
-        1    False
-        2    False
-        3    False
-        4    False
-        dtype: bool
-
-        `left` and `right` can be any scalar value:
-
-        >>> s = md.Series(['Alice', 'Bob', 'Carol', 'Eve'])
-        >>> s.between('Anna', 'Daniel').execute()
-        0    False
-        1     True
-        2     True
-        3    False
-        dtype: bool
-        """
-        if isinstance(inclusive, bool):  # pragma: no cover
-            # for pandas < 1.3.0
-            if inclusive:
-                inclusive = "both"
-            else:
-                inclusive = "neither"
-        if inclusive == "both":
-            lmask = self >= left
-            rmask = self <= right
-        elif inclusive == "left":
-            lmask = self >= left
-            rmask = self < right
-        elif inclusive == "right":
-            lmask = self > left
-            rmask = self <= right
-        elif inclusive == "neither":
-            lmask = self > left
-            rmask = self < right
-        else:
-            raise ValueError(
-                "Inclusive has to be either string of 'both',"
-                "'left', 'right', or 'neither'."
-            )
-
-        return lmask & rmask
-
     # def median(
     #     self, axis=None, skipna=True, out=None, overwrite_input=False, keepdims=False
     # ):
@@ -1589,18 +1536,6 @@ class BaseDataFrameData(HasShapeTileableData, _ToPandasMixin):
 
         return from_dataframe(self, dtype=dtype)
 
-    @staticmethod
-    def from_tensor(in_tensor, index=None, columns=None):
-        from .datasource.from_tensor import dataframe_from_tensor
-
-        return dataframe_from_tensor(in_tensor, index=index, columns=columns)
-
-    @staticmethod
-    def from_records(records, **kw):
-        from .datasource.from_records import from_records
-
-        return from_records(records, **kw)
-
     @property
     def index(self):
         from .datasource.index import from_tileable
@@ -1747,12 +1682,6 @@ class DataFrame(HasShapeTileable, _ToPandasMixin):
     def to_tensor(self):
         return self._data.to_tensor()
 
-    def from_tensor(self, in_tensor, index=None, columns=None):
-        return self._data.from_tensor(in_tensor, index=index, columns=columns)
-
-    def from_records(self, records, **kw):
-        return self._data.from_records(records, **kw)
-
     def __maxframe_tensor__(self, dtype=None, order="K"):
         return self._data.__maxframe_tensor__(dtype=dtype, order=order)
 
@@ -1772,6 +1701,14 @@ class DataFrame(HasShapeTileable, _ToPandasMixin):
             + [k for k in self.dtypes.index if isinstance(k, str) and k.isidentifier()]
         )
 
+    def __iter__(self):
+        # prevent being called by pandas to make sure `__eq__` works
+        prevent_called_from_pandas()
+        return iter(self.dtypes.index)
+
+    def __class_getitem__(cls, item):
+        return DataFrameType.from_getitem_args(item)
+
     @property
     def T(self):
         return self.transpose()

maxframe/dataframe/datasource/__init__.py

@@ -13,3 +13,21 @@
 # limitations under the License.
 
 from .core import PandasDataSourceOperator
+from .from_dict import dataframe_from_dict
+from .from_records import from_records
+from .from_tensor import dataframe_from_tensor, series_from_tensor
+
+
+def _install():
+    from ..core import DATAFRAME_TYPE, SERIES_TYPE
+
+    for t in DATAFRAME_TYPE:
+        t.from_dict = staticmethod(dataframe_from_dict)
+        t.from_records = staticmethod(from_records)
+        t.from_tensor = staticmethod(dataframe_from_tensor)
+    for t in SERIES_TYPE:
+        t.from_tensor = staticmethod(series_from_tensor)
+
+
+_install()
+del _install

maxframe/dataframe/datasource/core.py

@@ -18,6 +18,7 @@ from typing import List, MutableMapping, Optional, Union
 from ...serialization.serializables import Int64Field, StringField
 from ...utils import estimate_pandas_size
 from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import validate_dtype_backend
 
 
 class HeadOptimizedDataSource(DataFrameOperator, DataFrameOperatorMixin):
@@ -86,3 +87,8 @@ class PandasDataSourceOperator(DataFrameOperator):
         cls, ctx: MutableMapping[str, Union[int, float]], op: "PandasDataSourceOperator"
     ):
         ctx[op.outputs[0].key] = estimate_pandas_size(op.get_data())
+
+
+class DtypeBackendCompatibleMixin:
+    def __on_deserialize__(self):
+        self.dtype_backend = validate_dtype_backend(self.dtype_backend)

maxframe/dataframe/datasource/direct.py

@@ -0,0 +1,57 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pandas as pd
+
+
+def read_clipboard(sep=None, **kwargs):
+    """
+    Read text from clipboard and pass to :func:`~pandas.read_csv`.
+
+    Parses clipboard contents similar to how CSV files are parsed
+    using :func:`~pandas.read_csv`.
+
+    Parameters
+    ----------
+    sep : str, default '\\s+'
+        A string or regex delimiter. The default of ``'\\s+'`` denotes
+        one or more whitespace characters.
+
+    **kwargs
+        See :func:`~pandas.read_csv` for the full argument list.
+
+    Returns
+    -------
+    DataFrame
+        A parsed :class:`DataFrame` object.
+
+    See Also
+    --------
+    DataFrame.to_clipboard : Copy object to the system clipboard.
+    read_csv : Read a comma-separated values (csv) file into DataFrame.
+    read_fwf : Read a table of fixed-width formatted lines into DataFrame.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> df = md.DataFrame([[1, 2, 3], [4, 5, 6]], columns=['A', 'B', 'C'])
+    >>> df.to_clipboard()  # doctest: +SKIP
+    >>> md.read_clipboard()  # doctest: +SKIP.execute()
+         A  B  C
+    0    1  2  3
+    1    4  5  6
+    """
+    from ..initializer import DataFrame
+
+    return DataFrame(pd.read_clipboard(sep=sep, **kwargs))