maxframe 2.0.0b2__cp37-cp37m-win32.whl → 2.3.0rc1__cp37-cp37m-win32.whl

maxframe/core/graph/entity.py

@@ -26,7 +26,6 @@ from ...serialization.serializables import (
 )
 from ...serialization.serializables.core import SerializableSerializer
 from ...utils import extract_class_name, tokenize
-from ..operator import Fetch
 from .core import DAG
 
 
@@ -118,18 +117,22 @@ class SerializableGraph(Serializable):
     _successors = DictField("successors")
     _results = ListField("results")
     _graph_cls = StringField("graph_cls")
+    _extra_params = DictField("extra_params", default=None)
 
     @classmethod
     def from_graph(cls, graph: EntityGraph) -> "SerializableGraph":
-        return SerializableGraph(
+        kw = dict(
             _is_chunk=False,
-            _fetch_nodes=[chunk for chunk in graph if isinstance(chunk.op, Fetch)],
+            _fetch_nodes=[chunk for chunk in graph if chunk.is_fetch()],
             _nodes=graph._nodes,
             _predecessors=graph._predecessors,
             _successors=graph._successors,
             _results=graph.results,
             _graph_cls=extract_class_name(type(graph)),
         )
+        if hasattr(graph, "extra_params"):
+            kw["_extra_params"] = graph.extra_params
+        return SerializableGraph(**kw)
 
     def to_graph(self) -> EntityGraph:
         graph_cls = (
@@ -141,6 +144,8 @@ class SerializableGraph(Serializable):
         graph._nodes.update(self._nodes)
         graph._predecessors.update(self._predecessors)
         graph._successors.update(self._successors)
+        if self._extra_params:
+            graph.extra_params = self._extra_params
         return graph
 
 

maxframe/core/mode.py

@@ -42,6 +42,10 @@ def is_build_mode():
     return bool(getattr(_internal_mode, "build", False))
 
 
+def is_mock_mode():
+    return bool(getattr(_internal_mode, "mock", False))
+
+
 class _EnterModeFuncWrapper:
     def __init__(self, mode_name_to_value):
         self.mode_name_to_value = mode_name_to_value
@@ -86,10 +90,11 @@ class _EnterModeFuncWrapper:
         return _inner
 
 
-def enter_mode(kernel=None, build=None):
+def enter_mode(kernel=None, build=None, mock=None):
     mode_name_to_value = {
         "kernel": kernel,
         "build": build,
+        "mock": mock,
     }
     mode_name_to_value = {k: v for k, v in mode_name_to_value.items() if v is not None}
 

maxframe/core/operator/base.py

@@ -40,6 +40,7 @@ from ..entity.core import ENTITY_TYPE, Entity, EntityData
 from ..entity.output_types import OutputType
 from ..entity.tileables import Tileable
 from ..mode import enter_mode
+from .utils import rewrite_stop_iteration
 
 
 class OperatorMetaclass(SerializableMeta):
@@ -91,6 +92,10 @@ class SchedulingHint(Serializable):
     # id of gang scheduling for machine learning trainings
     gang_scheduling_id = StringField("gang_scheduling_id", default=None)
 
+    def __init__(self, *, pure_depends=None, **kwargs):
+        kwargs["_pure_depends"] = pure_depends or kwargs.get("_pure_depends") or []
+        super().__init__(**kwargs)
+
     @classproperty
     @lru_cache(1)
     def all_hint_names(cls):
@@ -306,7 +311,8 @@ class Operator(Base, OperatorLogicKeyGeneratorMixin, metaclass=OperatorMetaclass
             The replaced input object.
         """
         self.inputs[index] = replaced_input
-        self._set_inputs(self, self.inputs)
+        with rewrite_stop_iteration():
+            self._set_inputs(self, self.inputs)
 
     @property
     def inputs(self) -> List[Union[ENTITY_TYPE]]:
@@ -317,7 +323,8 @@ class Operator(Base, OperatorLogicKeyGeneratorMixin, metaclass=OperatorMetaclass
 
     @inputs.setter
     def inputs(self, vals):
-        self._set_inputs(self, vals)
+        with rewrite_stop_iteration():
+            self._set_inputs(self, vals)
 
     @property
     def output_limit(self) -> int:

maxframe/core/operator/core.py

@@ -18,6 +18,9 @@ from collections import defaultdict
 from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Type, Union
 
 import numpy as np
+import pandas as pd
+
+from .utils import rewrite_stop_iteration
 
 if TYPE_CHECKING:
     from .. import TileableGraph
@@ -82,7 +85,7 @@ class TileableOperatorMixin:
             nsplits = tuple(nsplits)
             shape = list(shape)
             for idx, (s, sp) in enumerate(zip(shape, nsplits)):
-                if not np.isnan(s):
+                if not pd.isna(s):
                     continue
                 s = sum(sp)
                 if not np.isnan(s):
@@ -118,11 +121,16 @@ class TileableOperatorMixin:
     def _new_tileables(
         self, inputs: List[TileableType], kws: List[dict] = None, **kw
     ) -> List[TileableType]:
+        assert (
+            isinstance(inputs, (list, tuple)) or inputs is None
+        ), f"{inputs} is not a list"
+
         output_limit = kw.pop("output_limit", None)
         if output_limit is None:
             output_limit = getattr(self, "output_limit")
 
-        self._set_inputs(self, inputs)
+        with rewrite_stop_iteration():
+            self._set_inputs(self, inputs)
         if self.gpu is None:
             self.gpu = self._check_if_gpu(self._inputs)
         if getattr(self, "_key", None) is None:

maxframe/core/operator/utils.py

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import contextlib
+import sys
+
 from ...typing_ import EntityType, TileableType
 from ..entity import TILEABLE_TYPE
 
@@ -53,3 +56,13 @@ def build_fetch(entity: EntityType, **kw) -> EntityType:
 
 def add_fetch_builder(entity_type, builder_func):
     _type_to_builder.append((entity_type, builder_func))
+
+
+@contextlib.contextmanager
+def rewrite_stop_iteration():
+    try:
+        yield
+    except StopIteration:
+        raise RuntimeError("Unexpected StopIteration happened.").with_traceback(
+            sys.exc_info()[2]
+        ) from None

maxframe/dataframe/__init__.py

@@ -21,6 +21,7 @@ from . import (
     fetch,
     groupby,
     indexing,
+    merge,
     misc,
     missing,
     reduction,
@@ -31,6 +32,7 @@ from . import (
     window,
 )
 from .datasource.date_range import date_range
+from .datasource.direct import read_clipboard
 from .datasource.from_index import series_from_index
 from .datasource.from_records import from_records
 from .datasource.from_tensor import dataframe_from_tensor, series_from_tensor
@@ -39,19 +41,18 @@ from .datasource.read_odps_query import read_odps_query
 from .datasource.read_odps_table import read_odps_table
 from .datasource.read_parquet import read_parquet
 from .datastore.to_odps import to_odps_table
-from .groupby import NamedAgg
 from .initializer import DataFrame, Index, Series, read_pandas
 from .merge import concat, merge
 from .misc.cut import cut
 from .misc.eval import maxframe_eval as eval  # pylint: disable=redefined-builtin
 from .misc.get_dummies import get_dummies
-from .misc.melt import melt
-from .misc.pivot import pivot
-from .misc.pivot_table import pivot_table
 from .misc.qcut import qcut
 from .misc.to_numeric import to_numeric
 from .missing import isna, isnull, notna, notnull
-from .reduction import CustomReduction, unique
+from .reduction import CustomReduction, NamedAgg, unique
+from .reshape.melt import melt
+from .reshape.pivot import pivot
+from .reshape.pivot_table import pivot_table
 from .tseries.to_datetime import to_datetime
 
 try:
@@ -80,3 +81,9 @@ del (
 )
 
 from_pandas = read_pandas
+
+# isort: off
+from .typing_ import register_pandas_typing_funcs
+
+register_pandas_typing_funcs()
+del register_pandas_typing_funcs

maxframe/dataframe/accessors/__init__.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from . import datetime_, dict_, list_, plotting, string_
+from . import datetime_, dict_, list_, plotting, string_, struct_

maxframe/dataframe/accessors/compat.py

@@ -0,0 +1,45 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+
+logger = logging.getLogger(__name__)
+
+
+class LegacySeriesMethodOperator(DataFrameOperator, DataFrameOperatorMixin):
+    _method_name = None
+    _method_cls = None
+
+    def __on_deserialize__(self):
+        cls = type(self)
+        local_fields = {
+            f
+            for f, name_hash in cls._FIELD_TO_NAME_HASH.items()
+            if name_hash == cls._NAME_HASH
+        }
+        kw = {
+            f: getattr(self, f)
+            for f in cls._FIELD_TO_NAME_HASH
+            if f not in local_fields and hasattr(self, f)
+        }
+
+        kw["method"] = self._method_name
+        kw["method_kwargs"] = {
+            f: getattr(self, f) for f in local_fields if hasattr(self, f)
+        }
+
+        logger.warning(f"Using deprecated operator class {cls.__name__}")
+        return self._method_cls(**kw)

maxframe/dataframe/accessors/datetime_/__init__.py

@@ -12,7 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .core import SeriesDatetimeMethod
+from .core import SeriesDatetimeMethod, datetime_method_to_handlers
+
+methods = set(datetime_method_to_handlers.keys())
+del datetime_method_to_handlers
 
 
 def _install():

maxframe/dataframe/accessors/dict_/contains.py

@@ -16,27 +16,16 @@ import pandas as pd
 import pyarrow as pa
 
 from .... import opcodes
-from ....core.entity.output_types import OutputType
 from ....serialization.serializables.field import AnyField
-from ...operators import DataFrameOperator, DataFrameOperatorMixin
+from .core import LegacySeriesDictOperator, SeriesDictMethod
 
 
-class SeriesDictContainsOperator(DataFrameOperator, DataFrameOperatorMixin):
+class SeriesDictContainsOperator(LegacySeriesDictOperator):
+    # operator class deprecated since v2.3.0
     _op_type_ = opcodes.SERIES_DICT_CONTAINS
+    _method_name = "contains"
     query_key = AnyField("query_key", default=None)
 
-    def __init__(self, **kw):
-        super().__init__(_output_types=[OutputType.series], **kw)
-
-    def __call__(self, series):
-        return self.new_series(
-            [series],
-            shape=series.shape,
-            dtype=pd.ArrowDtype(pa.bool_()),
-            index_value=series.index_value,
-            name=None,
-        )
-
 
 def series_dict_contains(series, query_key):
     """
@@ -78,4 +67,6 @@ def series_dict_contains(series, query_key):
     3     <NA>
     dtype: bool[pyarrow]
     """
-    return SeriesDictContainsOperator(query_key=query_key)(series)
+    return SeriesDictMethod(method="contains", method_kwargs=dict(query_key=query_key))(
+        series, dtype=pd.ArrowDtype(pa.bool_()), name=None
+    )

maxframe/dataframe/accessors/dict_/core.py

@@ -0,0 +1,48 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .... import opcodes
+from ....core import OutputType
+from ....serialization.serializables import DictField, StringField, TupleField
+from ....utils import no_default
+from ...operators import DataFrameOperator, DataFrameOperatorMixin
+from ..compat import LegacySeriesMethodOperator
+
+
+class SeriesDictMethod(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.SERIES_DICT_METHOD
+
+    method = StringField("method", default=None)
+    method_args = TupleField("method_args", default_factory=list)
+    method_kwargs = DictField("method_kwargs", default_factory=dict)
+
+    def __init__(self, output_types=None, **kw):
+        output_types = output_types or [OutputType.series]
+        kw["_output_types"] = kw.get("_output_types") or output_types
+        super().__init__(**kw)
+
+    def __call__(self, inp, dtype=None, name=no_default):
+        dtype = dtype or inp.dtype
+        name = inp.name if name is no_default else name
+        return self.new_series(
+            [inp],
+            shape=inp.shape,
+            dtype=dtype,
+            index_value=inp.index_value,
+            name=name,
+        )
+
+
+class LegacySeriesDictOperator(LegacySeriesMethodOperator):
+    _method_cls = SeriesDictMethod

maxframe/dataframe/accessors/dict_/getitem.py

@@ -15,30 +15,18 @@
 import pandas as pd
 
 from .... import opcodes
-from ....core.entity.output_types import OutputType
 from ....serialization.serializables.field import AnyField, BoolField
-from ...operators import DataFrameOperator, DataFrameOperatorMixin
+from .core import LegacySeriesDictOperator, SeriesDictMethod
 
 
-class SeriesDictGetItemOperator(DataFrameOperator, DataFrameOperatorMixin):
+class SeriesDictGetItemOperator(LegacySeriesDictOperator):
+    # operator class deprecated since v2.3.0
     _op_type_ = opcodes.SERIES_DICT_GETITEM
+    _method_name = "getitem"
     query_key = AnyField("query_key", default=None)
     default_value = AnyField("default_value", default=None)
     ignore_key_error = BoolField("ignore_key_error", default=False)
 
-    def __init__(self, **kw):
-        super().__init__(_output_types=[OutputType.series], **kw)
-
-    def __call__(self, series):
-        arrow_map_type = series.dtype.pyarrow_dtype
-        return self.new_series(
-            [series],
-            shape=series.shape,
-            dtype=pd.ArrowDtype(arrow_map_type.item_type),
-            index_value=series.index_value,
-            name=str(self.query_key),
-        )
-
 
 def series_dict_getitem(series, query_key, default_value=None):
     """
@@ -85,9 +73,14 @@ def series_dict_getitem(series, query_key, default_value=None):
     3    <NA>
     Name: k2, dtype: int64[pyarrow]
     """
-    return SeriesDictGetItemOperator(
+    method_kw = dict(
         query_key=query_key, default_value=default_value, ignore_key_error=True
-    )(series)
+    )
+    arrow_map_type = series.dtype.pyarrow_dtype
+    dtype = pd.ArrowDtype(arrow_map_type.item_type)
+    return SeriesDictMethod(method="getitem", method_kwargs=method_kw)(
+        series, name=str(query_key), dtype=dtype
+    )
 
 
 def series_dict_getitem_with_key_error(series, query_key):
@@ -139,6 +132,9 @@ def series_dict_getitem_with_key_error(series, query_key):
     3    <NA>
     Name: k1, dtype: int64[pyarrow]
     """
-    return SeriesDictGetItemOperator(
-        query_key=query_key, default_value=None, ignore_key_error=False
-    )(series)
+    method_kw = dict(query_key=query_key, default_value=None, ignore_key_error=False)
+    arrow_map_type = series.dtype.pyarrow_dtype
+    dtype = pd.ArrowDtype(arrow_map_type.item_type)
+    return SeriesDictMethod(method="getitem", method_kwargs=method_kw)(
+        series, name=str(query_key), dtype=dtype
+    )

maxframe/dataframe/accessors/dict_/length.py

@@ -16,24 +16,13 @@ import pandas as pd
 import pyarrow as pa
 
 from .... import opcodes
-from ....core.entity.output_types import OutputType
-from ...operators import DataFrameOperator, DataFrameOperatorMixin
+from .core import LegacySeriesDictOperator, SeriesDictMethod
 
 
-class SeriesDictLengthOperator(DataFrameOperator, DataFrameOperatorMixin):
+class SeriesDictLengthOperator(LegacySeriesDictOperator):
+    # operator class deprecated since v2.3.0
     _op_type_ = opcodes.SERIES_DICT_LENGTH
-
-    def __init__(self, **kw):
-        super().__init__(_output_types=[OutputType.series], **kw)
-
-    def __call__(self, series):
-        return self.new_series(
-            [series],
-            shape=series.shape,
-            index_value=series.index_value,
-            dtype=pd.ArrowDtype(pa.int64()),
-            name=None,
-        )
+    _method_name = "len"
 
 
 def series_dict_length(series):
@@ -70,4 +59,6 @@ def series_dict_length(series):
     3    <NA>
     dtype: int64[pyarrow]
     """
-    return SeriesDictLengthOperator()(series)
+    return SeriesDictMethod(method="len")(
+        series, name=None, dtype=pd.ArrowDtype(pa.int64())
+    )

maxframe/dataframe/accessors/dict_/remove.py

@@ -13,28 +13,17 @@
 # limitations under the License.
 
 from .... import opcodes
-from ....core.entity.output_types import OutputType
 from ....serialization.serializables.field import AnyField, BoolField
-from ...operators import DataFrameOperator, DataFrameOperatorMixin
+from .core import LegacySeriesDictOperator, SeriesDictMethod
 
 
-class SeriesDictRemoveOperator(DataFrameOperator, DataFrameOperatorMixin):
+class SeriesDictRemoveOperator(LegacySeriesDictOperator):
+    # operator class deprecated since v2.3.0
     _op_type_ = opcodes.SERIES_DICT_REMOVE
+    _method_name = "remove"
     query_key = AnyField("query_key", default=None)
     ignore_key_error = BoolField("ignore_key_error", default=False)
 
-    def __init__(self, **kw):
-        super().__init__(_output_types=[OutputType.series], **kw)
-
-    def __call__(self, series):
-        return self.new_series(
-            [series],
-            shape=series.shape,
-            index_value=series.index_value,
-            dtype=series.dtype,
-            name=series.name,
-        )
-
 
 def series_dict_remove(series, query_key, ignore_key_error: bool = False):
     """
@@ -82,6 +71,5 @@ def series_dict_remove(series, query_key, ignore_key_error: bool = False):
     3           <NA>
     dtype: map<string, int64>[pyarrow]
     """
-    return SeriesDictRemoveOperator(
-        query_key=query_key, ignore_key_error=ignore_key_error
-    )(series)
+    method_kwargs = dict(query_key=query_key, ignore_key_error=ignore_key_error)
+    return SeriesDictMethod(method="remove", method_kwargs=method_kwargs)(series)

maxframe/dataframe/accessors/dict_/setitem.py

@@ -13,30 +13,18 @@
 # limitations under the License.
 
 from .... import opcodes
-from ....core.entity.output_types import OutputType
 from ....serialization.serializables.field import AnyField
 from ...core import TILEABLE_TYPE
-from ...operators import DataFrameOperator, DataFrameOperatorMixin
+from .core import LegacySeriesDictOperator, SeriesDictMethod
 
 
-class SeriesDictSetItemOperator(DataFrameOperator, DataFrameOperatorMixin):
+class SeriesDictSetItemOperator(LegacySeriesDictOperator):
+    # operator class deprecated since v2.3.0
     _op_type_ = opcodes.SERIES_DICT_SETITEM
+    _method_name = "setitem"
     query_key = AnyField("query_key", default=None)
     value = AnyField("default_value", default=None)
 
-    def __init__(self, **kw):
-        super().__init__(_output_types=[OutputType.series], **kw)
-
-    def __call__(self, series):
-        new_series = self.new_series(
-            [series],
-            shape=series.shape,
-            index_value=series.index_value,
-            dtype=series.dtype,
-            name=series.name,
-        )
-        series.data = new_series.data
-
 
 def series_dict_setitem(series, query_key, value):
     """
@@ -67,7 +55,7 @@ def series_dict_setitem(series, query_key, value):
     >>> s = md.Series(
     ...     data=[[("k1", 1), ("k2", 2)], [("k1", 3)], None],
     ...     index=[1, 2, 3],
-    ...     dtype=map_(pa.string(), pa.int64()),
+    ...     dtype=dict_(pa.string(), pa.int64()),
     ... )
     >>> s.execute()
     1    [('k1', 1), ('k2', 2)]
@@ -86,4 +74,6 @@ def series_dict_setitem(series, query_key, value):
         raise NotImplementedError(
             "Set the value with a dataframe/series is not allowed yet"
         )
-    return SeriesDictSetItemOperator(query_key=query_key, value=value)(series)
+    method_kwargs = dict(query_key=query_key, value=value)
+    new_series = SeriesDictMethod(method="setitem", method_kwargs=method_kwargs)(series)
+    series.data = new_series.data