maxframe 2.0.0b2__cp37-cp37m-win32.whl → 2.2.0__cp37-cp37m-win32.whl

maxframe/config/config.py

@@ -18,7 +18,7 @@ import os
 import traceback
 import warnings
 from copy import deepcopy
-from typing import Any, Dict, Optional, Union
+from typing import Any, Callable, Dict, Optional, Union
 
 from odps.lib import tzlocal
 
@@ -33,13 +33,17 @@ from ..utils import get_python_tag
 from .validators import (
     ValidatorType,
     all_validator,
+    dtype_backend_validator,
     is_all_dict_keys_in,
     is_bool,
     is_dict,
     is_float,
+    is_great_than,
     is_in,
     is_integer,
+    is_less_than_or_equal_to,
     is_non_negative_integer,
+    is_notnull,
     is_null,
     is_numeric,
     is_string,
@@ -59,6 +63,7 @@ _DEFAULT_TASK_START_TIMEOUT = 60
 _DEFAULT_TASK_RESTART_TIMEOUT = 300
 _DEFAULT_LOGVIEW_HOURS = 24 * 30
 _DEFAULT_FUNCTION_RUNNING_OPTIONS = {"cpu": 1, "memory": "4GiB", "gpu": 0}
+_DEFAULT_MAX_MEMORY_CPU_RATIO = 12
 
 
 class OptionError(Exception):
@@ -66,11 +71,19 @@ class OptionError(Exception):
 
 
 class Redirection:
-    def __init__(self, item: str, warn: Optional[str] = None):
+    def __init__(
+        self,
+        item: str,
+        warn: Optional[str] = None,
+        getter: Callable = None,
+        setter: Callable = None,
+    ):
         self._items = item.split(".")
         self._warn = warn
         self._warned = True
         self._parent = None
+        self._getter = getter
+        self._setter = setter
 
     def bind(self, attr_dict):
         self._parent = attr_dict
@@ -88,6 +101,8 @@ class Redirection:
         conf = self._parent.root
         for it in self._items:
             conf = getattr(conf, it)
+        if callable(self._getter):
+            conf = self._getter(conf)
         return conf
 
     def setvalue(self, value: str, silent: bool = False) -> None:
@@ -97,6 +112,8 @@ class Redirection:
         conf = self._parent.root
         for it in self._items[:-1]:
             conf = getattr(conf, it)
+        if callable(self._setter):
+            value = self._setter(value)
         setattr(conf, self._items[-1], value)
 
 
@@ -251,9 +268,19 @@ class Config:
             self._remote_options.add(option)
 
     def redirect_option(
-        self, option: str, target: str, warn: str = _DEFAULT_REDIRECT_WARN
+        self,
+        option: str,
+        target: str,
+        warn: str = _DEFAULT_REDIRECT_WARN,
+        getter: Callable = None,
+        setter: Callable = None,
     ) -> None:
-        redir = Redirection(target, warn=warn.format(source=option, target=target))
+        redir = Redirection(
+            target,
+            warn=warn.format(source=option, target=target),
+            getter=getter,
+            setter=setter,
+        )
         self.register_option(option, redir)
 
     def unregister_option(self, option: str) -> None:
@@ -315,10 +342,18 @@ class Config:
 
 
 def _get_legal_local_tz_name() -> Optional[str]:
-    """Sometimes we may get illegal tz name from tzlocal.get_localzone()"""
-    tz_name = str(tzlocal.get_localzone())
-    if tz_name not in available_timezones():
-        return None
+    """
+    Sometimes we may get illegal tz name from tzlocal.get_localzone().
+    In some environments we can't get any tz name.
+    """
+    tz_name = None
+    try:
+        tz_name = str(tzlocal.get_localzone())
+        if tz_name not in available_timezones():
+            tz_name = None
+    except:
+        pass
+
     return tz_name
 
 
@@ -400,6 +435,9 @@ default_options.register_option(
 default_options.register_option(
     "session.quota_name", None, validator=is_null | is_string, remote=True
 )
+default_options.register_option(
+    "session.region_id", None, validator=is_null | is_string, remote=True
+)
 default_options.register_option(
     "session.enable_schema", None, validator=is_null | is_bool, remote=True
 )
@@ -446,7 +484,15 @@ default_options.register_option(
 )
 
 default_options.register_option("warn_duplicated_execution", False, validator=is_bool)
-default_options.register_option("dataframe.use_arrow_dtype", True, validator=is_bool)
+default_options.register_option(
+    "dataframe.dtype_backend", "numpy", validator=dtype_backend_validator
+)
+default_options.redirect_option(
+    "dataframe.use_arrow_dtype",
+    "dataframe.dtype_backend",
+    getter=lambda x: x == "pyarrow",
+    setter=lambda x: "pyarrow" if x else "numpy",
+)
 default_options.register_option(
     "dataframe.arrow_array.pandas_only", True, validator=is_bool
 )
@@ -464,6 +510,15 @@ default_options.register_option(
     validator=is_dict | is_all_dict_keys_in("cpu", "memory", "gpu"),
 )
 
+default_options.register_option(
+    "function.allowed_max_memory_cpu_ratio",
+    _DEFAULT_MAX_MEMORY_CPU_RATIO,
+    validator=is_integer
+    & is_notnull
+    & is_less_than_or_equal_to(_DEFAULT_MAX_MEMORY_CPU_RATIO)
+    & is_great_than(0),
+)
+
 ################
 # DPE Settings #
 ################
@@ -505,9 +560,15 @@ default_options.register_option(
 assume_finite = os.environ.get("SKLEARN_ASSUME_FINITE")
 if assume_finite is not None:
     assume_finite = bool(assume_finite)
+working_memory = os.environ.get("SKLEARN_WORKING_MEMORY")
+if working_memory is not None:
+    working_memory = int(working_memory)
 default_options.register_option(
     "learn.assume_finite", assume_finite, validator=is_null | is_bool
 )
+default_options.register_option(
+    "learn.working_memory", working_memory, validator=is_null | is_integer
+)
 
 _options_ctx_var = contextvars.ContextVar("_options_ctx_var")
 

maxframe/config/tests/test_validators.py

@@ -14,7 +14,11 @@
 
 import pytest
 
-from ..validators import is_positive_integer, simple_yaml_str_validator
+from ..validators import (
+    is_less_than_or_equal_to,
+    is_positive_integer,
+    simple_yaml_str_validator,
+)
 
 
 @pytest.mark.parametrize("value", ["a", "http://127.0.0.1:1234", "a-b#", "ab_", "123"])
@@ -32,3 +36,11 @@ def test_simple_yaml_str_validator_invalid(value):
 )
 def test_is_positive_integer_validator(value, valid):
     assert is_positive_integer(value) is valid
+
+
+@pytest.mark.parametrize(
+    "value,upper_bound,valid",
+    [(3, 5, True), (5, 5, True), (6, 5, False), (None, None, False), (None, 5, False)],
+)
+def test_is_less_than_or_equal_to_validator(value, upper_bound, valid):
+    assert is_less_than_or_equal_to(upper_bound)(value) is valid

maxframe/config/validators.py

@@ -12,9 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 from typing import Callable
 from urllib.parse import urlparse
 
+from .. import env
+from ..utils import str_to_bool
+
 ValidatorType = Callable[..., bool]
 
 
@@ -43,13 +47,22 @@ class Validator:
     def __or__(self, other):
         return OrValidator(self, other)
 
+    def __and__(self, other):
+        return AndValidator(self, other)
+
 
 class OrValidator(Validator):
     def __init__(self, lhs: Validator, rhs: Validator):
         super().__init__(lambda x: lhs(x) or rhs(x))
 
 
+class AndValidator(Validator):
+    def __init__(self, lhs: Validator, rhs: Validator):
+        super().__init__(lambda x: lhs(x) and rhs(x))
+
+
 is_null = Validator(lambda x: x is None)
+is_notnull = Validator(lambda x: x is not None)
 is_bool = Validator(lambda x: isinstance(x, bool))
 is_float = Validator(lambda x: isinstance(x, float))
 is_integer = Validator(lambda x: isinstance(x, int))
@@ -69,6 +82,30 @@ def is_all_dict_keys_in(*keys):
     return Validator(lambda x: x in keys_set)
 
 
+def is_less_than(upper_bound):
+    return Validator(
+        lambda x: is_notnull(x) and is_notnull(upper_bound) and x < upper_bound
+    )
+
+
+def is_less_than_or_equal_to(upper_bound):
+    return Validator(
+        lambda x: is_notnull(x) and is_notnull(upper_bound) and x <= upper_bound
+    )
+
+
+def is_great_than(lower_bound):
+    return Validator(
+        lambda x: is_notnull(x) and is_notnull(lower_bound) and x > lower_bound
+    )
+
+
+def is_great_than_or_equal_to(lower_bound):
+    return Validator(
+        lambda x: is_notnull(x) and is_notnull(lower_bound) and x >= lower_bound
+    )
+
+
 def _is_valid_cache_path(path: str) -> bool:
     """
     path should look like oss://oss_endpoint/oss_bucket/path
@@ -91,3 +128,15 @@ _invalid_char_in_yaml_str = {'"', "'", "\n", "\\"}
 def simple_yaml_str_validator(name: str) -> bool:
     chars = set(name)
     return len(_invalid_char_in_yaml_str & chars) == 0
+
+
+def dtype_backend_validator(name: str) -> bool:
+    from ..utils import pd_release_version
+
+    check_pd_version = not str_to_bool(os.getenv(env.MAXFRAME_INSIDE_TASK))
+    name = "pyarrow" if name == "arrow" else name
+    if name not in (None, "numpy", "pyarrow"):
+        return False
+    if check_pd_version and name == "pyarrow" and pd_release_version[:2] < (1, 5):
+        raise ValueError("Need pandas>=1.5 to use pyarrow backend")
+    return True

maxframe/conftest.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import contextlib
 import faulthandler
 import os
 from configparser import ConfigParser, NoOptionError, NoSectionError
@@ -87,17 +88,19 @@ def _get_bearer_token_env(test_config: ConfigParser, section_name: str) -> ODPS:
     )
 
 
-@pytest.fixture(scope="session")
-def odps_with_schema(test_config):
-    try:
-        return _get_bearer_token_env(test_config, "odps_with_schema")
-    except NoSectionError:
-        pytest.skip("Need to specify odps_with_schema section in test.conf")
-
-
-@pytest.fixture(scope="session", autouse=True)
-def odps_envs(test_config):
-    entry = _get_bearer_token_env(test_config, "odps")
+@contextlib.contextmanager
+def _enter_odps_envs(entry, drop_temp_tables=True):
+    stored_envs = {}
+    for env_name in (
+        "ODPS_BEARER_TOKEN",
+        "ODPS_PROJECT_NAME",
+        "ODPS_ENDPOINT",
+        "RAY_ISOLATION_UT_ENV",
+        "ODPS_TUNNEL_ENDPOINT",
+    ):
+        if env_name in os.environ:
+            stored_envs[env_name] = os.environ[env_name]
+            del os.environ[env_name]
 
     os.environ["ODPS_BEARER_TOKEN"] = entry.account.token
     os.environ["ODPS_PROJECT_NAME"] = entry.project
@@ -115,13 +118,37 @@ def odps_envs(test_config):
         os.environ.pop("ODPS_TUNNEL_ENDPOINT", None)
         os.environ.pop("RAY_ISOLATION_UT_ENV", None)
 
-        from .tests.utils import _test_tables_to_drop
+        for env_name, val in stored_envs.items():
+            os.environ[env_name] = val
+
+        if drop_temp_tables:
+            from .tests.utils import _test_tables_to_drop
+
+            for table_name in _test_tables_to_drop:
+                try:
+                    entry.delete_table(table_name, wait=False)
+                except:
+                    pass
+
 
-        for table_name in _test_tables_to_drop:
-            try:
-                entry.delete_table(table_name, wait=False)
-            except:
-                pass
+@pytest.fixture
+def odps_with_schema(test_config, request):
+    try:
+        entry = _get_bearer_token_env(test_config, "odps_with_schema")
+    except NoSectionError:
+        pytest.skip("Need to specify odps_with_schema section in test.conf")
+        raise
+
+    with _enter_odps_envs(entry, drop_temp_tables=False):
+        yield entry
+
+
+@pytest.fixture(scope="session", autouse=True)
+def odps_envs(test_config):
+    entry = _get_bearer_token_env(test_config, "odps")
+
+    with _enter_odps_envs(entry):
+        yield
 
 
 @pytest.fixture(scope="session")

maxframe/core/accessor.py

@@ -20,13 +20,13 @@ class BaseMaxFrameAccessor:
         self.obj = obj
 
     @classmethod
-    def _register(cls, name, func):
+    def _register(cls, name, func, is_property=False):
         @functools.wraps(func)
         def wrapped(self, *args, **kw):
             return func(self.obj, *args, **kw)
 
         wrapped.__name__ = name
-        setattr(cls, name, wrapped)
+        setattr(cls, name, wrapped if not is_property else property(wrapped))
         if hasattr(cls, "_api_count"):  # pragma: no branch
             cls._api_count += 1
 

maxframe/core/entity/core.py

@@ -53,6 +53,11 @@ class EntityData(Base):
     def is_sparse(self):
         return self.op.is_sparse()
 
+    def is_fetch(self):
+        from ..operator import Fetch
+
+        return isinstance(self.op, Fetch)
+
     issparse = is_sparse
 
     @property

maxframe/core/entity/tileables.py

@@ -312,7 +312,7 @@ class HasShapeTileableData(TileableData):
 
     @property
     def ndim(self):
-        return len(self.shape)
+        return len(self.shape) if self.shape is not None else np.nan
 
     def __len__(self):
         try:

maxframe/core/graph/entity.py

@@ -26,7 +26,6 @@ from ...serialization.serializables import (
 )
 from ...serialization.serializables.core import SerializableSerializer
 from ...utils import extract_class_name, tokenize
-from ..operator import Fetch
 from .core import DAG
 
 
@@ -123,7 +122,7 @@ class SerializableGraph(Serializable):
     def from_graph(cls, graph: EntityGraph) -> "SerializableGraph":
         return SerializableGraph(
             _is_chunk=False,
-            _fetch_nodes=[chunk for chunk in graph if isinstance(chunk.op, Fetch)],
+            _fetch_nodes=[chunk for chunk in graph if chunk.is_fetch()],
             _nodes=graph._nodes,
             _predecessors=graph._predecessors,
             _successors=graph._successors,

maxframe/core/operator/base.py

@@ -40,6 +40,7 @@ from ..entity.core import ENTITY_TYPE, Entity, EntityData
 from ..entity.output_types import OutputType
 from ..entity.tileables import Tileable
 from ..mode import enter_mode
+from .utils import rewrite_stop_iteration
 
 
 class OperatorMetaclass(SerializableMeta):
@@ -91,6 +92,10 @@ class SchedulingHint(Serializable):
     # id of gang scheduling for machine learning trainings
     gang_scheduling_id = StringField("gang_scheduling_id", default=None)
 
+    def __init__(self, *, pure_depends=None, **kwargs):
+        kwargs["_pure_depends"] = pure_depends or kwargs.get("_pure_depends") or []
+        super().__init__(**kwargs)
+
     @classproperty
     @lru_cache(1)
     def all_hint_names(cls):
@@ -306,7 +311,8 @@ class Operator(Base, OperatorLogicKeyGeneratorMixin, metaclass=OperatorMetaclass
             The replaced input object.
         """
         self.inputs[index] = replaced_input
-        self._set_inputs(self, self.inputs)
+        with rewrite_stop_iteration():
+            self._set_inputs(self, self.inputs)
 
     @property
     def inputs(self) -> List[Union[ENTITY_TYPE]]:
@@ -317,7 +323,8 @@ class Operator(Base, OperatorLogicKeyGeneratorMixin, metaclass=OperatorMetaclass
 
     @inputs.setter
     def inputs(self, vals):
-        self._set_inputs(self, vals)
+        with rewrite_stop_iteration():
+            self._set_inputs(self, vals)
 
     @property
     def output_limit(self) -> int:

maxframe/core/operator/core.py

@@ -18,6 +18,9 @@ from collections import defaultdict
 from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Type, Union
 
 import numpy as np
+import pandas as pd
+
+from .utils import rewrite_stop_iteration
 
 if TYPE_CHECKING:
     from .. import TileableGraph
@@ -82,7 +85,7 @@ class TileableOperatorMixin:
             nsplits = tuple(nsplits)
             shape = list(shape)
             for idx, (s, sp) in enumerate(zip(shape, nsplits)):
-                if not np.isnan(s):
+                if not pd.isna(s):
                     continue
                 s = sum(sp)
                 if not np.isnan(s):
@@ -118,11 +121,16 @@ class TileableOperatorMixin:
     def _new_tileables(
         self, inputs: List[TileableType], kws: List[dict] = None, **kw
     ) -> List[TileableType]:
+        assert (
+            isinstance(inputs, (list, tuple)) or inputs is None
+        ), f"{inputs} is not a list"
+
         output_limit = kw.pop("output_limit", None)
         if output_limit is None:
             output_limit = getattr(self, "output_limit")
 
-        self._set_inputs(self, inputs)
+        with rewrite_stop_iteration():
+            self._set_inputs(self, inputs)
         if self.gpu is None:
             self.gpu = self._check_if_gpu(self._inputs)
         if getattr(self, "_key", None) is None:

maxframe/core/operator/utils.py

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import contextlib
+import sys
+
 from ...typing_ import EntityType, TileableType
 from ..entity import TILEABLE_TYPE
 
@@ -53,3 +56,13 @@ def build_fetch(entity: EntityType, **kw) -> EntityType:
 
 def add_fetch_builder(entity_type, builder_func):
     _type_to_builder.append((entity_type, builder_func))
+
+
+@contextlib.contextmanager
+def rewrite_stop_iteration():
+    try:
+        yield
+    except StopIteration:
+        raise RuntimeError("Unexpected StopIteration happened.").with_traceback(
+            sys.exc_info()[2]
+        ) from None

maxframe/dataframe/__init__.py

@@ -21,6 +21,7 @@ from . import (
     fetch,
     groupby,
     indexing,
+    merge,
     misc,
     missing,
     reduction,
@@ -45,13 +46,13 @@ from .merge import concat, merge
 from .misc.cut import cut
 from .misc.eval import maxframe_eval as eval  # pylint: disable=redefined-builtin
 from .misc.get_dummies import get_dummies
-from .misc.melt import melt
-from .misc.pivot import pivot
-from .misc.pivot_table import pivot_table
 from .misc.qcut import qcut
 from .misc.to_numeric import to_numeric
 from .missing import isna, isnull, notna, notnull
 from .reduction import CustomReduction, unique
+from .reshape.melt import melt
+from .reshape.pivot import pivot
+from .reshape.pivot_table import pivot_table
 from .tseries.to_datetime import to_datetime
 
 try:
@@ -80,3 +81,9 @@ del (
 )
 
 from_pandas = read_pandas
+
+# isort: off
+from .typing_ import register_pandas_typing_funcs
+
+register_pandas_typing_funcs()
+del register_pandas_typing_funcs

maxframe/dataframe/accessors/__init__.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from . import datetime_, dict_, list_, plotting, string_
+from . import datetime_, dict_, list_, plotting, string_, struct_

maxframe/dataframe/accessors/compat.py

@@ -0,0 +1,45 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+
+logger = logging.getLogger(__name__)
+
+
+class LegacySeriesMethodOperator(DataFrameOperator, DataFrameOperatorMixin):
+    _method_name = None
+    _method_cls = None
+
+    def __on_deserialize__(self):
+        cls = type(self)
+        local_fields = {
+            f
+            for f, name_hash in cls._FIELD_TO_NAME_HASH.items()
+            if name_hash == cls._NAME_HASH
+        }
+        kw = {
+            f: getattr(self, f)
+            for f in cls._FIELD_TO_NAME_HASH
+            if f not in local_fields and hasattr(self, f)
+        }
+
+        kw["method"] = self._method_name
+        kw["method_kwargs"] = {
+            f: getattr(self, f) for f in local_fields if hasattr(self, f)
+        }
+
+        logger.warning(f"Using deprecated operator class {cls.__name__}")
+        return self._method_cls(**kw)

maxframe/dataframe/accessors/datetime_/__init__.py

@@ -12,7 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .core import SeriesDatetimeMethod
+from .core import SeriesDatetimeMethod, datetime_method_to_handlers
+
+methods = set(datetime_method_to_handlers.keys())
+del datetime_method_to_handlers
 
 
 def _install():

maxframe/dataframe/accessors/dict_/contains.py

@@ -16,27 +16,16 @@ import pandas as pd
 import pyarrow as pa
 
 from .... import opcodes
-from ....core.entity.output_types import OutputType
 from ....serialization.serializables.field import AnyField
-from ...operators import DataFrameOperator, DataFrameOperatorMixin
+from .core import LegacySeriesDictOperator, SeriesDictMethod
 
 
-class SeriesDictContainsOperator(DataFrameOperator, DataFrameOperatorMixin):
+class SeriesDictContainsOperator(LegacySeriesDictOperator):
+    # operator class deprecated since v2.3.0
     _op_type_ = opcodes.SERIES_DICT_CONTAINS
+    _method_name = "contains"
     query_key = AnyField("query_key", default=None)
 
-    def __init__(self, **kw):
-        super().__init__(_output_types=[OutputType.series], **kw)
-
-    def __call__(self, series):
-        return self.new_series(
-            [series],
-            shape=series.shape,
-            dtype=pd.ArrowDtype(pa.bool_()),
-            index_value=series.index_value,
-            name=None,
-        )
-
 
 def series_dict_contains(series, query_key):
     """
@@ -78,4 +67,6 @@ def series_dict_contains(series, query_key):
     3     <NA>
     dtype: bool[pyarrow]
     """
-    return SeriesDictContainsOperator(query_key=query_key)(series)
+    return SeriesDictMethod(method="contains", method_kwargs=dict(query_key=query_key))(
+        series, dtype=pd.ArrowDtype(pa.bool_()), name=None
+    )